From c3d82bea1d4f5b61ba4cf25b63fd58aea22612e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andy=28=EA=B9=80=EA=B7=9C=ED=9A=8C=29?= <48755156+KimKyuHoi@users.noreply.github.com> Date: Thu, 9 Apr 2026 18:37:13 +0900 Subject: [PATCH] feat(ABR): Monitor dropped frames to influence decisions (#9918) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close https://github.com/shaka-project/shaka-player/issues/745 The ratio is checked periodically, and when the threshold is exceeded, the current stream is 'disableStream' if possible, and a new decission is taken --------- Co-authored-by: Claude Code Co-authored-by: Álvaro Velad Galván Co-authored-by: Matthias Van Parijs --- demo/config.js | 12 +- externs/shaka/abr_manager.js | 21 ++- externs/shaka/player.js | 31 ++++- lib/abr/simple_abr_manager.js | 191 ++++++++++++++++++++++++++-- lib/player.js | 6 + lib/util/player_configuration.js | 4 + test/abr/simple_abr_manager_unit.js | 135 +++++++++++++++++++- 7 files changed, 388 insertions(+), 12 deletions(-) diff --git a/demo/config.js b/demo/config.js index 1a32c40b8..5e596f62a 100644 --- a/demo/config.js +++ b/demo/config.js @@ -375,7 +375,17 @@ shakaDemo.Config = class { /* canBeDecimal= */ true, /* canBeZero= */ true) .addBoolInput_('Prefer Network Information bandwidth', - 'abr.preferNetworkInformationBandwidth'); + 'abr.preferNetworkInformationBandwidth') + .addBoolInput_('Dropped Frames Protection Enabled', + 'abr.droppedFrames') + .addNumberInput_('Dropped Frames Threshold', + 'abr.advanced.droppedFramesThreshold', + /* canBeDecimal= */ true) + .addNumberInput_('Dropped Frames Interval', + 'abr.advanced.droppedFramesInterval', + /* canBeDecimal= */ true) + .addNumberInput_('Dropped Frames Ban Duration', + 'abr.advanced.droppedFramesBanDuration'); this.addRestrictionsSection_('abr', 'Adaptation Restrictions'); } diff --git a/externs/shaka/abr_manager.js b/externs/shaka/abr_manager.js index 46383646d..acd9ea6b3 100644 --- a/externs/shaka/abr_manager.js +++ b/externs/shaka/abr_manager.js @@ -28,13 +28,16 @@ shaka.extern.AbrManager = class { constructor() {} + /* eslint-disable @stylistic/max-len */ /** * Initializes the AbrManager. * * @param {shaka.extern.AbrManager.SwitchCallback} switchCallback + * @param {shaka.extern.AbrManager.DisableStreamCallback} disableStreamCallback * @exportDoc */ - init(switchCallback) {} + /* eslint-enable @stylistic/max-len */ + init(switchCallback, disableStreamCallback) {} /** * Stops any background timers and frees any objects held by this instance. @@ -181,6 +184,22 @@ shaka.extern.AbrManager = class { shaka.extern.AbrManager.SwitchCallback; +/** + * A callback into the Player that should be called when the AbrManager decides + * that the currently playing stream should be temporarily restricted. + * + * The first argument specifies the type of stream ('audio' or 'video'), + * and the second argument specifies the duration of the restriction in seconds. + * + * The exact behavior of the restriction (e.g. temporarily disabling the stream + * or otherwise penalizing it) is implementation-defined by the caller. + * + * @typedef {function(string, number)} + * @exportDoc + */ +shaka.extern.AbrManager.DisableStreamCallback; + + /** * A factory for creating the abr manager. * diff --git a/externs/shaka/player.js b/externs/shaka/player.js index 677fc9f0b..39f91446e 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -2528,6 +2528,7 @@ shaka.extern.AdsConfiguration; * cacheLoadThreshold: number, * minTimeToSwitch: number, * preferNetworkInformationBandwidth: boolean, + * droppedFrames: boolean, * }} * * @property {boolean} enabled @@ -2620,6 +2621,13 @@ shaka.extern.AdsConfiguration; * trust the information provided by the browser. *
* Defaults to false. + * @property {shaka.extern.DroppedFrameProtectionConfig} droppedFrameProtection + * Configuration for monitoring dropped frames and temporarily disabling + * streams that exceed a threshold. + * @property {boolean} droppedFrames + * Enable or disable dropped frames protection. + *
+ * Defaults to true. * @exportDoc */ shaka.extern.AbrConfiguration; @@ -2630,7 +2638,10 @@ shaka.extern.AbrConfiguration; * minTotalBytes: number, * minBytes: number, * fastHalfLife: number, - * slowHalfLife: number + * slowHalfLife: number, + * droppedFramesThreshold: number, + * droppedFramesInterval: number, + * droppedFramesBanDuration: number, * }} * * @property {number} minTotalBytes @@ -2658,6 +2669,24 @@ shaka.extern.AbrConfiguration; * new estimate. *
* Defaults to 5. + * @property {number} droppedFramesThreshold + * The dropThreshold represents the fraction of dropped frames relative to + * the total frames rendered during each check interval. For example, a value + * of 0.15 means that if 15% or more of the frames are dropped in that + * interval, the stream will be considered problematic and may be temporarily + * disabled. + *
+ * Defaults to 0.15. + * @property {number} droppedFramesInterval + * Interval in seconds to measure dropped frames and compare with the + * previous measurement. + *
+ * Defaults to 2. + * @property {number} droppedFramesBanDuration + * Duration in seconds to disable the stream after it exceeds the + * dropped frames threshold. + *
+ * Defaults to 30. * @exportDoc */ shaka.extern.AdvancedAbrConfiguration; diff --git a/lib/abr/simple_abr_manager.js b/lib/abr/simple_abr_manager.js index ec2bbe135..b4443a971 100644 --- a/lib/abr/simple_abr_manager.js +++ b/lib/abr/simple_abr_manager.js @@ -45,6 +45,9 @@ shaka.abr.SimpleAbrManager = class { /** @private {?shaka.extern.AbrManager.SwitchCallback} */ this.switch_ = null; + /** @private {?shaka.extern.AbrManager.DisableStreamCallback} */ + this.disableStreamCallback_ = null; + /** @private {boolean} */ this.enabled_ = false; @@ -138,6 +141,24 @@ shaka.abr.SimpleAbrManager = class { /** @private {?shaka.util.CmsdManager} */ this.cmsdManager_ = null; + + /** @private {shaka.util.Timer} */ + this.droppedFramePoller_ = null; + + /** @private {number} */ + this.lastDroppedFrames_ = 0; + + /** @private {number} */ + this.lastTotalFrames_ = 0; + + /** @private {number} */ + this.videoFrameCallbackId_ = 0; + + /** @private {number} */ + this.lastVideoWidth_ = 0; + + /** @private {number} */ + this.lastVideoHeight_ = 0; } @@ -153,19 +174,17 @@ shaka.abr.SimpleAbrManager = class { this.lastTimeChosenMs_ = null; this.mediaElement_ = null; - if (this.resizeObserver_) { - this.resizeObserver_.disconnect(); - this.resizeObserver_ = null; - } + this.resizeObserver_?.disconnect(); + this.resizeObserver_ = null; - if (this.resizeObserverTimer_) { - this.resizeObserverTimer_.stop(); - } + this.resizeObserverTimer_?.stop(); this.pictureInPictureWindow_ = null; this.cmsdManager_ = null; + this.stopDroppedFramePoller_(); + // Don't reset |startupComplete_|: if we've left the startup interval, we // can start using bandwidth estimates right away after init() is called. } @@ -185,8 +204,9 @@ shaka.abr.SimpleAbrManager = class { * @override * @export */ - init(switchCallback) { + init(switchCallback, disableStreamCallback) { this.switch_ = switchCallback; + this.disableStreamCallback_ = disableStreamCallback; } @@ -314,6 +334,7 @@ shaka.abr.SimpleAbrManager = class { */ enable() { this.enabled_ = true; + this.updateFramesInfo_(); if (this.variants_.length) { this.trySuggestStreams(); } @@ -452,6 +473,7 @@ shaka.abr.SimpleAbrManager = class { } this.pictureInPictureWindow_ = null; }); + this.startDroppedFramePoller_(); } @@ -473,6 +495,7 @@ shaka.abr.SimpleAbrManager = class { if (this.bandwidthEstimator_ && this.config_) { this.bandwidthEstimator_.configure(this.config_.advanced); } + this.startDroppedFramePoller_(); } @@ -520,6 +543,7 @@ shaka.abr.SimpleAbrManager = class { // them out before passing the choices on to StreamingEngine. this.switch_(chosenVariant, this.config_.clearBufferSwitch, this.config_.safeMarginSwitch); + this.updateFramesInfo_(); } } @@ -624,6 +648,157 @@ shaka.abr.SimpleAbrManager = class { return chosenVariant.video.width < newVariant.video.width || chosenVariant.video.height < newVariant.video.height; } + + /** + * Starts the dropped frame poller if the feature is enabled. + * + * @private + */ + startDroppedFramePoller_() { + this.stopDroppedFramePoller_(); + + const element = /** @type {!HTMLVideoElement} */ (this.mediaElement_); + if (!this.config_.droppedFrames || + !element || !element.getVideoPlaybackQuality) { + return; + } + + this.updateFramesInfo_(); + this.startVideoFrameCallback_(); + + this.droppedFramePoller_ = new shaka.util.Timer(() => { + this.checkDroppedFrames_(); + }).tickEvery(this.config_.advanced.droppedFramesInterval); + } + + /** + * Stops the dropped frame poller. + * + * @private + */ + stopDroppedFramePoller_() { + this.stopVideoFrameCallback_(); + this.droppedFramePoller_?.stop(); + this.droppedFramePoller_ = null; + } + + /** + * Checks the dropped frame rate since the last interval. If it exceeds the + * configured threshold, the current video stream is temporarily disabled. + * + * @private + */ + checkDroppedFrames_() { + // Skip checks paused state or ABR disabled or droppedFrames disabled. + if (this.mediaElement_.paused || !this.enabled_ || + !this.config_.droppedFrames) { + return; + } + + const element = /** @type {!HTMLVideoElement} */ (this.mediaElement_); + const info = element.getVideoPlaybackQuality(); + const currentDropped = info.droppedVideoFrames; + const currentTotal = info.totalVideoFrames; + + // Skip ban logic when playback rate is greater than 1x + // because frame drops are expected or when the total frames are 0. + if (this.mediaElement_.playbackRate > 1 || !currentTotal) { + this.lastDroppedFrames_ = currentDropped; + this.lastTotalFrames_ = currentTotal; + return; + } + + const deltaDropped = currentDropped - this.lastDroppedFrames_; + const deltaTotal = currentTotal - this.lastTotalFrames_; + + this.lastDroppedFrames_ = currentDropped; + this.lastTotalFrames_ = currentTotal; + + const dropRatio = deltaDropped / deltaTotal; + const droppedFramesThreshold = this.config_.advanced.droppedFramesThreshold; + if (dropRatio >= droppedFramesThreshold && this.disableStreamCallback_) { + shaka.log.warning( + 'Dropped frame ratio exceeded threshold: ' + dropRatio.toFixed(2) + + ' >= ' + droppedFramesThreshold + + '. Disabling current video stream.'); + + this.disableStreamCallback_( + 'video', this.config_.advanced.droppedFramesBanDuration); + } + } + + /** + * @private + */ + updateFramesInfo_() { + const element = /** @type {!HTMLVideoElement} */ (this.mediaElement_); + if (this.config_.droppedFrames && element?.getVideoPlaybackQuality) { + const info = element.getVideoPlaybackQuality(); + this.lastDroppedFrames_ = info.droppedVideoFrames; + this.lastTotalFrames_ = info.totalVideoFrames; + } + } + + /** + * @private + */ + startVideoFrameCallback_() { + const video = /** @type {!HTMLVideoElement} */ (this.mediaElement_); + if (!video || !('requestVideoFrameCallback' in video)) { + return; + } + + this.lastVideoWidth_ = 0; + this.lastVideoHeight_ = 0; + + const onVideoFrame = (now, metadata) => { + if (!this.mediaElement_ || this.mediaElement_ !== video) { + return; + } + + const width = metadata?.width || video.videoWidth; + const height = metadata?.height || video.videoHeight; + + if (width !== this.lastVideoWidth_ || height !== this.lastVideoHeight_) { + if (this.lastVideoWidth_ !== 0 || this.lastVideoHeight_ !== 0) { + shaka.log.debug('rVFC: video resolution changed from ' + + this.lastVideoWidth_ + 'x' + this.lastVideoHeight_ + + ' → ' + width + 'x' + height + '. Resetting frame counters.'); + this.updateFramesInfo_(); + } else { + shaka.log.v2('rVFC: initial resolution detected: ' + + width + 'x' + height); + } + + this.lastVideoWidth_ = width; + this.lastVideoHeight_ = height; + } + + this.videoFrameCallbackId_ = + video.requestVideoFrameCallback(onVideoFrame); + }; + + this.videoFrameCallbackId_ = + video.requestVideoFrameCallback(onVideoFrame); + + shaka.log.v1('rVFC: frame callback loop started.'); + } + + /** + * @private + */ + stopVideoFrameCallback_() { + if (this.videoFrameCallbackId_ !== 0 && this.mediaElement_ && + 'cancelVideoFrameCallback' in this.mediaElement_) { + const video = /** @type {!HTMLVideoElement} */ (this.mediaElement_); + video.cancelVideoFrameCallback(this.videoFrameCallbackId_); + shaka.log.v1('rVFC: frame callback loop stopped.'); + } + + this.videoFrameCallbackId_ = 0; + this.lastVideoWidth_ = 0; + this.lastVideoHeight_ = 0; + } }; diff --git a/lib/player.js b/lib/player.js index 96fbe4356..a3071f4ae 100644 --- a/lib/player.js +++ b/lib/player.js @@ -2963,6 +2963,12 @@ shaka.Player = class extends shaka.util.FakeEventTarget { this.abrManager_.init((variant, clearBuffer, safeMargin) => { return this.switch_(variant, clearBuffer, safeMargin); + }, (type, banDuration) => { + const currentVariant = this.streamingEngine_?.getCurrentVariant(); + const stream = currentVariant && currentVariant[type]; + if (stream) { + this.disableStream(stream, banDuration); + } }); this.abrManager_.setMediaElement(mediaElement); this.abrManager_.setCmsdManager(this.cmsdManager_); diff --git a/lib/util/player_configuration.js b/lib/util/player_configuration.js index ce77ac3c0..1683a8132 100644 --- a/lib/util/player_configuration.js +++ b/lib/util/player_configuration.js @@ -337,6 +337,9 @@ shaka.util.PlayerConfiguration = class { minBytes, fastHalfLife: 2, slowHalfLife: 5, + droppedFramesThreshold: 0.15, + droppedFramesInterval: 2, + droppedFramesBanDuration: 30, }, restrictToElementSize: false, restrictToScreenSize: false, @@ -346,6 +349,7 @@ shaka.util.PlayerConfiguration = class { cacheLoadThreshold: 5, minTimeToSwitch: 0, preferNetworkInformationBandwidth: false, + droppedFrames: true, }; const cmcd = { diff --git a/test/abr/simple_abr_manager_unit.js b/test/abr/simple_abr_manager_unit.js index 5b501bfb0..c34dce7bc 100644 --- a/test/abr/simple_abr_manager_unit.js +++ b/test/abr/simple_abr_manager_unit.js @@ -13,6 +13,8 @@ describe('SimpleAbrManager', () => { let config; /** @type {!jasmine.Spy} */ let switchCallback; + /** @type {!jasmine.Spy} */ + let restrictVideoCallback; /** @type {!shaka.abr.SimpleAbrManager} */ let abrManager; /** @type {shaka.extern.Manifest} */ @@ -23,6 +25,7 @@ describe('SimpleAbrManager', () => { beforeEach(() => { Date.now = () => 0; switchCallback = jasmine.createSpy('switchCallback'); + restrictVideoCallback = jasmine.createSpy('restrictVideoCallback'); // Keep unsorted. manifest = shaka.test.ManifestGenerator.generate((manifest) => { @@ -68,7 +71,8 @@ describe('SimpleAbrManager', () => { variants = manifest.variants; abrManager = new shaka.abr.SimpleAbrManager(); - abrManager.init(shaka.test.Util.spyFunc(switchCallback)); + abrManager.init(shaka.test.Util.spyFunc(switchCallback), + shaka.test.Util.spyFunc(restrictVideoCallback)); abrManager.configure(config); abrManager.setVariants(variants, false); }); @@ -396,4 +400,133 @@ describe('SimpleAbrManager', () => { chosen = abrManager.chooseVariant(); expect(chosen.id).toBe(10); }); + + describe('dropped frame protection', /** @suppress {accessControls} */ () => { + /** @type {!HTMLVideoElement} */ + let mockVideo; + + beforeEach(() => { + mockVideo = /** @type {!HTMLVideoElement} */ ( + document.createElement('video')); + Object.defineProperty(mockVideo, 'paused', { + get: () => false, + configurable: true, + }); + mockVideo.getVideoPlaybackQuality = () => ({ + droppedVideoFrames: 0, + totalVideoFrames: 0, + corruptedVideoFrames: 0, + creationTime: 0, + totalFrameDelay: 0, + }); + + config.droppedFrames = true; + config.advanced.droppedFramesThreshold = 0.15; + config.advanced.droppedFramesInterval = 2; + config.advanced.droppedFramesBanDuration = 30; + abrManager.configure(config); + abrManager.enable(); + abrManager.setMediaElement(mockVideo); + }); + + /** + * @param {number} dropped + * @param {number} total + * @return {!VideoPlaybackQuality} + */ + function makeQuality(dropped, total) { + return /** @type {!VideoPlaybackQuality} */ ({ + droppedVideoFrames: dropped, + totalVideoFrames: total, + corruptedVideoFrames: 0, + creationTime: 0, + totalFrameDelay: 0, + }); + } + + it('calls disableStreamCallback when drop ratio exceeds threshold', () => { + // Establish baseline counters. + mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100); + abrManager.checkDroppedFrames_(); + + // 20/100 new frames dropped = 20% > 15% threshold. + mockVideo.getVideoPlaybackQuality = () => makeQuality(20, 200); + abrManager.checkDroppedFrames_(); + + expect(restrictVideoCallback).toHaveBeenCalledWith('video', 30); + }); + + it('does not call disableStreamCallback when drop ratio is below threshold', + () => { + // Establish baseline counters. + mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100); + abrManager.checkDroppedFrames_(); + + // 10/100 new frames dropped = 10% < 15% threshold. + mockVideo.getVideoPlaybackQuality = () => makeQuality(10, 200); + abrManager.checkDroppedFrames_(); + + expect(restrictVideoCallback).not.toHaveBeenCalled(); + }); + + it('skips ban logic when playbackRate > 1', () => { + mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100); + abrManager.checkDroppedFrames_(); + + // High drop ratio at 2x speed — ban logic should be skipped. + mockVideo.playbackRate = 2; + mockVideo.getVideoPlaybackQuality = () => makeQuality(50, 200); + abrManager.checkDroppedFrames_(); + + expect(restrictVideoCallback).not.toHaveBeenCalled(); + }); + + it('resets counters when playbackRate > 1 and resumes to 1x', () => { + mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100); + abrManager.checkDroppedFrames_(); + + // 2x speed: counters are reset to current values (50, 200). + mockVideo.playbackRate = 2; + mockVideo.getVideoPlaybackQuality = () => makeQuality(50, 200); + abrManager.checkDroppedFrames_(); + + // Back to 1x: 5/100 new drops = 5% < 15% threshold. + mockVideo.playbackRate = 1; + mockVideo.getVideoPlaybackQuality = () => makeQuality(55, 300); + abrManager.checkDroppedFrames_(); + + expect(restrictVideoCallback).not.toHaveBeenCalled(); + }); + + it('skips check when paused', () => { + Object.defineProperty(mockVideo, 'paused', { + get: () => true, + configurable: true, + }); + + mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100); + abrManager.checkDroppedFrames_(); + + mockVideo.getVideoPlaybackQuality = () => makeQuality(50, 200); + abrManager.checkDroppedFrames_(); + + expect(restrictVideoCallback).not.toHaveBeenCalled(); + }); + + it('does not start poller when droppedFrames config is disabled', () => { + config.droppedFrames = false; + abrManager.configure(config); + abrManager.setMediaElement(mockVideo); + + // startDroppedFramePoller_() returns early, so no timer is created. + expect(abrManager.droppedFramePoller_).toBeNull(); + }); + + it('skips check when totalVideoFrames is 0', () => { + mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 0); + abrManager.checkDroppedFrames_(); + + expect(restrictVideoCallback).not.toHaveBeenCalled(); + }); + }); });