feat(ABR): Monitor dropped frames to influence decisions (#9918)

Close https://github.com/shaka-project/shaka-player/issues/745

The ratio is checked periodically, and when the threshold is exceeded,
the current stream is 'disableStream' if possible, and a new decission
is taken

---------

Co-authored-by: Claude Code <noreply@anthropic.com>
Co-authored-by: Álvaro Velad Galván <ladvan91@hotmail.com>
Co-authored-by: Matthias Van Parijs <matvp91@gmail.com>

demo/config.js

@@ -375,7 +375,17 @@ shakaDemo.Config = class {
             /* canBeDecimal= */ true,
             /* canBeZero= */ true)
         .addBoolInput_('Prefer Network Information bandwidth',
-            'abr.preferNetworkInformationBandwidth');
+            'abr.preferNetworkInformationBandwidth')
+        .addBoolInput_('Dropped Frames Protection Enabled',
+            'abr.droppedFrames')
+        .addNumberInput_('Dropped Frames Threshold',
+            'abr.advanced.droppedFramesThreshold',
+            /* canBeDecimal= */ true)
+        .addNumberInput_('Dropped Frames Interval',
+            'abr.advanced.droppedFramesInterval',
+            /* canBeDecimal= */ true)
+        .addNumberInput_('Dropped Frames Ban Duration',
+            'abr.advanced.droppedFramesBanDuration');
     this.addRestrictionsSection_('abr', 'Adaptation Restrictions');
   }
 

externs/shaka/abr_manager.js

@@ -28,13 +28,16 @@
 shaka.extern.AbrManager = class {
   constructor() {}
 
+  /* eslint-disable @stylistic/max-len */
   /**
    * Initializes the AbrManager.
    *
    * @param {shaka.extern.AbrManager.SwitchCallback} switchCallback
+   * @param {shaka.extern.AbrManager.DisableStreamCallback} disableStreamCallback
    * @exportDoc
    */
-  init(switchCallback) {}
+  /* eslint-enable @stylistic/max-len */
+  init(switchCallback, disableStreamCallback) {}
 
   /**
    * Stops any background timers and frees any objects held by this instance.
@@ -181,6 +184,22 @@ shaka.extern.AbrManager = class {
 shaka.extern.AbrManager.SwitchCallback;
 
 
+/**
+ * A callback into the Player that should be called when the AbrManager decides
+ * that the currently playing stream should be temporarily restricted.
+ *
+ * The first argument specifies the type of stream ('audio' or 'video'),
+ * and the second argument specifies the duration of the restriction in seconds.
+ *
+ * The exact behavior of the restriction (e.g. temporarily disabling the stream
+ * or otherwise penalizing it) is implementation-defined by the caller.
+ *
+ * @typedef {function(string, number)}
+ * @exportDoc
+ */
+shaka.extern.AbrManager.DisableStreamCallback;
+
+
 /**
  * A factory for creating the abr manager.
  *

externs/shaka/player.js

@@ -2528,6 +2528,7 @@ shaka.extern.AdsConfiguration;
  *   cacheLoadThreshold: number,
  *   minTimeToSwitch: number,
  *   preferNetworkInformationBandwidth: boolean,
+ *   droppedFrames: boolean,
  * }}
  *
  * @property {boolean} enabled
@@ -2620,6 +2621,13 @@ shaka.extern.AdsConfiguration;
  *   trust the information provided by the browser.
  *   <br>
  *   Defaults to <code>false</code>.
+ * @property {shaka.extern.DroppedFrameProtectionConfig} droppedFrameProtection
+ *   Configuration for monitoring dropped frames and temporarily disabling
+ *   streams that exceed a threshold.
+ * @property {boolean} droppedFrames
+ *   Enable or disable dropped frames protection.
+ *   <br>
+ *   Defaults to <code>true</code>.
  * @exportDoc
  */
 shaka.extern.AbrConfiguration;
@@ -2630,7 +2638,10 @@ shaka.extern.AbrConfiguration;
  *   minTotalBytes: number,
  *   minBytes: number,
  *   fastHalfLife: number,
- *   slowHalfLife: number
+ *   slowHalfLife: number,
+ *   droppedFramesThreshold: number,
+ *   droppedFramesInterval: number,
+ *   droppedFramesBanDuration: number,
  * }}
  *
  * @property {number} minTotalBytes
@@ -2658,6 +2669,24 @@ shaka.extern.AbrConfiguration;
  *   new estimate.
  *   <br>
  *   Defaults to <code>5</code>.
+ * @property {number} droppedFramesThreshold
+ *   The dropThreshold represents the fraction of dropped frames relative to
+ *   the total frames rendered during each check interval. For example, a value
+ *   of 0.15 means that if 15% or more of the frames are dropped in that
+ *   interval, the stream will be considered problematic and may be temporarily
+ *   disabled.
+ *   <br>
+ *   Defaults to <code>0.15</code>.
+ * @property {number} droppedFramesInterval
+ *   Interval in seconds to measure dropped frames and compare with the
+ *   previous measurement.
+ *   <br>
+ *   Defaults to <code>2</code>.
+ * @property {number} droppedFramesBanDuration
+ *   Duration in seconds to disable the stream after it exceeds the
+ *   dropped frames threshold.
+ *   <br>
+ *   Defaults to <code>30</code>.
  * @exportDoc
  */
 shaka.extern.AdvancedAbrConfiguration;

lib/abr/simple_abr_manager.js

@@ -45,6 +45,9 @@ shaka.abr.SimpleAbrManager = class {
     /** @private {?shaka.extern.AbrManager.SwitchCallback} */
     this.switch_ = null;
 
+    /** @private {?shaka.extern.AbrManager.DisableStreamCallback} */
+    this.disableStreamCallback_ = null;
+
     /** @private {boolean} */
     this.enabled_ = false;
 
@@ -138,6 +141,24 @@ shaka.abr.SimpleAbrManager = class {
 
     /** @private {?shaka.util.CmsdManager} */
     this.cmsdManager_ = null;
+
+    /** @private {shaka.util.Timer} */
+    this.droppedFramePoller_ = null;
+
+    /** @private {number} */
+    this.lastDroppedFrames_ = 0;
+
+    /** @private {number} */
+    this.lastTotalFrames_ = 0;
+
+    /** @private {number} */
+    this.videoFrameCallbackId_ = 0;
+
+    /** @private {number} */
+    this.lastVideoWidth_ = 0;
+
+    /** @private {number} */
+    this.lastVideoHeight_ = 0;
   }
 
 
@@ -153,19 +174,17 @@ shaka.abr.SimpleAbrManager = class {
     this.lastTimeChosenMs_ = null;
     this.mediaElement_ = null;
 
-    if (this.resizeObserver_) {
-      this.resizeObserver_.disconnect();
-      this.resizeObserver_ = null;
-    }
+    this.resizeObserver_?.disconnect();
+    this.resizeObserver_ = null;
 
-    if (this.resizeObserverTimer_) {
-      this.resizeObserverTimer_.stop();
-    }
+    this.resizeObserverTimer_?.stop();
 
     this.pictureInPictureWindow_ = null;
 
     this.cmsdManager_ = null;
 
+    this.stopDroppedFramePoller_();
+
     // Don't reset |startupComplete_|: if we've left the startup interval, we
     // can start using bandwidth estimates right away after init() is called.
   }
@@ -185,8 +204,9 @@ shaka.abr.SimpleAbrManager = class {
    * @override
    * @export
    */
-  init(switchCallback) {
+  init(switchCallback, disableStreamCallback) {
     this.switch_ = switchCallback;
+    this.disableStreamCallback_ = disableStreamCallback;
   }
 
 
@@ -314,6 +334,7 @@ shaka.abr.SimpleAbrManager = class {
    */
   enable() {
     this.enabled_ = true;
+    this.updateFramesInfo_();
     if (this.variants_.length) {
       this.trySuggestStreams();
     }
@@ -452,6 +473,7 @@ shaka.abr.SimpleAbrManager = class {
       }
       this.pictureInPictureWindow_ = null;
     });
+    this.startDroppedFramePoller_();
   }
 
 
@@ -473,6 +495,7 @@ shaka.abr.SimpleAbrManager = class {
     if (this.bandwidthEstimator_ && this.config_) {
       this.bandwidthEstimator_.configure(this.config_.advanced);
     }
+    this.startDroppedFramePoller_();
   }
 
 
@@ -520,6 +543,7 @@ shaka.abr.SimpleAbrManager = class {
       // them out before passing the choices on to StreamingEngine.
       this.switch_(chosenVariant, this.config_.clearBufferSwitch,
           this.config_.safeMarginSwitch);
+      this.updateFramesInfo_();
     }
   }
 
@@ -624,6 +648,157 @@ shaka.abr.SimpleAbrManager = class {
     return chosenVariant.video.width < newVariant.video.width ||
         chosenVariant.video.height < newVariant.video.height;
   }
+
+  /**
+   * Starts the dropped frame poller if the feature is enabled.
+   *
+   * @private
+   */
+  startDroppedFramePoller_() {
+    this.stopDroppedFramePoller_();
+
+    const element = /** @type {!HTMLVideoElement} */ (this.mediaElement_);
+    if (!this.config_.droppedFrames ||
+      !element || !element.getVideoPlaybackQuality) {
+      return;
+    }
+
+    this.updateFramesInfo_();
+    this.startVideoFrameCallback_();
+
+    this.droppedFramePoller_ = new shaka.util.Timer(() => {
+      this.checkDroppedFrames_();
+    }).tickEvery(this.config_.advanced.droppedFramesInterval);
+  }
+
+  /**
+   * Stops the dropped frame poller.
+   *
+   * @private
+   */
+  stopDroppedFramePoller_() {
+    this.stopVideoFrameCallback_();
+    this.droppedFramePoller_?.stop();
+    this.droppedFramePoller_ = null;
+  }
+
+  /**
+   * Checks the dropped frame rate since the last interval. If it exceeds the
+   * configured threshold, the current video stream is temporarily disabled.
+   *
+   * @private
+   */
+  checkDroppedFrames_() {
+    // Skip checks paused state or ABR disabled or droppedFrames disabled.
+    if (this.mediaElement_.paused || !this.enabled_ ||
+        !this.config_.droppedFrames) {
+      return;
+    }
+
+    const element = /** @type {!HTMLVideoElement} */ (this.mediaElement_);
+    const info = element.getVideoPlaybackQuality();
+    const currentDropped = info.droppedVideoFrames;
+    const currentTotal = info.totalVideoFrames;
+
+    // Skip ban logic when playback rate is greater than 1x
+    // because frame drops are expected or when the total frames are 0.
+    if (this.mediaElement_.playbackRate > 1 || !currentTotal) {
+      this.lastDroppedFrames_ = currentDropped;
+      this.lastTotalFrames_ = currentTotal;
+      return;
+    }
+
+    const deltaDropped = currentDropped - this.lastDroppedFrames_;
+    const deltaTotal = currentTotal - this.lastTotalFrames_;
+
+    this.lastDroppedFrames_ = currentDropped;
+    this.lastTotalFrames_ = currentTotal;
+
+    const dropRatio = deltaDropped / deltaTotal;
+    const droppedFramesThreshold = this.config_.advanced.droppedFramesThreshold;
+    if (dropRatio >= droppedFramesThreshold && this.disableStreamCallback_) {
+      shaka.log.warning(
+          'Dropped frame ratio exceeded threshold: ' + dropRatio.toFixed(2) +
+          ' >= ' + droppedFramesThreshold +
+          '. Disabling current video stream.');
+
+      this.disableStreamCallback_(
+          'video', this.config_.advanced.droppedFramesBanDuration);
+    }
+  }
+
+  /**
+   * @private
+   */
+  updateFramesInfo_() {
+    const element = /** @type {!HTMLVideoElement} */ (this.mediaElement_);
+    if (this.config_.droppedFrames && element?.getVideoPlaybackQuality) {
+      const info = element.getVideoPlaybackQuality();
+      this.lastDroppedFrames_ = info.droppedVideoFrames;
+      this.lastTotalFrames_ = info.totalVideoFrames;
+    }
+  }
+
+  /**
+   * @private
+   */
+  startVideoFrameCallback_() {
+    const video = /** @type {!HTMLVideoElement} */ (this.mediaElement_);
+    if (!video || !('requestVideoFrameCallback' in video)) {
+      return;
+    }
+
+    this.lastVideoWidth_ = 0;
+    this.lastVideoHeight_ = 0;
+
+    const onVideoFrame = (now, metadata) => {
+      if (!this.mediaElement_ || this.mediaElement_ !== video) {
+        return;
+      }
+
+      const width = metadata?.width || video.videoWidth;
+      const height = metadata?.height || video.videoHeight;
+
+      if (width !== this.lastVideoWidth_ || height !== this.lastVideoHeight_) {
+        if (this.lastVideoWidth_ !== 0 || this.lastVideoHeight_ !== 0) {
+          shaka.log.debug('rVFC: video resolution changed from ' +
+              this.lastVideoWidth_ + 'x' + this.lastVideoHeight_ +
+              ' → ' + width + 'x' + height + '. Resetting frame counters.');
+          this.updateFramesInfo_();
+        } else {
+          shaka.log.v2('rVFC: initial resolution detected: ' +
+              width + 'x' + height);
+        }
+
+        this.lastVideoWidth_ = width;
+        this.lastVideoHeight_ = height;
+      }
+
+      this.videoFrameCallbackId_ =
+          video.requestVideoFrameCallback(onVideoFrame);
+    };
+
+    this.videoFrameCallbackId_ =
+        video.requestVideoFrameCallback(onVideoFrame);
+
+    shaka.log.v1('rVFC: frame callback loop started.');
+  }
+
+  /**
+   * @private
+   */
+  stopVideoFrameCallback_() {
+    if (this.videoFrameCallbackId_ !== 0 && this.mediaElement_ &&
+        'cancelVideoFrameCallback' in this.mediaElement_) {
+      const video = /** @type {!HTMLVideoElement} */ (this.mediaElement_);
+      video.cancelVideoFrameCallback(this.videoFrameCallbackId_);
+      shaka.log.v1('rVFC: frame callback loop stopped.');
+    }
+
+    this.videoFrameCallbackId_ = 0;
+    this.lastVideoWidth_ = 0;
+    this.lastVideoHeight_ = 0;
+  }
 };
 
 

lib/player.js

@@ -2963,6 +2963,12 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
 
     this.abrManager_.init((variant, clearBuffer, safeMargin) => {
       return this.switch_(variant, clearBuffer, safeMargin);
+    }, (type, banDuration) => {
+      const currentVariant = this.streamingEngine_?.getCurrentVariant();
+      const stream = currentVariant && currentVariant[type];
+      if (stream) {
+        this.disableStream(stream, banDuration);
+      }
     });
     this.abrManager_.setMediaElement(mediaElement);
     this.abrManager_.setCmsdManager(this.cmsdManager_);

lib/util/player_configuration.js

@@ -337,6 +337,9 @@ shaka.util.PlayerConfiguration = class {
         minBytes,
         fastHalfLife: 2,
         slowHalfLife: 5,
+        droppedFramesThreshold: 0.15,
+        droppedFramesInterval: 2,
+        droppedFramesBanDuration: 30,
       },
       restrictToElementSize: false,
       restrictToScreenSize: false,
@@ -346,6 +349,7 @@ shaka.util.PlayerConfiguration = class {
       cacheLoadThreshold: 5,
       minTimeToSwitch: 0,
       preferNetworkInformationBandwidth: false,
+      droppedFrames: true,
     };
 
     const cmcd = {

test/abr/simple_abr_manager_unit.js

@@ -13,6 +13,8 @@ describe('SimpleAbrManager', () => {
   let config;
   /** @type {!jasmine.Spy} */
   let switchCallback;
+  /** @type {!jasmine.Spy} */
+  let restrictVideoCallback;
   /** @type {!shaka.abr.SimpleAbrManager} */
   let abrManager;
   /** @type {shaka.extern.Manifest} */
@@ -23,6 +25,7 @@ describe('SimpleAbrManager', () => {
   beforeEach(() => {
     Date.now = () => 0;
     switchCallback = jasmine.createSpy('switchCallback');
+    restrictVideoCallback = jasmine.createSpy('restrictVideoCallback');
 
     // Keep unsorted.
     manifest = shaka.test.ManifestGenerator.generate((manifest) => {
@@ -68,7 +71,8 @@ describe('SimpleAbrManager', () => {
     variants = manifest.variants;
 
     abrManager = new shaka.abr.SimpleAbrManager();
-    abrManager.init(shaka.test.Util.spyFunc(switchCallback));
+    abrManager.init(shaka.test.Util.spyFunc(switchCallback),
+        shaka.test.Util.spyFunc(restrictVideoCallback));
     abrManager.configure(config);
     abrManager.setVariants(variants, false);
   });
@@ -396,4 +400,133 @@ describe('SimpleAbrManager', () => {
     chosen = abrManager.chooseVariant();
     expect(chosen.id).toBe(10);
   });
+
+  describe('dropped frame protection', /** @suppress {accessControls} */ () => {
+    /** @type {!HTMLVideoElement} */
+    let mockVideo;
+
+    beforeEach(() => {
+      mockVideo = /** @type {!HTMLVideoElement} */ (
+        document.createElement('video'));
+      Object.defineProperty(mockVideo, 'paused', {
+        get: () => false,
+        configurable: true,
+      });
+      mockVideo.getVideoPlaybackQuality = () => ({
+        droppedVideoFrames: 0,
+        totalVideoFrames: 0,
+        corruptedVideoFrames: 0,
+        creationTime: 0,
+        totalFrameDelay: 0,
+      });
+
+      config.droppedFrames = true;
+      config.advanced.droppedFramesThreshold = 0.15;
+      config.advanced.droppedFramesInterval = 2;
+      config.advanced.droppedFramesBanDuration = 30;
+      abrManager.configure(config);
+      abrManager.enable();
+      abrManager.setMediaElement(mockVideo);
+    });
+
+    /**
+     * @param {number} dropped
+     * @param {number} total
+     * @return {!VideoPlaybackQuality}
+     */
+    function makeQuality(dropped, total) {
+      return /** @type {!VideoPlaybackQuality} */ ({
+        droppedVideoFrames: dropped,
+        totalVideoFrames: total,
+        corruptedVideoFrames: 0,
+        creationTime: 0,
+        totalFrameDelay: 0,
+      });
+    }
+
+    it('calls disableStreamCallback when drop ratio exceeds threshold', () => {
+      // Establish baseline counters.
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100);
+      abrManager.checkDroppedFrames_();
+
+      // 20/100 new frames dropped = 20% > 15% threshold.
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(20, 200);
+      abrManager.checkDroppedFrames_();
+
+      expect(restrictVideoCallback).toHaveBeenCalledWith('video', 30);
+    });
+
+    it('does not call disableStreamCallback when drop ratio is below threshold',
+        () => {
+          // Establish baseline counters.
+          mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100);
+          abrManager.checkDroppedFrames_();
+
+          // 10/100 new frames dropped = 10% < 15% threshold.
+          mockVideo.getVideoPlaybackQuality = () => makeQuality(10, 200);
+          abrManager.checkDroppedFrames_();
+
+          expect(restrictVideoCallback).not.toHaveBeenCalled();
+        });
+
+    it('skips ban logic when playbackRate > 1', () => {
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100);
+      abrManager.checkDroppedFrames_();
+
+      // High drop ratio at 2x speed — ban logic should be skipped.
+      mockVideo.playbackRate = 2;
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(50, 200);
+      abrManager.checkDroppedFrames_();
+
+      expect(restrictVideoCallback).not.toHaveBeenCalled();
+    });
+
+    it('resets counters when playbackRate > 1 and resumes to 1x', () => {
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100);
+      abrManager.checkDroppedFrames_();
+
+      // 2x speed: counters are reset to current values (50, 200).
+      mockVideo.playbackRate = 2;
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(50, 200);
+      abrManager.checkDroppedFrames_();
+
+      // Back to 1x: 5/100 new drops = 5% < 15% threshold.
+      mockVideo.playbackRate = 1;
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(55, 300);
+      abrManager.checkDroppedFrames_();
+
+      expect(restrictVideoCallback).not.toHaveBeenCalled();
+    });
+
+    it('skips check when paused', () => {
+      Object.defineProperty(mockVideo, 'paused', {
+        get: () => true,
+        configurable: true,
+      });
+
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 100);
+      abrManager.checkDroppedFrames_();
+
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(50, 200);
+      abrManager.checkDroppedFrames_();
+
+      expect(restrictVideoCallback).not.toHaveBeenCalled();
+    });
+
+    it('does not start poller when droppedFrames config is disabled', () => {
+      config.droppedFrames = false;
+      abrManager.configure(config);
+      abrManager.setMediaElement(mockVideo);
+
+      // startDroppedFramePoller_() returns early, so no timer is created.
+      expect(abrManager.droppedFramePoller_).toBeNull();
+    });
+
+    it('skips check when totalVideoFrames is 0', () => {
+      mockVideo.getVideoPlaybackQuality = () => makeQuality(0, 0);
+      abrManager.checkDroppedFrames_();
+
+      expect(restrictVideoCallback).not.toHaveBeenCalled();
+    });
+  });
 });