Audio adaptation

SimpleAbrManager will still choose the middle audio stream by default,
but will adjust audio up or down once video has been chosen.  Video
will be downgraded before audio, and audio will be downgraded only as
a last resort.

For audio-only content or content with one video stream and many audio
streams, audio will be chosen to fill available bandwidth, just as it
already was for video.

Change-Id: Ia4a0e9607d860713a5ad7887ac6a9aa04767eb25

lib/abr/simple_abr_manager.js

@@ -122,21 +122,21 @@ shaka.abr.SimpleAbrManager.prototype.chooseStreams = function(
     this.streamSetsByType_[type] = streamSetsByType[type];
   }
 
+  var audioStream = this.streamsByType_['audio'];
+  var videoStream = this.streamsByType_['video'];
+
   // Choose streams for the specific types requested.
   var chosen = {};
 
   if ('audio' in streamSetsByType) {
-    var audioStream = this.chooseAudioStream_();
-    if (audioStream) {
-      chosen['audio'] = audioStream;
-      this.streamsByType_['audio'] = audioStream;
-    } else {
-      delete this.streamsByType_['audio'];
-    }
+    // Choose middle audio Stream as a default until we decide on video.
+    audioStream = this.getMiddleAudioStream_();
   }
 
   if ('video' in streamSetsByType) {
-    var videoStream = this.chooseVideoStream_();
+    // Choose the best video Stream assuming the bandwidth requirements of the
+    // audio Stream.
+    videoStream = this.chooseOneStream_('video', audioStream);
     if (videoStream) {
       chosen['video'] = videoStream;
       this.streamsByType_['video'] = videoStream;
@@ -144,6 +144,16 @@ shaka.abr.SimpleAbrManager.prototype.chooseStreams = function(
       delete this.streamsByType_['video'];
     }
   }
+  if ('audio' in streamSetsByType) {
+    // Refine the choice of audio now that video has been chosen.
+    audioStream = this.chooseOneStream_('audio', videoStream);
+    if (audioStream) {
+      chosen['audio'] = audioStream;
+      this.streamsByType_['audio'] = audioStream;
+    } else {
+      delete this.streamsByType_['audio'];
+    }
+  }
 
   if ('text' in streamSetsByType) {
     // We don't adapt text, so just choose stream 0.
@@ -242,15 +252,23 @@ shaka.abr.SimpleAbrManager.prototype.suggestStreams_ = function() {
 shaka.abr.SimpleAbrManager.prototype.chooseStreams_ = function() {
   var streamsByType = {};
 
-  // Choose audio Stream.
-  var audioStream = this.chooseAudioStream_();
+  // Choose middle audio Stream as a default to decide on video.
+  var audioStream = this.getMiddleAudioStream_();
+
+  // Choose the best video Stream assuming the bandwidth requirements of the
+  // middle audio Stream.
+  var videoStream = this.chooseOneStream_('video', audioStream);
+
+  // Refine the choice of audio up or down based on the bandwidth left over by
+  // the video Stream we chose.  If we aren't using all of our bandwidth, we may
+  // move to higher quality audio.  If we are already on the lowest video and
+  // we still need too much bandwidth, we may move to lower quality audio.
+  audioStream = this.chooseOneStream_('audio', videoStream);
+
   if (audioStream) {
     streamsByType['audio'] = audioStream;
     this.streamsByType_['audio'] = audioStream;
   }
-
-  // Choose video Stream.
-  var videoStream = this.chooseVideoStream_();
   if (videoStream) {
     streamsByType['video'] = videoStream;
     this.streamsByType_['video'] = videoStream;
@@ -262,12 +280,14 @@ shaka.abr.SimpleAbrManager.prototype.chooseStreams_ = function() {
 
 
 /**
- * Chooses which audio Stream to switch to.
+ * Returns the middle audio Stream, which is the default when changing video
+ * Streams.  Once the video Stream has been chosen, the choice of audio Stream
+ * will be refined.
  *
  * @return {?shakaExtern.Stream}
  * @private
  */
-shaka.abr.SimpleAbrManager.prototype.chooseAudioStream_ = function() {
+shaka.abr.SimpleAbrManager.prototype.getMiddleAudioStream_ = function() {
   // Alias.
   var SimpleAbrManager = shaka.abr.SimpleAbrManager;
 
@@ -277,50 +297,57 @@ shaka.abr.SimpleAbrManager.prototype.chooseAudioStream_ = function() {
     return null;
   var audioStreams = SimpleAbrManager.sortStreamsByBandwidth_(audioStreamSet);
 
-  // Just pick the middle one.
-  // TODO: Implement better audio adaptation.
+  // Return the middle one, rounding up.
+  // For example, for 3 streams, the middle is stream index 1, or floor(3/2).
+  // For 2 streams (0 and 1) the middle rounding up is stream 1, or floor(2/2).
   return audioStreams[Math.floor(audioStreams.length / 2)];
 };
 
 
 /**
- * Chooses which video Stream to switch to.
+ * Chooses a Stream of the desired type assuming another Stream has already
+ * been chosen.
  *
+ * @param {string} type
+ * @param {?shakaExtern.Stream} otherStream
  * @return {?shakaExtern.Stream}
  * @private
  */
-shaka.abr.SimpleAbrManager.prototype.chooseVideoStream_ = function() {
+shaka.abr.SimpleAbrManager.prototype.chooseOneStream_ =
+    function(type, otherStream) {
+  // TODO: Come up with a better name for chooseOneStream_, which is not
+  // descriptive enough at the call sites.
+
   // Alias.
   var SimpleAbrManager = shaka.abr.SimpleAbrManager;
 
-  // Get sorted video Streams.
-  var videoStreamSet = this.streamSetsByType_['video'];
-  if (!videoStreamSet)
+  // Get sorted Streams.
+  var streamSet = this.streamSetsByType_[type];
+  if (!streamSet)
     return null;
-  var videoStreams = SimpleAbrManager.sortStreamsByBandwidth_(videoStreamSet);
+  var streams = SimpleAbrManager.sortStreamsByBandwidth_(streamSet);
 
-  var audioStream = this.streamsByType_['audio'];
-  var audioBandwidth = (audioStream && audioStream.bandwidth) || 0;
+  var otherBandwidth = (otherStream && otherStream.bandwidth) || 0;
   var currentBandwidth = this.bandwidthEstimator_.getBandwidthEstimate();
 
   // Start by assuming that we will use the first Stream.
-  var chosen = videoStreams[0];
+  var chosen = streams[0];
 
-  for (var i = 0; i < videoStreams.length; ++i) {
-    var stream = videoStreams[i];
-    var nextStream = (i + 1 < videoStreams.length) ?
-                     videoStreams[i + 1] :
+  for (var i = 0; i < streams.length; ++i) {
+    var stream = streams[i];
+    var nextStream = (i + 1 < streams.length) ?
+                     streams[i + 1] :
                      {bandwidth: Infinity};
 
     // Ignore Streams which don't have bandwidth information.
     if (!stream.bandwidth) continue;
 
-    var minBandwidth = (stream.bandwidth + audioBandwidth) /
+    var minBandwidth = (stream.bandwidth + otherBandwidth) /
                        SimpleAbrManager.BANDWIDTH_DOWNGRADE_TARGET_;
-    var maxBandwidth = (nextStream.bandwidth + audioBandwidth) /
+    var maxBandwidth = (nextStream.bandwidth + otherBandwidth) /
                        SimpleAbrManager.BANDWIDTH_UPGRADE_TARGET_;
     shaka.log.v2('Bandwidth ranges:',
-                 ((stream.bandwidth + audioBandwidth) / 1e6).toFixed(3),
+                 ((stream.bandwidth + otherBandwidth) / 1e6).toFixed(3),
                  (minBandwidth / 1e6).toFixed(3),
                  (maxBandwidth / 1e6).toFixed(3));
 

test/abr/simple_abr_manager_unit.js

@@ -21,6 +21,7 @@ describe('SimpleAbrManager', function() {
   var audioStreamSet;
   var videoStreamSet;
   var streamSetsByType;
+  var sufficientBWMultiplier = 1.06;
 
   beforeAll(function() {
     jasmine.clock().install();
@@ -122,10 +123,12 @@ describe('SimpleAbrManager', function() {
 
     // Simulate some segments being downloaded just above the desired
     // bandwidth.
-    var bytesPerSecond = 1.1 * (audioBandwidth + videoBandwidth) / 8.0;
+    var bytesPerSecond =
+        sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
 
     var bandwidthKbps = (audioBandwidth + videoBandwidth) / 1000.0;
-    var description = 'picks correct Stream at ' + bandwidthKbps + ' kbps';
+    var description =
+        'picks correct video Stream at ' + bandwidthKbps + ' kbps';
 
     it(description, function() {
       abrManager.chooseStreams(streamSetsByType);
@@ -147,10 +150,73 @@ describe('SimpleAbrManager', function() {
     });
   });
 
+  [5e5, 6e5].forEach(function(audioBandwidth) {
+    var videoBandwidth = 1e6;
+
+    // Simulate some segments being downloaded just above the desired
+    // bandwidth.
+    var bytesPerSecond =
+        sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
+
+    var bandwidthKbps = (audioBandwidth + videoBandwidth) / 1000.0;
+    var description =
+        'picks correct audio Stream at ' + bandwidthKbps + ' kbps';
+
+    it(description, function() {
+      abrManager.chooseStreams(streamSetsByType);
+
+      abrManager.segmentDownloaded(0, 1000, bytesPerSecond);
+      abrManager.segmentDownloaded(1000, 2000, bytesPerSecond);
+
+      abrManager.enable();
+
+      // Make another call to segmentDownloaded() so switchCallback() is
+      // called.
+      abrManager.segmentDownloaded(3000, 4000, bytesPerSecond);
+
+      expect(switchCallback).toHaveBeenCalled();
+      expect(switchCallback.calls.argsFor(0)[0]).toEqual({
+        'audio': jasmine.objectContaining({bandwidth: audioBandwidth}),
+        'video': jasmine.objectContaining({bandwidth: videoBandwidth})
+      });
+    });
+  });
+
+  it('picks lowest audio Stream when there is insufficient bandwidth',
+      function() {
+        // The lowest audio track will only be chosen if needed to fit the
+        // the lowest video track.
+        var audioBandwidth = 4e5;
+        var videoBandwidth = 5e5;
+
+        // Simulate some segments being downloaded just above the desired
+        // bandwidth.
+        var bytesPerSecond =
+            sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
+
+        abrManager.chooseStreams(streamSetsByType);
+
+        abrManager.segmentDownloaded(0, 1000, bytesPerSecond);
+        abrManager.segmentDownloaded(1000, 2000, bytesPerSecond);
+
+        abrManager.enable();
+
+        // Make another call to segmentDownloaded() so switchCallback() is
+        // called.
+        abrManager.segmentDownloaded(3000, 4000, bytesPerSecond);
+
+        expect(switchCallback).toHaveBeenCalled();
+        expect(switchCallback.calls.argsFor(0)[0]).toEqual({
+          'audio': jasmine.objectContaining({bandwidth: audioBandwidth}),
+          'video': jasmine.objectContaining({bandwidth: videoBandwidth})
+        });
+      });
+
   it('does not call switchCallback() if not enabled', function() {
     var audioBandwidth = 5e5;
     var videoBandwidth = 2e6;
-    var bytesPerSecond = 1.1 * (audioBandwidth + videoBandwidth) / 8.0;
+    var bytesPerSecond =
+        sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
 
     abrManager.chooseStreams(streamSetsByType);
 
@@ -164,7 +230,8 @@ describe('SimpleAbrManager', function() {
   it('does not call switchCallback() in switch interval', function() {
     var audioBandwidth = 5e5;
     var videoBandwidth = 3e6;
-    var bytesPerSecond = 1.1 * (audioBandwidth + videoBandwidth) / 8.0;
+    var bytesPerSecond =
+        sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
 
     abrManager.chooseStreams(streamSetsByType);
 
@@ -180,10 +247,16 @@ describe('SimpleAbrManager', function() {
     // Simulate drop in bandwidth.
     audioBandwidth = 5e5;
     videoBandwidth = 1e6;
-    bytesPerSecond = 0.9 * (audioBandwidth + videoBandwidth) / 8.0;
+    bytesPerSecond =
+        sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
 
+    abrManager.segmentDownloaded(4000, 5000, bytesPerSecond);
     abrManager.segmentDownloaded(5000, 6000, bytesPerSecond);
+    abrManager.segmentDownloaded(6000, 7000, bytesPerSecond);
     abrManager.segmentDownloaded(7000, 8000, bytesPerSecond);
+    abrManager.segmentDownloaded(8000, 9000, bytesPerSecond);
+    abrManager.segmentDownloaded(9000, 10000, bytesPerSecond);
+    abrManager.segmentDownloaded(10000, 11000, bytesPerSecond);
 
     // Stay inside switch interval.
     shaka.test.Util.fakeEventLoop(
@@ -208,7 +281,8 @@ describe('SimpleAbrManager', function() {
     // upgrade.
     var audioBandwidth = 5e5;
     var videoBandwidth = 4e6;
-    var bytesPerSecond = 1.1 * (audioBandwidth + videoBandwidth) / 8.0;
+    var bytesPerSecond =
+        sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
 
     abrManager.chooseStreams(streamSetsByType);
 
@@ -231,7 +305,8 @@ describe('SimpleAbrManager', function() {
     // downgrade.
     var audioBandwidth = 5e5;
     var videoBandwidth = 5e5;
-    var bytesPerSecond = 1.1 * (audioBandwidth + videoBandwidth) / 8.0;
+    var bytesPerSecond =
+        sufficientBWMultiplier * (audioBandwidth + videoBandwidth) / 8.0;
 
     // Set the default high so that the initial choice will be high-quality.
     abrManager.setDefaultEstimate(4e6);