diff --git a/lib/msf/msf_parser.js b/lib/msf/msf_parser.js
index 7d13538e8..4ffd44830 100644
--- a/lib/msf/msf_parser.js
+++ b/lib/msf/msf_parser.js
@@ -94,6 +94,15 @@ shaka.msf.MSFParser = class {
 
     /** @private {boolean} */
     this.isFirstVideoSegment_ = true;
+
+    /**
+     * Tracks the first segment start time for each content type.
+     * Used to delay locking the presentation timeline until all expected
+     * stream types have received their first segment, preventing premature
+     * eviction of segments from streams with earlier timestamps.
+     * @private {!Map<string, number>}
+     */
+    this.firstSegmentStartTimes_ = new Map();
   }
 
   /**
@@ -637,15 +646,45 @@ shaka.msf.MSFParser = class {
 
         reference.setSegmentData(obj.data);
 
-        stream.segmentIndex.mergeAndEvict(
-            [reference],
-            this.presentationTimeline_.getSegmentAvailabilityStart());
+        // Track the first segment start time per content type.
+        // We must not lock the presentation timeline until all expected
+        // stream types have received at least one segment, otherwise
+        // streams with earlier timestamps get their segments evicted
+        // immediately by mergeAndEvict (e.g. video starts 4s before audio).
+        if (!this.firstSegmentStartTimes_.has(type)) {
+          this.firstSegmentStartTimes_.set(type, startTime);
+        }
+
+        const timelineLocked = this.presentationTimeline_.isStartTimeLocked();
+
+        // Before the timeline is locked, pass 0 as the eviction start to
+        // prevent mergeAndEvict from discarding segments prematurely.
+        const evictionStart = timelineLocked ?
+            this.presentationTimeline_.getSegmentAvailabilityStart() : 0;
+        stream.segmentIndex.mergeAndEvict([reference], evictionStart);
 
         this.presentationTimeline_.notifySegments([reference]);
-        if (!this.presentationTimeline_.isStartTimeLocked()) {
-          this.presentationTimeline_.lockStartTime();
-          this.presentationTimeline_.setSegmentAvailabilityDuration(
-              Math.max(0.5, duration));
+
+        if (!timelineLocked) {
+          // Only lock once we have first segments from all expected types.
+          const hasAudio = this.audioStreams_.length > 0;
+          const hasVideo = this.videoStreams_.length > 0;
+          const gotAudio = this.firstSegmentStartTimes_.has(
+              ContentType.AUDIO);
+          const gotVideo = this.firstSegmentStartTimes_.has(
+              ContentType.VIDEO);
+          const allReady = (!hasAudio || gotAudio) && (!hasVideo || gotVideo);
+
+          if (allReady) {
+            // Set availability duration wide enough to cover the gap
+            // between the earliest and latest stream start times, so
+            // segments from streams with earlier timestamps aren't evicted.
+            const starts = [...this.firstSegmentStartTimes_.values()];
+            const gap = Math.max(...starts) - Math.min(...starts);
+            this.presentationTimeline_.lockStartTime();
+            this.presentationTimeline_.setSegmentAvailabilityDuration(
+                Math.max(0.5, duration, gap + duration));
+          }
         }
 
         if (!this.config_.disableText &&