[DRAFT] Mark fragments with empty appends with gap

src/controller/base-stream-controller.ts

@@ -1862,7 +1862,7 @@ export default class BaseStreamController
         fatal: false,
         error,
         frag,
-        reason: `Found no media in msn ${frag.sn} of level "${level.url}"`,
+        reason: error.message,
       });
       if (!this.hls) {
         return;

src/controller/fragment-tracker.ts

@@ -1,6 +1,7 @@
 import { Events } from '../events';
 import { Fragment, Part } from '../loader/fragment';
 import { PlaylistLevelType } from '../types/loader';
+import { ErrorDetails, ErrorTypes } from '../errors';
 import type { SourceBufferName } from '../types/buffer';
 import type {
   FragmentBufferedRange,
@@ -294,6 +295,23 @@ export class FragmentTracker implements ComponentAPI {
         break;
       }
     }
+    if (buffered.time.length === 0) {
+      // Nothing found in buffer, mark as gap
+      fragment.gap = true;
+      this.removeFragment(fragment);
+      this.fragBuffered(fragment, true);
+      const error = new Error(
+        `No media appended for msn ${fragment.sn} of level "${fragment.level}"`,
+      );
+      this.hls.trigger(Events.ERROR, {
+        type: ErrorTypes.MEDIA_ERROR,
+        details: ErrorDetails.FRAG_PARSING_ERROR,
+        fatal: false,
+        error,
+        frag: fragment,
+        reason: error.message,
+      });
+    }
     return buffered;
   }
 

src/controller/stream-controller.ts

@@ -8,7 +8,7 @@ import { ElementaryStreamTypes, Fragment } from '../loader/fragment';
 import TransmuxerInterface from '../demux/transmuxer-interface';
 import { ChunkMetadata } from '../types/transmuxer';
 import GapController, { MAX_START_GAP_JUMP } from './gap-controller';
-import { ErrorDetails } from '../errors';
+import { ErrorDetails, ErrorTypes } from '../errors';
 import type { NetworkComponentAPI } from '../types/component-api';
 import type Hls from '../hls';
 import type { Level } from '../types/level';
@@ -1348,6 +1348,18 @@ export default class StreamController
     this.fragPrevious = null;
     this.nextLoadPosition = frag.start;
     this.state = State.IDLE;
+    // Frag parsing error will force a level switch
+    const error = new Error(
+      `Backtrack for msn ${frag.sn} of level "${frag.level}"`,
+    );
+    this.hls.trigger(Events.ERROR, {
+      type: ErrorTypes.MEDIA_ERROR,
+      details: ErrorDetails.FRAG_PARSING_ERROR,
+      fatal: false,
+      error,
+      frag,
+      reason: error.message,
+    });
   }
 
   private checkFragmentChanged() {

src/remux/passthrough-remuxer.ts

@@ -8,7 +8,7 @@ import {
   patchEncyptionData,
 } from '../utils/mp4-tools';
 import {
-  getDuration,
+  getSampleData,
   getStartDTS,
   offsetStartDTS,
   parseInitSegment,
@@ -40,6 +40,7 @@ class PassThroughRemuxer implements Remuxer {
   private initPTS: RationalTimestamp | null = null;
   private initTracks?: TrackSet;
   private lastEndTime: number | null = null;
+  private isVideoContiguous: boolean = false;
 
   public destroy() {}
 
@@ -49,6 +50,7 @@ class PassThroughRemuxer implements Remuxer {
   }
 
   public resetNextTimestamp() {
+    this.isVideoContiguous = false;
     this.lastEndTime = null;
   }
 
@@ -168,7 +170,10 @@ class PassThroughRemuxer implements Remuxer {
       this.emitInitSegment = false;
     }
 
-    const duration = getDuration(data, initData);
+    const { duration, firstKeyFrame, sampleCount } = getSampleData(
+      data,
+      initData,
+    );
     const startDTS = getStartDTS(initData, data);
     const decodeTime = startDTS === null ? timeOffset : startDTS;
     if (
@@ -212,6 +217,7 @@ class PassThroughRemuxer implements Remuxer {
       type += 'video';
     }
 
+    const independent = firstKeyFrame !== -1;
     const track: RemuxedTrack = {
       data1: data,
       startPTS: startTime,
@@ -225,8 +231,33 @@ class PassThroughRemuxer implements Remuxer {
       dropped: 0,
     };
 
-    result.audio = track.type === 'audio' ? track : undefined;
-    result.video = track.type !== 'audio' ? track : undefined;
+    result.audio = hasAudio && !hasVideo ? track : undefined;
+    result.video = hasVideo ? track : undefined;
+    if (hasVideo && sampleCount) {
+      track.nb = sampleCount;
+      (track.dropped =
+        this.isVideoContiguous || firstKeyFrame === 0
+          ? 0
+          : firstKeyFrame === -1
+            ? sampleCount
+            : firstKeyFrame),
+        (track.independent = independent);
+      track.firstKeyFrame = firstKeyFrame;
+      if (independent && firstKeyFrame) {
+        track.firstKeyFramePTS =
+          startTime + (duration * firstKeyFrame) / sampleCount;
+      }
+      if (!this.isVideoContiguous) {
+        result.independent = independent;
+      }
+      this.isVideoContiguous ||= independent;
+      if (track.dropped) {
+        logger.warn(
+          `fmp4 does not start with IDR: firstIDR ${firstKeyFrame}/${sampleCount} dropped: ${track.dropped} pts: ${track.firstKeyFramePTS || 'NA'}`,
+        );
+      }
+    }
+
     result.initSegment = initSegment;
     result.id3 = flushTextTrackMetadataCueSamples(
       id3Track,

src/utils/mp4-tools.ts

@@ -476,7 +476,7 @@
 
 function skipBERInteger(bytes: Uint8Array, i: number): number {
   const limit = i + 5;
   while (bytes[i++] & 0x80 && i < limit) {}
   return i;
 }
 
@@ -634,10 +634,19 @@
      unsigned int(32)  default_sample_flags
   }
  */
-export function getDuration(data: Uint8Array, initData: InitData) {
+export function getSampleData(
+  data: Uint8Array,
+  initData: InitData,
+): {
+  duration: number;
+  firstKeyFrame?: number;
+  sampleCount?: number;
+} {
   let rawDuration = 0;
   let videoDuration = 0;
   let audioDuration = 0;
+  let firstKeyFrame: number | undefined;
+  let sampleCount: number | undefined;
   const trafs = findBox(data, ['moof', 'traf']);
   for (let i = 0; i < trafs.length; i++) {
     const traf = trafs[i];
@@ -670,12 +679,63 @@
     const timescale = track.timescale || 90e3;
     const truns = findBox(traf, ['trun']);
     for (let j = 0; j < truns.length; j++) {
-      rawDuration = computeRawDurationFromSamples(truns[j]);
+      const trun = truns[j];
+      rawDuration = computeRawDurationFromSamples(trun);
       if (!rawDuration && sampleDuration) {
-        const sampleCount = readUint32(truns[j], 4);
+        const sampleCount = readUint32(trun, 4);
         rawDuration = sampleDuration * sampleCount;
       }
       if (track.type === ElementaryStreamTypes.VIDEO) {
+        firstKeyFrame = -1;
+        const dataOffsetPresent = trun[3] & 0x01;
+        const firstSampleFlagsPresent = trun[3] & 0x04;
+        const sampleDurationPresent = trun[2] & 0x01;
+        const sampleSizePresent = trun[2] & 0x02;
+        const sampleFlagsPresent = trun[2] & 0x04;
+        const sampleCompositionTimeOffsetPresent = trun[2] & 0x08;
+        sampleCount = readUint32(trun, 4);
+        let offset = 8;
+        let remaining = sampleCount;
+        if (dataOffsetPresent) {
+          offset += 4;
+        }
+        if (firstSampleFlagsPresent && sampleCount) {
+          const isNonSyncSample = trun[offset + 1] & 0x01;
+          if (!isNonSyncSample) {
+            firstKeyFrame = 0;
+          }
+          offset += 4;
+          if (sampleDurationPresent) {
+            offset += 4;
+          }
+          if (sampleSizePresent) {
+            offset += 4;
+          }
+          if (sampleCompositionTimeOffsetPresent) {
+            offset += 4;
+          }
+          remaining--;
+        }
+        while (remaining--) {
+          if (sampleDurationPresent) {
+            offset += 4;
+          }
+          if (sampleSizePresent) {
+            offset += 4;
+          }
+          if (sampleFlagsPresent) {
+            const isNonSyncSample = trun[offset + 1] & 0x01;
+            if (!isNonSyncSample) {
+              if (firstKeyFrame === -1) {
+                firstKeyFrame = sampleCount - (remaining + 1);
+              }
+            }
+            offset += 4;
+          }
+          if (sampleCompositionTimeOffsetPresent) {
+            offset += 4;
+          }
+        }
         videoDuration += rawDuration / timescale;
       } else if (track.type === ElementaryStreamTypes.AUDIO) {
         audioDuration += rawDuration / timescale;
@@ -686,7 +746,7 @@
     // If duration samples are not available in the traf use sidx subsegment_duration
     let sidxMinStart = Infinity;
     let sidxMaxEnd = 0;
-    let sidxDuration = 0;
+    let duration = 0;
     const sidxs = findBox(data, ['sidx']);
     for (let i = 0; i < sidxs.length; i++) {
       const sidx = parseSegmentIndex(sidxs[i]);
@@ -703,17 +763,25 @@
           sidxMaxEnd,
           subSegmentDuration + sidx.earliestPresentationTime / sidx.timescale,
         );
-        sidxDuration = sidxMaxEnd - sidxMinStart;
+        duration = sidxMaxEnd - sidxMinStart;
       }
     }
-    if (sidxDuration && Number.isFinite(sidxDuration)) {
-      return sidxDuration;
+    if (duration && Number.isFinite(duration)) {
+      return {
+        duration,
+      };
     }
   }
   if (videoDuration) {
-    return videoDuration;
+    return {
+      duration: videoDuration,
+      firstKeyFrame,
+      sampleCount,
+    };
   }
-  return audioDuration;
+  return {
+    duration: audioDuration,
+  };
 }
 
 /*
@@ -736,7 +804,7 @@
      }[ sample_count ]
   }
  */
-export function computeRawDurationFromSamples(trun): number {
+export function computeRawDurationFromSamples(trun: Uint8Array): number {
   const flags = readUint32(trun, 0);
   // Flags are at offset 0, non-optional sample_count is at offset 4. Therefore we start 8 bytes in.
   // Each field is an int32, which is 4 bytes