Bug 1347518 - Part 1: Use SEI recovery point to mark keyframe. r=jesup, a=lizzard
authorJean-Yves Avenard <jyavenard@mozilla.com>
Mon, 21 Aug 2017 18:34:38 +0200
changeset 655630 fb8967e983b19fe45dafda52e81809d938ca93ca
parent 655629 9e2fda5c1b955de6c341a3d41c30751eee5fd833
child 655631 e070c1cca1be6f9c51f70ea9ac6f557c6dd76e8f
push id76942
push userbmo:scwwu@mozilla.com
push dateWed, 30 Aug 2017 08:27:15 +0000
reviewersjesup, lizzard
bugs1347518
milestone56.0
Bug 1347518 - Part 1: Use SEI recovery point to mark keyframe. r=jesup, a=lizzard Some streams do not include any IDR frames. However, they do include SEI recovery point markers. We use those to mark keyframes. MozReview-Commit-ID: IHfUx7fEgEJ
media/libstagefright/binding/H264.cpp
media/libstagefright/binding/include/mp4_demuxer/H264.h
--- a/media/libstagefright/binding/H264.cpp
+++ b/media/libstagefright/binding/H264.cpp
@@ -800,19 +800,26 @@ H264::GetFrameType(const mozilla::MediaR
     }
     if (!nalLen) {
       continue;
     }
     const uint8_t* p = reader.Read(nalLen);
     if (!p) {
       return FrameType::INVALID;
     }
-    if ((p[0] & 0x1f) == H264_NAL_IDR_SLICE) {
+    int8_t nalType = *p & 0x1f;
+    if (nalType == H264_NAL_IDR_SLICE) {
       // IDR NAL.
       return FrameType::I_FRAME;
+    } else if (nalType == H264_NAL_SEI) {
+      RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen);
+      SEIRecoveryData data;
+      if (DecodeRecoverySEI(decodedNAL, data)) {
+        return FrameType::I_FRAME;
+      }
     }
   }
 
   return FrameType::OTHER;
 }
 
 /* static */ already_AddRefed<mozilla::MediaByteBuffer>
 H264::ExtractExtraData(const mozilla::MediaRawData* aSample)
@@ -970,12 +977,80 @@ H264::CompareExtraData(const mozilla::Me
       return false;
     }
     ++it1;
     ++it2;
   }
   return true;
 }
 
+static inline bool
+ReadSEIInt(ByteReader& aBr, uint32_t& aOutput)
+{
+  uint8_t tmpByte;
+
+  aOutput = 0;
+  if (!aBr.CanRead8()) {
+    return false;
+  }
+  tmpByte = aBr.ReadU8();
+  while (tmpByte == 0xFF) {
+    aOutput += 255;
+    if (!aBr.CanRead8()) {
+      return false;
+    }
+    tmpByte = aBr.ReadU8();
+  }
+  aOutput += tmpByte;   // this is the last byte
+  return true;
+}
+
+/* static */ bool
+H264::DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI,
+                        SEIRecoveryData& aDest)
+{
+  if (!aSEI) {
+    return false;
+  }
+  // sei_rbsp() as per 7.3.2.3 Supplemental enhancement information RBSP syntax
+  ByteReader br(aSEI);
+
+  do {
+    // sei_message() as per
+    // 7.3.2.3.1 Supplemental enhancement information message syntax
+    uint32_t payloadType = 0;
+    if (!ReadSEIInt(br, payloadType)) {
+      return false;
+    }
+
+    uint32_t payloadSize = 0;
+    if (!ReadSEIInt(br, payloadSize)) {
+      return false;
+    }
+
+    // sei_payload(payloadType, payloadSize) as per
+    // D.1 SEI payload syntax.
+    const uint8_t* p = br.Read(payloadSize);
+    if (!p) {
+      return false;
+    }
+    if (payloadType == 6) { // SEI_RECOVERY_POINT
+      if (payloadSize == 0) {
+        // Invalid content, ignore.
+        continue;
+      }
+      // D.1.7 Recovery point SEI message syntax
+      BitReader br(p, payloadSize * 8);
+      aDest.recovery_frame_cnt = br.ReadUE();
+      aDest.exact_match_flag = br.ReadBit();
+      aDest.broken_link_flag = br.ReadBit();
+      aDest.changing_slice_group_idc = br.ReadBits(2);
+      return true;
+    }
+  } while(br.CanRead8() && br.PeekU8() != 0x80); // more_rbsp_data() msg[offset] != 0x80
+  // ignore the trailing bits rbsp_trailing_bits();
+  return false;
+}
+
 #undef READUE
 #undef READSE
 
 } // namespace mp4_demuxer
--- a/media/libstagefright/binding/include/mp4_demuxer/H264.h
+++ b/media/libstagefright/binding/include/mp4_demuxer/H264.h
@@ -399,16 +399,51 @@ struct SPSData
 
   bool scaling_matrix_present;
   uint8_t scaling_matrix4x4[6][16];
   uint8_t scaling_matrix8x8[6][64];
 
   SPSData();
 };
 
+struct SEIRecoveryData
+{
+  /*
+    recovery_frame_cnt specifies the recovery point of output pictures in output
+    order. All decoded pictures in output order are indicated to be correct or
+    approximately correct in content starting at the output order position of
+    the reference picture having the frame_num equal to the frame_num of the VCL
+    NAL units for the current access unit incremented by recovery_frame_cnt in
+    modulo MaxFrameNum arithmetic. recovery_frame_cnt shall be in the range of 0
+    to MaxFrameNum − 1, inclusive.
+  */
+  uint32_t recovery_frame_cnt = 0;
+  /*
+    exact_match_flag indicates whether decoded pictures at and subsequent to the
+    specified recovery point in output order derived by starting the decoding
+    process at the access unit associated with the recovery point SEI message
+    shall be an exact match to the pictures that would be produced by starting
+    the decoding process at the location of a previous IDR access unit in the
+    NAL unit stream. The value 0 indicates that the match need not be exact and
+    the value 1 indicates that the match shall be exact.
+  */
+  bool exact_match_flag = false;
+  /*
+    broken_link_flag indicates the presence or absence of a broken link in the
+    NAL unit stream at the location of the recovery point SEI message */
+  bool broken_link_flag = false;
+  /*
+    changing_slice_group_idc equal to 0 indicates that decoded pictures are
+    correct or approximately correct in content at and subsequent to the
+    recovery point in output order when all macroblocks of the primary coded
+    pictures are decoded within the changing slice group period
+  */
+  uint8_t changing_slice_group_idc = 0;
+};
+
 class H264
 {
 public:
   /* Check if out of band extradata contains a SPS NAL */
   static bool HasSPS(const mozilla::MediaByteBuffer* aExtraData);
   // Extract SPS and PPS NALs from aSample by looking into each NALs.
   // aSample must be in AVCC format.
   static already_AddRefed<mozilla::MediaByteBuffer> ExtractExtraData(
@@ -449,13 +484,17 @@ private:
   static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit(
     const uint8_t* aNAL, size_t aLength);
   /* Decode SPS NAL RBSP and fill SPSData structure */
   static bool DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest);
   static bool vui_parameters(BitReader& aBr, SPSData& aDest);
   // Read HRD parameters, all data is ignored.
   static void hrd_parameters(BitReader& aBr);
   static uint8_t NumSPS(const mozilla::MediaByteBuffer* aExtraData);
+  // Decode SEI payload and return true if the SEI NAL indicates a recovery
+  // point.
+  static bool DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI,
+                                SEIRecoveryData& aDest);
 };
 
 } // namespace mp4_demuxer
 
 #endif // MP4_DEMUXER_H264_H_