commit ddafaa057aca13f8875b3a4b8f33fd4aab7dfea8
parent 57c1ff3d5f4c7f5a5bd559c09352bef51aecdfe1
Author: Andrew Osmond <aosmond@gmail.com>
Date: Fri, 2 Jan 2026 03:48:20 +0000
Bug 2008270 - Consolidate encoder functionality that extracts AVCC extradata from AnnexB samples. r=media-playback-reviewers,jolin,alwu
This will be useful for FFmpegVideoEncoder on Android which needs to do
a similar function. This patch also makes the extraction/transformation
a bit safer by validating the position of the SPS/PPS NAL units before
extracting the necessary information.
Differential Revision: https://phabricator.services.mozilla.com/D277717
Diffstat:
6 files changed, 117 insertions(+), 53 deletions(-)
diff --git a/dom/media/gtest/TestMediaDataEncoder.cpp b/dom/media/gtest/TestMediaDataEncoder.cpp
@@ -426,7 +426,7 @@ static void H264EncodesTest(Usage aUsage,
GET_OR_RETURN_ON_ERROR(Encode(e, 1UL, aFrameSource));
EXPECT_EQ(output.Length(), 1UL);
EXPECT_TRUE(isAVCC ? AnnexB::IsAVCC(output[0])
- : AnnexB::IsAnnexB(output[0]));
+ : AnnexB::IsAnnexB(*output[0]));
WaitForShutdown(e);
output.Clear();
@@ -461,7 +461,7 @@ static void H264EncodesTest(Usage aUsage,
}
} else {
for (auto frame : output) {
- EXPECT_TRUE(AnnexB::IsAnnexB(frame));
+ EXPECT_TRUE(AnnexB::IsAnnexB(*frame));
}
}
@@ -549,7 +549,7 @@ static void H264EncodeBatchTest(
}
} else {
for (auto frame : output) {
- EXPECT_TRUE(AnnexB::IsAnnexB(frame));
+ EXPECT_TRUE(AnnexB::IsAnnexB(*frame));
}
}
@@ -740,7 +740,7 @@ TEST_F(MediaDataEncoderTest, H264AVCC) {
GET_OR_RETURN_ON_ERROR(Encode(e, NUM_FRAMES, mData));
EXPECT_EQ(output.Length(), NUM_FRAMES);
for (auto frame : output) {
- EXPECT_FALSE(AnnexB::IsAnnexB(frame));
+ EXPECT_FALSE(AnnexB::IsAnnexB(*frame));
if (frame->mKeyframe) {
// The extradata may be included at the beginning, whenever it changes,
// or with every keyframe to support robust seeking or decoder resets.
diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp
@@ -334,6 +334,19 @@ void AnnexB::ParseNALEntries(const Span<const uint8_t>& aSpan,
}
/* static */
+size_t AnnexB::FindNalType(const Span<const uint8_t>& aSpan,
+ const nsTArray<AnnexB::NALEntry>& aNalEntries,
+ NAL_TYPES aType, size_t aStartIndex) {
+ for (size_t i = aStartIndex; i < aNalEntries.Length(); ++i) {
+ uint8_t nalUnitType = aSpan[aNalEntries[i].mOffset] & 0x1f;
+ if (nalUnitType == aType) {
+ return i;
+ }
+ }
+ return SIZE_MAX;
+}
+
+/* static */
bool AnnexB::FindAllNalTypes(const Span<const uint8_t>& aSpan,
const nsTArray<NAL_TYPES>& aTypes) {
nsTArray<AnnexB::NALEntry> nalEntries;
@@ -382,12 +395,43 @@ static Result<mozilla::Ok, nsresult> ParseNALUnits(ByteWriter<BigEndian>& aBw,
return Ok();
}
+/* static */
+RefPtr<MediaByteBuffer> AnnexB::ExtractExtraDataForAVCC(
+ const Span<const uint8_t>& aSpan) {
+ if (!IsAnnexB(aSpan)) {
+ return nullptr;
+ }
+
+ nsTArray<NALEntry> paramSets;
+ ParseNALEntries(aSpan, paramSets);
+
+ size_t spsIndex =
+ FindNalType(aSpan, paramSets, H264_NAL_SPS, /* aStartIndex */ 0);
+ if (spsIndex == SIZE_MAX) {
+ return nullptr;
+ }
+
+ size_t ppsIndex =
+ FindNalType(aSpan, paramSets, H264_NAL_PPS, /* aStartIndex */ 0);
+ if (ppsIndex == SIZE_MAX) {
+ return nullptr;
+ }
+
+ auto avcc = MakeRefPtr<MediaByteBuffer>();
+ const auto& spsEntry = paramSets.ElementAt(spsIndex);
+ const auto& ppsEntry = paramSets.ElementAt(ppsIndex);
+ const auto sps = aSpan.Subspan(spsEntry.mOffset, spsEntry.mSize);
+ const auto pps = aSpan.Subspan(ppsEntry.mOffset, ppsEntry.mSize);
+ H264::WriteExtraData(avcc, sps[1], sps[2], sps[3], sps, pps);
+ return avcc;
+}
+
bool AnnexB::ConvertSampleToAVCC(mozilla::MediaRawData* aSample,
const RefPtr<MediaByteBuffer>& aAVCCHeader) {
if (IsAVCC(aSample)) {
return ConvertAVCCTo4BytesAVCC(aSample).isOk();
}
- if (!IsAnnexB(aSample)) {
+ if (!IsAnnexB(*aSample)) {
// Not AnnexB, nothing to convert.
return true;
}
@@ -433,7 +477,7 @@ Result<mozilla::Ok, nsresult> AnnexB::ConvertSampleToHVCC(
if (IsHVCC(aSample)) {
return ConvertHVCCTo4BytesHVCC(aSample);
}
- if (!IsAnnexB(aSample)) {
+ if (!IsAnnexB(*aSample)) {
// Not AnnexB, nothing to convert.
return Ok();
}
@@ -489,11 +533,11 @@ bool AnnexB::IsHVCC(const mozilla::MediaRawData* aSample) {
}
/* static */
-bool AnnexB::IsAnnexB(const mozilla::MediaRawData* aSample) {
- if (aSample->Size() < 4) {
+bool AnnexB::IsAnnexB(const Span<const uint8_t>& aSpan) {
+ if (aSpan.Length() < 4) {
return false;
}
- uint32_t header = mozilla::BigEndian::readUint32(aSample->Data());
+ uint32_t header = mozilla::BigEndian::readUint32(aSpan.Elements());
return header == 0x00000001 || (header >> 8) == 0x000001;
}
diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.h b/dom/media/platforms/agnostic/bytestreams/AnnexB.h
@@ -39,6 +39,9 @@ class AnnexB {
static mozilla::Result<mozilla::Ok, nsresult> ConvertHVCCSampleToAnnexB(
mozilla::MediaRawData* aSample, bool aAddSPS = true);
+ // Extract extradata for AVCC from an Annex B sample.
+ static RefPtr<MediaByteBuffer> ExtractExtraDataForAVCC(
+ const Span<const uint8_t>& aSpan);
// Convert a sample from Annex B to AVCC.
// an AVCC extradata must not be set.
static bool ConvertSampleToAVCC(
@@ -66,7 +69,7 @@ class AnnexB {
// Returns true if format is HVCC and sample has valid extradata.
static bool IsHVCC(const mozilla::MediaRawData* aSample);
// Returns true if format is AnnexB.
- static bool IsAnnexB(const mozilla::MediaRawData* aSample);
+ static bool IsAnnexB(const Span<const uint8_t>& aSpan);
// Parse NAL entries from the bytes stream to know the offset and the size of
// each NAL in the bytes stream.
@@ -78,6 +81,10 @@ class AnnexB {
const nsTArray<NAL_TYPES>& aTypes);
private:
+ static size_t FindNalType(const Span<const uint8_t>& aSpan,
+ const nsTArray<AnnexB::NALEntry>& aNalEntries,
+ NAL_TYPES aType, size_t aStartIndex);
+
// AVCC box parser helper.
static mozilla::Result<mozilla::Ok, nsresult> ConvertSPSOrPPS(
mozilla::BufferReader& aReader, uint8_t aCount,
diff --git a/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp b/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp
@@ -204,7 +204,7 @@ TEST(AnnexB, AVCCToAnnexBConversion)
<< "AnnexB sample should be the same size as the AVCC sample -- the 4 "
"byte NAL length data (AVCC) is replaced with 4 bytes of NAL "
"separator (AnnexB)";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawDataClone))
<< "The sample should be AnnexB following conversion";
}
@@ -221,7 +221,7 @@ TEST(AnnexB, AVCCToAnnexBConversion)
"byte NAL length data (AVCC) is replaced with 4 bytes of NAL "
"separator (AnnexB) and SPS data is not added as the frame is not a "
"keyframe";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawDataClone))
<< "The sample should be AnnexB following conversion";
}
@@ -235,7 +235,7 @@ TEST(AnnexB, AVCCToAnnexBConversion)
EXPECT_GT(rawDataClone->Size(), rawData->Size())
<< "AnnexB sample should be larger than the AVCC sample because we've "
"added SPS data";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawDataClone))
<< "The sample should be AnnexB following conversion";
// We could verify the SPS and PPS data we add, but we don't have great
// tooling to do so. Consider doing so in future.
@@ -273,7 +273,7 @@ TEST(AnnexB, AVCCToAnnexBConversion)
EXPECT_EQ(rawCryptoDataClone->mCrypto.mEncryptedSizes[0],
rawCryptoData->mCrypto.mEncryptedSizes[0])
<< "Conversion should not affect encrypted sizes";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawCryptoDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawCryptoDataClone))
<< "The sample should be AnnexB following conversion";
}
}
@@ -291,7 +291,7 @@ TEST(AnnexB, HVCCToAnnexBConversion)
<< "AnnexB sample should be the same size as the HVCC sample -- the 4 "
"byte NAL length data (HVCC) is replaced with 4 bytes of NAL "
"separator (AnnexB)";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawDataClone))
<< "The sample should be AnnexB following conversion";
}
{
@@ -307,7 +307,7 @@ TEST(AnnexB, HVCCToAnnexBConversion)
"byte NAL length data (HVCC) is replaced with 4 bytes of NAL "
"separator (AnnexB) and SPS data is not added as the frame is not a "
"keyframe";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawDataClone))
<< "The sample should be AnnexB following conversion";
}
{
@@ -320,7 +320,7 @@ TEST(AnnexB, HVCCToAnnexBConversion)
EXPECT_GT(rawDataClone->Size(), rawData->Size())
<< "AnnexB sample should be larger than the HVCC sample because we've "
"added SPS data";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawDataClone))
<< "The sample should be AnnexB following conversion";
// We could verify the SPS and PPS data we add, but we don't have great
// tooling to do so. Consider doing so in future.
@@ -357,7 +357,7 @@ TEST(AnnexB, HVCCToAnnexBConversion)
EXPECT_EQ(rawCryptoDataClone->mCrypto.mEncryptedSizes[0],
rawCryptoData->mCrypto.mEncryptedSizes[0])
<< "Conversion should not affect encrypted sizes";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawCryptoDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawCryptoDataClone))
<< "The sample should be AnnexB following conversion";
}
}
@@ -873,6 +873,39 @@ TEST(H264, CreateNewExtraData)
EXPECT_TRUE(res.isErr());
}
+TEST(H264, AnnexBExtractExtraDataForAVCC)
+{
+ // First create an AnnexB config
+ const uint8_t annexBBytesBuffer[]{// AnnexB delimiter
+ 0x00, 0x00, 0x00, 0x01,
+ // SPS NAL unit
+ 0x67, 0x64, 0x00, 0x1F,
+ // AnnexB delimiter
+ 0x00, 0x00, 0x00, 0x01,
+ // PPS NAL unit
+ 0x68, 0xCE};
+ auto annexBExtradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ annexBExtradata->AppendElements(annexBBytesBuffer,
+ std::size(annexBBytesBuffer));
+
+ // Extract the AVCC extradata from the AnnexB bytestream
+ auto avccExtradata = AnnexB::ExtractExtraDataForAVCC(*annexBExtradata);
+ ASSERT_TRUE(!!avccExtradata);
+
+ // Now parse that extradata to make sure it matches the original AnnexB
+ auto res = AVCCConfig::Parse(avccExtradata);
+ ASSERT_TRUE(res.isOk());
+ auto avcc = res.unwrap();
+ EXPECT_EQ(avcc.NumSPS(), 1u);
+ EXPECT_EQ(avcc.NumSPSExt(), 0u);
+ EXPECT_EQ(avcc.NumPPS(), 1u);
+ EXPECT_EQ(avcc.NALUSize(), 4);
+ EXPECT_EQ(avcc.mConfigurationVersion, 1u);
+ EXPECT_EQ(avcc.mAVCProfileIndication, 100u);
+ EXPECT_EQ(avcc.mProfileCompatibility, 0u);
+ EXPECT_EQ(avcc.mAVCLevelIndication, 31u);
+}
+
TEST(H265, HVCCParsingSuccess)
{
{
@@ -1171,7 +1204,7 @@ TEST(H265, AnnexBToHVCC)
Result<Ok, nsresult> result =
AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ false);
EXPECT_TRUE(result.isOk()) << "HVCC to AnnexB Conversion should succeed";
- EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ EXPECT_TRUE(AnnexB::IsAnnexB(*rawDataClone))
<< "The sample should be AnnexB following conversion";
auto rv = AnnexB::ConvertSampleToHVCC(rawDataClone);
diff --git a/dom/media/platforms/android/AndroidDataEncoder.cpp b/dom/media/platforms/android/AndroidDataEncoder.cpp
@@ -279,20 +279,7 @@ static RefPtr<MediaByteBuffer> ExtractCodecConfig(
if (!aAsAVCC) {
return config;
}
- // Convert to avcC.
- nsTArray<AnnexB::NALEntry> paramSets;
- AnnexB::ParseNALEntries(
- Span<const uint8_t>(config->Elements(), config->Length()), paramSets);
-
- auto avcc = MakeRefPtr<MediaByteBuffer>();
- AnnexB::NALEntry& sps = paramSets.ElementAt(0);
- AnnexB::NALEntry& pps = paramSets.ElementAt(1);
- const uint8_t* spsPtr = config->Elements() + sps.mOffset;
- H264::WriteExtraData(
- avcc, spsPtr[1], spsPtr[2], spsPtr[3],
- Span<const uint8_t>(spsPtr, sps.mSize),
- Span<const uint8_t>(config->Elements() + pps.mOffset, pps.mSize));
- return avcc;
+ return AnnexB::ExtractExtraDataForAVCC(*config);
}
void AndroidDataEncoder::ProcessOutput(
@@ -339,8 +326,15 @@ void AndroidDataEncoder::ProcessOutput(
if (size > 0) {
if ((flags & java::sdk::MediaCodec::BUFFER_FLAG_CODEC_CONFIG) != 0) {
- mConfigData = ExtractCodecConfig(aBuffer, offset, size,
- IsAVCC(mConfig.mCodecSpecific));
+ auto configData = ExtractCodecConfig(aBuffer, offset, size,
+ IsAVCC(mConfig.mCodecSpecific));
+ if (configData) {
+ mConfigData = std::move(configData);
+ } else {
+ MOZ_ASSERT_UNREACHABLE("Bad config data!");
+ Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "fail to extract codec config"_ns));
+ }
return;
}
RefPtr<MediaRawData> output;
diff --git a/dom/media/platforms/wmf/WMFDataEncoderUtils.cpp b/dom/media/platforms/wmf/WMFDataEncoderUtils.cpp
@@ -96,28 +96,14 @@ EncodeSupportSet CanCreateWMFEncoder(const EncoderConfig& aConfig) {
static already_AddRefed<MediaByteBuffer> ParseH264Parameters(
const nsTArray<uint8_t>& aHeader, const bool aAsAnnexB) {
+ if (!aAsAnnexB) {
+ return AnnexB::ExtractExtraDataForAVCC(aHeader).forget();
+ }
size_t length = aHeader.Length();
auto annexB = MakeRefPtr<MediaByteBuffer>(length);
PodCopy(annexB->Elements(), aHeader.Elements(), length);
annexB->SetLength(length);
- if (aAsAnnexB) {
- return annexB.forget();
- }
-
- // Convert to avcC.
- nsTArray<AnnexB::NALEntry> paramSets;
- AnnexB::ParseNALEntries(
- Span<const uint8_t>(annexB->Elements(), annexB->Length()), paramSets);
-
- auto avcc = MakeRefPtr<MediaByteBuffer>();
- AnnexB::NALEntry& sps = paramSets.ElementAt(0);
- AnnexB::NALEntry& pps = paramSets.ElementAt(1);
- const uint8_t* spsPtr = annexB->Elements() + sps.mOffset;
- H264::WriteExtraData(
- avcc, spsPtr[1], spsPtr[2], spsPtr[3],
- Span<const uint8_t>(spsPtr, sps.mSize),
- Span<const uint8_t>(annexB->Elements() + pps.mOffset, pps.mSize));
- return avcc.forget();
+ return annexB.forget();
}
static uint32_t GetProfile(H264_PROFILE aProfileLevel) {
