H264.cpp (50098B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #include "H264.h" 6 7 #include <limits> 8 9 #include "AnnexB.h" 10 #include "BitReader.h" 11 #include "BitWriter.h" 12 #include "BufferReader.h" 13 #include "ByteStreamsUtils.h" 14 #include "ByteWriter.h" 15 #include "MediaInfo.h" 16 #include "mozilla/PodOperations.h" 17 #include "mozilla/Result.h" 18 #include "mozilla/Try.h" 19 20 #define READSE(var, min, max) \ 21 { \ 22 int32_t val = br.ReadSE(); \ 23 if (val < min || val > max) { \ 24 return false; \ 25 } \ 26 aDest.var = val; \ 27 } 28 29 #define READUE(var, max) \ 30 { \ 31 uint32_t uval = br.ReadUE(); \ 32 if (uval > max) { \ 33 return false; \ 34 } \ 35 aDest.var = uval; \ 36 } 37 38 mozilla::LazyLogModule gH264("H264"); 39 40 #define LOG(msg, ...) MOZ_LOG(gH264, LogLevel::Debug, (msg, ##__VA_ARGS__)) 41 42 namespace mozilla { 43 44 // Default scaling lists (per spec). 45 // ITU H264: 46 // Table 7-2 – Assignment of mnemonic names to scaling list indices and 47 // specification of fall-back rule 48 static const uint8_t Default_4x4_Intra[16] = {6, 13, 13, 20, 20, 20, 28, 28, 49 28, 28, 32, 32, 32, 37, 37, 42}; 50 51 static const uint8_t Default_4x4_Inter[16] = {10, 14, 14, 20, 20, 20, 24, 24, 52 24, 24, 27, 27, 27, 30, 30, 34}; 53 54 static const uint8_t Default_8x8_Intra[64] = { 55 6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23, 56 23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 57 27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31, 58 31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42}; 59 60 static const uint8_t Default_8x8_Inter[64] = { 61 9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21, 62 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24, 63 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27, 64 27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35}; 65 66 namespace detail { 67 static void scaling_list(BitReader& aBr, uint8_t* aScalingList, 68 int aSizeOfScalingList, const uint8_t* aDefaultList, 69 const uint8_t* aFallbackList) { 70 int32_t lastScale = 8; 71 int32_t nextScale = 8; 72 int32_t deltaScale; 73 74 // (pic|seq)_scaling_list_present_flag[i] 75 if (!aBr.ReadBit()) { 76 if (aFallbackList) { 77 memcpy(aScalingList, aFallbackList, aSizeOfScalingList); 78 } 79 return; 80 } 81 82 for (int i = 0; i < aSizeOfScalingList; i++) { 83 if (nextScale != 0) { 84 deltaScale = aBr.ReadSE(); 85 nextScale = (lastScale + deltaScale + 256) % 256; 86 if (!i && !nextScale) { 87 memcpy(aScalingList, aDefaultList, aSizeOfScalingList); 88 return; 89 } 90 } 91 aScalingList[i] = (nextScale == 0) ? lastScale : nextScale; 92 lastScale = aScalingList[i]; 93 } 94 } 95 } // namespace detail. 96 97 template <size_t N> 98 static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N], 99 const uint8_t (&aDefaultList)[N], 100 const uint8_t (&aFallbackList)[N]) { 101 detail::scaling_list(aBr, aScalingList, N, aDefaultList, aFallbackList); 102 } 103 104 template <size_t N> 105 static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N], 106 const uint8_t (&aDefaultList)[N]) { 107 detail::scaling_list(aBr, aScalingList, N, aDefaultList, nullptr); 108 } 109 110 SPSData::SPSData() { 111 PodZero(this); 112 // Default values when they aren't defined as per ITU-T H.264 (2014/02). 113 chroma_format_idc = 1; 114 video_format = 5; 115 colour_primaries = 2; 116 transfer_characteristics = 2; 117 sample_ratio = 1.0; 118 memset(scaling_matrix4x4, 16, sizeof(scaling_matrix4x4)); 119 memset(scaling_matrix8x8, 16, sizeof(scaling_matrix8x8)); 120 } 121 122 bool SPSData::operator==(const SPSData& aOther) const { 123 return this->valid && aOther.valid && !memcmp(this, &aOther, sizeof(SPSData)); 124 } 125 126 bool SPSData::operator!=(const SPSData& aOther) const { 127 return !(operator==(aOther)); 128 } 129 130 static PrimaryID GetPrimaryID(int aPrimary) { 131 if (aPrimary < 1 || aPrimary > 22 || aPrimary == 3) { 132 return PrimaryID::INVALID; 133 } 134 if (aPrimary > 12 && aPrimary < 22) { 135 return PrimaryID::INVALID; 136 } 137 return static_cast<PrimaryID>(aPrimary); 138 } 139 140 static TransferID GetTransferID(int aTransfer) { 141 if (aTransfer < 1 || aTransfer > 18 || aTransfer == 3) { 142 return TransferID::INVALID; 143 } 144 return static_cast<TransferID>(aTransfer); 145 } 146 147 static MatrixID GetMatrixID(int aMatrix) { 148 if (aMatrix < 0 || aMatrix > 11 || aMatrix == 3) { 149 return MatrixID::INVALID; 150 } 151 return static_cast<MatrixID>(aMatrix); 152 } 153 154 gfx::YUVColorSpace SPSData::ColorSpace() const { 155 // Bitfield, note that guesses with higher values take precedence over 156 // guesses with lower values. 157 enum Guess { 158 GUESS_BT601 = 1 << 0, 159 GUESS_BT709 = 1 << 1, 160 GUESS_BT2020 = 1 << 2, 161 }; 162 163 uint32_t guess = 0; 164 165 switch (GetPrimaryID(colour_primaries)) { 166 case PrimaryID::BT709: 167 guess |= GUESS_BT709; 168 break; 169 case PrimaryID::BT470M: 170 case PrimaryID::BT470BG: 171 case PrimaryID::SMPTE170M: 172 case PrimaryID::SMPTE240M: 173 guess |= GUESS_BT601; 174 break; 175 case PrimaryID::BT2020: 176 guess |= GUESS_BT2020; 177 break; 178 case PrimaryID::FILM: 179 case PrimaryID::SMPTEST428_1: 180 case PrimaryID::SMPTEST431_2: 181 case PrimaryID::SMPTEST432_1: 182 case PrimaryID::EBU_3213_E: 183 case PrimaryID::INVALID: 184 case PrimaryID::UNSPECIFIED: 185 break; 186 } 187 188 switch (GetTransferID(transfer_characteristics)) { 189 case TransferID::BT709: 190 guess |= GUESS_BT709; 191 break; 192 case TransferID::GAMMA22: 193 case TransferID::GAMMA28: 194 case TransferID::SMPTE170M: 195 case TransferID::SMPTE240M: 196 guess |= GUESS_BT601; 197 break; 198 case TransferID::BT2020_10: 199 case TransferID::BT2020_12: 200 guess |= GUESS_BT2020; 201 break; 202 case TransferID::LINEAR: 203 case TransferID::LOG: 204 case TransferID::LOG_SQRT: 205 case TransferID::IEC61966_2_4: 206 case TransferID::BT1361_ECG: 207 case TransferID::IEC61966_2_1: 208 case TransferID::SMPTEST2084: 209 case TransferID::SMPTEST428_1: 210 case TransferID::ARIB_STD_B67: 211 case TransferID::INVALID: 212 case TransferID::UNSPECIFIED: 213 break; 214 } 215 216 switch (GetMatrixID(matrix_coefficients)) { 217 case MatrixID::BT709: 218 guess |= GUESS_BT709; 219 break; 220 case MatrixID::BT470BG: 221 case MatrixID::SMPTE170M: 222 case MatrixID::SMPTE240M: 223 guess |= GUESS_BT601; 224 break; 225 case MatrixID::BT2020_NCL: 226 case MatrixID::BT2020_CL: 227 guess |= GUESS_BT2020; 228 break; 229 case MatrixID::RGB: 230 case MatrixID::FCC: 231 case MatrixID::YCOCG: 232 case MatrixID::YDZDX: 233 case MatrixID::INVALID: 234 case MatrixID::UNSPECIFIED: 235 break; 236 } 237 238 // Removes lowest bit until only a single bit remains. 239 while (guess & (guess - 1)) { 240 guess &= guess - 1; 241 } 242 if (!guess) { 243 // A better default to BT601 which should die a slow death. 244 guess = GUESS_BT709; 245 } 246 247 switch (guess) { 248 case GUESS_BT601: 249 return gfx::YUVColorSpace::BT601; 250 case GUESS_BT709: 251 return gfx::YUVColorSpace::BT709; 252 case GUESS_BT2020: 253 return gfx::YUVColorSpace::BT2020; 254 default: 255 MOZ_CRASH("not possible to get here but makes compiler happy"); 256 } 257 } 258 259 gfx::ColorDepth SPSData::ColorDepth() const { 260 if (bit_depth_luma_minus8 != 0 && bit_depth_luma_minus8 != 2 && 261 bit_depth_luma_minus8 != 4) { 262 // We don't know what that is, just assume 8 bits to prevent decoding 263 // regressions if we ever encounter those. 264 return gfx::ColorDepth::COLOR_8; 265 } 266 return gfx::ColorDepthForBitDepth(bit_depth_luma_minus8 + 8); 267 } 268 269 // SPSNAL and SPSNALIterator do not own their data. 270 class SPSNAL { 271 public: 272 SPSNAL(const uint8_t* aPtr, size_t aLength) { 273 MOZ_ASSERT(aPtr); 274 275 if (aLength == 0 || (*aPtr & 0x1f) != H264_NAL_SPS) { 276 return; 277 } 278 mDecodedNAL = H264::DecodeNALUnit(aPtr, aLength); 279 if (mDecodedNAL) { 280 mLength = BitReader::GetBitLength(mDecodedNAL); 281 } 282 } 283 284 SPSNAL() = default; 285 286 bool IsValid() const { return mDecodedNAL; } 287 288 bool operator==(const SPSNAL& aOther) const { 289 if (!mDecodedNAL || !aOther.mDecodedNAL) { 290 return false; 291 } 292 293 SPSData decodedSPS1; 294 SPSData decodedSPS2; 295 if (!GetSPSData(decodedSPS1) || !aOther.GetSPSData(decodedSPS2)) { 296 // Couldn't decode one SPS, perform a binary comparison 297 if (mLength != aOther.mLength) { 298 return false; 299 } 300 MOZ_ASSERT(mLength / 8 <= mDecodedNAL->Length()); 301 302 if (memcmp(mDecodedNAL->Elements(), aOther.mDecodedNAL->Elements(), 303 mLength / 8) != 0) { 304 return false; 305 } 306 307 uint32_t remaining = mLength - (mLength & ~7); 308 309 BitReader b1(mDecodedNAL->Elements() + mLength / 8, remaining); 310 BitReader b2(aOther.mDecodedNAL->Elements() + mLength / 8, remaining); 311 for (uint32_t i = 0; i < remaining; i++) { 312 if (b1.ReadBit() != b2.ReadBit()) { 313 return false; 314 } 315 } 316 return true; 317 } 318 319 return decodedSPS1 == decodedSPS2; 320 } 321 322 bool operator!=(const SPSNAL& aOther) const { return !(operator==(aOther)); } 323 324 bool GetSPSData(SPSData& aDest) const { 325 return H264::DecodeSPS(mDecodedNAL, aDest); 326 } 327 328 private: 329 RefPtr<mozilla::MediaByteBuffer> mDecodedNAL; 330 uint32_t mLength = 0; 331 }; 332 333 class SPSNALIterator { 334 public: 335 explicit SPSNALIterator(const mozilla::MediaByteBuffer* aExtraData) 336 : mExtraDataPtr(aExtraData->Elements()), mReader(aExtraData) { 337 if (!mReader.Read(5)) { 338 return; 339 } 340 341 auto res = mReader.ReadU8(); 342 mNumSPS = res.isOk() ? res.unwrap() & 0x1f : 0; 343 if (mNumSPS == 0) { 344 return; 345 } 346 mValid = true; 347 } 348 349 SPSNALIterator& operator++() { 350 if (mEOS || !mValid) { 351 return *this; 352 } 353 if (--mNumSPS == 0) { 354 mEOS = true; 355 } 356 auto res = mReader.ReadU16(); 357 uint16_t length = res.isOk() ? res.unwrap() : 0; 358 if (length == 0 || !mReader.Read(length)) { 359 mEOS = true; 360 } 361 return *this; 362 } 363 364 explicit operator bool() const { return mValid && !mEOS; } 365 366 SPSNAL operator*() const { 367 MOZ_ASSERT(bool(*this)); 368 BufferReader reader(mExtraDataPtr + mReader.Offset(), mReader.Remaining()); 369 370 auto res = reader.ReadU16(); 371 uint16_t length = res.isOk() ? res.unwrap() : 0; 372 const uint8_t* ptr = reader.Read(length); 373 if (!ptr || !length) { 374 return SPSNAL(); 375 } 376 return SPSNAL(ptr, length); 377 } 378 379 private: 380 const uint8_t* mExtraDataPtr; 381 BufferReader mReader; 382 bool mValid = false; 383 bool mEOS = false; 384 uint8_t mNumSPS = 0; 385 }; 386 387 /* static */ Result<int, nsresult> H264::ExtractSVCTemporalId( 388 const uint8_t* aData, size_t aLength) { 389 nsTArray<AnnexB::NALEntry> paramSets; 390 AnnexB::ParseNALEntries(Span<const uint8_t>(aData, aLength), paramSets); 391 392 BufferReader reader(aData, aLength); 393 394 // Discard what's needed to find the correct NAL. 395 int i = 0; 396 while (paramSets[i].mSize < 4) { 397 i++; 398 } 399 reader.Read(paramSets[i].mOffset); 400 401 uint8_t byte = MOZ_TRY(reader.ReadU8()); 402 uint8_t nalUnitType = byte & 0x1f; 403 if (nalUnitType == H264_NAL_PREFIX || nalUnitType == H264_NAL_SLICE_EXT) { 404 bool svcExtensionFlag = false; 405 byte = MOZ_TRY(reader.ReadU8()); 406 svcExtensionFlag = byte & 0x80; 407 if (svcExtensionFlag) { 408 // Discard the first byte, and find the temporal id in the second byte 409 MOZ_TRY(reader.ReadU8()); 410 byte = MOZ_TRY(reader.ReadU8()); 411 int temporalId = (byte & 0xE0) >> 5; 412 return temporalId; 413 } 414 } 415 return 0; 416 } 417 418 /* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::DecodeNALUnit( 419 const uint8_t* aNAL, size_t aLength) { 420 MOZ_ASSERT(aNAL); 421 422 if (aLength < 4) { 423 return nullptr; 424 } 425 426 RefPtr<mozilla::MediaByteBuffer> rbsp = new mozilla::MediaByteBuffer; 427 BufferReader reader(aNAL, aLength); 428 auto res = reader.ReadU8(); 429 if (res.isErr()) { 430 return nullptr; 431 } 432 uint8_t nal_unit_type = res.unwrap() & 0x1f; 433 uint32_t nalUnitHeaderBytes = 1; 434 if (nal_unit_type == H264_NAL_PREFIX || nal_unit_type == H264_NAL_SLICE_EXT || 435 nal_unit_type == H264_NAL_SLICE_EXT_DVC) { 436 bool svc_extension_flag = false; 437 bool avc_3d_extension_flag = false; 438 if (nal_unit_type != H264_NAL_SLICE_EXT_DVC) { 439 res = reader.PeekU8(); 440 if (res.isErr()) { 441 return nullptr; 442 } 443 svc_extension_flag = res.unwrap() & 0x80; 444 } else { 445 res = reader.PeekU8(); 446 if (res.isErr()) { 447 return nullptr; 448 } 449 avc_3d_extension_flag = res.unwrap() & 0x80; 450 } 451 if (svc_extension_flag) { 452 nalUnitHeaderBytes += 3; 453 } else if (avc_3d_extension_flag) { 454 nalUnitHeaderBytes += 2; 455 } else { 456 nalUnitHeaderBytes += 3; 457 } 458 } 459 if (!reader.Read(nalUnitHeaderBytes - 1)) { 460 return nullptr; 461 } 462 uint32_t lastbytes = 0xffff; 463 while (reader.Remaining()) { 464 auto res = reader.ReadU8(); 465 if (res.isErr()) { 466 return nullptr; 467 } 468 uint8_t byte = res.unwrap(); 469 if ((lastbytes & 0xffff) == 0 && byte == 0x03) { 470 // reset last two bytes, to detect the 0x000003 sequence again. 471 lastbytes = 0xffff; 472 } else { 473 rbsp->AppendElement(byte); 474 } 475 lastbytes = (lastbytes << 8) | byte; 476 } 477 return rbsp.forget(); 478 } 479 480 // The reverse of DecodeNALUnit. To allow the distinction between Annex B (that 481 // uses 0x000001 as marker) and AVCC, the pattern 0x00 0x00 0x0n (where n is 482 // between 0 and 3) can't be found in the bytestream. A 0x03 byte is inserted 483 // after the second 0. Eg. 0x00 0x00 0x00 becomes 0x00 0x00 0x03 0x00 484 /* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::EncodeNALUnit( 485 const uint8_t* aNAL, size_t aLength) { 486 MOZ_ASSERT(aNAL); 487 RefPtr<MediaByteBuffer> rbsp = new MediaByteBuffer(); 488 BufferReader reader(aNAL, aLength); 489 490 auto res = reader.ReadU8(); 491 if (res.isErr()) { 492 return rbsp.forget(); 493 } 494 rbsp->AppendElement(res.unwrap()); 495 496 res = reader.ReadU8(); 497 if (res.isErr()) { 498 return rbsp.forget(); 499 } 500 rbsp->AppendElement(res.unwrap()); 501 502 while ((res = reader.ReadU8()).isOk()) { 503 uint8_t val = res.unwrap(); 504 if (val <= 0x03 && rbsp->ElementAt(rbsp->Length() - 2) == 0 && 505 rbsp->ElementAt(rbsp->Length() - 1) == 0) { 506 rbsp->AppendElement(0x03); 507 } 508 rbsp->AppendElement(val); 509 } 510 return rbsp.forget(); 511 } 512 513 static int32_t ConditionDimension(double aValue) { 514 // This will exclude NaNs and too-big values. 515 if (aValue > 1.0 && aValue <= double(INT32_MAX) / 2) { 516 return int32_t(aValue); 517 } 518 return 0; 519 } 520 521 /* static */ 522 bool H264::DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest) { 523 if (!aSPS) { 524 return false; 525 } 526 BitReader br(aSPS, BitReader::GetBitLength(aSPS)); 527 528 aDest.profile_idc = br.ReadBits(8); 529 aDest.constraint_set0_flag = br.ReadBit(); 530 aDest.constraint_set1_flag = br.ReadBit(); 531 aDest.constraint_set2_flag = br.ReadBit(); 532 aDest.constraint_set3_flag = br.ReadBit(); 533 aDest.constraint_set4_flag = br.ReadBit(); 534 aDest.constraint_set5_flag = br.ReadBit(); 535 br.ReadBits(2); // reserved_zero_2bits 536 aDest.level_idc = br.ReadBits(8); 537 READUE(seq_parameter_set_id, MAX_SPS_COUNT - 1); 538 539 if (aDest.profile_idc == 100 || aDest.profile_idc == 110 || 540 aDest.profile_idc == 122 || aDest.profile_idc == 244 || 541 aDest.profile_idc == 44 || aDest.profile_idc == 83 || 542 aDest.profile_idc == 86 || aDest.profile_idc == 118 || 543 aDest.profile_idc == 128 || aDest.profile_idc == 138 || 544 aDest.profile_idc == 139 || aDest.profile_idc == 134) { 545 READUE(chroma_format_idc, 3); 546 if (aDest.chroma_format_idc == 3) { 547 aDest.separate_colour_plane_flag = br.ReadBit(); 548 } 549 READUE(bit_depth_luma_minus8, 6); 550 READUE(bit_depth_chroma_minus8, 6); 551 br.ReadBit(); // qpprime_y_zero_transform_bypass_flag 552 aDest.seq_scaling_matrix_present_flag = br.ReadBit(); 553 if (aDest.seq_scaling_matrix_present_flag) { 554 scaling_list(br, aDest.scaling_matrix4x4[0], Default_4x4_Intra, 555 Default_4x4_Intra); 556 scaling_list(br, aDest.scaling_matrix4x4[1], Default_4x4_Intra, 557 aDest.scaling_matrix4x4[0]); 558 scaling_list(br, aDest.scaling_matrix4x4[2], Default_4x4_Intra, 559 aDest.scaling_matrix4x4[1]); 560 scaling_list(br, aDest.scaling_matrix4x4[3], Default_4x4_Inter, 561 Default_4x4_Inter); 562 scaling_list(br, aDest.scaling_matrix4x4[4], Default_4x4_Inter, 563 aDest.scaling_matrix4x4[3]); 564 scaling_list(br, aDest.scaling_matrix4x4[5], Default_4x4_Inter, 565 aDest.scaling_matrix4x4[4]); 566 567 scaling_list(br, aDest.scaling_matrix8x8[0], Default_8x8_Intra, 568 Default_8x8_Intra); 569 scaling_list(br, aDest.scaling_matrix8x8[1], Default_8x8_Inter, 570 Default_8x8_Inter); 571 if (aDest.chroma_format_idc == 3) { 572 scaling_list(br, aDest.scaling_matrix8x8[2], Default_8x8_Intra, 573 aDest.scaling_matrix8x8[0]); 574 scaling_list(br, aDest.scaling_matrix8x8[3], Default_8x8_Inter, 575 aDest.scaling_matrix8x8[1]); 576 scaling_list(br, aDest.scaling_matrix8x8[4], Default_8x8_Intra, 577 aDest.scaling_matrix8x8[2]); 578 scaling_list(br, aDest.scaling_matrix8x8[5], Default_8x8_Inter, 579 aDest.scaling_matrix8x8[3]); 580 } 581 } 582 } else if (aDest.profile_idc == 183) { 583 aDest.chroma_format_idc = 0; 584 } else { 585 // default value if chroma_format_idc isn't set. 586 aDest.chroma_format_idc = 1; 587 } 588 READUE(log2_max_frame_num, 12); 589 aDest.log2_max_frame_num += 4; 590 READUE(pic_order_cnt_type, 2); 591 if (aDest.pic_order_cnt_type == 0) { 592 READUE(log2_max_pic_order_cnt_lsb, 12); 593 aDest.log2_max_pic_order_cnt_lsb += 4; 594 } else if (aDest.pic_order_cnt_type == 1) { 595 aDest.delta_pic_order_always_zero_flag = br.ReadBit(); 596 READSE(offset_for_non_ref_pic, -231, 230); 597 READSE(offset_for_top_to_bottom_field, -231, 230); 598 uint32_t num_ref_frames_in_pic_order_cnt_cycle = br.ReadUE(); 599 for (uint32_t i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; i++) { 600 br.ReadSE(); // offset_for_ref_frame[i] 601 } 602 } 603 aDest.max_num_ref_frames = br.ReadUE(); 604 aDest.gaps_in_frame_num_allowed_flag = br.ReadBit(); 605 aDest.pic_width_in_mbs = br.ReadUE() + 1; 606 aDest.pic_height_in_map_units = br.ReadUE() + 1; 607 aDest.frame_mbs_only_flag = br.ReadBit(); 608 if (!aDest.frame_mbs_only_flag) { 609 aDest.pic_height_in_map_units *= 2; 610 aDest.mb_adaptive_frame_field_flag = br.ReadBit(); 611 } 612 aDest.direct_8x8_inference_flag = br.ReadBit(); 613 aDest.frame_cropping_flag = br.ReadBit(); 614 if (aDest.frame_cropping_flag) { 615 aDest.frame_crop_left_offset = br.ReadUE(); 616 aDest.frame_crop_right_offset = br.ReadUE(); 617 aDest.frame_crop_top_offset = br.ReadUE(); 618 aDest.frame_crop_bottom_offset = br.ReadUE(); 619 } 620 621 aDest.sample_ratio = 1.0f; 622 aDest.vui_parameters_present_flag = br.ReadBit(); 623 if (aDest.vui_parameters_present_flag) { 624 if (!vui_parameters(br, aDest)) { 625 return false; 626 } 627 } 628 629 // Calculate common values. 630 631 uint8_t ChromaArrayType = 632 aDest.separate_colour_plane_flag ? 0 : aDest.chroma_format_idc; 633 // Calculate width. 634 uint32_t CropUnitX = 1; 635 uint32_t SubWidthC = aDest.chroma_format_idc == 3 ? 1 : 2; 636 if (ChromaArrayType != 0) { 637 CropUnitX = SubWidthC; 638 } 639 640 // Calculate Height 641 uint32_t CropUnitY = 2 - aDest.frame_mbs_only_flag; 642 uint32_t SubHeightC = aDest.chroma_format_idc <= 1 ? 2 : 1; 643 if (ChromaArrayType != 0) { 644 CropUnitY *= SubHeightC; 645 } 646 647 uint32_t width = aDest.pic_width_in_mbs * 16; 648 uint32_t height = aDest.pic_height_in_map_units * 16; 649 if (aDest.frame_crop_left_offset <= 650 std::numeric_limits<int32_t>::max() / 4 / CropUnitX && 651 aDest.frame_crop_right_offset <= 652 std::numeric_limits<int32_t>::max() / 4 / CropUnitX && 653 aDest.frame_crop_top_offset <= 654 std::numeric_limits<int32_t>::max() / 4 / CropUnitY && 655 aDest.frame_crop_bottom_offset <= 656 std::numeric_limits<int32_t>::max() / 4 / CropUnitY && 657 (aDest.frame_crop_left_offset + aDest.frame_crop_right_offset) * 658 CropUnitX < 659 width && 660 (aDest.frame_crop_top_offset + aDest.frame_crop_bottom_offset) * 661 CropUnitY < 662 height) { 663 aDest.crop_left = aDest.frame_crop_left_offset * CropUnitX; 664 aDest.crop_right = aDest.frame_crop_right_offset * CropUnitX; 665 aDest.crop_top = aDest.frame_crop_top_offset * CropUnitY; 666 aDest.crop_bottom = aDest.frame_crop_bottom_offset * CropUnitY; 667 } else { 668 // Nonsensical value, ignore them. 669 aDest.crop_left = aDest.crop_right = aDest.crop_top = aDest.crop_bottom = 0; 670 } 671 672 aDest.pic_width = width - aDest.crop_left - aDest.crop_right; 673 aDest.pic_height = height - aDest.crop_top - aDest.crop_bottom; 674 675 aDest.interlaced = !aDest.frame_mbs_only_flag; 676 677 // Determine display size. 678 if (aDest.sample_ratio > 1.0) { 679 // Increase the intrinsic width 680 aDest.display_width = ConditionDimension( 681 AssertedCast<double>(aDest.pic_width) * aDest.sample_ratio); 682 aDest.display_height = aDest.pic_height; 683 } else { 684 // Increase the intrinsic height 685 aDest.display_width = aDest.pic_width; 686 aDest.display_height = ConditionDimension( 687 AssertedCast<double>(aDest.pic_height) / aDest.sample_ratio); 688 } 689 690 aDest.valid = true; 691 692 return true; 693 } 694 695 /* static */ 696 bool H264::vui_parameters(BitReader& aBr, SPSData& aDest) { 697 aDest.aspect_ratio_info_present_flag = aBr.ReadBit(); 698 if (aDest.aspect_ratio_info_present_flag) { 699 aDest.aspect_ratio_idc = aBr.ReadBits(8); 700 aDest.sar_width = aDest.sar_height = 0; 701 702 // From E.2.1 VUI parameters semantics (ITU-T H.264 02/2014) 703 switch (aDest.aspect_ratio_idc) { 704 case 0: 705 // Unspecified 706 break; 707 case 1: 708 /* 709 1:1 710 7680x4320 16:9 frame without horizontal overscan 711 3840x2160 16:9 frame without horizontal overscan 712 1280x720 16:9 frame without horizontal overscan 713 1920x1080 16:9 frame without horizontal overscan (cropped from 714 1920x1088) 640x480 4:3 frame without horizontal overscan 715 */ 716 aDest.sample_ratio = 1.0f; 717 break; 718 case 2: 719 /* 720 12:11 721 720x576 4:3 frame with horizontal overscan 722 352x288 4:3 frame without horizontal overscan 723 */ 724 aDest.sample_ratio = 12.0 / 11.0; 725 break; 726 case 3: 727 /* 728 10:11 729 720x480 4:3 frame with horizontal overscan 730 352x240 4:3 frame without horizontal overscan 731 */ 732 aDest.sample_ratio = 10.0 / 11.0; 733 break; 734 case 4: 735 /* 736 16:11 737 720x576 16:9 frame with horizontal overscan 738 528x576 4:3 frame without horizontal overscan 739 */ 740 aDest.sample_ratio = 16.0 / 11.0; 741 break; 742 case 5: 743 /* 744 40:33 745 720x480 16:9 frame with horizontal overscan 746 528x480 4:3 frame without horizontal overscan 747 */ 748 aDest.sample_ratio = 40.0 / 33.0; 749 break; 750 case 6: 751 /* 752 24:11 753 352x576 4:3 frame without horizontal overscan 754 480x576 16:9 frame with horizontal overscan 755 */ 756 aDest.sample_ratio = 24.0 / 11.0; 757 break; 758 case 7: 759 /* 760 20:11 761 352x480 4:3 frame without horizontal overscan 762 480x480 16:9 frame with horizontal overscan 763 */ 764 aDest.sample_ratio = 20.0 / 11.0; 765 break; 766 case 8: 767 /* 768 32:11 769 352x576 16:9 frame without horizontal overscan 770 */ 771 aDest.sample_ratio = 32.0 / 11.0; 772 break; 773 case 9: 774 /* 775 80:33 776 352x480 16:9 frame without horizontal overscan 777 */ 778 aDest.sample_ratio = 80.0 / 33.0; 779 break; 780 case 10: 781 /* 782 18:11 783 480x576 4:3 frame with horizontal overscan 784 */ 785 aDest.sample_ratio = 18.0 / 11.0; 786 break; 787 case 11: 788 /* 789 15:11 790 480x480 4:3 frame with horizontal overscan 791 */ 792 aDest.sample_ratio = 15.0 / 11.0; 793 break; 794 case 12: 795 /* 796 64:33 797 528x576 16:9 frame with horizontal overscan 798 */ 799 aDest.sample_ratio = 64.0 / 33.0; 800 break; 801 case 13: 802 /* 803 160:99 804 528x480 16:9 frame without horizontal overscan 805 */ 806 aDest.sample_ratio = 160.0 / 99.0; 807 break; 808 case 14: 809 /* 810 4:3 811 1440x1080 16:9 frame without horizontal overscan 812 */ 813 aDest.sample_ratio = 4.0 / 3.0; 814 break; 815 case 15: 816 /* 817 3:2 818 1280x1080 16:9 frame without horizontal overscan 819 */ 820 aDest.sample_ratio = 3.2 / 2.0; 821 break; 822 case 16: 823 /* 824 2:1 825 960x1080 16:9 frame without horizontal overscan 826 */ 827 aDest.sample_ratio = 2.0 / 1.0; 828 break; 829 case 255: 830 /* Extended_SAR */ 831 aDest.sar_width = aBr.ReadBits(16); 832 aDest.sar_height = aBr.ReadBits(16); 833 if (aDest.sar_width && aDest.sar_height) { 834 aDest.sample_ratio = float(aDest.sar_width) / float(aDest.sar_height); 835 } 836 break; 837 default: 838 break; 839 } 840 } 841 842 if (aBr.ReadBit()) { // overscan_info_present_flag 843 aDest.overscan_appropriate_flag = aBr.ReadBit(); 844 } 845 846 if (aBr.ReadBit()) { // video_signal_type_present_flag 847 aDest.video_format = aBr.ReadBits(3); 848 aDest.video_full_range_flag = aBr.ReadBit(); 849 aDest.colour_description_present_flag = aBr.ReadBit(); 850 if (aDest.colour_description_present_flag) { 851 aDest.colour_primaries = aBr.ReadBits(8); 852 aDest.transfer_characteristics = aBr.ReadBits(8); 853 aDest.matrix_coefficients = aBr.ReadBits(8); 854 } 855 } 856 857 aDest.chroma_loc_info_present_flag = aBr.ReadBit(); 858 if (aDest.chroma_loc_info_present_flag) { 859 BitReader& br = aBr; // so that macro READUE works 860 READUE(chroma_sample_loc_type_top_field, 5); 861 READUE(chroma_sample_loc_type_bottom_field, 5); 862 } 863 864 bool timing_info_present_flag = aBr.ReadBit(); 865 if (timing_info_present_flag) { 866 aBr.ReadBits(32); // num_units_in_tick 867 aBr.ReadBits(32); // time_scale 868 aBr.ReadBit(); // fixed_frame_rate_flag 869 } 870 return true; 871 } 872 873 /* static */ 874 bool H264::DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData, 875 SPSData& aDest) { 876 SPSNALIterator it(aExtraData); 877 if (!it) { 878 return false; 879 } 880 return (*it).GetSPSData(aDest); 881 } 882 883 /* static */ 884 bool H264::EnsureSPSIsSane(SPSData& aSPS) { 885 bool valid = true; 886 static const float default_aspect = 4.0f / 3.0f; 887 if (aSPS.sample_ratio <= 0.0f || aSPS.sample_ratio > 6.0f) { 888 if (aSPS.pic_width && aSPS.pic_height) { 889 aSPS.sample_ratio = (float)aSPS.pic_width / (float)aSPS.pic_height; 890 } else { 891 aSPS.sample_ratio = default_aspect; 892 } 893 aSPS.display_width = aSPS.pic_width; 894 aSPS.display_height = aSPS.pic_height; 895 valid = false; 896 } 897 if (aSPS.max_num_ref_frames > 16) { 898 aSPS.max_num_ref_frames = 16; 899 valid = false; 900 } 901 return valid; 902 } 903 904 /* static */ 905 uint32_t H264::ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData) { 906 uint32_t maxRefFrames = 4; 907 // Retrieve video dimensions from H264 SPS NAL. 908 SPSData spsdata; 909 if (DecodeSPSFromExtraData(aExtraData, spsdata)) { 910 // max_num_ref_frames determines the size of the sliding window 911 // we need to queue that many frames in order to guarantee proper 912 // pts frames ordering. Use a minimum of 4 to ensure proper playback of 913 // non compliant videos. 914 maxRefFrames = 915 std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u); 916 } 917 return maxRefFrames; 918 } 919 920 /* static */ H264::FrameType H264::GetFrameType( 921 const mozilla::MediaRawData* aSample) { 922 auto avcc = AVCCConfig::Parse(aSample); 923 if (avcc.isErr()) { 924 // We must have a valid AVCC frame with extradata. 925 return FrameType::INVALID; 926 } 927 MOZ_ASSERT(aSample->Data()); 928 929 int nalLenSize = avcc.unwrap().NALUSize(); 930 931 BufferReader reader(aSample->Data(), aSample->Size()); 932 933 while (reader.Remaining() >= nalLenSize) { 934 uint32_t nalLen = 0; 935 switch (nalLenSize) { 936 case 1: 937 nalLen = reader.ReadU8().unwrapOr(0); 938 break; 939 case 2: 940 nalLen = reader.ReadU16().unwrapOr(0); 941 break; 942 case 3: 943 nalLen = reader.ReadU24().unwrapOr(0); 944 break; 945 case 4: 946 nalLen = reader.ReadU32().unwrapOr(0); 947 break; 948 default: 949 MOZ_ASSERT_UNREACHABLE("NAL length is up to 4 bytes"); 950 } 951 if (!nalLen) { 952 continue; 953 } 954 const uint8_t* p = reader.Read(nalLen); 955 if (!p) { 956 return FrameType::INVALID; 957 } 958 int8_t nalType = AssertedCast<int8_t>(*p & 0x1f); 959 if (nalType == H264_NAL_IDR_SLICE) { 960 // IDR NAL. 961 return FrameType::I_FRAME_IDR; 962 } 963 if (nalType == H264_NAL_SEI) { 964 RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen); 965 SEIRecoveryData data; 966 if (DecodeRecoverySEI(decodedNAL, data)) { 967 // When both conditions are true, it means that starting decoding from 968 // an SEI frame will produce the same result as starting from an IDR 969 // frame, with no frame difference. If only one condition is true, it is 970 // not a true IDR substitute and may cause minor visual artifacts. 971 // However, since some video streams are incorrectly muxed without 972 // proper attributes, allowing playback with a few visual imperfections 973 // is preferable to failing to play them at all. 974 return (data.recovery_frame_cnt == 0 || data.exact_match_flag == 0) 975 ? FrameType::I_FRAME_IDR 976 : FrameType::I_FRAME_OTHER; 977 } 978 } else if (nalType == H264_NAL_SLICE) { 979 RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen); 980 if (DecodeISlice(decodedNAL)) { 981 return FrameType::I_FRAME_OTHER; 982 } 983 } 984 } 985 986 return FrameType::OTHER; 987 } 988 989 /* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::ExtractExtraData( 990 const mozilla::MediaRawData* aSample) { 991 auto avcc = AVCCConfig::Parse(aSample); 992 MOZ_ASSERT(avcc.isOk()); 993 994 RefPtr<mozilla::MediaByteBuffer> extradata = new mozilla::MediaByteBuffer; 995 996 // SPS content 997 nsTArray<uint8_t> sps; 998 ByteWriter<BigEndian> spsw(sps); 999 int numSps = 0; 1000 // PPS content 1001 nsTArray<uint8_t> pps; 1002 ByteWriter<BigEndian> ppsw(pps); 1003 int numPps = 0; 1004 1005 int nalLenSize = avcc.unwrap().NALUSize(); 1006 1007 size_t sampleSize = aSample->Size(); 1008 if (aSample->mCrypto.IsEncrypted()) { 1009 // The content is encrypted, we can only parse the non-encrypted data. 1010 MOZ_ASSERT(aSample->mCrypto.mPlainSizes.Length() > 0); 1011 if (aSample->mCrypto.mPlainSizes.Length() == 0 || 1012 aSample->mCrypto.mPlainSizes[0] > sampleSize) { 1013 // This is invalid content. 1014 return nullptr; 1015 } 1016 sampleSize = aSample->mCrypto.mPlainSizes[0]; 1017 } 1018 1019 BufferReader reader(aSample->Data(), sampleSize); 1020 1021 nsTArray<SPSData> SPSTable; 1022 // If we encounter SPS with the same id but different content, we will stop 1023 // attempting to detect duplicates. 1024 bool checkDuplicate = true; 1025 1026 // Find SPS and PPS NALUs in AVCC data 1027 while (reader.Remaining() > nalLenSize) { 1028 uint32_t nalLen = 0; 1029 switch (nalLenSize) { 1030 case 1: 1031 (void)reader.ReadU8().map( 1032 [&](uint8_t x) mutable { return nalLen = x; }); 1033 break; 1034 case 2: 1035 (void)reader.ReadU16().map( 1036 [&](uint16_t x) mutable { return nalLen = x; }); 1037 break; 1038 case 3: 1039 (void)reader.ReadU24().map( 1040 [&](uint32_t x) mutable { return nalLen = x; }); 1041 break; 1042 case 4: 1043 (void)reader.ReadU32().map( 1044 [&](uint32_t x) mutable { return nalLen = x; }); 1045 break; 1046 default: 1047 MOZ_ASSERT_UNREACHABLE("NAL length size is at most 4 bytes"); 1048 } 1049 const uint8_t* p = reader.Read(nalLen); 1050 if (!p) { 1051 // The read failed, but we may already have some SPS + PPS data so 1052 // break out of reading and process what we have, if any. 1053 break; 1054 } 1055 uint8_t nalType = *p & 0x1f; 1056 1057 if (nalType == H264_NAL_SPS) { 1058 RefPtr<mozilla::MediaByteBuffer> sps = DecodeNALUnit(p, nalLen); 1059 SPSData data; 1060 if (!DecodeSPS(sps, data)) { 1061 // Invalid SPS, ignore. 1062 continue; 1063 } 1064 uint8_t spsId = data.seq_parameter_set_id; 1065 if (spsId >= SPSTable.Length()) { 1066 if (!SPSTable.SetLength(spsId + 1, fallible)) { 1067 // OOM. 1068 return nullptr; 1069 } 1070 } 1071 if (checkDuplicate && SPSTable[spsId].valid && SPSTable[spsId] == data) { 1072 // Duplicate ignore. 1073 continue; 1074 } 1075 if (SPSTable[spsId].valid) { 1076 // We already have detected a SPS with this Id. Just to be safe we 1077 // disable SPS duplicate detection. 1078 checkDuplicate = false; 1079 } else { 1080 SPSTable[spsId] = data; 1081 } 1082 numSps++; 1083 if (!spsw.WriteU16(nalLen) || !spsw.Write(p, nalLen)) { 1084 return extradata.forget(); 1085 } 1086 } else if (nalType == H264_NAL_PPS) { 1087 numPps++; 1088 if (!ppsw.WriteU16(nalLen) || !ppsw.Write(p, nalLen)) { 1089 return extradata.forget(); 1090 } 1091 } 1092 } 1093 1094 // We ignore PPS data if we didn't find a SPS as we would be unable to 1095 // decode it anyway. 1096 numPps = numSps ? numPps : 0; 1097 1098 if (numSps && sps.Length() > 5) { 1099 extradata->AppendElement(1); // version 1100 extradata->AppendElement(sps[3]); // profile 1101 extradata->AppendElement(sps[4]); // profile compat 1102 extradata->AppendElement(sps[5]); // level 1103 extradata->AppendElement(0xfc | 3); // nal size - 1 1104 extradata->AppendElement(0xe0 | numSps); 1105 extradata->AppendElements(sps.Elements(), sps.Length()); 1106 extradata->AppendElement(numPps); 1107 if (numPps) { 1108 extradata->AppendElements(pps.Elements(), pps.Length()); 1109 } 1110 } 1111 1112 return extradata.forget(); 1113 } 1114 1115 /* static */ 1116 uint8_t H264::NumSPS(const mozilla::MediaByteBuffer* aExtraData) { 1117 auto avcc = AVCCConfig::Parse(aExtraData); 1118 return avcc.isErr() ? 0 : avcc.unwrap().NumSPS(); 1119 } 1120 1121 /* static */ 1122 bool H264::HasSPS(const mozilla::MediaByteBuffer* aExtraData) { 1123 return H264::NumSPS(aExtraData) > 0; 1124 } 1125 1126 /* static */ 1127 bool H264::CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1, 1128 const mozilla::MediaByteBuffer* aExtraData2) { 1129 if (aExtraData1 == aExtraData2) { 1130 return true; 1131 } 1132 uint8_t numSPS = NumSPS(aExtraData1); 1133 if (numSPS == 0 || numSPS != NumSPS(aExtraData2)) { 1134 return false; 1135 } 1136 1137 // We only compare if the SPS are the same as the various H264 decoders can 1138 // deal with in-band change of PPS. 1139 1140 SPSNALIterator it1(aExtraData1); 1141 SPSNALIterator it2(aExtraData2); 1142 1143 while (it1 && it2) { 1144 if (*it1 != *it2) { 1145 return false; 1146 } 1147 ++it1; 1148 ++it2; 1149 } 1150 return true; 1151 } 1152 1153 static inline Result<Ok, nsresult> ReadSEIInt(BufferReader& aBr, 1154 uint32_t& aOutput) { 1155 aOutput = 0; 1156 uint8_t tmpByte = MOZ_TRY(aBr.ReadU8()); 1157 while (tmpByte == 0xFF) { 1158 aOutput += 255; 1159 tmpByte = MOZ_TRY(aBr.ReadU8()); 1160 } 1161 aOutput += tmpByte; // this is the last byte 1162 return Ok(); 1163 } 1164 1165 /* static */ 1166 bool H264::DecodeISlice(const mozilla::MediaByteBuffer* aSlice) { 1167 if (!aSlice) { 1168 return false; 1169 } 1170 1171 // According to ITU-T Rec H.264 Table 7.3.3, read the slice type from 1172 // slice_header, and the slice type 2 and 7 are representing I slice. 1173 BitReader br(aSlice); 1174 // Skip `first_mb_in_slice` 1175 br.ReadUE(); 1176 // The value of slice type can go from 0 to 9, but the value between 5 to 1177 // 9 are actually equal to 0 to 4. 1178 const uint32_t sliceType = br.ReadUE() % 5; 1179 return sliceType == SLICE_TYPES::I_SLICE || sliceType == SI_SLICE; 1180 } 1181 1182 /* static */ 1183 bool H264::DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI, 1184 SEIRecoveryData& aDest) { 1185 if (!aSEI) { 1186 return false; 1187 } 1188 // sei_rbsp() as per 7.3.2.3 Supplemental enhancement information RBSP syntax 1189 BufferReader br(aSEI); 1190 1191 do { 1192 // sei_message() as per 1193 // 7.3.2.3.1 Supplemental enhancement information message syntax 1194 uint32_t payloadType = 0; 1195 if (ReadSEIInt(br, payloadType).isErr()) { 1196 return false; 1197 } 1198 1199 uint32_t payloadSize = 0; 1200 if (ReadSEIInt(br, payloadSize).isErr()) { 1201 return false; 1202 } 1203 1204 // sei_payload(payloadType, payloadSize) as per 1205 // D.1 SEI payload syntax. 1206 const uint8_t* p = br.Read(payloadSize); 1207 if (!p) { 1208 return false; 1209 } 1210 if (payloadType == 6) { // SEI_RECOVERY_POINT 1211 if (payloadSize == 0) { 1212 // Invalid content, ignore. 1213 continue; 1214 } 1215 // D.1.7 Recovery point SEI message syntax 1216 BitReader br(p, payloadSize * 8); 1217 aDest.recovery_frame_cnt = br.ReadUE(); 1218 aDest.exact_match_flag = br.ReadBit(); 1219 aDest.broken_link_flag = br.ReadBit(); 1220 aDest.changing_slice_group_idc = br.ReadBits(2); 1221 return true; 1222 } 1223 } while (br.PeekU8().isOk() && 1224 br.PeekU8().unwrap() != 1225 0x80); // more_rbsp_data() msg[offset] != 0x80 1226 // ignore the trailing bits rbsp_trailing_bits(); 1227 return false; 1228 } 1229 1230 /*static */ already_AddRefed<mozilla::MediaByteBuffer> H264::CreateExtraData( 1231 uint8_t aProfile, uint8_t aConstraints, H264_LEVEL aLevel, 1232 const gfx::IntSize& aSize) { 1233 // SPS of a 144p video. 1234 const uint8_t originSPS[] = {0x4d, 0x40, 0x0c, 0xe8, 0x80, 0x80, 0x9d, 1235 0x80, 0xb5, 0x01, 0x01, 0x01, 0x40, 0x00, 1236 0x00, 0x00, 0x40, 0x00, 0x00, 0x0f, 0x03, 1237 0xc5, 0x0a, 0x44, 0x80}; 1238 1239 RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer(); 1240 extraData->AppendElements(originSPS, sizeof(originSPS)); 1241 BitReader br(extraData, BitReader::GetBitLength(extraData)); 1242 1243 RefPtr<MediaByteBuffer> sps = new MediaByteBuffer(); 1244 BitWriter bw(sps); 1245 1246 br.ReadBits(8); // Skip original profile_idc 1247 bw.WriteU8(aProfile); 1248 br.ReadBits(8); // Skip original constraint flags && reserved_zero_2bits 1249 aConstraints = 1250 aConstraints & ~0x3; // Ensure reserved_zero_2bits are set to 0 1251 bw.WriteBits(aConstraints, 8); 1252 br.ReadBits(8); // Skip original level_idc 1253 bw.WriteU8(static_cast<uint8_t>(aLevel)); 1254 bw.WriteUE(br.ReadUE()); // seq_parameter_set_id (0 stored on 1 bit) 1255 1256 if (aProfile == 100 || aProfile == 110 || aProfile == 122 || 1257 aProfile == 244 || aProfile == 44 || aProfile == 83 || aProfile == 86 || 1258 aProfile == 118 || aProfile == 128 || aProfile == 138 || 1259 aProfile == 139 || aProfile == 134) { 1260 bw.WriteUE(1); // chroma_format_idc -> always set to 4:2:0 chroma format 1261 bw.WriteUE(0); // bit_depth_luma_minus8 -> always 8 bits here 1262 bw.WriteUE(0); // bit_depth_chroma_minus8 -> always 8 bits here 1263 bw.WriteBit(false); // qpprime_y_zero_transform_bypass_flag 1264 bw.WriteBit(false); // seq_scaling_matrix_present_flag 1265 } 1266 1267 bw.WriteBits(br.ReadBits(11), 1268 11); // log2_max_frame_num to gaps_in_frame_num_allowed_flag 1269 1270 // skip over original exp-golomb encoded width/height 1271 br.ReadUE(); // skip width 1272 br.ReadUE(); // skip height 1273 uint32_t width = aSize.width; 1274 uint32_t widthNeeded = width % 16 != 0 ? (width / 16 + 1) * 16 : width; 1275 uint32_t height = aSize.height; 1276 uint32_t heightNeeded = height % 16 != 0 ? (height / 16 + 1) * 16 : height; 1277 bw.WriteUE(widthNeeded / 16 - 1); 1278 bw.WriteUE(heightNeeded / 16 - 1); 1279 bw.WriteBit(br.ReadBit()); // write frame_mbs_only_flag 1280 bw.WriteBit(br.ReadBit()); // write direct_8x8_inference_flag; 1281 if (widthNeeded != width || heightNeeded != height) { 1282 // Write cropping value 1283 bw.WriteBit(true); // skip frame_cropping_flag 1284 bw.WriteUE(0); // frame_crop_left_offset 1285 bw.WriteUE((widthNeeded - width) / 2); // frame_crop_right_offset 1286 bw.WriteUE(0); // frame_crop_top_offset 1287 bw.WriteUE((heightNeeded - height) / 2); // frame_crop_bottom_offset 1288 } else { 1289 bw.WriteBit(false); // skip frame_cropping_flag 1290 } 1291 br.ReadBit(); // skip frame_cropping_flag; 1292 // Write the remainings of the original sps (vui_parameters which sets an 1293 // aspect ration of 1.0) 1294 while (br.BitsLeft()) { 1295 bw.WriteBit(br.ReadBit()); 1296 } 1297 bw.CloseWithRbspTrailing(); 1298 1299 RefPtr<MediaByteBuffer> encodedSPS = 1300 EncodeNALUnit(sps->Elements(), sps->Length()); 1301 extraData->Clear(); 1302 1303 const uint8_t PPS[] = {0xeb, 0xef, 0x20}; 1304 1305 WriteExtraData( 1306 extraData, aProfile, aConstraints, static_cast<uint8_t>(aLevel), 1307 Span<const uint8_t>(encodedSPS->Elements(), encodedSPS->Length()), 1308 Span<const uint8_t>(PPS, sizeof(PPS))); 1309 1310 return extraData.forget(); 1311 } 1312 1313 void H264::WriteExtraData(MediaByteBuffer* aDestExtraData, 1314 const uint8_t aProfile, const uint8_t aConstraints, 1315 const uint8_t aLevel, const Span<const uint8_t> aSPS, 1316 const Span<const uint8_t> aPPS) { 1317 aDestExtraData->AppendElement(1); 1318 aDestExtraData->AppendElement(aProfile); 1319 aDestExtraData->AppendElement(aConstraints); 1320 aDestExtraData->AppendElement(aLevel); 1321 aDestExtraData->AppendElement(3); // nalLENSize-1 1322 aDestExtraData->AppendElement(1); // numPPS 1323 uint8_t c[2]; 1324 mozilla::BigEndian::writeUint16(&c[0], aSPS.Length() + 1); 1325 aDestExtraData->AppendElements(c, 2); 1326 aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_SPS); 1327 aDestExtraData->AppendElements(aSPS.Elements(), aSPS.Length()); 1328 1329 aDestExtraData->AppendElement(1); // numPPS 1330 mozilla::BigEndian::writeUint16(&c[0], aPPS.Length() + 1); 1331 aDestExtraData->AppendElements(c, 2); 1332 aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_PPS); 1333 aDestExtraData->AppendElements(aPPS.Elements(), aPPS.Length()); 1334 } 1335 1336 /* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse( 1337 const mozilla::MediaRawData* aSample) { 1338 if (!aSample || aSample->Size() < 3) { 1339 return mozilla::Err(NS_ERROR_FAILURE); 1340 } 1341 if (aSample->mTrackInfo && 1342 !aSample->mTrackInfo->mMimeType.EqualsLiteral("video/avc")) { 1343 LOG("Only allow 'video/avc' (mimeType=%s)", 1344 aSample->mTrackInfo->mMimeType.get()); 1345 return mozilla::Err(NS_ERROR_FAILURE); 1346 } 1347 return AVCCConfig::Parse(aSample->mExtraData); 1348 } 1349 1350 /* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse( 1351 const mozilla::MediaByteBuffer* aExtraData) { 1352 if (!aExtraData || aExtraData->Length() < 7) { 1353 return mozilla::Err(NS_ERROR_FAILURE); 1354 } 1355 AVCCConfig avcc{}; 1356 BitReader reader(aExtraData); 1357 1358 avcc.mConfigurationVersion = reader.ReadBits(8); 1359 if (avcc.mConfigurationVersion != 1) { 1360 LOG("Invalid configuration version %u", avcc.mConfigurationVersion); 1361 return mozilla::Err(NS_ERROR_FAILURE); 1362 } 1363 avcc.mAVCProfileIndication = reader.ReadBits(8); 1364 avcc.mProfileCompatibility = reader.ReadBits(8); 1365 avcc.mAVCLevelIndication = reader.ReadBits(8); 1366 (void)reader.ReadBits(6); // reserved 1367 avcc.mLengthSizeMinusOne = reader.ReadBits(2); 1368 (void)reader.ReadBits(3); // reserved 1369 const uint8_t numSPS = reader.ReadBits(5); 1370 for (uint8_t idx = 0; idx < numSPS; idx++) { 1371 if (reader.BitsLeft() < 16) { 1372 LOG("Aborting parsing, not enough bits (16) for SPS length!"); 1373 return mozilla::Err(NS_ERROR_FAILURE); 1374 } 1375 uint16_t sequenceParameterSetLength = reader.ReadBits(16); 1376 uint32_t spsBitsLength = sequenceParameterSetLength * 8; 1377 const uint8_t* spsPtr = aExtraData->Elements() + reader.BitCount() / 8; 1378 if (reader.AdvanceBits(spsBitsLength) < spsBitsLength) { 1379 LOG("Aborting parsing, SPS NALU size (%u bits) is larger than remaining!", 1380 spsBitsLength); 1381 return mozilla::Err(NS_ERROR_FAILURE); 1382 } 1383 H264NALU nalu(spsPtr, sequenceParameterSetLength); 1384 if (nalu.mNalUnitType != H264_NAL_SPS) { 1385 LOG("Aborting parsing, expect SPS but got incorrect NALU type (%d)!", 1386 nalu.mNalUnitType); 1387 return mozilla::Err(NS_ERROR_FAILURE); 1388 } 1389 avcc.mSPSs.AppendElement(nalu); 1390 } 1391 if (reader.BitsLeft() < 8) { 1392 LOG("Missing numOfPictureParameterSets"); 1393 return mozilla::Err(NS_ERROR_FAILURE); 1394 } 1395 const uint8_t numPPS = reader.ReadBits(8); 1396 for (uint8_t idx = 0; idx < numPPS; idx++) { 1397 if (reader.BitsLeft() < 16) { 1398 LOG("Aborting parsing, not enough bits (16) for PPS length!"); 1399 return mozilla::Err(NS_ERROR_FAILURE); 1400 } 1401 uint16_t pictureParameterSetLength = reader.ReadBits(16); 1402 uint32_t ppsBitsLength = pictureParameterSetLength * 8; 1403 const uint8_t* ppsPtr = aExtraData->Elements() + reader.BitCount() / 8; 1404 if (reader.AdvanceBits(ppsBitsLength) < ppsBitsLength) { 1405 LOG("Aborting parsing, PPS NALU size (%u bits) is larger than remaining!", 1406 ppsBitsLength); 1407 return mozilla::Err(NS_ERROR_FAILURE); 1408 } 1409 H264NALU nalu(ppsPtr, pictureParameterSetLength); 1410 if (nalu.mNalUnitType != H264_NAL_PPS) { 1411 LOG("Aborting parsing, expect PPS but got incorrect NALU type (%d)!", 1412 nalu.mNalUnitType); 1413 return mozilla::Err(NS_ERROR_FAILURE); 1414 } 1415 avcc.mPPSs.AppendElement(nalu); 1416 } 1417 1418 // The AVCDecoderConfigurationRecord syntax requires that the SPSExt must be 1419 // present if AVCProfileIndication is not 66 (Baseline), 77 (Main), or 88 1420 // (Extended). However, in practice, many H.264 streams in the wild omit the 1421 // SPSExt fields, especially when default values are used (e.g., 1422 // chroma_format_idc = 1 for 4:2:0 chroma format, bit_depth_luma_minus8 = 0, 1423 // and bit_depth_chroma_minus8 = 0) Therefore, parsing this part is not 1424 // mandatory and fail to parse this part will not cause an actual error. 1425 // Instead, we will simply clear the incorrect result. 1426 if (avcc.mAVCProfileIndication != 66 && avcc.mAVCProfileIndication != 77 && 1427 avcc.mAVCProfileIndication != 88 && reader.BitsLeft() >= 32) { 1428 (void)reader.ReadBits(6); // reserved 1429 avcc.mChromaFormat = Some(reader.ReadBits(2)); 1430 (void)reader.ReadBits(5); // reserved 1431 avcc.mBitDepthLumaMinus8 = Some(reader.ReadBits(3)); 1432 (void)reader.ReadBits(5); // reserved 1433 avcc.mBitDepthChromaMinus8 = Some(reader.ReadBits(3)); 1434 const uint8_t numOfSequenceParameterSetExt = reader.ReadBits(8); 1435 for (uint8_t idx = 0; idx < numOfSequenceParameterSetExt; idx++) { 1436 if (reader.BitsLeft() < 16) { 1437 LOG("Aborting parsing, not enough bits (16) for SPSExt length!"); 1438 break; 1439 } 1440 uint16_t sequenceParameterSetExtLength = reader.ReadBits(16); 1441 uint32_t spsExtBitsLength = sequenceParameterSetExtLength * 8; 1442 const uint8_t* spsExtPtr = aExtraData->Elements() + reader.BitCount() / 8; 1443 if (reader.AdvanceBits(spsExtBitsLength) < spsExtBitsLength) { 1444 LOG("Aborting parsing, SPS Ext NALU size (%u bits) is larger than " 1445 "remaining!", 1446 spsExtBitsLength); 1447 break; 1448 } 1449 H264NALU nalu(spsExtPtr, sequenceParameterSetExtLength); 1450 if (nalu.mNalUnitType != H264_NAL_SPS_EXT) { 1451 LOG("Aborting parsing, expect SPSExt but got incorrect NALU type " 1452 "(%d)!", 1453 nalu.mNalUnitType); 1454 break; 1455 } 1456 avcc.mSPSExts.AppendElement(nalu); 1457 } 1458 if (avcc.mSPSExts.Length() != numOfSequenceParameterSetExt) { 1459 LOG("Failed to parse all SPSExt, and soft fail."); 1460 } 1461 } 1462 return avcc; 1463 } 1464 1465 already_AddRefed<mozilla::MediaByteBuffer> AVCCConfig::CreateNewExtraData() 1466 const { 1467 auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); 1468 BitWriter writer(extradata); 1469 writer.WriteBits(mConfigurationVersion, 8); 1470 writer.WriteBits(mAVCProfileIndication, 8); 1471 writer.WriteBits(mProfileCompatibility, 8); 1472 writer.WriteBits(mAVCLevelIndication, 8); 1473 writer.WriteBits(0x111111, 6); // reserved 1474 writer.WriteBits(mLengthSizeMinusOne, 2); 1475 writer.WriteBits(0x111, 3); // reserved 1476 writer.WriteBits(mSPSs.Length(), 5); 1477 for (const auto& nalu : mSPSs) { 1478 writer.WriteBits(nalu.mNALU.Length(), 16); 1479 MOZ_DIAGNOSTIC_ASSERT(writer.BitCount() % 8 == 0); 1480 extradata->AppendElements(nalu.mNALU.Elements(), nalu.mNALU.Length()); 1481 writer.AdvanceBytes(nalu.mNALU.Length()); 1482 } 1483 writer.WriteBits(mPPSs.Length(), 8); 1484 for (const auto& nalu : mPPSs) { 1485 writer.WriteBits(nalu.mNALU.Length(), 16); 1486 MOZ_DIAGNOSTIC_ASSERT(writer.BitCount() % 8 == 0); 1487 extradata->AppendElements(nalu.mNALU.Elements(), nalu.mNALU.Length()); 1488 writer.AdvanceBytes(nalu.mNALU.Length()); 1489 } 1490 if (mAVCProfileIndication != 66 && mAVCProfileIndication != 77 && 1491 mAVCProfileIndication != 88 && mChromaFormat.isSome() && 1492 mBitDepthLumaMinus8.isSome() && mBitDepthChromaMinus8.isSome()) { 1493 writer.WriteBits(0x111111, 6); // reserved 1494 writer.WriteBits(*mChromaFormat, 2); 1495 writer.WriteBits(0x11111, 5); // reserved 1496 writer.WriteBits(*mBitDepthLumaMinus8, 3); 1497 writer.WriteBits(0x11111, 5); // reserved 1498 writer.WriteBits(*mBitDepthChromaMinus8, 3); 1499 writer.WriteBits(mSPSExts.Length(), 8); 1500 for (const auto& nalu : mSPSExts) { 1501 writer.WriteBits(nalu.mNALU.Length(), 16); 1502 MOZ_DIAGNOSTIC_ASSERT(writer.BitCount() % 8 == 0); 1503 extradata->AppendElements(nalu.mNALU.Elements(), nalu.mNALU.Length()); 1504 writer.AdvanceBytes(nalu.mNALU.Length()); 1505 } 1506 } 1507 return AVCCConfig::Parse(extradata).isOk() ? extradata.forget() : nullptr; 1508 } 1509 1510 H264NALU::H264NALU(const uint8_t* aData, uint32_t aByteCount) 1511 : mNALU(aData, aByteCount) { 1512 // Per 7.3.1 NAL unit syntax 1513 BitReader reader(aData, aByteCount * 8); 1514 (void)reader.ReadBit(); // forbidden_zero_bit 1515 (void)reader.ReadBits(2); // nal_ref_idc 1516 mNalUnitType = reader.ReadBits(5); 1517 } 1518 1519 #undef READUE 1520 #undef READSE 1521 1522 } // namespace mozilla 1523 1524 #undef LOG