MediaChangeMonitor.cpp (53747B)
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "MediaChangeMonitor.h" 8 9 #include "Adts.h" 10 #include "AnnexB.h" 11 #include "GeckoProfiler.h" 12 #include "H264.h" 13 #include "H265.h" 14 #include "ImageContainer.h" 15 #include "MP4Decoder.h" 16 #include "MediaInfo.h" 17 #include "PDMFactory.h" 18 #include "VPXDecoder.h" 19 #include "nsPrintfCString.h" 20 #ifdef MOZ_AV1 21 # include "AOMDecoder.h" 22 #endif 23 #include "gfxUtils.h" 24 #include "mozilla/ProfilerMarkers.h" 25 #include "mozilla/StaticPrefs_media.h" 26 #include "mozilla/TaskQueue.h" 27 28 namespace mozilla { 29 30 extern LazyLogModule gMediaDecoderLog; 31 32 #define LOG(x, ...) \ 33 MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (x, ##__VA_ARGS__)) 34 35 #define LOGV(x, ...) \ 36 MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, (x, ##__VA_ARGS__)) 37 38 // Gets the pixel aspect ratio from the decoded video size and the rendered 39 // size. 40 inline double GetPixelAspectRatio(const gfx::IntSize& aImage, 41 const gfx::IntSize& aDisplay) { 42 if (MOZ_UNLIKELY(aImage.IsEmpty() || aDisplay.IsEmpty())) { 43 return 0.0; 44 } 45 return (static_cast<double>(aDisplay.Width()) / aImage.Width()) / 46 (static_cast<double>(aDisplay.Height()) / aImage.Height()); 47 } 48 49 // Returns the render size based on the PAR and the new image size. 50 inline gfx::IntSize ApplyPixelAspectRatio(double aPixelAspectRatio, 51 const gfx::IntSize& aImage) { 52 // No need to apply PAR, or an invalid PAR. 53 if (aPixelAspectRatio == 1.0 || MOZ_UNLIKELY(aPixelAspectRatio <= 0)) { 54 return aImage; 55 } 56 double width = aImage.Width() * aPixelAspectRatio; 57 // Ignore values that would cause overflow. 58 if (MOZ_UNLIKELY(width > std::numeric_limits<int32_t>::max())) { 59 return aImage; 60 } 61 return gfx::IntSize(static_cast<int32_t>(width), aImage.Height()); 62 } 63 64 static bool IsBeingProfiledOrLogEnabled() { 65 return MOZ_LOG_TEST(gMediaDecoderLog, LogLevel::Info) || 66 profiler_thread_is_being_profiled_for_markers(); 67 } 68 69 // H264ChangeMonitor is used to ensure that only AVCC or AnnexB is fed to the 70 // underlying MediaDataDecoder. The H264ChangeMonitor allows playback of content 71 // where the SPS NAL may not be provided in the init segment (e.g. AVC3 or Annex 72 // B) H264ChangeMonitor will monitor the input data, and will delay creation of 73 // the MediaDataDecoder until a SPS and PPS NALs have been extracted. 74 75 class H264ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { 76 public: 77 explicit H264ChangeMonitor(const CreateDecoderParams& aParams) 78 : mCurrentConfig(aParams.VideoConfig()), 79 mFullParsing(aParams.mOptions.contains( 80 CreateDecoderParams::Option::FullH264Parsing)) 81 #ifdef MOZ_WMF_MEDIA_ENGINE 82 , 83 mIsMediaEnginePlayback(aParams.mMediaEngineId.isSome()) 84 #endif 85 { 86 if (CanBeInstantiated()) { 87 UpdateConfigFromExtraData(mCurrentConfig.mExtraData); 88 auto avcc = AVCCConfig::Parse(mCurrentConfig.mExtraData); 89 if (avcc.isOk() && avcc.unwrap().NALUSize() != 4) { 90 // `CheckForChange()` will use `AnnexB::ConvertSampleToAVCC()` to change 91 // NAL units into 4-byte. 92 // `AVCDecoderConfigurationRecord.lengthSizeMinusOne` in the config 93 // should be modified too. 94 mCurrentConfig.mExtraData->ReplaceElementAt(4, 0xfc | 3); 95 } 96 } 97 } 98 99 bool CanBeInstantiated() const override { 100 return H264::HasSPS(mCurrentConfig.mExtraData); 101 } 102 103 MediaResult CheckForChange(MediaRawData* aSample) override { 104 // To be usable we need to convert the sample to 4 bytes NAL size AVCC. 105 if (!AnnexB::ConvertSampleToAVCC(aSample)) { 106 // We need AVCC content to be able to later parse the SPS. 107 // This is a no-op if the data is already AVCC. 108 return MediaResult(NS_ERROR_OUT_OF_MEMORY, 109 RESULT_DETAIL("ConvertSampleToAVCC")); 110 } 111 112 if (!AnnexB::IsAVCC(aSample)) { 113 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 114 RESULT_DETAIL("Invalid H264 content")); 115 } 116 117 RefPtr<MediaByteBuffer> extra_data = 118 aSample->mKeyframe || !mGotSPS || mFullParsing 119 ? H264::ExtractExtraData(aSample) 120 : nullptr; 121 122 if (!H264::HasSPS(extra_data) && !H264::HasSPS(mCurrentConfig.mExtraData)) { 123 // We don't have inband data and the original config didn't contain a SPS. 124 // We can't decode this content. 125 return NS_ERROR_NOT_INITIALIZED; 126 } 127 128 mGotSPS = true; 129 130 if (!H264::HasSPS(extra_data)) { 131 // This sample doesn't contain inband SPS/PPS 132 // We now check if the out of band one has changed. 133 // This scenario can currently only occur on Android with devices that can 134 // recycle a decoder. 135 bool hasOutOfBandExtraData = H264::HasSPS(aSample->mExtraData); 136 if (!hasOutOfBandExtraData || !mPreviousExtraData || 137 H264::CompareExtraData(aSample->mExtraData, mPreviousExtraData)) { 138 if (hasOutOfBandExtraData && !mPreviousExtraData) { 139 // We are decoding the first sample, store the out of band sample's 140 // extradata so that we can check for future change. 141 mPreviousExtraData = aSample->mExtraData; 142 } 143 return NS_OK; 144 } 145 extra_data = aSample->mExtraData; 146 } else { 147 // A situation where inband SPS exists in the sample. 148 #ifdef MOZ_WMF_MEDIA_ENGINE 149 extra_data = MergeParameterSetsWhenInbandSPSExists(extra_data); 150 #endif 151 if (H264::CompareExtraData(extra_data, mCurrentConfig.mExtraData)) { 152 return NS_OK; 153 } 154 } 155 156 // Store the sample's extradata so we don't trigger a false positive 157 // with the out of band test on the next sample. 158 mPreviousExtraData = aSample->mExtraData; 159 UpdateConfigFromExtraData(extra_data); 160 161 if (IsBeingProfiledOrLogEnabled()) { 162 nsPrintfCString msg( 163 "H264ChangeMonitor::CheckForChange has detected a " 164 "change in the stream and will request a new decoder"); 165 LOG("%s", msg.get()); 166 PROFILER_MARKER_TEXT("H264 Stream Change", MEDIA_PLAYBACK, {}, msg); 167 } 168 return NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER; 169 } 170 171 const TrackInfo& Config() const override { return mCurrentConfig; } 172 173 MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion, 174 MediaRawData* aSample, 175 bool aNeedKeyFrame) override { 176 MOZ_DIAGNOSTIC_ASSERT( 177 aConversion == MediaDataDecoder::ConversionRequired::kNeedAnnexB || 178 aConversion == MediaDataDecoder::ConversionRequired::kNeedAVCC, 179 "Conversion must be either AVCC or AnnexB"); 180 181 aSample->mExtraData = mCurrentConfig.mExtraData; 182 aSample->mTrackInfo = mTrackInfo; 183 184 bool appendExtradata = aNeedKeyFrame; 185 #ifdef MOZ_WMF_MEDIA_ENGINE 186 // The error SPR_E_INVALID_H264_SLICE_HEADERS is caused by the media engine 187 // being unable to handle an IDR frame without a valid SPS. Therefore, we 188 // ensure that SPS should always be presented in the bytestream for all IDR 189 // frames. 190 if (mIsMediaEnginePlayback && 191 H264::GetFrameType(aSample) == H264::FrameType::I_FRAME_IDR) { 192 RefPtr<MediaByteBuffer> extradata = H264::ExtractExtraData(aSample); 193 appendExtradata = aNeedKeyFrame || !H264::HasSPS(extradata); 194 LOG("%s need to append extradata for IDR sample [%" PRId64 ",%" PRId64 195 "]", 196 appendExtradata ? "Do" : "No", aSample->mTime.ToMicroseconds(), 197 aSample->GetEndTime().ToMicroseconds()); 198 } 199 #endif 200 201 if (aConversion == MediaDataDecoder::ConversionRequired::kNeedAnnexB) { 202 auto res = AnnexB::ConvertAVCCSampleToAnnexB(aSample, appendExtradata); 203 if (res.isErr()) { 204 return MediaResult(res.unwrapErr(), 205 RESULT_DETAIL("ConvertSampleToAnnexB")); 206 } 207 } 208 209 return NS_OK; 210 } 211 212 private: 213 void UpdateConfigFromExtraData(MediaByteBuffer* aExtraData) { 214 SPSData spsdata; 215 if (H264::DecodeSPSFromExtraData(aExtraData, spsdata) && 216 spsdata.pic_width > 0 && spsdata.pic_height > 0) { 217 H264::EnsureSPSIsSane(spsdata); 218 mCurrentConfig.mImage.width = spsdata.pic_width; 219 mCurrentConfig.mImage.height = spsdata.pic_height; 220 mCurrentConfig.mDisplay.width = spsdata.display_width; 221 mCurrentConfig.mDisplay.height = spsdata.display_height; 222 mCurrentConfig.mColorDepth = spsdata.ColorDepth(); 223 mCurrentConfig.mColorSpace = Some(spsdata.ColorSpace()); 224 // spsdata.colour_primaries has the same values as 225 // gfx::CICP::ColourPrimaries. 226 mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries( 227 static_cast<gfx::CICP::ColourPrimaries>(spsdata.colour_primaries), 228 gMediaDecoderLog); 229 // spsdata.transfer_characteristics has the same values as 230 // gfx::CICP::TransferCharacteristics. 231 mCurrentConfig.mTransferFunction = gfxUtils::CicpToTransferFunction( 232 static_cast<gfx::CICP::TransferCharacteristics>( 233 spsdata.transfer_characteristics)); 234 mCurrentConfig.mColorRange = spsdata.video_full_range_flag 235 ? gfx::ColorRange::FULL 236 : gfx::ColorRange::LIMITED; 237 } 238 mCurrentConfig.mExtraData = aExtraData; 239 mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++); 240 } 241 242 #ifdef MOZ_WMF_MEDIA_ENGINE 243 // Merge aExtraData, containing in-band parameter set updates having at least 244 // one SPS, with mCurrentConfig.mExtraData, and return the merged 245 // AVCDecoderConfigurationRecord. The existing PPSs (if any) are retained iff 246 // no new PPSs are in aExtraData. Partial updates of only some SPSs or of 247 // only some PPSs are not yet supported. 248 RefPtr<MediaByteBuffer> MergeParameterSetsWhenInbandSPSExists( 249 MediaByteBuffer* aExtraData) const { 250 // TODO : consider to enable this for other decoders if necessary in bug 251 // 1973611 252 if (!mIsMediaEnginePlayback) { 253 return aExtraData; 254 } 255 256 auto res = AVCCConfig::Parse(aExtraData); 257 MOZ_ASSERT(res.isOk()); 258 auto avccNew = res.unwrap(); 259 if (avccNew.NumPPS() != 0) { 260 // New extradata already has PPS. 261 return aExtraData; 262 } 263 264 // A case where the new extradata includes an SPS change but lacks a 265 // PPS. This implies that the PPS might be present in the previous 266 // extradata, making it a candidate for reuse. This refinement could 267 // potentially resolve the DRM_E_H264_SH_PPS_NOT_FOUND error. 268 res = AVCCConfig::Parse(mCurrentConfig.mExtraData); 269 if (res.isErr()) { 270 return aExtraData; 271 } 272 const auto avccOld = res.unwrap(); 273 if (avccOld.NumPPS() == 0) { 274 // Still no PPS, there is nothing we can do. 275 return aExtraData; 276 } 277 278 // Reuse the previous PPS then generate a new extradata. 279 MOZ_ASSERT(avccNew.NumPPS() == 0 && avccOld.NumPPS() != 0); 280 avccNew.mPPSs.AppendElements(avccOld.mPPSs); 281 if (RefPtr<MediaByteBuffer> newExtraData = avccNew.CreateNewExtraData()) { 282 LOG("Refining extradata by inserting PPS to ensure both SPS and PPS " 283 "are present"); 284 return newExtraData; 285 } 286 return aExtraData; 287 } 288 #endif 289 290 VideoInfo mCurrentConfig; 291 uint32_t mStreamID = 0; 292 const bool mFullParsing; 293 #ifdef MOZ_WMF_MEDIA_ENGINE 294 // True if the playback is performed by Windows Media Foundation Engine. 295 const bool mIsMediaEnginePlayback; 296 #endif 297 bool mGotSPS = false; 298 RefPtr<TrackInfoSharedPtr> mTrackInfo; 299 RefPtr<MediaByteBuffer> mPreviousExtraData; 300 }; 301 302 class HEVCChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { 303 public: 304 explicit HEVCChangeMonitor(const VideoInfo& aInfo) : mCurrentConfig(aInfo) { 305 const bool canBeInstantiated = CanBeInstantiated(); 306 if (canBeInstantiated) { 307 UpdateConfigFromExtraData(aInfo.mExtraData); 308 } 309 LOG("created HEVCChangeMonitor, CanBeInstantiated=%d", canBeInstantiated); 310 } 311 312 bool CanBeInstantiated() const override { 313 auto rv = HVCCConfig::Parse(mCurrentConfig.mExtraData); 314 if (rv.isErr()) { 315 return false; 316 } 317 return rv.unwrap().HasSPS(); 318 } 319 320 MediaResult CheckForChange(MediaRawData* aSample) override { 321 // To be usable we need to convert the sample to 4 bytes NAL size HVCC. 322 if (auto rv = AnnexB::ConvertSampleToHVCC(aSample); rv.isErr()) { 323 // We need HVCC content to be able to later parse the SPS. 324 // This is a no-op if the data is already HVCC. 325 nsPrintfCString msg("Failed to convert to HVCC"); 326 LOG("%s", msg.get()); 327 return MediaResult(rv.unwrapErr(), msg); 328 } 329 330 if (!AnnexB::IsHVCC(aSample)) { 331 nsPrintfCString msg("Invalid HVCC content"); 332 LOG("%s", msg.get()); 333 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, msg); 334 } 335 336 RefPtr<MediaByteBuffer> extraData = 337 aSample->mKeyframe || !mSPS.IsEmpty() 338 ? H265::ExtractHVCCExtraData(aSample) 339 : nullptr; 340 if (!extraData || extraData->IsEmpty()) { 341 // No inband parameter set in sample bitstream. Try out-of-band extradata. 342 extraData = aSample->mExtraData; 343 } 344 // Sample doesn't contain any SPS and we already have SPS, do nothing. 345 auto curConfig = HVCCConfig::Parse(mCurrentConfig.mExtraData); 346 if ((!extraData || extraData->IsEmpty()) && curConfig.unwrap().HasSPS()) { 347 LOG("No SPS in sample. Use existing config"); 348 return NS_OK; 349 } 350 351 auto rv = HVCCConfig::Parse(extraData); 352 // Ignore a corrupted extradata. 353 if (rv.isErr()) { 354 LOG("Ignore corrupted extradata"); 355 return NS_OK; 356 } 357 const HVCCConfig newConfig = rv.unwrap(); 358 LOGV("Current config: %s, new config: %s", 359 curConfig.isOk() ? curConfig.inspect().ToString().get() : "invalid", 360 newConfig.ToString().get()); 361 362 if (!newConfig.HasSPS() && !curConfig.unwrap().HasSPS()) { 363 // We don't have inband data and the original config didn't contain a SPS. 364 // We can't decode this content. 365 LOG("No sps found, waiting for initialization"); 366 return NS_ERROR_NOT_INITIALIZED; 367 } 368 369 if (H265::CompareExtraData(extraData, mCurrentConfig.mExtraData)) { 370 LOG("No config changed"); 371 return NS_OK; 372 } 373 UpdateConfigFromExtraData(extraData); 374 375 if (IsBeingProfiledOrLogEnabled()) { 376 nsPrintfCString msg( 377 "HEVCChangeMonitor::CheckForChange has detected a change in the " 378 "stream and will request a new decoder"); 379 LOG("%s", msg.get()); 380 PROFILER_MARKER_TEXT("HEVC Stream Change", MEDIA_PLAYBACK, {}, msg); 381 } 382 return NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER; 383 } 384 385 const TrackInfo& Config() const override { return mCurrentConfig; } 386 387 MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion, 388 MediaRawData* aSample, 389 bool aNeedKeyFrame) override { 390 MOZ_DIAGNOSTIC_ASSERT( 391 aConversion == MediaDataDecoder::ConversionRequired::kNeedAnnexB || 392 aConversion == MediaDataDecoder::ConversionRequired::kNeedHVCC); 393 MOZ_DIAGNOSTIC_ASSERT(AnnexB::IsHVCC(aSample)); 394 395 aSample->mExtraData = mCurrentConfig.mExtraData; 396 aSample->mTrackInfo = mTrackInfo; 397 398 bool appendExtradata = aNeedKeyFrame; 399 if (aSample->mCrypto.IsEncrypted() && !mReceivedFirstEncryptedSample) { 400 LOG("Detected first encrypted sample [%" PRId64 ",%" PRId64 401 "], keyframe=%d", 402 aSample->mTime.ToMicroseconds(), 403 aSample->GetEndTime().ToMicroseconds(), aSample->mKeyframe); 404 mReceivedFirstEncryptedSample = true; 405 appendExtradata = true; 406 } 407 408 if (aConversion == MediaDataDecoder::ConversionRequired::kNeedAnnexB) { 409 auto res = AnnexB::ConvertHVCCSampleToAnnexB(aSample, appendExtradata); 410 if (res.isErr()) { 411 return MediaResult(res.unwrapErr(), 412 RESULT_DETAIL("ConvertSampleToAnnexB")); 413 } 414 } 415 return NS_OK; 416 } 417 418 bool IsHardwareAccelerated(nsACString& aFailureReason) const override { 419 // We only support HEVC via hardware decoding. 420 return true; 421 } 422 423 void Flush() override { mReceivedFirstEncryptedSample = false; } 424 425 private: 426 void UpdateConfigFromExtraData(MediaByteBuffer* aExtraData) { 427 auto rv = HVCCConfig::Parse(aExtraData); 428 MOZ_ASSERT(rv.isOk()); 429 const auto hvcc = rv.unwrap(); 430 431 // If there are any new SPS/PPS/VPS, update the current stored ones. 432 if (auto nalu = hvcc.GetFirstAvaiableNALU(H265NALU::NAL_TYPES::SPS_NUT)) { 433 mSPS.Clear(); 434 mSPS.AppendElements(nalu->mNALU); 435 if (auto rv = H265::DecodeSPSFromSPSNALU(*nalu); rv.isOk()) { 436 const auto sps = rv.unwrap(); 437 mCurrentConfig.mImage.width = sps.GetImageSize().Width(); 438 mCurrentConfig.mImage.height = sps.GetImageSize().Height(); 439 if (const auto& vui = sps.vui_parameters; 440 vui && vui->HasValidAspectRatio()) { 441 mCurrentConfig.mDisplay = ApplyPixelAspectRatio( 442 vui->GetPixelAspectRatio(), mCurrentConfig.mImage); 443 } else { 444 mCurrentConfig.mDisplay.width = sps.GetDisplaySize().Width(); 445 mCurrentConfig.mDisplay.height = sps.GetDisplaySize().Height(); 446 } 447 mCurrentConfig.mColorDepth = sps.ColorDepth(); 448 mCurrentConfig.mColorSpace = Some(sps.ColorSpace()); 449 mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries( 450 static_cast<gfx::CICP::ColourPrimaries>(sps.ColorPrimaries()), 451 gMediaDecoderLog); 452 mCurrentConfig.mTransferFunction = gfxUtils::CicpToTransferFunction( 453 static_cast<gfx::CICP::TransferCharacteristics>( 454 sps.TransferFunction())); 455 mCurrentConfig.mColorRange = sps.IsFullColorRange() 456 ? gfx::ColorRange::FULL 457 : gfx::ColorRange::LIMITED; 458 } 459 } 460 if (auto nalu = hvcc.GetFirstAvaiableNALU(H265NALU::NAL_TYPES::PPS_NUT)) { 461 mPPS.Clear(); 462 mPPS.AppendElements(nalu->mNALU); 463 } 464 if (auto nalu = hvcc.GetFirstAvaiableNALU(H265NALU::NAL_TYPES::VPS_NUT)) { 465 mVPS.Clear(); 466 mVPS.AppendElements(nalu->mNALU); 467 } 468 if (auto nalu = 469 hvcc.GetFirstAvaiableNALU(H265NALU::NAL_TYPES::PREFIX_SEI_NUT)) { 470 mSEI.Clear(); 471 mSEI.AppendElements(nalu->mNALU); 472 } 473 474 // Construct a new extradata. A situation we encountered previously involved 475 // the initial extradata containing all required NALUs, while the inband 476 // extradata included only an SPS without the PPS or VPS. If we replace the 477 // extradata with the inband version alone, we risk losing the VPS and PPS, 478 // leading to decoder initialization failure on macOS. To avoid this, we 479 // should update only the differing NALUs, ensuring all essential 480 // information remains in the extradata. 481 MOZ_ASSERT(!mSPS.IsEmpty()); // SPS is something MUST to have 482 nsTArray<H265NALU> nalus; 483 // Append NALU by the order of NALU type. If we don't do so, it would cause 484 // an error on the FFmpeg decoder on Linux. 485 if (!mVPS.IsEmpty()) { 486 nalus.AppendElement(H265NALU(mVPS.Elements(), mVPS.Length())); 487 } 488 nalus.AppendElement(H265NALU(mSPS.Elements(), mSPS.Length())); 489 if (!mPPS.IsEmpty()) { 490 nalus.AppendElement(H265NALU(mPPS.Elements(), mPPS.Length())); 491 } 492 if (!mSEI.IsEmpty()) { 493 nalus.AppendElement(H265NALU(mSEI.Elements(), mSEI.Length())); 494 } 495 mCurrentConfig.mExtraData = H265::CreateNewExtraData(hvcc, nalus); 496 mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++); 497 LOG("Updated extradata, hasSPS=%d, hasPPS=%d, hasVPS=%d, hasSEI=%d", 498 !mSPS.IsEmpty(), !mPPS.IsEmpty(), !mVPS.IsEmpty(), !mSEI.IsEmpty()); 499 } 500 501 VideoInfo mCurrentConfig; 502 503 // Full bytes content for nalu. 504 nsTArray<uint8_t> mSPS; 505 nsTArray<uint8_t> mPPS; 506 nsTArray<uint8_t> mVPS; 507 nsTArray<uint8_t> mSEI; 508 509 uint32_t mStreamID = 0; 510 RefPtr<TrackInfoSharedPtr> mTrackInfo; 511 512 // This ensures the first encrypted sample always includes all necessary 513 // information for decoding, as some decoders, such as MediaEngine, require 514 // SPS/PPS to be appended during the clearlead-to-encrypted transition. 515 bool mReceivedFirstEncryptedSample = false; 516 }; 517 518 class VPXChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { 519 public: 520 explicit VPXChangeMonitor(const VideoInfo& aInfo) 521 : mCurrentConfig(aInfo), 522 mCodec(VPXDecoder::IsVP8(aInfo.mMimeType) ? VPXDecoder::Codec::VP8 523 : VPXDecoder::Codec::VP9), 524 mPixelAspectRatio(GetPixelAspectRatio(aInfo.mImage, aInfo.mDisplay)) { 525 mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++); 526 527 if (mCurrentConfig.mExtraData && !mCurrentConfig.mExtraData->IsEmpty()) { 528 // If we're passed VP codec configuration, store it so that we can 529 // instantiate the decoder on init. 530 VPXDecoder::VPXStreamInfo vpxInfo; 531 vpxInfo.mImage = mCurrentConfig.mImage; 532 vpxInfo.mDisplay = mCurrentConfig.mDisplay; 533 VPXDecoder::ReadVPCCBox(vpxInfo, mCurrentConfig.mExtraData); 534 mInfo = Some(vpxInfo); 535 536 mCurrentConfig.mTransferFunction = Some(vpxInfo.TransferFunction()); 537 mCurrentConfig.mColorPrimaries = Some(vpxInfo.ColorPrimaries()); 538 mCurrentConfig.mColorSpace = Some(vpxInfo.ColorSpace()); 539 } 540 } 541 542 bool CanBeInstantiated() const override { 543 if (mCodec == VPXDecoder::Codec::VP8 && mCurrentConfig.mImage.IsEmpty()) { 544 // libvpx VP8 decoder via FFmpeg requires the image size to be set when 545 // initializing. 546 return false; 547 } 548 549 // We want to see at least one sample before we create a decoder so that we 550 // can create the vpcC content on mCurrentConfig.mExtraData. 551 return mInfo || mCurrentConfig.mCrypto.IsEncrypted(); 552 } 553 554 MediaResult CheckForChange(MediaRawData* aSample) override { 555 // Don't look at encrypted content. 556 if (aSample->mCrypto.IsEncrypted()) { 557 return NS_OK; 558 } 559 auto dataSpan = Span<const uint8_t>(aSample->Data(), aSample->Size()); 560 561 // We don't trust the keyframe flag as set on the MediaRawData. 562 VPXDecoder::VPXStreamInfo info; 563 if (!VPXDecoder::GetStreamInfo(dataSpan, info, mCodec)) { 564 return NS_ERROR_DOM_MEDIA_DECODE_ERR; 565 } 566 567 // For both VP8 and VP9, we only look for resolution changes 568 // on keyframes. Other resolution changes are invalid. 569 if (!info.mKeyFrame) { 570 return NS_OK; 571 } 572 573 nsresult rv = NS_OK; 574 if (mInfo) { 575 if (mInfo.ref().IsCompatible(info)) { 576 return rv; 577 } 578 579 // The VPX bitstream does not contain color primary or transfer function 580 // info, so copy over the old values (in case they are used). 581 info.mColorPrimaries = mInfo.ref().mColorPrimaries; 582 info.mTransferFunction = mInfo.ref().mTransferFunction; 583 584 // We can't properly determine the image rect once we've had a resolution 585 // change. 586 mCurrentConfig.ResetImageRect(); 587 PROFILER_MARKER_TEXT( 588 "VPX Stream Change", MEDIA_PLAYBACK, {}, 589 "VPXChangeMonitor::CheckForChange has detected a change in the " 590 "stream and will request a new decoder"); 591 rv = NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER; 592 } else if (mCurrentConfig.mImage != info.mImage || 593 mCurrentConfig.mDisplay != info.mDisplay) { 594 // We can't properly determine the image rect if we're changing 595 // resolution based on sample information. 596 mCurrentConfig.ResetImageRect(); 597 PROFILER_MARKER_TEXT("VPX Stream Init Discrepancy", MEDIA_PLAYBACK, {}, 598 "VPXChangeMonitor::CheckForChange has detected a " 599 "discrepancy between initialization data and stream " 600 "content and will request a new decoder"); 601 rv = NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER; 602 } 603 604 LOG("Detect inband %s resolution changes, image (%" PRId32 ",%" PRId32 605 ")->(%" PRId32 ",%" PRId32 "), display (%" PRId32 ",%" PRId32 606 ")->(%" PRId32 ",%" PRId32 " %s)", 607 mCodec == VPXDecoder::Codec::VP9 ? "VP9" : "VP8", 608 mCurrentConfig.mImage.Width(), mCurrentConfig.mImage.Height(), 609 info.mImage.Width(), info.mImage.Height(), 610 mCurrentConfig.mDisplay.Width(), mCurrentConfig.mDisplay.Height(), 611 info.mDisplay.Width(), info.mDisplay.Height(), 612 info.mDisplayAndImageDifferent ? "specified" : "unspecified"); 613 614 bool imageSizeEmpty = mCurrentConfig.mImage.IsEmpty(); 615 mInfo = Some(info); 616 mCurrentConfig.mImage = info.mImage; 617 if (imageSizeEmpty || info.mDisplayAndImageDifferent) { 618 // If the flag to change the display size is set in the sequence, we 619 // set our original values to begin rescaling according to the new values. 620 mCurrentConfig.mDisplay = info.mDisplay; 621 mPixelAspectRatio = GetPixelAspectRatio(info.mImage, info.mDisplay); 622 } else { 623 mCurrentConfig.mDisplay = 624 ApplyPixelAspectRatio(mPixelAspectRatio, info.mImage); 625 } 626 627 mCurrentConfig.mColorDepth = gfx::ColorDepthForBitDepth(info.mBitDepth); 628 mCurrentConfig.mColorSpace = Some(info.ColorSpace()); 629 630 // VPX bitstream doesn't specify color primaries, transfer function, or 631 // level. Keep the values that were set upon class construction. 632 // 633 // If a video changes colorspaces away from BT2020, we won't clear 634 // mTransferFunction, in case the video changes back to BT2020 and we 635 // need the value again. 636 637 mCurrentConfig.mColorRange = info.ColorRange(); 638 if (mCodec == VPXDecoder::Codec::VP9) { 639 mCurrentConfig.mExtraData->ClearAndRetainStorage(); 640 VPXDecoder::GetVPCCBox(mCurrentConfig.mExtraData, info); 641 } 642 mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++); 643 644 return rv; 645 } 646 647 const TrackInfo& Config() const override { return mCurrentConfig; } 648 649 MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion, 650 MediaRawData* aSample, 651 bool aNeedKeyFrame) override { 652 aSample->mTrackInfo = mTrackInfo; 653 654 return NS_OK; 655 } 656 657 private: 658 VideoInfo mCurrentConfig; 659 const VPXDecoder::Codec mCodec; 660 Maybe<VPXDecoder::VPXStreamInfo> mInfo; 661 uint32_t mStreamID = 0; 662 RefPtr<TrackInfoSharedPtr> mTrackInfo; 663 double mPixelAspectRatio; 664 }; 665 666 #ifdef MOZ_AV1 667 class AV1ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { 668 public: 669 explicit AV1ChangeMonitor(const VideoInfo& aInfo) 670 : mCurrentConfig(aInfo), 671 mPixelAspectRatio(GetPixelAspectRatio(aInfo.mImage, aInfo.mDisplay)) { 672 mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++); 673 674 if (mCurrentConfig.mExtraData && !mCurrentConfig.mExtraData->IsEmpty()) { 675 // If we're passed AV1 codec configuration, store it so that we can 676 // instantiate a decoder in MediaChangeMonitor::Create. 677 AOMDecoder::AV1SequenceInfo seqInfo; 678 MediaResult seqHdrResult; 679 AOMDecoder::TryReadAV1CBox(mCurrentConfig.mExtraData, seqInfo, 680 seqHdrResult); 681 // If the av1C box doesn't include a sequence header specifying image 682 // size, keep the one provided by VideoInfo. 683 if (seqHdrResult.Code() != NS_OK) { 684 seqInfo.mImage = mCurrentConfig.mImage; 685 } 686 687 UpdateConfig(seqInfo); 688 } 689 } 690 691 bool CanBeInstantiated() const override { 692 // We want to have enough codec configuration to determine whether hardware 693 // decoding can be used before creating a decoder. The av1C box or a 694 // sequence header from a sample will contain this information. 695 return mInfo || mCurrentConfig.mCrypto.IsEncrypted(); 696 } 697 698 void UpdateConfig(const AOMDecoder::AV1SequenceInfo& aInfo) { 699 mInfo = Some(aInfo); 700 mCurrentConfig.mColorDepth = gfx::ColorDepthForBitDepth(aInfo.mBitDepth); 701 mCurrentConfig.mColorSpace = gfxUtils::CicpToColorSpace( 702 aInfo.mColorSpace.mMatrix, aInfo.mColorSpace.mPrimaries, 703 gMediaDecoderLog); 704 mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries( 705 aInfo.mColorSpace.mPrimaries, gMediaDecoderLog); 706 mCurrentConfig.mTransferFunction = 707 gfxUtils::CicpToTransferFunction(aInfo.mColorSpace.mTransfer); 708 mCurrentConfig.mColorRange = aInfo.mColorSpace.mRange; 709 710 if (mCurrentConfig.mImage != mInfo->mImage) { 711 gfx::IntSize newDisplay = 712 ApplyPixelAspectRatio(mPixelAspectRatio, aInfo.mImage); 713 LOG("AV1ChangeMonitor detected a resolution change in-band, image " 714 "(%" PRIu32 ",%" PRIu32 ")->(%" PRIu32 ",%" PRIu32 715 "), display (%" PRIu32 ",%" PRIu32 ")->(%" PRIu32 ",%" PRIu32 716 " from PAR)", 717 mCurrentConfig.mImage.Width(), mCurrentConfig.mImage.Height(), 718 aInfo.mImage.Width(), aInfo.mImage.Height(), 719 mCurrentConfig.mDisplay.Width(), mCurrentConfig.mDisplay.Height(), 720 newDisplay.Width(), newDisplay.Height()); 721 mCurrentConfig.mImage = aInfo.mImage; 722 mCurrentConfig.mDisplay = newDisplay; 723 mCurrentConfig.ResetImageRect(); 724 } 725 726 bool wroteSequenceHeader = false; 727 // Our headers should all be around the same size. 728 mCurrentConfig.mExtraData->ClearAndRetainStorage(); 729 AOMDecoder::WriteAV1CBox(aInfo, mCurrentConfig.mExtraData.get(), 730 wroteSequenceHeader); 731 // Header should always be written ReadSequenceHeaderInfo succeeds. 732 MOZ_ASSERT(wroteSequenceHeader); 733 } 734 735 MediaResult CheckForChange(MediaRawData* aSample) override { 736 // Don't look at encrypted content. 737 if (aSample->mCrypto.IsEncrypted()) { 738 return NS_OK; 739 } 740 auto dataSpan = Span<const uint8_t>(aSample->Data(), aSample->Size()); 741 742 // We don't trust the keyframe flag as set on the MediaRawData. 743 AOMDecoder::AV1SequenceInfo info; 744 MediaResult seqHdrResult = 745 AOMDecoder::ReadSequenceHeaderInfo(dataSpan, info); 746 nsresult seqHdrCode = seqHdrResult.Code(); 747 if (seqHdrCode == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { 748 return NS_OK; 749 } 750 if (seqHdrCode != NS_OK) { 751 LOG("AV1ChangeMonitor::CheckForChange read a corrupted sample: %s", 752 seqHdrResult.Description().get()); 753 return seqHdrResult; 754 } 755 756 nsresult rv = NS_OK; 757 if (mInfo.isSome() && 758 (mInfo->mProfile != info.mProfile || 759 mInfo->ColorDepth() != info.ColorDepth() || 760 mInfo->mMonochrome != info.mMonochrome || 761 mInfo->mSubsamplingX != info.mSubsamplingX || 762 mInfo->mSubsamplingY != info.mSubsamplingY || 763 mInfo->mChromaSamplePosition != info.mChromaSamplePosition || 764 mInfo->mImage != info.mImage)) { 765 PROFILER_MARKER_TEXT( 766 "AV1 Stream Change", MEDIA_PLAYBACK, {}, 767 "AV1ChangeMonitor::CheckForChange has detected a change in a " 768 "stream and will request a new decoder"); 769 LOG("AV1ChangeMonitor detected a change and requests a new decoder"); 770 rv = NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER; 771 } 772 773 UpdateConfig(info); 774 775 if (rv == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) { 776 mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++); 777 } 778 return rv; 779 } 780 781 const TrackInfo& Config() const override { return mCurrentConfig; } 782 783 MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion, 784 MediaRawData* aSample, 785 bool aNeedKeyFrame) override { 786 aSample->mTrackInfo = mTrackInfo; 787 return NS_OK; 788 } 789 790 private: 791 VideoInfo mCurrentConfig; 792 Maybe<AOMDecoder::AV1SequenceInfo> mInfo; 793 uint32_t mStreamID = 0; 794 RefPtr<TrackInfoSharedPtr> mTrackInfo; 795 double mPixelAspectRatio; 796 }; 797 #endif 798 799 class AACCodecChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { 800 public: 801 explicit AACCodecChangeMonitor(const AudioInfo& aInfo) 802 : mCurrentConfig(aInfo), mIsADTS(IsADTS(aInfo)) {} 803 804 bool CanBeInstantiated() const override { return true; } 805 806 MediaResult CheckForChange(MediaRawData* aSample) override { 807 bool isADTS = 808 ADTS::FrameHeader::MatchesSync(Span{aSample->Data(), aSample->Size()}); 809 if (isADTS != mIsADTS) { 810 if (mIsADTS) { 811 if (!MakeAACSpecificConfig()) { 812 LOG("Failed to make AAC specific config"); 813 return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR); 814 } 815 LOG("Reconfiguring decoder adts -> raw aac, with maked AAC specific " 816 "config: %zu bytes", 817 mCurrentConfig.mCodecSpecificConfig 818 .as<AudioCodecSpecificBinaryBlob>() 819 .mBinaryBlob->Length()); 820 } else { 821 LOG("Reconfiguring decoder raw aac -> adts"); 822 // Remove AAC specific config to configure a ADTS decoder. 823 mCurrentConfig.mCodecSpecificConfig = 824 AudioCodecSpecificVariant{NoCodecSpecificData{}}; 825 } 826 827 mIsADTS = isADTS; 828 return MediaResult(NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER); 829 } 830 return NS_OK; 831 } 832 833 const TrackInfo& Config() const override { return mCurrentConfig; } 834 835 MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion, 836 MediaRawData* aSample, 837 bool aNeedKeyFrame) override { 838 return NS_OK; 839 } 840 841 private: 842 static bool IsADTS(const AudioInfo& aInfo) { 843 return !aInfo.mCodecSpecificConfig.is<AacCodecSpecificData>() && 844 !aInfo.mCodecSpecificConfig.is<AudioCodecSpecificBinaryBlob>(); 845 } 846 847 bool MakeAACSpecificConfig() { 848 MOZ_ASSERT(IsADTS(mCurrentConfig)); 849 // If profile is not set, default to AAC-LC 850 const uint8_t aacObjectType = 851 mCurrentConfig.mProfile ? mCurrentConfig.mProfile : 2; 852 auto r = ADTS::MakeSpecificConfig(aacObjectType, mCurrentConfig.mRate, 853 mCurrentConfig.mChannels); 854 if (r.isErr()) { 855 return false; 856 } 857 mCurrentConfig.mCodecSpecificConfig = 858 AudioCodecSpecificVariant{AudioCodecSpecificBinaryBlob{r.unwrap()}}; 859 return true; 860 } 861 862 AudioInfo mCurrentConfig; 863 bool mIsADTS; 864 }; 865 866 MediaChangeMonitor::MediaChangeMonitor( 867 PDMFactory* aPDMFactory, 868 UniquePtr<CodecChangeMonitor>&& aCodecChangeMonitor, 869 MediaDataDecoder* aDecoder, const CreateDecoderParams& aParams) 870 : mChangeMonitor(std::move(aCodecChangeMonitor)), 871 mPDMFactory(aPDMFactory), 872 mCurrentConfig(aParams.mConfig.Clone()), 873 mDecoder(aDecoder), 874 mParams(aParams) {} 875 876 /* static */ 877 RefPtr<PlatformDecoderModule::CreateDecoderPromise> MediaChangeMonitor::Create( 878 PDMFactory* aPDMFactory, const CreateDecoderParams& aParams) { 879 LOG("MediaChangeMonitor::Create, params = %s", aParams.ToString().get()); 880 UniquePtr<CodecChangeMonitor> changeMonitor; 881 if (aParams.IsVideo()) { 882 const VideoInfo& config = aParams.VideoConfig(); 883 if (VPXDecoder::IsVPX(config.mMimeType)) { 884 changeMonitor = MakeUnique<VPXChangeMonitor>(config); 885 #ifdef MOZ_AV1 886 } else if (AOMDecoder::IsAV1(config.mMimeType)) { 887 changeMonitor = MakeUnique<AV1ChangeMonitor>(config); 888 #endif 889 } else if (MP4Decoder::IsHEVC(config.mMimeType)) { 890 changeMonitor = MakeUnique<HEVCChangeMonitor>(config); 891 } else { 892 MOZ_ASSERT(MP4Decoder::IsH264(config.mMimeType)); 893 changeMonitor = MakeUnique<H264ChangeMonitor>(aParams); 894 } 895 } else { 896 MOZ_ASSERT(MP4Decoder::IsAAC(aParams.AudioConfig().mMimeType)); 897 changeMonitor = MakeUnique<AACCodecChangeMonitor>(aParams.AudioConfig()); 898 } 899 900 // The change monitor may have an updated track config. E.g. the h264 monitor 901 // may update the config after parsing extra data in the VideoInfo. Create a 902 // new set of params with the updated track info from our monitor and the 903 // other params for aParams and use that going forward. 904 const CreateDecoderParams updatedParams{changeMonitor->Config(), aParams}; 905 LOG("updated params = %s", updatedParams.ToString().get()); 906 907 RefPtr<MediaChangeMonitor> instance = new MediaChangeMonitor( 908 aPDMFactory, std::move(changeMonitor), nullptr, updatedParams); 909 910 if (instance->mChangeMonitor->CanBeInstantiated()) { 911 RefPtr<PlatformDecoderModule::CreateDecoderPromise> p = 912 instance->CreateDecoder()->Then( 913 GetCurrentSerialEventTarget(), __func__, 914 [instance = RefPtr{instance}] { 915 return PlatformDecoderModule::CreateDecoderPromise:: 916 CreateAndResolve(instance, __func__); 917 }, 918 [](const MediaResult& aError) { 919 return PlatformDecoderModule::CreateDecoderPromise:: 920 CreateAndReject(aError, __func__); 921 }); 922 return p; 923 } 924 925 return PlatformDecoderModule::CreateDecoderPromise::CreateAndResolve( 926 instance, __func__); 927 } 928 929 MediaChangeMonitor::~MediaChangeMonitor() = default; 930 931 RefPtr<MediaDataDecoder::InitPromise> MediaChangeMonitor::Init() { 932 mThread = GetCurrentSerialEventTarget(); 933 if (mDecoder) { 934 RefPtr<InitPromise> p = mInitPromise.Ensure(__func__); 935 RefPtr<MediaChangeMonitor> self = this; 936 mDecoder->Init() 937 ->Then(GetCurrentSerialEventTarget(), __func__, 938 [self, this](InitPromise::ResolveOrRejectValue&& aValue) { 939 mInitPromiseRequest.Complete(); 940 if (aValue.IsResolve()) { 941 mDecoderInitialized = true; 942 mConversionRequired = Some(mDecoder->NeedsConversion()); 943 mCanRecycleDecoder = Some(CanRecycleDecoder()); 944 if (mPendingSeekThreshold) { 945 mDecoder->SetSeekThreshold(*mPendingSeekThreshold); 946 mPendingSeekThreshold.reset(); 947 } 948 } 949 return mInitPromise.ResolveOrRejectIfExists(std::move(aValue), 950 __func__); 951 }) 952 ->Track(mInitPromiseRequest); 953 return p; 954 } 955 956 // We haven't been able to initialize a decoder due to missing 957 // extradata. 958 return MediaDataDecoder::InitPromise::CreateAndResolve(TrackType::kVideoTrack, 959 __func__); 960 } 961 962 RefPtr<MediaDataDecoder::DecodePromise> MediaChangeMonitor::Decode( 963 MediaRawData* aSample) { 964 AssertOnThread(); 965 MOZ_RELEASE_ASSERT(mFlushPromise.IsEmpty(), 966 "Flush operation didn't complete"); 967 968 MOZ_RELEASE_ASSERT( 969 !mDecodePromiseRequest.Exists() && !mInitPromiseRequest.Exists(), 970 "Can't request a new decode until previous one completed"); 971 972 MediaResult rv = CheckForChange(aSample); 973 974 if (rv == NS_ERROR_NOT_INITIALIZED) { 975 // We are missing the required init data to create the decoder. 976 if (mParams.mOptions.contains( 977 CreateDecoderParams::Option::ErrorIfNoInitializationData)) { 978 // This frame can't be decoded and should be treated as an error. 979 return DecodePromise::CreateAndReject(rv, __func__); 980 } 981 // Swallow the frame, and await delivery of init data. 982 return DecodePromise::CreateAndResolve(DecodedData(), __func__); 983 } 984 if (rv == NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER) { 985 // The decoder is pending initialization. 986 RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__); 987 return p; 988 } 989 990 if (NS_FAILED(rv)) { 991 return DecodePromise::CreateAndReject(rv, __func__); 992 } 993 994 if (mNeedKeyframe && !aSample->mKeyframe) { 995 return DecodePromise::CreateAndResolve(DecodedData(), __func__); 996 } 997 998 rv = mChangeMonitor->PrepareSample(*mConversionRequired, aSample, 999 mNeedKeyframe); 1000 if (NS_FAILED(rv)) { 1001 return DecodePromise::CreateAndReject(rv, __func__); 1002 } 1003 1004 mNeedKeyframe = false; 1005 1006 return mDecoder->Decode(aSample); 1007 } 1008 1009 RefPtr<MediaDataDecoder::FlushPromise> MediaChangeMonitor::Flush() { 1010 AssertOnThread(); 1011 mDecodePromiseRequest.DisconnectIfExists(); 1012 mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); 1013 mNeedKeyframe = true; 1014 mChangeMonitor->Flush(); 1015 mPendingFrames.Clear(); 1016 1017 MOZ_RELEASE_ASSERT(mFlushPromise.IsEmpty(), "Previous flush didn't complete"); 1018 1019 /* 1020 When we detect a change of content in the byte stream, we first drain the 1021 current decoder (1), flush (2), shut it down (3) create a new decoder (4) 1022 and initialize it (5). It is possible for MediaChangeMonitor::Flush to be 1023 called during any of those times. If during (1): 1024 - mDrainRequest will not be empty. 1025 - The old decoder can still be used, with the current extradata as 1026 stored in mCurrentConfig.mExtraData. 1027 1028 If during (2): 1029 - mFlushRequest will not be empty. 1030 - The old decoder can still be used, with the current extradata as 1031 stored in mCurrentConfig.mExtraData. 1032 1033 If during (3): 1034 - mShutdownRequest won't be empty. 1035 - mDecoder is empty. 1036 - The old decoder is no longer referenced by the MediaChangeMonitor. 1037 1038 If during (4): 1039 - mDecoderRequest won't be empty. 1040 - mDecoder is not set. Steps will continue to (5) to set and initialize it 1041 1042 If during (5): 1043 - mInitPromiseRequest won't be empty. 1044 - mDecoder is set but not usable yet. 1045 */ 1046 1047 if (mDrainRequest.Exists() || mFlushRequest.Exists() || 1048 mShutdownRequest.Exists() || mDecoderRequest.Exists() || 1049 mInitPromiseRequest.Exists()) { 1050 // We let the current decoder complete and will resume after. 1051 RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__); 1052 return p; 1053 } 1054 if (mDecoder && mDecoderInitialized) { 1055 return mDecoder->Flush(); 1056 } 1057 return FlushPromise::CreateAndResolve(true, __func__); 1058 } 1059 1060 RefPtr<MediaDataDecoder::DecodePromise> MediaChangeMonitor::Drain() { 1061 AssertOnThread(); 1062 MOZ_RELEASE_ASSERT(!mDrainRequest.Exists()); 1063 mNeedKeyframe = true; 1064 if (mDecoder) { 1065 return mDecoder->Drain(); 1066 } 1067 return DecodePromise::CreateAndResolve(DecodedData(), __func__); 1068 } 1069 1070 RefPtr<ShutdownPromise> MediaChangeMonitor::Shutdown() { 1071 AssertOnThread(); 1072 mInitPromiseRequest.DisconnectIfExists(); 1073 mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); 1074 mDecodePromiseRequest.DisconnectIfExists(); 1075 mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); 1076 mDrainRequest.DisconnectIfExists(); 1077 mFlushRequest.DisconnectIfExists(); 1078 mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); 1079 mShutdownRequest.DisconnectIfExists(); 1080 1081 if (mShutdownPromise) { 1082 // We have a shutdown in progress, return that promise instead as we can't 1083 // shutdown a decoder twice. 1084 RefPtr<ShutdownPromise> p = std::move(mShutdownPromise); 1085 return p; 1086 } 1087 return ShutdownDecoder(); 1088 } 1089 1090 RefPtr<ShutdownPromise> MediaChangeMonitor::ShutdownDecoder() { 1091 AssertOnThread(); 1092 mConversionRequired.reset(); 1093 if (mDecoder) { 1094 MutexAutoLock lock(mMutex); 1095 RefPtr<MediaDataDecoder> decoder = std::move(mDecoder); 1096 return decoder->Shutdown(); 1097 } 1098 return ShutdownPromise::CreateAndResolve(true, __func__); 1099 } 1100 1101 bool MediaChangeMonitor::IsHardwareAccelerated( 1102 nsACString& aFailureReason) const { 1103 if (mDecoder) { 1104 return mDecoder->IsHardwareAccelerated(aFailureReason); 1105 } 1106 #ifdef MOZ_APPLEMEDIA 1107 // On mac, we can assume H264 is hardware accelerated for now. 1108 // This allows MediaCapabilities to report that playback will be smooth. 1109 // Which will always be. 1110 return true; 1111 #else 1112 return mChangeMonitor->IsHardwareAccelerated(aFailureReason); 1113 #endif 1114 } 1115 1116 void MediaChangeMonitor::SetSeekThreshold(const media::TimeUnit& aTime) { 1117 GetCurrentSerialEventTarget()->Dispatch(NS_NewRunnableFunction( 1118 "MediaChangeMonitor::SetSeekThreshold", 1119 [self = RefPtr<MediaChangeMonitor>(this), time = aTime, this] { 1120 // During the shutdown. 1121 if (mShutdownPromise) { 1122 return; 1123 } 1124 if (mDecoder && mDecoderInitialized) { 1125 mDecoder->SetSeekThreshold(time); 1126 } else { 1127 mPendingSeekThreshold = Some(time); 1128 } 1129 })); 1130 } 1131 1132 RefPtr<MediaChangeMonitor::CreateDecoderPromise> 1133 MediaChangeMonitor::CreateDecoder() { 1134 mCurrentConfig = mChangeMonitor->Config().Clone(); 1135 CreateDecoderParams currentParams = {*mCurrentConfig, mParams}; 1136 currentParams.mWrappers -= media::Wrapper::MediaChangeMonitor; 1137 LOG("MediaChangeMonitor::CreateDecoder, current params = %s", 1138 currentParams.ToString().get()); 1139 RefPtr<CreateDecoderPromise> p = 1140 mPDMFactory->CreateDecoder(currentParams) 1141 ->Then( 1142 GetCurrentSerialEventTarget(), __func__, 1143 [self = RefPtr{this}, this](RefPtr<MediaDataDecoder>&& aDecoder) { 1144 MutexAutoLock lock(mMutex); 1145 mDecoder = std::move(aDecoder); 1146 DDLINKCHILD("decoder", mDecoder.get()); 1147 return CreateDecoderPromise::CreateAndResolve(true, __func__); 1148 }, 1149 [self = RefPtr{this}](const MediaResult& aError) { 1150 return CreateDecoderPromise::CreateAndReject(aError, __func__); 1151 }); 1152 1153 mDecoderInitialized = false; 1154 mNeedKeyframe = true; 1155 1156 return p; 1157 } 1158 1159 MediaResult MediaChangeMonitor::CreateDecoderAndInit(MediaRawData* aSample) { 1160 MOZ_ASSERT(mThread && mThread->IsOnCurrentThread()); 1161 1162 MediaResult rv = mChangeMonitor->CheckForChange(aSample); 1163 if (!NS_SUCCEEDED(rv) && rv != NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) { 1164 return rv; 1165 } 1166 1167 if (!mChangeMonitor->CanBeInstantiated()) { 1168 // Nothing found yet, will try again later. 1169 return NS_ERROR_NOT_INITIALIZED; 1170 } 1171 1172 CreateDecoder() 1173 ->Then( 1174 GetCurrentSerialEventTarget(), __func__, 1175 [self = RefPtr{this}, this, sample = RefPtr{aSample}] { 1176 mDecoderRequest.Complete(); 1177 mDecoder->Init() 1178 ->Then( 1179 GetCurrentSerialEventTarget(), __func__, 1180 [self, sample, this](const TrackType aTrackType) { 1181 mInitPromiseRequest.Complete(); 1182 mDecoderInitialized = true; 1183 mConversionRequired = Some(mDecoder->NeedsConversion()); 1184 mCanRecycleDecoder = Some(CanRecycleDecoder()); 1185 1186 if (mPendingSeekThreshold) { 1187 mDecoder->SetSeekThreshold(*mPendingSeekThreshold); 1188 mPendingSeekThreshold.reset(); 1189 } 1190 1191 if (!mFlushPromise.IsEmpty()) { 1192 // A Flush is pending, abort the current operation. 1193 mFlushPromise.Resolve(true, __func__); 1194 return; 1195 } 1196 1197 DecodeFirstSample(sample); 1198 }, 1199 [self, this](const MediaResult& aError) { 1200 mInitPromiseRequest.Complete(); 1201 1202 if (!mFlushPromise.IsEmpty()) { 1203 // A Flush is pending, abort the current operation. 1204 mFlushPromise.Reject(aError, __func__); 1205 return; 1206 } 1207 1208 mDecodePromise.RejectIfExists( 1209 MediaResult( 1210 aError.Code(), 1211 RESULT_DETAIL("Unable to initialize decoder")), 1212 __func__); 1213 }) 1214 ->Track(mInitPromiseRequest); 1215 }, 1216 [self = RefPtr{this}, this](const MediaResult& aError) { 1217 mDecoderRequest.Complete(); 1218 if (!mFlushPromise.IsEmpty()) { 1219 // A Flush is pending, abort the current operation. 1220 mFlushPromise.Reject(aError, __func__); 1221 return; 1222 } 1223 mDecodePromise.RejectIfExists( 1224 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 1225 RESULT_DETAIL("Unable to create decoder")), 1226 __func__); 1227 }) 1228 ->Track(mDecoderRequest); 1229 return NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER; 1230 } 1231 1232 bool MediaChangeMonitor::CanRecycleDecoder() const { 1233 MOZ_ASSERT(mDecoder); 1234 return StaticPrefs::media_decoder_recycle_enabled() && 1235 mDecoder->SupportDecoderRecycling(); 1236 } 1237 1238 void MediaChangeMonitor::DecodeFirstSample(MediaRawData* aSample) { 1239 // We feed all the data to AnnexB decoder as a non-keyframe could contain 1240 // the SPS/PPS when used with WebRTC and this data is needed by the decoder. 1241 if (mNeedKeyframe && !aSample->mKeyframe && 1242 *mConversionRequired != ConversionRequired::kNeedAnnexB) { 1243 mDecodePromise.Resolve(std::move(mPendingFrames), __func__); 1244 mPendingFrames = DecodedData(); 1245 return; 1246 } 1247 1248 MediaResult rv = mChangeMonitor->PrepareSample(*mConversionRequired, aSample, 1249 mNeedKeyframe); 1250 1251 if (NS_FAILED(rv)) { 1252 mDecodePromise.Reject(rv, __func__); 1253 return; 1254 } 1255 1256 if (aSample->mKeyframe) { 1257 mNeedKeyframe = false; 1258 } 1259 1260 RefPtr<MediaChangeMonitor> self = this; 1261 mDecoder->Decode(aSample) 1262 ->Then( 1263 GetCurrentSerialEventTarget(), __func__, 1264 [self, this](MediaDataDecoder::DecodedData&& aResults) { 1265 mDecodePromiseRequest.Complete(); 1266 mPendingFrames.AppendElements(std::move(aResults)); 1267 mDecodePromise.ResolveIfExists(std::move(mPendingFrames), __func__); 1268 mPendingFrames = DecodedData(); 1269 }, 1270 [self, this](const MediaResult& aError) { 1271 mDecodePromiseRequest.Complete(); 1272 mDecodePromise.RejectIfExists(aError, __func__); 1273 }) 1274 ->Track(mDecodePromiseRequest); 1275 } 1276 1277 MediaResult MediaChangeMonitor::CheckForChange(MediaRawData* aSample) { 1278 if (!mDecoder) { 1279 return CreateDecoderAndInit(aSample); 1280 } 1281 1282 MediaResult rv = mChangeMonitor->CheckForChange(aSample); 1283 1284 if (NS_SUCCEEDED(rv) || rv != NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) { 1285 return rv; 1286 } 1287 1288 if (*mCanRecycleDecoder) { 1289 // Do not recreate the decoder, reuse it. 1290 mNeedKeyframe = true; 1291 return NS_OK; 1292 } 1293 1294 // The content has changed, signal to drain the current decoder and once done 1295 // create a new one. 1296 DrainThenFlushDecoder(aSample); 1297 return NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER; 1298 } 1299 1300 void MediaChangeMonitor::DrainThenFlushDecoder(MediaRawData* aPendingSample) { 1301 AssertOnThread(); 1302 MOZ_ASSERT(mDecoderInitialized); 1303 RefPtr<MediaRawData> sample = aPendingSample; 1304 RefPtr<MediaChangeMonitor> self = this; 1305 mDecoder->Drain() 1306 ->Then( 1307 GetCurrentSerialEventTarget(), __func__, 1308 [self, sample, this](MediaDataDecoder::DecodedData&& aResults) { 1309 mDrainRequest.Complete(); 1310 if (!mFlushPromise.IsEmpty()) { 1311 // A Flush is pending, abort the current operation. 1312 mFlushPromise.Resolve(true, __func__); 1313 return; 1314 } 1315 if (aResults.Length() > 0) { 1316 mPendingFrames.AppendElements(std::move(aResults)); 1317 DrainThenFlushDecoder(sample); 1318 return; 1319 } 1320 // We've completed the draining operation, we can now flush the 1321 // decoder. 1322 FlushThenShutdownDecoder(sample); 1323 }, 1324 [self, this](const MediaResult& aError) { 1325 mDrainRequest.Complete(); 1326 if (!mFlushPromise.IsEmpty()) { 1327 // A Flush is pending, abort the current operation. 1328 mFlushPromise.Reject(aError, __func__); 1329 return; 1330 } 1331 mDecodePromise.RejectIfExists(aError, __func__); 1332 }) 1333 ->Track(mDrainRequest); 1334 } 1335 1336 void MediaChangeMonitor::FlushThenShutdownDecoder( 1337 MediaRawData* aPendingSample) { 1338 AssertOnThread(); 1339 MOZ_ASSERT(mDecoderInitialized); 1340 RefPtr<MediaRawData> sample = aPendingSample; 1341 RefPtr<MediaChangeMonitor> self = this; 1342 mDecoder->Flush() 1343 ->Then( 1344 GetCurrentSerialEventTarget(), __func__, 1345 [self, sample, this]() { 1346 mFlushRequest.Complete(); 1347 1348 if (!mFlushPromise.IsEmpty()) { 1349 // A Flush is pending, abort the current operation. 1350 mFlushPromise.Resolve(true, __func__); 1351 return; 1352 } 1353 1354 mShutdownPromise = ShutdownDecoder(); 1355 mShutdownPromise 1356 ->Then( 1357 GetCurrentSerialEventTarget(), __func__, 1358 [self, sample, this]() { 1359 mShutdownRequest.Complete(); 1360 mShutdownPromise = nullptr; 1361 1362 if (!mFlushPromise.IsEmpty()) { 1363 // A Flush is pending, abort the current 1364 // operation. 1365 mFlushPromise.Resolve(true, __func__); 1366 return; 1367 } 1368 1369 MediaResult rv = CreateDecoderAndInit(sample); 1370 if (rv == NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER) { 1371 // All good so far, will continue later. 1372 return; 1373 } 1374 MOZ_ASSERT(NS_FAILED(rv)); 1375 mDecodePromise.RejectIfExists(rv, __func__); 1376 return; 1377 }, 1378 [] { MOZ_CRASH("Can't reach here'"); }) 1379 ->Track(mShutdownRequest); 1380 }, 1381 [self, this](const MediaResult& aError) { 1382 mFlushRequest.Complete(); 1383 if (!mFlushPromise.IsEmpty()) { 1384 // A Flush is pending, abort the current operation. 1385 mFlushPromise.Reject(aError, __func__); 1386 return; 1387 } 1388 mDecodePromise.RejectIfExists(aError, __func__); 1389 }) 1390 ->Track(mFlushRequest); 1391 } 1392 1393 MediaDataDecoder* MediaChangeMonitor::GetDecoderOnNonOwnerThread() const { 1394 MutexAutoLock lock(mMutex); 1395 return mDecoder; 1396 } 1397 1398 #undef LOG 1399 1400 } // namespace mozilla