ExternalEngineStateMachine.cpp (55599B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #include "ExternalEngineStateMachine.h" 6 7 #include "PerformanceRecorder.h" 8 #ifdef MOZ_WMF_MEDIA_ENGINE 9 # include "MFMediaEngineDecoderModule.h" 10 # include "mozilla/MFMediaEngineChild.h" 11 # include "mozilla/StaticPrefs_media.h" 12 #endif 13 #include "VideoUtils.h" 14 #include "mozilla/AppShutdown.h" 15 #include "mozilla/Atomics.h" 16 #include "mozilla/ClearOnShutdown.h" 17 #include "mozilla/ProfilerLabels.h" 18 #include "mozilla/StaticMutex.h" 19 #include "mozilla/UniquePtr.h" 20 #include "mozilla/glean/DomMediaPlatformsWmfMetrics.h" 21 #include "nsPrintfCString.h" 22 #include "nsThreadUtils.h" 23 24 namespace mozilla { 25 26 extern LazyLogModule gMediaDecoderLog; 27 28 #define FMT(x, ...) \ 29 "Decoder=%p, State=%s, " x, mDecoderID, GetStateStr(), ##__VA_ARGS__ 30 #define LOG(x, ...) \ 31 DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, "Decoder=%p, State=%s, " x, \ 32 mDecoderID, GetStateStr(), ##__VA_ARGS__) 33 #define LOGV(x, ...) \ 34 DDMOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, "Decoder=%p, State=%s, " x, \ 35 mDecoderID, GetStateStr(), ##__VA_ARGS__) 36 #define LOGW(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get()) 37 #define LOGE(x, ...) \ 38 NS_DebugBreak(NS_DEBUG_WARNING, \ 39 nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, \ 40 __FILE__, __LINE__) 41 42 const char* ExternalEngineEventToStr(ExternalEngineEvent aEvent) { 43 #define EVENT_TO_STR(event) \ 44 case ExternalEngineEvent::event: \ 45 return #event 46 switch (aEvent) { 47 EVENT_TO_STR(LoadedMetaData); 48 EVENT_TO_STR(LoadedFirstFrame); 49 EVENT_TO_STR(LoadedData); 50 EVENT_TO_STR(Waiting); 51 EVENT_TO_STR(Playing); 52 EVENT_TO_STR(Seeked); 53 EVENT_TO_STR(BufferingStarted); 54 EVENT_TO_STR(BufferingEnded); 55 EVENT_TO_STR(Timeupdate); 56 EVENT_TO_STR(Ended); 57 EVENT_TO_STR(RequestForAudio); 58 EVENT_TO_STR(RequestForVideo); 59 EVENT_TO_STR(AudioEnough); 60 EVENT_TO_STR(VideoEnough); 61 default: 62 MOZ_ASSERT_UNREACHABLE("Undefined event!"); 63 return "Undefined"; 64 } 65 #undef EVENT_TO_STR 66 } 67 68 /** 69 * This class monitors the amount of crash happened for a remote engine 70 * process. It the amount of crash of the remote process exceeds the defined 71 * threshold, then `ShouldRecoverProcess()` will return false to indicate that 72 * we should not keep spawning that remote process because it's too easy to 73 * crash. 74 * 75 * In addition, we also have another mechanism in the media format reader 76 * (MFR) to detect crash amount of remote processes, but that would only 77 * happen during the decoding process. The main reason to choose using this 78 * simple monitor, instead of the mechanism in the MFR is because that 79 * mechanism can't detect every crash happening in the remote process, such as 80 * crash happening during initializing the remote engine, or setting the CDM 81 * pipepline, which can happen prior to decoding. 82 */ 83 class ProcessCrashMonitor final { 84 public: 85 static void NotifyCrash() { 86 StaticMutexAutoLock lock(sMutex); 87 auto* monitor = ProcessCrashMonitor::EnsureInstance(); 88 if (!monitor) { 89 return; 90 } 91 monitor->mCrashNums++; 92 } 93 static bool ShouldRecoverProcess() { 94 StaticMutexAutoLock lock(sMutex); 95 auto* monitor = ProcessCrashMonitor::EnsureInstance(); 96 if (!monitor) { 97 return false; 98 } 99 return monitor->mCrashNums <= monitor->mMaxCrashes; 100 } 101 102 private: 103 ProcessCrashMonitor() : mCrashNums(0) { 104 #ifdef MOZ_WMF_MEDIA_ENGINE 105 mMaxCrashes = StaticPrefs::media_wmf_media_engine_max_crashes(); 106 #else 107 mMaxCrashes = 0; 108 #endif 109 }; 110 ProcessCrashMonitor(const ProcessCrashMonitor&) = delete; 111 ProcessCrashMonitor& operator=(const ProcessCrashMonitor&) = delete; 112 113 static ProcessCrashMonitor* EnsureInstance() { 114 if (sIsShutdown) { 115 return nullptr; 116 } 117 if (!sCrashMonitor) { 118 sCrashMonitor.reset(new ProcessCrashMonitor()); 119 GetMainThreadSerialEventTarget()->Dispatch( 120 NS_NewRunnableFunction("ProcessCrashMonitor::EnsureInstance", [&] { 121 RunOnShutdown( 122 [&] { 123 StaticMutexAutoLock lock(sMutex); 124 sCrashMonitor.reset(); 125 sIsShutdown = true; 126 }, 127 ShutdownPhase::XPCOMShutdown); 128 })); 129 } 130 return sCrashMonitor.get(); 131 } 132 133 static StaticMutex sMutex; 134 static UniquePtr<ProcessCrashMonitor> sCrashMonitor; 135 static Atomic<bool> sIsShutdown; 136 137 uint32_t mCrashNums; 138 uint32_t mMaxCrashes; 139 }; 140 141 StaticMutex ProcessCrashMonitor::sMutex; 142 constinit UniquePtr<ProcessCrashMonitor> ProcessCrashMonitor::sCrashMonitor; 143 Atomic<bool> ProcessCrashMonitor::sIsShutdown{false}; 144 145 /* static */ 146 const char* ExternalEngineStateMachine::StateToStr(State aNextState) { 147 #define STATE_TO_STR(state) \ 148 case State::state: \ 149 return #state 150 switch (aNextState) { 151 STATE_TO_STR(InitEngine); 152 STATE_TO_STR(ReadingMetadata); 153 STATE_TO_STR(RunningEngine); 154 STATE_TO_STR(SeekingData); 155 STATE_TO_STR(ShutdownEngine); 156 STATE_TO_STR(RecoverEngine); 157 default: 158 MOZ_ASSERT_UNREACHABLE("Undefined state!"); 159 return "Undefined"; 160 } 161 #undef STATE_TO_STR 162 } 163 164 const char* ExternalEngineStateMachine::GetStateStr() const { 165 return StateToStr(mState.mName); 166 } 167 168 static bool IsBeingProfiledOrLogEnabled() { 169 return MOZ_LOG_TEST(gMediaDecoderLog, LogLevel::Info) || 170 profiler_thread_is_being_profiled_for_markers(); 171 } 172 173 void ExternalEngineStateMachine::ChangeStateTo(State aNextState) { 174 if (IsBeingProfiledOrLogEnabled()) { 175 nsPrintfCString msg("Change state : '%s' -> '%s' (play-state=%d)", 176 StateToStr(mState.mName), StateToStr(aNextState), 177 mPlayState.Ref()); 178 LOG("%s", msg.get()); 179 PROFILER_MARKER_TEXT("EESM::ChangeStateTo", MEDIA_PLAYBACK, {}, msg); 180 } 181 // Assert the possible state transitions. 182 MOZ_ASSERT_IF( 183 mState.IsReadingMetadata(), 184 aNextState == State::InitEngine || aNextState == State::ShutdownEngine); 185 MOZ_ASSERT_IF(mState.IsInitEngine(), aNextState == State::RunningEngine || 186 aNextState == State::ShutdownEngine); 187 MOZ_ASSERT_IF(mState.IsRunningEngine(), 188 aNextState == State::SeekingData || 189 aNextState == State::ShutdownEngine || 190 aNextState == State::RecoverEngine); 191 MOZ_ASSERT_IF(mState.IsSeekingData(), 192 aNextState == State::RunningEngine || 193 aNextState == State::ShutdownEngine || 194 aNextState == State::RecoverEngine); 195 MOZ_ASSERT_IF(mState.IsShutdownEngine(), aNextState == State::ShutdownEngine); 196 MOZ_ASSERT_IF( 197 mState.IsRecoverEngine(), 198 aNextState == State::SeekingData || aNextState == State::ShutdownEngine); 199 if (aNextState == State::SeekingData) { 200 mState = StateObject({StateObject::SeekingData()}); 201 } else if (aNextState == State::InitEngine) { 202 mState = StateObject({StateObject::InitEngine()}); 203 } else if (aNextState == State::RunningEngine) { 204 mState = StateObject({StateObject::RunningEngine()}); 205 } else if (aNextState == State::ShutdownEngine) { 206 mState = StateObject({StateObject::ShutdownEngine()}); 207 } else if (aNextState == State::RecoverEngine) { 208 mState = StateObject({StateObject::RecoverEngine()}); 209 } else { 210 MOZ_ASSERT_UNREACHABLE("Wrong state!"); 211 } 212 NotifyAudibleStateChangeIfNeeded(); 213 } 214 215 ExternalEngineStateMachine::ExternalEngineStateMachine( 216 MediaDecoder* aDecoder, MediaFormatReader* aReader) 217 : MediaDecoderStateMachineBase(aDecoder, aReader) { 218 LOG("Created ExternalEngineStateMachine"); 219 MOZ_ASSERT(mState.IsReadingMetadata()); 220 ReadMetadata(); 221 } 222 223 ExternalEngineStateMachine::~ExternalEngineStateMachine() { 224 LOG("ExternalEngineStateMachine is destroyed"); 225 } 226 227 void ExternalEngineStateMachine::InitEngine() { 228 MOZ_ASSERT(mState.IsInitEngine() || mState.IsRecoverEngine()); 229 #ifdef MOZ_WMF_MEDIA_ENGINE 230 mEngine.reset(new MFMediaEngineWrapper(this, mFrameStats)); 231 #endif 232 if (mEngine) { 233 MOZ_ASSERT(mInfo); 234 if (IsBeingProfiledOrLogEnabled()) { 235 nsPrintfCString msg{"mMinimizePreroll %d IsEncryptedCustomIdent %d", 236 mMinimizePreroll, mReader->IsEncryptedCustomIdent()}; 237 LOG("Init engine, %s", msg.get()); 238 PROFILER_MARKER_TEXT("EESM::InitEngine", MEDIA_PLAYBACK, {}, msg); 239 } 240 auto* state = mState.AsInitEngine(); 241 ExternalPlaybackEngine::InitFlagSet flags; 242 if (mMinimizePreroll) { 243 flags += ExternalPlaybackEngine::InitFlag::ShouldPreload; 244 } 245 if (mReader->IsEncryptedCustomIdent()) { 246 flags += ExternalPlaybackEngine::InitFlag::EncryptedCustomIdent; 247 } 248 state->mInitPromise = mEngine->Init(*mInfo, flags); 249 state->mInitPromise 250 ->Then(OwnerThread(), __func__, this, 251 &ExternalEngineStateMachine::OnEngineInitSuccess, 252 &ExternalEngineStateMachine::OnEngineInitFailure) 253 ->Track(state->mEngineInitRequest); 254 } 255 } 256 257 void ExternalEngineStateMachine::OnEngineInitSuccess() { 258 AssertOnTaskQueue(); 259 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnEngineInitSuccess", 260 MEDIA_PLAYBACK); 261 MOZ_ASSERT(mState.IsInitEngine() || mState.IsRecoverEngine()); 262 if (IsBeingProfiledOrLogEnabled()) { 263 nsPrintfCString msg("Initialized the external playback engine %" PRIu64, 264 mEngine->Id()); 265 LOG("%s", msg.get()); 266 PROFILER_MARKER_TEXT("EESM::OnEngineInitSuccess", MEDIA_PLAYBACK, {}, msg); 267 } 268 auto* state = mState.AsInitEngine(); 269 state->mEngineInitRequest.Complete(); 270 mReader->UpdateMediaEngineId(mEngine->Id()); 271 state->mInitPromise = nullptr; 272 if (mState.IsInitEngine()) { 273 StartRunningEngine(); 274 return; 275 } 276 // We just recovered from CDM process crash, seek to previous position. 277 SeekTarget target(mCurrentPosition.Ref(), SeekTarget::Type::Accurate); 278 Seek(target); 279 } 280 281 void ExternalEngineStateMachine::OnEngineInitFailure() { 282 AssertOnTaskQueue(); 283 MOZ_ASSERT(mState.IsInitEngine() || mState.IsRecoverEngine()); 284 LOGE("Failed to initialize the external playback engine"); 285 PROFILER_MARKER_UNTYPED("EESM::OnEngineInitFailure", MEDIA_PLAYBACK); 286 auto* state = mState.AsInitEngine(); 287 state->mEngineInitRequest.Complete(); 288 state->mInitPromise = nullptr; 289 // Even if we failed to initialize the media engine, we still want to try 290 // again with the normal state machine, so don't return a fatal error, return 291 // NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR instead. 292 ReportTelemetry(NS_ERROR_DOM_MEDIA_MEDIA_ENGINE_INITIALIZATION_ERR); 293 DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR, 294 __func__)); 295 } 296 297 void ExternalEngineStateMachine::ReadMetadata() { 298 MOZ_ASSERT(NS_IsMainThread()); 299 MOZ_ASSERT(mState.IsReadingMetadata()); 300 PROFILER_MARKER_UNTYPED("EESM::ReadMetadata", MEDIA_PLAYBACK); 301 (void)OwnerThread()->Dispatch(NS_NewRunnableFunction( 302 "ExternalEngineStateMachine::ReadMetadata", 303 [self = RefPtr<ExternalEngineStateMachine>{this}, this] { 304 mReader->ReadMetadata() 305 ->Then(OwnerThread(), __func__, this, 306 &ExternalEngineStateMachine::OnMetadataRead, 307 &ExternalEngineStateMachine::OnMetadataNotRead) 308 ->Track(mState.AsReadingMetadata()->mMetadataRequest); 309 })); 310 } 311 312 void ExternalEngineStateMachine::OnMetadataRead(MetadataHolder&& aMetadata) { 313 AssertOnTaskQueue(); 314 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnMetadataRead", 315 MEDIA_PLAYBACK); 316 MOZ_ASSERT(mState.IsReadingMetadata()); 317 LOG("OnMetadataRead"); 318 319 mState.AsReadingMetadata()->mMetadataRequest.Complete(); 320 mInfo.emplace(*aMetadata.mInfo); 321 mMediaSeekable = Info().mMediaSeekable; 322 mMediaSeekableOnlyInBufferedRanges = 323 Info().mMediaSeekableOnlyInBufferedRanges; 324 325 if (!IsFormatSupportedByExternalEngine(*mInfo)) { 326 // The external engine doesn't support the type, try to notify the decoder 327 // to use our own state machine again. Not a real "error", because it would 328 // fallback to another state machine. 329 DecodeError( 330 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); 331 return; 332 } 333 334 #ifdef MOZ_WMF_MEDIA_ENGINE 335 // Only support encrypted playback. Not a real "error", because it would 336 // fallback to another state machine. 337 if ((!mInfo->IsEncrypted() && !mReader->IsEncryptedCustomIdent()) && 338 StaticPrefs::media_wmf_media_engine_enabled() == 2) { 339 LOG("External engine only supports encrypted playback by the pref"); 340 DecodeError( 341 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); 342 return; 343 } 344 #endif 345 346 if (Info().mMetadataDuration.isSome()) { 347 mDuration = Info().mMetadataDuration; 348 } else if (Info().mUnadjustedMetadataEndTime.isSome()) { 349 const media::TimeUnit unadjusted = Info().mUnadjustedMetadataEndTime.ref(); 350 const media::TimeUnit adjustment = Info().mStartTime; 351 mInfo->mMetadataDuration.emplace(unadjusted - adjustment); 352 mDuration = Info().mMetadataDuration; 353 } 354 355 // If we don't know the duration by this point, we assume infinity, per spec. 356 if (mDuration.Ref().isNothing()) { 357 mDuration = Some(media::TimeUnit::FromInfinity()); 358 } 359 MOZ_ASSERT(mDuration.Ref().isSome()); 360 361 if (mInfo->HasVideo()) { 362 mVideoDisplay = mInfo->mVideo.mDisplay; 363 } 364 365 if (IsBeingProfiledOrLogEnabled()) { 366 nsPrintfCString msg( 367 "a=%s, v=%s, size=[%dx%d], duration=%s, encrypted=%d, " 368 "IsEncryptedCustomIdent=%d", 369 mInfo->HasAudio() ? mInfo->mAudio.mMimeType.get() : "none", 370 mInfo->HasVideo() ? mInfo->mVideo.mMimeType.get() : "none", 371 mVideoDisplay.width, mVideoDisplay.height, 372 mDuration.Ref()->ToString().get(), mInfo->IsEncrypted(), 373 mReader->IsEncryptedCustomIdent()); 374 LOG("Metadata loaded : %s", msg.get()); 375 PROFILER_MARKER_TEXT("EESM::OnMetadataRead", MEDIA_PLAYBACK, {}, msg); 376 } 377 378 mMetadataLoadedEvent.Notify(std::move(aMetadata.mInfo), 379 std::move(aMetadata.mTags), 380 MediaDecoderEventVisibility::Observable); 381 ChangeStateTo(State::InitEngine); 382 InitEngine(); 383 } 384 385 void ExternalEngineStateMachine::OnMetadataNotRead(const MediaResult& aError) { 386 AssertOnTaskQueue(); 387 MOZ_ASSERT(mState.IsReadingMetadata()); 388 LOGE("Decode metadata failed, shutting down decoder"); 389 PROFILER_MARKER_UNTYPED("EESM::OnMetadataNotRead", MEDIA_PLAYBACK); 390 mState.AsReadingMetadata()->mMetadataRequest.Complete(); 391 ReportTelemetry(aError); 392 DecodeError(aError); 393 } 394 395 bool ExternalEngineStateMachine::IsFormatSupportedByExternalEngine( 396 const MediaInfo& aInfo) { 397 AssertOnTaskQueue(); 398 MOZ_ASSERT(mState.IsReadingMetadata()); 399 #ifdef MOZ_WMF_MEDIA_ENGINE 400 const bool audioSupported = 401 !aInfo.HasAudio() || 402 MFMediaEngineDecoderModule::SupportsConfig(aInfo.mAudio); 403 const bool videoSupported = 404 !aInfo.HasVideo() || 405 MFMediaEngineDecoderModule::SupportsConfig(aInfo.mVideo); 406 LOG("audio=%s (supported=%d), video=%s(supported=%d)", 407 aInfo.HasAudio() ? aInfo.mAudio.mMimeType.get() : "none", audioSupported, 408 aInfo.HasVideo() ? aInfo.mVideo.mMimeType.get() : "none", videoSupported); 409 return audioSupported && videoSupported; 410 #else 411 return false; 412 #endif 413 } 414 415 RefPtr<MediaDecoder::SeekPromise> ExternalEngineStateMachine::InvokeSeek( 416 const SeekTarget& aTarget) { 417 return InvokeAsync( 418 OwnerThread(), __func__, 419 [self = RefPtr<ExternalEngineStateMachine>(this), this, 420 target = aTarget]() -> RefPtr<MediaDecoder::SeekPromise> { 421 AssertOnTaskQueue(); 422 if (!mEngine || !mEngine->IsInited()) { 423 LOG("Can't perform seek (%" PRId64 ") now, add a pending seek task", 424 target.GetTime().ToMicroseconds()); 425 // We haven't added any pending seek before 426 if (mPendingSeek.mPromise.IsEmpty()) { 427 mPendingTasks.AppendElement(NS_NewRunnableFunction( 428 "ExternalEngineStateMachine::InvokeSeek", 429 [self = RefPtr{this}, this] { 430 if (!mPendingSeek.Exists()) { 431 return; 432 } 433 Seek(*mPendingSeek.mTarget) 434 ->Then(OwnerThread(), __func__, 435 [self = RefPtr{this}, 436 this](const MediaDecoder::SeekPromise:: 437 ResolveOrRejectValue& aVal) { 438 mPendingSeekRequest.Complete(); 439 if (aVal.IsResolve()) { 440 mPendingSeek.Resolve(__func__); 441 } else { 442 mPendingSeek.RejectIfExists(__func__); 443 } 444 mPendingSeek = SeekJob(); 445 }) 446 ->Track(mPendingSeekRequest); 447 })); 448 } else { 449 // Reject previous pending promise, as we will create a new one 450 LOG("Replace previous pending seek with a new one"); 451 mPendingSeek.RejectIfExists(__func__); 452 mPendingSeekRequest.DisconnectIfExists(); 453 } 454 mPendingSeek.mTarget = Some(target); 455 return mPendingSeek.mPromise.Ensure(__func__); 456 } 457 if (mPendingSeek.Exists()) { 458 LOG("Discard pending seek because another new seek happens"); 459 mPendingSeek.RejectIfExists(__func__); 460 mPendingSeek = SeekJob(); 461 mPendingSeekRequest.DisconnectIfExists(); 462 } 463 return self->Seek(target); 464 }); 465 } 466 467 RefPtr<MediaDecoder::SeekPromise> ExternalEngineStateMachine::Seek( 468 const SeekTarget& aTarget) { 469 AssertOnTaskQueue(); 470 if (!mState.IsRunningEngine() && !mState.IsSeekingData() && 471 !mState.IsRecoverEngine()) { 472 MOZ_ASSERT(false, "Can't seek due to unsupported state."); 473 return MediaDecoder::SeekPromise::CreateAndReject(true, __func__); 474 } 475 // We don't support these type of seek, because they're depending on the 476 // implementation of the external engine, which might not be supported. 477 if (aTarget.IsNextFrame() || aTarget.IsVideoOnly()) { 478 return MediaDecoder::SeekPromise::CreateAndReject(true, __func__); 479 } 480 if (IsBeingProfiledOrLogEnabled()) { 481 nsPrintfCString msg("Start seeking to %" PRId64, 482 aTarget.GetTime().ToMicroseconds()); 483 LOG("%s", msg.get()); 484 PROFILER_MARKER_TEXT("EESM::Seek", MEDIA_PLAYBACK, {}, msg); 485 } 486 auto* state = mState.AsSeekingData(); 487 if (!state) { 488 // We're in other states, so change the state to seeking. 489 ChangeStateTo(State::SeekingData); 490 state = mState.AsSeekingData(); 491 } 492 state->SetTarget(aTarget); 493 494 // Update related status. 495 mSentPlaybackEndedEvent = false; 496 mOnPlaybackEvent.Notify(MediaPlaybackEvent::SeekStarted); 497 mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING); 498 499 // Notify the external playback engine about seeking. After the engine changes 500 // its current time, it would send `seeked` event. 501 mEngine->Seek(aTarget.GetTime()); 502 state->mWaitingEngineSeeked = true; 503 SeekReader(); 504 return state->mSeekJob.mPromise.Ensure(__func__); 505 } 506 507 void ExternalEngineStateMachine::SeekReader() { 508 AssertOnTaskQueue(); 509 MOZ_ASSERT(mState.IsSeekingData()); 510 auto* state = mState.AsSeekingData(); 511 512 // Reset the reader first and ask it to perform a demuxer seek. 513 ResetDecode(); 514 state->mWaitingReaderSeeked = true; 515 if (IsBeingProfiledOrLogEnabled()) { 516 nsPrintfCString msg("Seek reader to %" PRId64, 517 state->GetTargetTime().ToMicroseconds()); 518 LOG("%s", msg.get()); 519 PROFILER_MARKER_TEXT("EESM::SeekReader", MEDIA_PLAYBACK, {}, msg); 520 } 521 mReader->Seek(state->mSeekJob.mTarget.ref()) 522 ->Then(OwnerThread(), __func__, this, 523 &ExternalEngineStateMachine::OnSeekResolved, 524 &ExternalEngineStateMachine::OnSeekRejected) 525 ->Track(state->mSeekRequest); 526 } 527 528 void ExternalEngineStateMachine::OnSeekResolved(const media::TimeUnit& aUnit) { 529 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnSeekResolved", 530 MEDIA_PLAYBACK); 531 AssertOnTaskQueue(); 532 MOZ_ASSERT(mState.IsSeekingData()); 533 auto* state = mState.AsSeekingData(); 534 535 LOG("OnReaderSeekResolved"); 536 PROFILER_MARKER_UNTYPED("EESM::OnReaderSeekResolved", MEDIA_PLAYBACK); 537 state->mSeekRequest.Complete(); 538 state->mWaitingReaderSeeked = false; 539 540 // Start sending new data to the external playback engine. 541 if (HasAudio()) { 542 mHasEnoughAudio = false; 543 OnRequestAudio(); 544 } 545 if (HasVideo()) { 546 mHasEnoughVideo = false; 547 OnRequestVideo(); 548 } 549 CheckIfSeekCompleted(); 550 } 551 552 void ExternalEngineStateMachine::OnSeekRejected( 553 const SeekRejectValue& aReject) { 554 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnSeekRejected", 555 MEDIA_PLAYBACK); 556 AssertOnTaskQueue(); 557 MOZ_ASSERT(mState.IsSeekingData()); 558 auto* state = mState.AsSeekingData(); 559 560 LOG("OnReaderSeekRejected"); 561 PROFILER_MARKER_UNTYPED("EESM::OnReaderSeekRejected", MEDIA_PLAYBACK); 562 state->mSeekRequest.Complete(); 563 if (aReject.mError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { 564 LOG("OnSeekRejected reason=WAITING_FOR_DATA type=%s", 565 MediaData::EnumValueToString(aReject.mType)); 566 MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA, 567 !IsRequestingAudioData()); 568 MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA, 569 !IsRequestingVideoData()); 570 MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA, 571 !IsWaitingAudioData()); 572 MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA, 573 !IsWaitingVideoData()); 574 575 // Fire 'waiting' to notify the player that we are waiting for data. 576 mOnNextFrameStatus.Notify( 577 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING); 578 WaitForData(aReject.mType); 579 return; 580 } 581 582 if (aReject.mError == NS_ERROR_DOM_MEDIA_END_OF_STREAM) { 583 EndOfStream(aReject.mType); 584 return; 585 } 586 587 MOZ_ASSERT(NS_FAILED(aReject.mError), 588 "Cancels should also disconnect mSeekRequest"); 589 state->RejectIfExists(__func__); 590 ReportTelemetry(aReject.mError); 591 DecodeError(aReject.mError); 592 } 593 594 bool ExternalEngineStateMachine::IsSeeking() { 595 AssertOnTaskQueue(); 596 const auto* state = mState.AsSeekingData(); 597 return state && state->IsSeeking(); 598 } 599 600 void ExternalEngineStateMachine::CheckIfSeekCompleted() { 601 AssertOnTaskQueue(); 602 MOZ_ASSERT(mState.IsSeekingData()); 603 auto* state = mState.AsSeekingData(); 604 if (state->mWaitingEngineSeeked || state->mWaitingReaderSeeked) { 605 LOG("Seek hasn't been completed yet, waitEngineSeeked=%d, " 606 "waitReaderSeeked=%d", 607 state->mWaitingEngineSeeked, state->mWaitingReaderSeeked); 608 return; 609 } 610 611 // As seeking should be accurate and we can't control the exact timing inside 612 // the external media engine. We always set the newCurrentTime = seekTime 613 // so that the updated HTMLMediaElement.currentTime will always be the seek 614 // target. 615 if (state->GetTargetTime() != mCurrentPosition) { 616 LOG("Force adjusting current time (%" PRId64 617 ") to match to target (%" PRId64 ")", 618 mCurrentPosition.Ref().ToMicroseconds(), 619 state->GetTargetTime().ToMicroseconds()); 620 mCurrentPosition = state->GetTargetTime(); 621 } 622 623 LOG("Seek completed"); 624 PROFILER_MARKER_TEXT( 625 "EESM::SeekCompleted", MEDIA_PLAYBACK, {}, 626 nsPrintfCString("currentTime %" PRId64, 627 mCurrentPosition.Ref().ToMicroseconds())); 628 state->Resolve(__func__); 629 mOnPlaybackEvent.Notify(MediaPlaybackEvent::Invalidate); 630 mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE); 631 StartRunningEngine(); 632 } 633 634 void ExternalEngineStateMachine::ResetDecode() { 635 AssertOnTaskQueue(); 636 if (!mInfo) { 637 return; 638 } 639 640 LOG("ResetDecode"); 641 MediaFormatReader::TrackSet tracks; 642 if (HasVideo()) { 643 mVideoDataRequest.DisconnectIfExists(); 644 mVideoWaitRequest.DisconnectIfExists(); 645 tracks += TrackInfo::kVideoTrack; 646 } 647 if (HasAudio()) { 648 mAudioDataRequest.DisconnectIfExists(); 649 mAudioWaitRequest.DisconnectIfExists(); 650 tracks += TrackInfo::kAudioTrack; 651 } 652 mReader->ResetDecode(tracks); 653 } 654 655 RefPtr<GenericPromise> ExternalEngineStateMachine::InvokeSetSink( 656 const RefPtr<AudioDeviceInfo>& aSink) { 657 MOZ_ASSERT(NS_IsMainThread()); 658 // TODO : can media engine support this? 659 return GenericPromise::CreateAndReject(NS_ERROR_FAILURE, __func__); 660 } 661 662 RefPtr<ShutdownPromise> ExternalEngineStateMachine::Shutdown() { 663 AssertOnTaskQueue(); 664 if (mState.IsShutdownEngine()) { 665 LOG("Already shutdown"); 666 return mState.AsShutdownEngine()->mShutdown; 667 } 668 669 LOG("Shutdown"); 670 ChangeStateTo(State::ShutdownEngine); 671 ResetDecode(); 672 673 mAudioDataRequest.DisconnectIfExists(); 674 mVideoDataRequest.DisconnectIfExists(); 675 mAudioWaitRequest.DisconnectIfExists(); 676 mVideoWaitRequest.DisconnectIfExists(); 677 678 mDuration.DisconnectAll(); 679 mCurrentPosition.DisconnectAll(); 680 mIsAudioDataAudible.DisconnectAll(); 681 682 mMetadataManager.Disconnect(); 683 684 mSetCDMProxyPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_ABORT_ERR, __func__); 685 mSetCDMProxyRequest.DisconnectIfExists(); 686 687 mPendingSeek.RejectIfExists(__func__); 688 mPendingSeekRequest.DisconnectIfExists(); 689 690 mPendingTasks.Clear(); 691 692 if (mEngine) { 693 mEngine->Shutdown(); 694 } 695 696 auto* state = mState.AsShutdownEngine(); 697 state->mShutdown = mReader->Shutdown()->Then( 698 OwnerThread(), __func__, [self = RefPtr{this}, this]() { 699 LOG("Shutting down state machine task queue"); 700 return OwnerThread()->BeginShutdown(); 701 }); 702 return state->mShutdown; 703 } 704 705 void ExternalEngineStateMachine::BufferedRangeUpdated() { 706 AssertOnTaskQueue(); 707 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::BufferedRangeUpdated", 708 MEDIA_PLAYBACK); 709 710 // While playing an unseekable stream of unknown duration, mDuration 711 // is updated as we play. But if data is being downloaded 712 // faster than played, mDuration won't reflect the end of playable data 713 // since we haven't played the frame at the end of buffered data. So update 714 // mDuration here as new data is downloaded to prevent such a lag. 715 if (mBuffered.Ref().IsInvalid()) { 716 return; 717 } 718 719 bool exists; 720 media::TimeUnit end{mBuffered.Ref().GetEnd(&exists)}; 721 if (!exists) { 722 return; 723 } 724 725 // Use estimated duration from buffer ranges when mDuration is unknown or 726 // the estimated duration is larger. 727 if (mDuration.Ref().isNothing() || mDuration.Ref()->IsInfinite() || 728 end > mDuration.Ref().ref()) { 729 mDuration = Some(end); 730 DDLOG(DDLogCategory::Property, "duration_us", 731 mDuration.Ref()->ToMicroseconds()); 732 } 733 } 734 735 #define PERFORM_WHEN_ALLOW(Func) \ 736 do { \ 737 if (mState.IsShutdownEngine() || mHasFatalError || \ 738 AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) { \ 739 return; \ 740 } \ 741 /* Initialzation is not done yet, postpone the operation */ \ 742 if (!mEngine || !mEngine->IsInited()) { \ 743 LOG("%s is called before init", __func__); \ 744 mPendingTasks.AppendElement(NewRunnableMethod( \ 745 __func__, this, &ExternalEngineStateMachine::Func)); \ 746 return; \ 747 } \ 748 } while (false) 749 750 void ExternalEngineStateMachine::SetPlaybackRate(double aPlaybackRate) { 751 AssertOnTaskQueue(); 752 // TODO : consider to make `mPlaybackRate` a mirror to fit other usages like 753 // `mVolume` and `mPreservesPitch`. 754 mPlaybackRate = aPlaybackRate; 755 PlaybackRateChanged(); 756 } 757 758 void ExternalEngineStateMachine::PlaybackRateChanged() { 759 AssertOnTaskQueue(); 760 PERFORM_WHEN_ALLOW(PlaybackRateChanged); 761 MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || 762 mState.IsSeekingData()); 763 mEngine->SetPlaybackRate(mPlaybackRate); 764 } 765 766 void ExternalEngineStateMachine::VolumeChanged() { 767 AssertOnTaskQueue(); 768 PERFORM_WHEN_ALLOW(VolumeChanged); 769 MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || 770 mState.IsSeekingData()); 771 mEngine->SetVolume(mVolume); 772 } 773 774 void ExternalEngineStateMachine::PreservesPitchChanged() { 775 AssertOnTaskQueue(); 776 PERFORM_WHEN_ALLOW(PreservesPitchChanged); 777 MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || 778 mState.IsSeekingData()); 779 mEngine->SetPreservesPitch(mPreservesPitch); 780 } 781 782 void ExternalEngineStateMachine::PlayStateChanged() { 783 AssertOnTaskQueue(); 784 PERFORM_WHEN_ALLOW(PlayStateChanged); 785 MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || 786 mState.IsSeekingData()); 787 if (mPlayState == MediaDecoder::PLAY_STATE_PLAYING) { 788 mEngine->Play(); 789 } else if (mPlayState == MediaDecoder::PLAY_STATE_PAUSED) { 790 mEngine->Pause(); 791 } 792 NotifyAudibleStateChangeIfNeeded(); 793 } 794 795 void ExternalEngineStateMachine::LoopingChanged() { 796 AssertOnTaskQueue(); 797 PERFORM_WHEN_ALLOW(LoopingChanged); 798 MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || 799 mState.IsSeekingData()); 800 mEngine->SetLooping(mLooping); 801 } 802 803 #undef PERFORM_WHEN_ALLOW 804 805 void ExternalEngineStateMachine::EndOfStream(MediaData::Type aType) { 806 AssertOnTaskQueue(); 807 MOZ_ASSERT(mState.IsRunningEngine() || mState.IsSeekingData()); 808 static auto DataTypeToTrackType = [](const MediaData::Type& aType) { 809 if (aType == MediaData::Type::VIDEO_DATA) { 810 return TrackInfo::TrackType::kVideoTrack; 811 } 812 if (aType == MediaData::Type::AUDIO_DATA) { 813 return TrackInfo::TrackType::kAudioTrack; 814 } 815 return TrackInfo::TrackType::kUndefinedTrack; 816 }; 817 mEngine->NotifyEndOfStream(DataTypeToTrackType(aType)); 818 } 819 820 void ExternalEngineStateMachine::WaitForData(MediaData::Type aType) { 821 AssertOnTaskQueue(); 822 MOZ_ASSERT(mState.IsRunningEngine() || mState.IsSeekingData()); 823 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::WaitForData", 824 MEDIA_PLAYBACK); 825 MOZ_ASSERT(aType == MediaData::Type::AUDIO_DATA || 826 aType == MediaData::Type::VIDEO_DATA); 827 828 LOG("WaitForData"); 829 RefPtr<ExternalEngineStateMachine> self = this; 830 if (aType == MediaData::Type::AUDIO_DATA) { 831 MOZ_ASSERT(HasAudio()); 832 mReader->WaitForData(MediaData::Type::AUDIO_DATA) 833 ->Then( 834 OwnerThread(), __func__, 835 [self, this](MediaData::Type aType) { 836 AUTO_PROFILER_LABEL( 837 "ExternalEngineStateMachine::WaitForData:AudioResolved", 838 MEDIA_PLAYBACK); 839 MOZ_ASSERT(aType == MediaData::Type::AUDIO_DATA); 840 LOG("Done waiting for audio data"); 841 mAudioWaitRequest.Complete(); 842 MaybeFinishWaitForData(); 843 }, 844 [self, this](const WaitForDataRejectValue& aRejection) { 845 AUTO_PROFILER_LABEL( 846 "ExternalEngineStateMachine::WaitForData:AudioRejected", 847 MEDIA_PLAYBACK); 848 mAudioWaitRequest.Complete(); 849 DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); 850 }) 851 ->Track(mAudioWaitRequest); 852 } else { 853 MOZ_ASSERT(HasVideo()); 854 mReader->WaitForData(MediaData::Type::VIDEO_DATA) 855 ->Then( 856 OwnerThread(), __func__, 857 [self, this](MediaData::Type aType) { 858 AUTO_PROFILER_LABEL( 859 "ExternalEngineStateMachine::WaitForData:VideoResolved", 860 MEDIA_PLAYBACK); 861 MOZ_ASSERT(aType == MediaData::Type::VIDEO_DATA); 862 LOG("Done waiting for video data"); 863 mVideoWaitRequest.Complete(); 864 MaybeFinishWaitForData(); 865 }, 866 [self, this](const WaitForDataRejectValue& aRejection) { 867 AUTO_PROFILER_LABEL( 868 "ExternalEngineStateMachine::WaitForData:VideoRejected", 869 MEDIA_PLAYBACK); 870 mVideoWaitRequest.Complete(); 871 DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); 872 }) 873 ->Track(mVideoWaitRequest); 874 } 875 } 876 877 void ExternalEngineStateMachine::MaybeFinishWaitForData() { 878 AssertOnTaskQueue(); 879 MOZ_ASSERT(mState.IsRunningEngine() || mState.IsSeekingData()); 880 881 bool isWaitingForAudio = HasAudio() && mAudioWaitRequest.Exists(); 882 bool isWaitingForVideo = HasVideo() && mVideoWaitRequest.Exists(); 883 if (isWaitingForAudio || isWaitingForVideo) { 884 LOG("Still waiting for data (waitAudio=%d, waitVideo=%d)", 885 isWaitingForAudio, isWaitingForVideo); 886 return; 887 } 888 889 LOG("Finished waiting for data"); 890 if (mState.IsSeekingData()) { 891 SeekReader(); 892 return; 893 } 894 if (HasAudio()) { 895 RunningEngineUpdate(MediaData::Type::AUDIO_DATA); 896 } 897 if (HasVideo()) { 898 RunningEngineUpdate(MediaData::Type::VIDEO_DATA); 899 } 900 } 901 902 void ExternalEngineStateMachine::StartRunningEngine() { 903 ChangeStateTo(State::RunningEngine); 904 // Manually check the play state because the engine might be recovered from 905 // crash or just get recreated, so PlayStateChanged() won't be triggered. 906 if (mPlayState == MediaDecoder::PLAY_STATE_PLAYING) { 907 mEngine->Play(); 908 } 909 if (HasAudio()) { 910 RunningEngineUpdate(MediaData::Type::AUDIO_DATA); 911 } 912 if (HasVideo()) { 913 RunningEngineUpdate(MediaData::Type::VIDEO_DATA); 914 } 915 // Run tasks which was called before the engine is ready. 916 if (!mPendingTasks.IsEmpty()) { 917 for (auto& task : mPendingTasks) { 918 (void)OwnerThread()->Dispatch(task.forget()); 919 } 920 mPendingTasks.Clear(); 921 } 922 } 923 924 void ExternalEngineStateMachine::RunningEngineUpdate(MediaData::Type aType) { 925 AssertOnTaskQueue(); 926 MOZ_ASSERT(mState.IsRunningEngine() || mState.IsSeekingData()); 927 if (aType == MediaData::Type::AUDIO_DATA && !mHasEnoughAudio) { 928 OnRequestAudio(); 929 } 930 if (aType == MediaData::Type::VIDEO_DATA && !mHasEnoughVideo) { 931 OnRequestVideo(); 932 } 933 } 934 935 void ExternalEngineStateMachine::OnRequestAudio() { 936 AssertOnTaskQueue(); 937 MOZ_ASSERT(mState.IsRunningEngine() || mState.IsSeekingData()); 938 939 if (!HasAudio()) { 940 return; 941 } 942 943 if (IsRequestingAudioData() || mAudioWaitRequest.Exists() || IsSeeking()) { 944 LOGV( 945 "No need to request audio, isRequesting=%d, waitingAudio=%d, " 946 "isSeeking=%d", 947 IsRequestingAudioData(), mAudioWaitRequest.Exists(), IsSeeking()); 948 return; 949 } 950 951 PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestData); 952 RefPtr<ExternalEngineStateMachine> self = this; 953 mReader->RequestAudioData() 954 ->Then( 955 OwnerThread(), __func__, 956 [this, self, perfRecorder(std::move(perfRecorder))]( 957 const RefPtr<AudioData>& aAudio) mutable { 958 perfRecorder.Record(); 959 mAudioDataRequest.Complete(); 960 AUTO_PROFILER_LABEL( 961 "ExternalEngineStateMachine::OnRequestAudio:Resolved", 962 MEDIA_PLAYBACK); 963 MOZ_ASSERT(aAudio); 964 RunningEngineUpdate(MediaData::Type::AUDIO_DATA); 965 }, 966 [this, self](const MediaResult& aError) { 967 mAudioDataRequest.Complete(); 968 AUTO_PROFILER_LABEL( 969 "ExternalEngineStateMachine::OnRequestAudio:Rejected", 970 MEDIA_PLAYBACK); 971 LOG("OnRequestAudio ErrorName=%s Message=%s", 972 aError.ErrorName().get(), aError.Message().get()); 973 switch (aError.Code()) { 974 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: 975 WaitForData(MediaData::Type::AUDIO_DATA); 976 break; 977 case NS_ERROR_DOM_MEDIA_CANCELED: 978 OnRequestAudio(); 979 break; 980 case NS_ERROR_DOM_MEDIA_END_OF_STREAM: 981 LOG("Reach to the end, no more audio data"); 982 EndOfStream(MediaData::Type::AUDIO_DATA); 983 break; 984 case NS_ERROR_DOM_MEDIA_REMOTE_CRASHED_MF_CDM_ERR: 985 // We will handle the process crash in `NotifyErrorInternal()` 986 // so here just silently ignore this. 987 break; 988 default: 989 ReportTelemetry(aError); 990 DecodeError(aError); 991 } 992 }) 993 ->Track(mAudioDataRequest); 994 } 995 996 void ExternalEngineStateMachine::OnRequestVideo() { 997 AssertOnTaskQueue(); 998 MOZ_ASSERT(mState.IsRunningEngine() || mState.IsSeekingData()); 999 1000 if (!HasVideo()) { 1001 return; 1002 } 1003 1004 if (IsTrackingVideoData() || IsSeeking()) { 1005 LOGV( 1006 "No need to request video, isRequesting=%d, waitingVideo=%d, " 1007 "isSeeking=%d", 1008 IsRequestingVideoData(), mVideoWaitRequest.Exists(), IsSeeking()); 1009 return; 1010 } 1011 1012 PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestData, 1013 Info().mVideo.mImage.height); 1014 RefPtr<ExternalEngineStateMachine> self = this; 1015 mReader->RequestVideoData(GetVideoThreshold(), false) 1016 ->Then( 1017 OwnerThread(), __func__, 1018 [this, self, perfRecorder(std::move(perfRecorder))]( 1019 const RefPtr<VideoData>& aVideo) mutable { 1020 perfRecorder.Record(); 1021 mVideoDataRequest.Complete(); 1022 AUTO_PROFILER_LABEL( 1023 "ExternalEngineStateMachine::OnRequestVideo:Resolved", 1024 MEDIA_PLAYBACK); 1025 MOZ_ASSERT(aVideo); 1026 if (!mHasReceivedFirstDecodedVideoFrame) { 1027 mHasReceivedFirstDecodedVideoFrame = true; 1028 OnLoadedFirstFrame(); 1029 } 1030 RunningEngineUpdate(MediaData::Type::VIDEO_DATA); 1031 // Send image to PIP window. 1032 if (mSecondaryVideoContainer.Ref()) { 1033 mSecondaryVideoContainer.Ref()->SetCurrentFrame( 1034 mVideoDisplay, aVideo->mImage, TimeStamp::Now(), 1035 media::TimeUnit::Invalid(), aVideo->mTime); 1036 } else { 1037 mVideoFrameContainer->SetCurrentFrame( 1038 mVideoDisplay, aVideo->mImage, TimeStamp::Now(), 1039 media::TimeUnit::Invalid(), aVideo->mTime); 1040 } 1041 }, 1042 [this, self](const MediaResult& aError) { 1043 mVideoDataRequest.Complete(); 1044 AUTO_PROFILER_LABEL( 1045 "ExternalEngineStateMachine::OnRequestVideo:Rejected", 1046 MEDIA_PLAYBACK); 1047 LOG("OnRequestVideo ErrorName=%s Message=%s", 1048 aError.ErrorName().get(), aError.Message().get()); 1049 switch (aError.Code()) { 1050 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: 1051 WaitForData(MediaData::Type::VIDEO_DATA); 1052 break; 1053 case NS_ERROR_DOM_MEDIA_CANCELED: 1054 OnRequestVideo(); 1055 break; 1056 case NS_ERROR_DOM_MEDIA_END_OF_STREAM: 1057 LOG("Reach to the end, no more video data"); 1058 EndOfStream(MediaData::Type::VIDEO_DATA); 1059 break; 1060 case NS_ERROR_DOM_MEDIA_REMOTE_CRASHED_MF_CDM_ERR: 1061 // We will handle the process crash in `NotifyErrorInternal()` 1062 // so here just silently ignore this. 1063 break; 1064 default: 1065 ReportTelemetry(aError); 1066 DecodeError(aError); 1067 } 1068 }) 1069 ->Track(mVideoDataRequest); 1070 } 1071 1072 void ExternalEngineStateMachine::OnLoadedFirstFrame() { 1073 AssertOnTaskQueue(); 1074 // We will wait until receive the first video frame. 1075 if (mInfo->HasVideo() && !mHasReceivedFirstDecodedVideoFrame) { 1076 LOG("Hasn't received first decoded video frame"); 1077 return; 1078 } 1079 LOG("OnLoadedFirstFrame"); 1080 MediaDecoderEventVisibility visibility = 1081 mSentFirstFrameLoadedEvent ? MediaDecoderEventVisibility::Suppressed 1082 : MediaDecoderEventVisibility::Observable; 1083 mSentFirstFrameLoadedEvent = true; 1084 mFirstFrameLoadedEvent.Notify(UniquePtr<MediaInfo>(new MediaInfo(Info())), 1085 visibility); 1086 mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE); 1087 } 1088 1089 void ExternalEngineStateMachine::OnLoadedData() { 1090 AssertOnTaskQueue(); 1091 // In case the external engine doesn't send the first frame loaded event 1092 // correctly. 1093 LOG("OnLoadedData"); 1094 if (!mSentFirstFrameLoadedEvent) { 1095 OnLoadedFirstFrame(); 1096 } 1097 mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE); 1098 } 1099 1100 void ExternalEngineStateMachine::OnWaiting() { 1101 AssertOnTaskQueue(); 1102 LOG("OnWaiting"); 1103 mOnNextFrameStatus.Notify( 1104 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING); 1105 } 1106 1107 void ExternalEngineStateMachine::OnPlaying() { 1108 AssertOnTaskQueue(); 1109 LOG("OnPlaying"); 1110 mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE); 1111 } 1112 1113 void ExternalEngineStateMachine::OnSeeked() { 1114 AssertOnTaskQueue(); 1115 if (!mState.IsSeekingData()) { 1116 LOG("Engine Seeking has been completed, ignore the event"); 1117 return; 1118 } 1119 MOZ_ASSERT(mState.IsSeekingData()); 1120 1121 const auto currentTime = mEngine->GetCurrentPosition(); 1122 auto* state = mState.AsSeekingData(); 1123 if (IsBeingProfiledOrLogEnabled()) { 1124 nsPrintfCString msg("target=%" PRId64 ", currentTime=%" PRId64, 1125 state->GetTargetTime().ToMicroseconds(), 1126 currentTime.ToMicroseconds()); 1127 LOG("OnEngineSeeked : %s", msg.get()); 1128 PROFILER_MARKER_TEXT("EESM::OnEngineSeeked", MEDIA_PLAYBACK, {}, msg); 1129 } 1130 // It's possible to receive multiple seeked event if we seek the engine 1131 // before the previous seeking finishes, so we would wait until the last 1132 // seeking is finished. 1133 if (currentTime >= state->GetTargetTime()) { 1134 state->mWaitingEngineSeeked = false; 1135 CheckIfSeekCompleted(); 1136 } 1137 } 1138 1139 void ExternalEngineStateMachine::OnBufferingStarted() { 1140 AssertOnTaskQueue(); 1141 mOnNextFrameStatus.Notify( 1142 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING); 1143 if (HasAudio()) { 1144 WaitForData(MediaData::Type::AUDIO_DATA); 1145 } 1146 if (HasVideo()) { 1147 WaitForData(MediaData::Type::VIDEO_DATA); 1148 } 1149 if (IsBeingProfiledOrLogEnabled()) { 1150 nsPrintfCString msg("hasAudio=%d, hasVideo=%d", HasAudio(), HasVideo()); 1151 LOG("OnBufferingStarted : %s", msg.get()); 1152 PROFILER_MARKER_TEXT("EESM::OnBufferingStarted", MEDIA_PLAYBACK, {}, msg); 1153 } 1154 } 1155 1156 void ExternalEngineStateMachine::OnBufferingEnded() { 1157 AssertOnTaskQueue(); 1158 LOG("OnBufferingEnded"); 1159 PROFILER_MARKER_UNTYPED("EESM::OnBufferingEnded", MEDIA_PLAYBACK); 1160 mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE); 1161 } 1162 1163 void ExternalEngineStateMachine::OnEnded() { 1164 AssertOnTaskQueue(); 1165 if (mSentPlaybackEndedEvent) { 1166 return; 1167 } 1168 LOG("Playback is ended"); 1169 PROFILER_MARKER_UNTYPED("EESM::OnEnded", MEDIA_PLAYBACK); 1170 mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE); 1171 mOnPlaybackEvent.Notify(MediaPlaybackEvent::PlaybackEnded); 1172 mSentPlaybackEndedEvent = true; 1173 } 1174 1175 void ExternalEngineStateMachine::OnTimeupdate() { 1176 AssertOnTaskQueue(); 1177 if (IsSeeking()) { 1178 return; 1179 } 1180 mCurrentPosition = mEngine->GetCurrentPosition(); 1181 if (mDuration.Ref().ref() < mCurrentPosition.Ref()) { 1182 mDuration = Some(mCurrentPosition.Ref()); 1183 } 1184 if (IsBeingProfiledOrLogEnabled()) { 1185 nsPrintfCString msg("current time=%" PRId64 ", duration=%" PRId64, 1186 mCurrentPosition.Ref().ToMicroseconds(), 1187 mDuration.Ref()->ToMicroseconds()); 1188 LOG("OnTimeupdate, %s", msg.get()); 1189 PROFILER_MARKER_TEXT("EESM::OnTimeupdate", MEDIA_PLAYBACK, {}, msg); 1190 } 1191 } 1192 1193 void ExternalEngineStateMachine::NotifyEventInternal( 1194 ExternalEngineEvent aEvent) { 1195 AssertOnTaskQueue(); 1196 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::NotifyEventInternal", 1197 MEDIA_PLAYBACK); 1198 if (mState.IsShutdownEngine()) { 1199 return; 1200 } 1201 PROFILER_MARKER_TEXT("EESM::NotifyEventInternal", MEDIA_PLAYBACK, {}, 1202 nsPrintfCString("%s", ExternalEngineEventToStr(aEvent))); 1203 switch (aEvent) { 1204 case ExternalEngineEvent::LoadedMetaData: 1205 // We read metadata by ourselves, ignore this if there is any. 1206 MOZ_ASSERT(mInfo); 1207 break; 1208 case ExternalEngineEvent::LoadedFirstFrame: 1209 OnLoadedFirstFrame(); 1210 break; 1211 case ExternalEngineEvent::LoadedData: 1212 OnLoadedData(); 1213 break; 1214 case ExternalEngineEvent::Waiting: 1215 OnWaiting(); 1216 break; 1217 case ExternalEngineEvent::Playing: 1218 OnPlaying(); 1219 break; 1220 case ExternalEngineEvent::Seeked: 1221 OnSeeked(); 1222 break; 1223 case ExternalEngineEvent::BufferingStarted: 1224 OnBufferingStarted(); 1225 break; 1226 case ExternalEngineEvent::BufferingEnded: 1227 OnBufferingEnded(); 1228 break; 1229 case ExternalEngineEvent::Timeupdate: 1230 OnTimeupdate(); 1231 break; 1232 case ExternalEngineEvent::Ended: 1233 OnEnded(); 1234 break; 1235 case ExternalEngineEvent::RequestForAudio: 1236 mHasEnoughAudio = false; 1237 if (ShouldRunEngineUpdateForRequest()) { 1238 RunningEngineUpdate(MediaData::Type::AUDIO_DATA); 1239 } 1240 break; 1241 case ExternalEngineEvent::RequestForVideo: 1242 mHasEnoughVideo = false; 1243 if (ShouldRunEngineUpdateForRequest()) { 1244 RunningEngineUpdate(MediaData::Type::VIDEO_DATA); 1245 } 1246 break; 1247 case ExternalEngineEvent::AudioEnough: 1248 mHasEnoughAudio = true; 1249 break; 1250 case ExternalEngineEvent::VideoEnough: 1251 mHasEnoughVideo = true; 1252 break; 1253 default: 1254 MOZ_ASSERT_UNREACHABLE("Undefined event!"); 1255 break; 1256 } 1257 } 1258 1259 bool ExternalEngineStateMachine::ShouldRunEngineUpdateForRequest() { 1260 // Running engine update will request new data, which could be run on 1261 // `RunningEngine` or `SeekingData` state. However, in `SeekingData` we should 1262 // only request new data after finishing reader seek, otherwise the reader 1263 // would start requesting data from a wrong position. 1264 return mState.IsRunningEngine() || 1265 (mState.AsSeekingData() && 1266 !mState.AsSeekingData()->mWaitingReaderSeeked); 1267 } 1268 1269 void ExternalEngineStateMachine::NotifyErrorInternal( 1270 const MediaResult& aError) { 1271 AssertOnTaskQueue(); 1272 LOG("Engine error: %s", aError.Description().get()); 1273 PROFILER_MARKER_TEXT("EESM::NotifyErrorInternal", MEDIA_PLAYBACK, {}, 1274 aError.Description()); 1275 if (aError == NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR) { 1276 // The external engine doesn't support the type, try to notify the decoder 1277 // to use our own state machine again. 1278 ReportTelemetry(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR); 1279 DecodeError( 1280 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); 1281 } else if (aError == NS_ERROR_DOM_MEDIA_REMOTE_CRASHED_MF_CDM_ERR) { 1282 ReportTelemetry(NS_ERROR_DOM_MEDIA_REMOTE_CRASHED_MF_CDM_ERR); 1283 RecoverFromCDMProcessCrashIfNeeded(); 1284 } else if (mState.IsInitEngine() && mKeySystem.IsEmpty()) { 1285 // If any error occurs during media engine initialization, we should attempt 1286 // to use another state machine for playback. Unless the key system is 1287 // already set, it indicates that playback can only be initiated via the 1288 // media engine. In this case, we will propagate the error and refrain 1289 // from trying another state machine. 1290 LOG("Error happened on the engine initialization, the media engine " 1291 "playback might not be supported"); 1292 ReportTelemetry(NS_ERROR_DOM_MEDIA_MEDIA_ENGINE_INITIALIZATION_ERR); 1293 DecodeError( 1294 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); 1295 } else { 1296 ReportTelemetry(aError); 1297 DecodeError(aError); 1298 } 1299 } 1300 1301 void ExternalEngineStateMachine::NotifyResizingInternal(uint32_t aWidth, 1302 uint32_t aHeight) { 1303 if (IsBeingProfiledOrLogEnabled()) { 1304 nsPrintfCString msg("video resize from [%d,%d] to [%d,%d]", 1305 mVideoDisplay.width, mVideoDisplay.height, aWidth, 1306 aHeight); 1307 LOG("%s", msg.get()); 1308 PROFILER_MARKER_TEXT("EESM::NotifyResizingInternal", MEDIA_PLAYBACK, {}, 1309 msg); 1310 } 1311 mVideoDisplay = gfx::IntSize{aWidth, aHeight}; 1312 } 1313 1314 void ExternalEngineStateMachine::RecoverFromCDMProcessCrashIfNeeded() { 1315 AssertOnTaskQueue(); 1316 if (mState.IsRecoverEngine()) { 1317 return; 1318 } 1319 ProcessCrashMonitor::NotifyCrash(); 1320 if (!ProcessCrashMonitor::ShouldRecoverProcess()) { 1321 LOG("CDM process has crashed too many times, abort recovery"); 1322 DecodeError( 1323 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); 1324 return; 1325 } 1326 1327 if (mState.IsInitEngine()) { 1328 LOG("Failed on the engine initialization, the media engine playback might " 1329 "not be supported"); 1330 DecodeError( 1331 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); 1332 return; 1333 } 1334 1335 if (IsBeingProfiledOrLogEnabled()) { 1336 nsPrintfCString msg( 1337 "CDM process crashed, recover the engine again (last time=%" PRId64 ")", 1338 mCurrentPosition.Ref().ToMicroseconds()); 1339 LOG("%s", msg.get()); 1340 PROFILER_MARKER_TEXT("EESM::RecoverFromCDMProcessCrashIfNeeded", 1341 MEDIA_PLAYBACK, {}, msg); 1342 } 1343 ChangeStateTo(State::RecoverEngine); 1344 if (HasVideo()) { 1345 mVideoDataRequest.DisconnectIfExists(); 1346 mVideoWaitRequest.DisconnectIfExists(); 1347 } 1348 if (HasAudio()) { 1349 mAudioDataRequest.DisconnectIfExists(); 1350 mAudioWaitRequest.DisconnectIfExists(); 1351 } 1352 // Ask the reader to shutdown current decoders which are no longer available 1353 // due to the remote process crash. 1354 mReader->ReleaseResources(); 1355 InitEngine(); 1356 } 1357 1358 media::TimeUnit ExternalEngineStateMachine::GetVideoThreshold() { 1359 AssertOnTaskQueue(); 1360 if (auto* state = mState.AsSeekingData()) { 1361 return state->GetTargetTime(); 1362 } 1363 return mCurrentPosition.Ref(); 1364 } 1365 1366 void ExternalEngineStateMachine::UpdateSecondaryVideoContainer() { 1367 AssertOnTaskQueue(); 1368 LOG("UpdateSecondaryVideoContainer=%p", mSecondaryVideoContainer.Ref().get()); 1369 mOnSecondaryVideoContainerInstalled.Notify(mSecondaryVideoContainer.Ref()); 1370 } 1371 1372 RefPtr<SetCDMPromise> ExternalEngineStateMachine::SetCDMProxy( 1373 CDMProxy* aProxy) { 1374 if (mState.IsShutdownEngine()) { 1375 return SetCDMPromise::CreateAndReject(NS_ERROR_FAILURE, __func__); 1376 } 1377 1378 if (!mEngine || !mEngine->IsInited()) { 1379 LOG("SetCDMProxy is called before init"); 1380 mReader->SetEncryptedCustomIdent(); 1381 mPendingTasks.AppendElement(NS_NewRunnableFunction( 1382 "ExternalEngineStateMachine::SetCDMProxy", 1383 [self = RefPtr{this}, proxy = RefPtr{aProxy}, this] { 1384 SetCDMProxy(proxy) 1385 ->Then(OwnerThread(), __func__, 1386 [self = RefPtr{this}, 1387 this](const SetCDMPromise::ResolveOrRejectValue& aVal) { 1388 mSetCDMProxyRequest.Complete(); 1389 if (aVal.IsResolve()) { 1390 mSetCDMProxyPromise.Resolve(true, __func__); 1391 } else { 1392 mSetCDMProxyPromise.Reject(NS_ERROR_DOM_MEDIA_CDM_ERR, 1393 __func__); 1394 } 1395 }) 1396 ->Track(mSetCDMProxyRequest); 1397 })); 1398 return mSetCDMProxyPromise.Ensure(__func__); 1399 } 1400 1401 // TODO : set CDM proxy again if we recreate the media engine after crash. 1402 mKeySystem = NS_ConvertUTF16toUTF8(aProxy->KeySystem()); 1403 if (IsBeingProfiledOrLogEnabled()) { 1404 nsPrintfCString msg("SetCDMProxy=%p (key-system=%s)", aProxy, 1405 mKeySystem.get()); 1406 LOG("%s", msg.get()); 1407 PROFILER_MARKER_TEXT("EESM::SetCDMProxy", MEDIA_PLAYBACK, {}, msg); 1408 } 1409 MOZ_DIAGNOSTIC_ASSERT(mEngine); 1410 // TODO : we should check the result of setting CDM proxy in the MFCDM process 1411 if (!mEngine->SetCDMProxy(aProxy)) { 1412 LOG("Failed to set CDM proxy on the engine"); 1413 return SetCDMPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CDM_ERR, __func__); 1414 } 1415 return MediaDecoderStateMachineBase::SetCDMProxy(aProxy); 1416 } 1417 1418 nsresult ExternalEngineStateMachine::IsCDMProxySupported(CDMProxy* aProxy) { 1419 #ifdef MOZ_WMF_CDM 1420 MOZ_ASSERT(aProxy); 1421 1422 // The CDM needs to be hosted in the same process of the external engine, and 1423 // only WMFCDM meets this requirement. 1424 if (!aProxy->AsWMFCDMProxy()) { 1425 return NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR; 1426 } 1427 1428 // 1=enabled encrypted and clear, 2=enabled encrypted 1429 if (StaticPrefs::media_wmf_media_engine_enabled() != 1 && 1430 StaticPrefs::media_wmf_media_engine_enabled() != 2) { 1431 return NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR; 1432 } 1433 1434 return NS_OK; 1435 #else 1436 return NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR; 1437 #endif 1438 } 1439 1440 void ExternalEngineStateMachine::ReportTelemetry(const MediaResult& aError) { 1441 glean::mfcdm::ErrorExtra extraData; 1442 extraData.errorName = Some(aError.ErrorName()); 1443 extraData.currentState = Some(nsAutoCString{StateToStr(mState.mName)}); 1444 nsAutoCString resolution; 1445 if (mInfo) { 1446 if (mInfo->HasAudio()) { 1447 extraData.audioCodec = Some(mInfo->mAudio.mMimeType); 1448 } 1449 if (mInfo->HasVideo()) { 1450 extraData.videoCodec = Some(mInfo->mVideo.mMimeType); 1451 DetermineResolutionForTelemetry(*mInfo, resolution); 1452 extraData.resolution = Some(resolution); 1453 } 1454 } 1455 if (!mKeySystem.IsEmpty()) { 1456 extraData.keySystem = Some(mKeySystem); 1457 } 1458 if (auto platformErrorCode = aError.GetPlatformErrorCode()) { 1459 extraData.platformError = platformErrorCode; 1460 } 1461 glean::mfcdm::error.Record(Some(extraData)); 1462 if (MOZ_LOG_TEST(gMediaDecoderLog, LogLevel::Debug)) { 1463 nsPrintfCString logMessage{"MFCDM Error event, error=%s", 1464 aError.ErrorName().get()}; 1465 if (auto platformErrorCode = aError.GetPlatformErrorCode()) { 1466 logMessage.Append(nsPrintfCString{", hr=%x", *platformErrorCode}); 1467 } 1468 if (mInfo) { 1469 if (mInfo->HasAudio()) { 1470 logMessage.Append( 1471 nsPrintfCString{", audio=%s", mInfo->mAudio.mMimeType.get()}); 1472 } 1473 if (mInfo->HasVideo()) { 1474 logMessage.Append(nsPrintfCString{", video=%s, resolution=%s", 1475 mInfo->mVideo.mMimeType.get(), 1476 resolution.get()}); 1477 } 1478 } 1479 if (!mKeySystem.IsEmpty()) { 1480 logMessage.Append(nsPrintfCString{", keySystem=%s", mKeySystem.get()}); 1481 } 1482 LOG("%s", logMessage.get()); 1483 } 1484 } 1485 1486 void ExternalEngineStateMachine::DecodeError(const MediaResult& aError) { 1487 if (aError != NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA || 1488 aError != NS_ERROR_DOM_MEDIA_CANCELED) { 1489 mHasFatalError = true; 1490 } 1491 MediaDecoderStateMachineBase ::DecodeError(aError); 1492 } 1493 1494 void ExternalEngineStateMachine::NotifyAudibleStateChangeIfNeeded() { 1495 // Only perform a simple check because we can't access audio data from the 1496 // external engine. 1497 mIsAudioDataAudible = mInfo && HasAudio() && 1498 mPlayState == MediaDecoder::PLAY_STATE_PLAYING && 1499 mState.IsRunningEngine(); 1500 } 1501 1502 #undef FMT 1503 #undef LOG 1504 #undef LOGV 1505 #undef LOGW 1506 #undef LOGE 1507 1508 } // namespace mozilla