MediaTrackGraph.cpp (156511B)
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ 2 /* This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 #include "AudioCaptureTrack.h" 7 #include "AudioDeviceInfo.h" 8 #include "AudioNodeExternalInputTrack.h" 9 #include "AudioNodeTrack.h" 10 #include "AudioSegment.h" 11 #include "CrossGraphPort.h" 12 #include "CubebDeviceEnumerator.h" 13 #include "ForwardedInputTrack.h" 14 #include "ImageContainer.h" 15 #include "MediaTrackGraphImpl.h" 16 #include "VideoSegment.h" 17 #include "mozilla/Attributes.h" 18 #include "mozilla/Logging.h" 19 #include "nsContentUtils.h" 20 #include "nsGlobalWindowInner.h" 21 #include "nsPrintfCString.h" 22 #include "nsServiceManagerUtils.h" 23 #include "prerror.h" 24 #if defined(MOZ_WEBRTC) 25 # include "MediaEngineWebRTCAudio.h" 26 #endif // MOZ_WEBRTC 27 #include <algorithm> 28 29 #include "GeckoProfiler.h" 30 #include "GraphRunner.h" 31 #include "MediaTrackListener.h" 32 #include "Tracing.h" 33 #include "UnderrunHandler.h" 34 #include "VideoFrameContainer.h" 35 #include "VideoUtils.h" 36 #include "mozilla/AbstractThread.h" 37 #include "mozilla/CycleCollectedJSRuntime.h" 38 #include "mozilla/Preferences.h" 39 #include "mozilla/StaticPrefs_dom.h" 40 #include "mozilla/StaticPrefs_media.h" 41 #include "mozilla/dom/BaseAudioContextBinding.h" 42 #include "mozilla/dom/Document.h" 43 #include "mozilla/dom/WorkletThread.h" 44 #include "mozilla/media/MediaUtils.h" 45 #include "transport/runnable_utils.h" 46 #include "webaudio/blink/DenormalDisabler.h" 47 #include "webaudio/blink/HRTFDatabaseLoader.h" 48 49 using namespace mozilla::layers; 50 using namespace mozilla::dom; 51 using namespace mozilla::gfx; 52 using namespace mozilla::media; 53 54 namespace mozilla { 55 56 using AudioDeviceID = CubebUtils::AudioDeviceID; 57 using IsInShutdown = MediaTrack::IsInShutdown; 58 59 LazyLogModule gMediaTrackGraphLog("MediaTrackGraph"); 60 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg) 61 62 NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() { 63 return mNativeInputTrack.get(); 64 } 65 66 DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack( 67 CubebUtils::AudioDeviceID aID) { 68 if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) { 69 return mNativeInputTrack.get(); 70 } 71 for (const RefPtr<NonNativeInputTrack>& t : mNonNativeInputTracks) { 72 if (t->mDeviceId == aID) { 73 return t.get(); 74 } 75 } 76 return nullptr; 77 } 78 79 NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() { 80 if (mNonNativeInputTracks.IsEmpty()) { 81 return nullptr; 82 } 83 return mNonNativeInputTracks[0].get(); 84 } 85 86 void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) { 87 if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) { 88 MOZ_ASSERT(!mNativeInputTrack); 89 mNativeInputTrack = native; 90 } else { 91 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack(); 92 MOZ_ASSERT(nonNative); 93 struct DeviceTrackComparator { 94 public: 95 bool Equals(const RefPtr<NonNativeInputTrack>& aTrack, 96 CubebUtils::AudioDeviceID aDeviceId) const { 97 return aTrack->mDeviceId == aDeviceId; 98 } 99 }; 100 MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId, 101 DeviceTrackComparator())); 102 mNonNativeInputTracks.AppendElement(nonNative); 103 } 104 } 105 106 void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) { 107 if (aTrack->AsNativeInputTrack()) { 108 MOZ_ASSERT(mNativeInputTrack); 109 MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack()); 110 mNativeInputTrack = nullptr; 111 } else { 112 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack(); 113 MOZ_ASSERT(nonNative); 114 DebugOnly<bool> removed = mNonNativeInputTracks.RemoveElement(nonNative); 115 MOZ_ASSERT(removed); 116 } 117 } 118 119 /** 120 * A hash table containing the graph instances, one per Window ID, 121 * sample rate, and device ID combination. 122 */ 123 124 struct MediaTrackGraphImpl::Lookup final { 125 HashNumber Hash() const { 126 return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID); 127 } 128 const uint64_t mWindowID; 129 const TrackRate mSampleRate; 130 const CubebUtils::AudioDeviceID mOutputDeviceID; 131 }; 132 133 // Implicit to support GraphHashSet.lookup(*graph). 134 MOZ_IMPLICIT MediaTrackGraphImpl::operator MediaTrackGraphImpl::Lookup() const { 135 return {mWindowID, mSampleRate, PrimaryOutputDeviceID()}; 136 } 137 138 namespace { 139 struct GraphHasher { // for HashSet 140 using Lookup = const MediaTrackGraphImpl::Lookup; 141 142 static HashNumber hash(const Lookup& aLookup) { return aLookup.Hash(); } 143 144 static bool match(const MediaTrackGraphImpl* aGraph, const Lookup& aLookup) { 145 return aGraph->mWindowID == aLookup.mWindowID && 146 aGraph->GraphRate() == aLookup.mSampleRate && 147 aGraph->PrimaryOutputDeviceID() == aLookup.mOutputDeviceID; 148 } 149 }; 150 151 // The weak reference to the graph is removed when its last track is removed. 152 using GraphHashSet = 153 HashSet<MediaTrackGraphImpl*, GraphHasher, InfallibleAllocPolicy>; 154 GraphHashSet* Graphs() { 155 MOZ_ASSERT(NS_IsMainThread()); 156 static GraphHashSet sGraphs(4); // 4 is minimum HashSet capacity 157 return &sGraphs; 158 } 159 } // anonymous namespace 160 161 static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode, 162 MediaSegment* aSegment, 163 MediaSegment* aRawSegment) { 164 if (aDisabledMode == DisabledTrackMode::ENABLED) { 165 return; 166 } 167 if (aDisabledMode == DisabledTrackMode::SILENCE_BLACK) { 168 aSegment->ReplaceWithDisabled(); 169 if (aRawSegment) { 170 aRawSegment->ReplaceWithDisabled(); 171 } 172 } else if (aDisabledMode == DisabledTrackMode::SILENCE_FREEZE) { 173 aSegment->ReplaceWithNull(); 174 if (aRawSegment) { 175 aRawSegment->ReplaceWithNull(); 176 } 177 } else { 178 MOZ_CRASH("Unsupported mode"); 179 } 180 } 181 182 MediaTrackGraphImpl::~MediaTrackGraphImpl() { 183 MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(), 184 "All tracks should have been destroyed by messages from the main " 185 "thread"); 186 LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this)); 187 LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl")); 188 } 189 190 void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) { 191 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 192 aTrack->mStartTime = mProcessedTime; 193 194 if (aTrack->IsSuspended()) { 195 mSuspendedTracks.AppendElement(aTrack); 196 LOG(LogLevel::Debug, 197 ("%p: Adding media track %p, in the suspended track array", this, 198 aTrack)); 199 } else { 200 mTracks.AppendElement(aTrack); 201 LOG(LogLevel::Debug, ("%p: Adding media track %p, count %zu", this, aTrack, 202 mTracks.Length())); 203 } 204 205 SetTrackOrderDirty(); 206 } 207 208 void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) { 209 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 210 // Remove references in mTrackUpdates before we allow aTrack to die. 211 // Pending updates are not needed (since the main thread has already given 212 // up the track) so we will just drop them. 213 { 214 MonitorAutoLock lock(mMonitor); 215 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) { 216 if (mTrackUpdates[i].mTrack == aTrack) { 217 mTrackUpdates[i].mTrack = nullptr; 218 } 219 } 220 } 221 222 // Ensure that mFirstCycleBreaker is updated when necessary. 223 SetTrackOrderDirty(); 224 225 UnregisterAllAudioOutputs(aTrack); 226 227 if (aTrack->IsSuspended()) { 228 mSuspendedTracks.RemoveElement(aTrack); 229 } else { 230 mTracks.RemoveElement(aTrack); 231 } 232 233 LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack, 234 mTracks.Length())); 235 236 NS_RELEASE(aTrack); // probably destroying it 237 } 238 239 TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking( 240 const MediaTrack* aTrack, GraphTime aTime) const { 241 MOZ_ASSERT( 242 aTime <= mStateComputedTime, 243 "Don't ask about times where we haven't made blocking decisions yet"); 244 return std::max<TrackTime>( 245 0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime); 246 } 247 248 void MediaTrackGraphImpl::UpdateCurrentTimeForTracks( 249 GraphTime aPrevCurrentTime) { 250 MOZ_ASSERT(OnGraphThread()); 251 for (MediaTrack* track : AllTracks()) { 252 // Shouldn't have already notified of ended *and* have output! 253 MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime, 254 !track->mNotifiedEnded); 255 256 // Calculate blocked time and fire Blocked/Unblocked events 257 GraphTime blockedTime = mStateComputedTime - track->mStartBlocking; 258 NS_ASSERTION(blockedTime >= 0, "Error in blocking time"); 259 track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime, 260 blockedTime); 261 LOG(LogLevel::Verbose, 262 ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track, 263 MediaTimeToSeconds(track->mStartTime), 264 MediaTimeToSeconds(blockedTime))); 265 track->mStartBlocking = mStateComputedTime; 266 267 TrackTime trackCurrentTime = 268 track->GraphTimeToTrackTime(mStateComputedTime); 269 if (track->mEnded) { 270 MOZ_ASSERT(track->GetEnd() <= trackCurrentTime); 271 if (!track->mNotifiedEnded) { 272 // Playout of this track ended and listeners have not been notified. 273 track->mNotifiedEnded = true; 274 SetTrackOrderDirty(); 275 for (const auto& listener : track->mTrackListeners) { 276 listener->NotifyOutput(this, track->GetEnd()); 277 listener->NotifyEnded(this); 278 } 279 } 280 } else { 281 for (const auto& listener : track->mTrackListeners) { 282 listener->NotifyOutput(this, trackCurrentTime); 283 } 284 } 285 } 286 } 287 288 template <typename C, typename Chunk> 289 void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack, 290 C& aSegment, 291 TrackTime aStart, 292 TrackTime aEnd) { 293 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 294 MOZ_ASSERT(aTrack); 295 296 TrackTime offset = 0; 297 for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded(); 298 chunk.Next()) { 299 if (offset >= aEnd) { 300 break; 301 } 302 offset += chunk->GetDuration(); 303 if (chunk->IsNull() || offset < aStart) { 304 continue; 305 } 306 const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle(); 307 if (principalHandle != aSegment.GetLastPrincipalHandle()) { 308 aSegment.SetLastPrincipalHandle(principalHandle); 309 LOG(LogLevel::Debug, 310 ("%p: MediaTrack %p, principalHandle " 311 "changed in %sChunk with duration %lld", 312 this, aTrack, 313 aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video", 314 (long long)chunk->GetDuration())); 315 for (const auto& listener : aTrack->mTrackListeners) { 316 listener->NotifyPrincipalHandleChanged(this, principalHandle); 317 } 318 } 319 } 320 } 321 322 void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) { 323 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 324 for (MediaTrack* track : AllTracks()) { 325 TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime); 326 TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime); 327 if (!track->mSegment) { 328 continue; 329 } 330 if (track->mType == MediaSegment::AUDIO) { 331 ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>( 332 track, *track->GetData<AudioSegment>(), iterationStart, iterationEnd); 333 } else if (track->mType == MediaSegment::VIDEO) { 334 ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>( 335 track, *track->GetData<VideoSegment>(), iterationStart, iterationEnd); 336 } else { 337 MOZ_CRASH("Unknown track type"); 338 } 339 } 340 } 341 342 GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack, 343 GraphTime aEndBlockingDecisions) { 344 // Ended tracks can't underrun. ProcessedMediaTracks also can't cause 345 // underrun currently, since we'll always be able to produce data for them 346 // unless they block on some other track. 347 if (aTrack->mEnded || aTrack->AsProcessedTrack()) { 348 return aEndBlockingDecisions; 349 } 350 // This track isn't ended or suspended. We don't need to call 351 // TrackTimeToGraphTime since an underrun is the only thing that can block 352 // it. 353 GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime; 354 #ifdef DEBUG 355 if (bufferEnd < mProcessedTime) { 356 LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, " 357 "bufferEnd %f < mProcessedTime %f (%" PRId64 358 " < %" PRId64 "), TrackTime %" PRId64, 359 this, aTrack, MediaTimeToSeconds(bufferEnd), 360 MediaTimeToSeconds(mProcessedTime), bufferEnd, 361 mProcessedTime, aTrack->GetEnd())); 362 NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran"); 363 } 364 #endif 365 return std::min(bufferEnd, aEndBlockingDecisions); 366 } 367 368 namespace { 369 // Value of mCycleMarker for unvisited tracks in cycle detection. 370 const uint32_t NOT_VISITED = UINT32_MAX; 371 // Value of mCycleMarker for ordered tracks in muted cycles. 372 const uint32_t IN_MUTED_CYCLE = 1; 373 } // namespace 374 375 bool MediaTrackGraphImpl::AudioTrackPresent() { 376 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 377 378 bool audioTrackPresent = false; 379 for (MediaTrack* track : mTracks) { 380 if (track->AsAudioNodeTrack()) { 381 audioTrackPresent = true; 382 break; 383 } 384 385 if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) { 386 audioTrackPresent = true; 387 break; 388 } 389 } 390 391 // We may not have audio input device when we only have AudioNodeTracks. But 392 // if audioTrackPresent is false, we must have no input device. 393 MOZ_DIAGNOSTIC_ASSERT_IF( 394 !audioTrackPresent, 395 !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()); 396 397 return audioTrackPresent; 398 } 399 400 void MediaTrackGraphImpl::CheckDriver() { 401 MOZ_ASSERT(OnGraphThread()); 402 // An offline graph has only one driver. 403 if (!mRealtime) { 404 return; 405 } 406 407 AudioCallbackDriver* audioCallbackDriver = 408 CurrentDriver()->AsAudioCallbackDriver(); 409 if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) { 410 for (PendingResumeOperation& op : mPendingResumeOperations) { 411 op.Apply(this); 412 } 413 mPendingResumeOperations.Clear(); 414 } 415 416 // Note that this looks for any audio tracks, input or output, and switches 417 // to a SystemClockDriver if there are none active or no resume operations 418 // to make any active. 419 bool needAudioCallbackDriver = 420 !mPendingResumeOperations.IsEmpty() || AudioTrackPresent(); 421 if (!needAudioCallbackDriver) { 422 if (audioCallbackDriver && audioCallbackDriver->IsStarted()) { 423 SwitchAtNextIteration( 424 new SystemClockDriver(this, CurrentDriver(), mSampleRate)); 425 } 426 return; 427 } 428 429 NativeInputTrack* native = 430 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); 431 CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr; 432 uint32_t inputChannelCount = AudioInputChannelCount(inputDevice); 433 AudioInputType inputPreference = AudioInputDevicePreference(inputDevice); 434 Maybe<AudioInputProcessingParamsRequest> processingRequest = 435 ToMaybeRef(native).map([](auto& native) { 436 return native.UpdateRequestedProcessingParams(); 437 }); 438 439 uint32_t primaryOutputChannelCount = PrimaryOutputChannelCount(); 440 if (!audioCallbackDriver) { 441 if (primaryOutputChannelCount > 0) { 442 AudioCallbackDriver* driver = new AudioCallbackDriver( 443 this, CurrentDriver(), mSampleRate, primaryOutputChannelCount, 444 inputChannelCount, PrimaryOutputDeviceID(), inputDevice, 445 inputPreference, processingRequest); 446 SwitchAtNextIteration(driver); 447 } 448 return; 449 } 450 451 // Check if a new driver is required for a configuration change. Generally, 452 // a driver switch is explicitly triggered by an event (e.g., setting 453 // the AudioDestinationNode channelCount), but if an HTMLMediaElement is 454 // directly playing back via another HTMLMediaElement, the number of channels 455 // of the media determines how many channels to output, and it can change 456 // dynamically. 457 if (primaryOutputChannelCount != audioCallbackDriver->OutputChannelCount() || 458 inputDevice != audioCallbackDriver->InputDeviceID() || 459 inputChannelCount != audioCallbackDriver->InputChannelCount() || 460 inputPreference != audioCallbackDriver->InputDevicePreference()) { 461 AudioCallbackDriver* driver = new AudioCallbackDriver( 462 this, CurrentDriver(), mSampleRate, primaryOutputChannelCount, 463 inputChannelCount, PrimaryOutputDeviceID(), inputDevice, 464 inputPreference, processingRequest); 465 SwitchAtNextIteration(driver); 466 return; 467 } 468 469 if (processingRequest && 470 processingRequest->mGeneration != 471 audioCallbackDriver->RequestedInputProcessingParams().mGeneration) { 472 LOG(LogLevel::Debug, 473 ("%p: Setting on the fly requested processing params %s (Gen %d)", this, 474 CubebUtils::ProcessingParamsToString(processingRequest->mParams).get(), 475 processingRequest->mGeneration)); 476 audioCallbackDriver->RequestInputProcessingParams(*processingRequest); 477 } 478 } 479 480 void MediaTrackGraphImpl::UpdateTrackOrder() { 481 if (!mTrackOrderDirty) { 482 return; 483 } 484 485 mTrackOrderDirty = false; 486 487 // The algorithm for finding cycles is based on Tim Leslie's iterative 488 // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly 489 // connected components (SCC) algorithm. There are variations (a) to 490 // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to 491 // re-run the algorithm over SCCs with breaks at DelayNodes. 492 // 493 // [1] http://www.timl.id.au/?p=327 494 // [2] 495 // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582 496 // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707 497 // 498 // There are two stacks. One for the depth-first search (DFS), 499 mozilla::LinkedList<MediaTrack> dfsStack; 500 // and another for tracks popped from the DFS stack, but still being 501 // considered as part of SCCs involving tracks on the stack. 502 mozilla::LinkedList<MediaTrack> sccStack; 503 504 // An index into mTracks for the next track found with no unsatisfied 505 // upstream dependencies. 506 uint32_t orderedTrackCount = 0; 507 508 for (uint32_t i = 0; i < mTracks.Length(); ++i) { 509 MediaTrack* t = mTracks[i]; 510 ProcessedMediaTrack* pt = t->AsProcessedTrack(); 511 if (pt) { 512 // The dfsStack initially contains a list of all processed tracks in 513 // unchanged order. 514 dfsStack.insertBack(t); 515 pt->mCycleMarker = NOT_VISITED; 516 } else { 517 // SourceMediaTracks have no inputs and so can be ordered now. 518 mTracks[orderedTrackCount] = t; 519 ++orderedTrackCount; 520 } 521 } 522 523 // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a 524 // counter to label mCycleMarker on the next visited track in the DFS 525 // uniquely in the set of visited tracks that are still being considered. 526 // 527 // In this implementation, the counter descends so that the values are 528 // strictly greater than the values that mCycleMarker takes when the track 529 // has been ordered (0 or IN_MUTED_CYCLE). 530 // 531 // Each new track labelled, as the DFS searches upstream, receives a value 532 // less than those used for all other tracks being considered. 533 uint32_t nextStackMarker = NOT_VISITED - 1; 534 // Reset list of DelayNodes in cycles stored at the tail of mTracks. 535 mFirstCycleBreaker = mTracks.Length(); 536 537 // Rearrange dfsStack order as required to DFS upstream and pop tracks 538 // in processing order to place in mTracks. 539 while (auto pt = static_cast<ProcessedMediaTrack*>(dfsStack.getFirst())) { 540 const auto& inputs = pt->mInputs; 541 MOZ_ASSERT(pt->AsProcessedTrack()); 542 if (pt->mCycleMarker == NOT_VISITED) { 543 // Record the position on the visited stack, so that any searches 544 // finding this track again know how much of the stack is in the cycle. 545 pt->mCycleMarker = nextStackMarker; 546 --nextStackMarker; 547 // Not-visited input tracks should be processed first. 548 // SourceMediaTracks have already been ordered. 549 for (uint32_t i = inputs.Length(); i--;) { 550 if (inputs[i]->GetSource()->IsSuspended()) { 551 continue; 552 } 553 auto input = inputs[i]->GetSource()->AsProcessedTrack(); 554 if (input && input->mCycleMarker == NOT_VISITED) { 555 // It can be that this track has an input which is from a suspended 556 // AudioContext. 557 if (input->isInList()) { 558 input->remove(); 559 dfsStack.insertFront(input); 560 } 561 } 562 } 563 continue; 564 } 565 566 // Returning from DFS. Pop from dfsStack. 567 pt->remove(); 568 569 // cycleStackMarker keeps track of the highest marker value on any 570 // upstream track, if any, found receiving input, directly or indirectly, 571 // from the visited stack (and so from |ps|, making a cycle). In a 572 // variation from Tarjan's SCC algorithm, this does not include |ps| 573 // unless it is part of the cycle. 574 uint32_t cycleStackMarker = 0; 575 for (uint32_t i = inputs.Length(); i--;) { 576 if (inputs[i]->GetSource()->IsSuspended()) { 577 continue; 578 } 579 auto input = inputs[i]->GetSource()->AsProcessedTrack(); 580 if (input) { 581 cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker); 582 } 583 } 584 585 if (cycleStackMarker <= IN_MUTED_CYCLE) { 586 // All inputs have been ordered and their stack markers have been removed. 587 // This track is not part of a cycle. It can be processed next. 588 pt->mCycleMarker = 0; 589 mTracks[orderedTrackCount] = pt; 590 ++orderedTrackCount; 591 continue; 592 } 593 594 // A cycle has been found. Record this track for ordering when all 595 // tracks in this SCC have been popped from the DFS stack. 596 sccStack.insertFront(pt); 597 598 if (cycleStackMarker > pt->mCycleMarker) { 599 // Cycles have been found that involve tracks that remain on the stack. 600 // Leave mCycleMarker indicating the most downstream (last) track on 601 // the stack known to be part of this SCC. In this way, any searches on 602 // other paths that find |ps| will know (without having to traverse from 603 // this track again) that they are part of this SCC (i.e. part of an 604 // intersecting cycle). 605 pt->mCycleMarker = cycleStackMarker; 606 continue; 607 } 608 609 // |pit| is the root of an SCC involving no other tracks on dfsStack, the 610 // complete SCC has been recorded, and tracks in this SCC are part of at 611 // least one cycle. 612 MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker); 613 // If there are DelayNodes in this SCC, then they may break the cycles. 614 bool haveDelayNode = false; 615 auto next = sccStack.getFirst(); 616 // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker. 617 // (There may be other tracks later in sccStack from other incompletely 618 // searched SCCs, involving tracks still on dfsStack.) 619 // 620 // DelayNodes in cycles must behave differently from those not in cycles, 621 // so all DelayNodes in the SCC must be identified. 622 while (next && static_cast<ProcessedMediaTrack*>(next)->mCycleMarker <= 623 cycleStackMarker) { 624 auto nt = next->AsAudioNodeTrack(); 625 // Get next before perhaps removing from list below. 626 next = next->getNext(); 627 if (nt && nt->Engine()->AsDelayNodeEngine()) { 628 haveDelayNode = true; 629 // DelayNodes break cycles by producing their output in a 630 // preprocessing phase; they do not need to be ordered before their 631 // consumers. Order them at the tail of mTracks so that they can be 632 // handled specially. Do so now, so that DFS ignores them. 633 nt->remove(); 634 nt->mCycleMarker = 0; 635 --mFirstCycleBreaker; 636 mTracks[mFirstCycleBreaker] = nt; 637 } 638 } 639 auto after_scc = next; 640 while ((next = sccStack.getFirst()) != after_scc) { 641 next->remove(); 642 auto removed = static_cast<ProcessedMediaTrack*>(next); 643 if (haveDelayNode) { 644 // Return tracks to the DFS stack again (to order and detect cycles 645 // without delayNodes). Any of these tracks that are still inputs 646 // for tracks on the visited stack must be returned to the front of 647 // the stack to be ordered before their dependents. We know that none 648 // of these tracks need input from tracks on the visited stack, so 649 // they can all be searched and ordered before the current stack head 650 // is popped. 651 removed->mCycleMarker = NOT_VISITED; 652 dfsStack.insertFront(removed); 653 } else { 654 // Tracks in cycles without any DelayNodes must be muted, and so do 655 // not need input and can be ordered now. They must be ordered before 656 // their consumers so that their muted output is available. 657 removed->mCycleMarker = IN_MUTED_CYCLE; 658 mTracks[orderedTrackCount] = removed; 659 ++orderedTrackCount; 660 } 661 } 662 } 663 664 MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker); 665 } 666 667 TrackTime MediaTrackGraphImpl::PlayAudio(const TrackAndVolume& aOutput, 668 GraphTime aPlayedTime, 669 uint32_t aOutputChannelCount) { 670 MOZ_ASSERT(OnGraphThread()); 671 MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode"); 672 673 TrackTime ticksWritten = 0; 674 675 ticksWritten = 0; 676 MediaTrack* track = aOutput.mTrack; 677 AudioSegment* audio = track->GetData<AudioSegment>(); 678 AudioSegment output; 679 680 TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime); 681 682 // We don't update Track->mTracksStartTime here to account for time spent 683 // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after 684 // the blocked period has completed. But we do need to make sure we play 685 // from the right offsets in the track buffer, even if we've already 686 // written silence for some amount of blocked time after the current time. 687 GraphTime t = aPlayedTime; 688 while (t < mStateComputedTime) { 689 bool blocked = t >= track->mStartBlocking; 690 GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking; 691 NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!"); 692 693 // Check how many ticks of sound we can provide if we are blocked some 694 // time in the middle of this cycle. 695 TrackTime toWrite = end - t; 696 697 if (blocked) { 698 output.InsertNullDataAtStart(toWrite); 699 ticksWritten += toWrite; 700 LOG(LogLevel::Verbose, 701 ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for " 702 "%f to %f (%" PRId64 " to %" PRId64 ")", 703 this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end), 704 offset, offset + toWrite)); 705 } else { 706 TrackTime endTicksNeeded = offset + toWrite; 707 TrackTime endTicksAvailable = audio->GetDuration(); 708 709 if (endTicksNeeded <= endTicksAvailable) { 710 LOG(LogLevel::Verbose, 711 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f " 712 "(samples %" PRId64 " to %" PRId64 ")", 713 this, track, toWrite, MediaTimeToSeconds(t), 714 MediaTimeToSeconds(end), offset, endTicksNeeded)); 715 output.AppendSlice(*audio, offset, endTicksNeeded); 716 ticksWritten += toWrite; 717 offset = endTicksNeeded; 718 } else { 719 // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not 720 // ended."); If we are at the end of the track, maybe write the 721 // remaining samples, and pad with/output silence. 722 if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) { 723 output.AppendSlice(*audio, offset, endTicksAvailable); 724 725 LOG(LogLevel::Verbose, 726 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f " 727 "(samples %" PRId64 " to %" PRId64 ")", 728 this, track, toWrite, MediaTimeToSeconds(t), 729 MediaTimeToSeconds(end), offset, endTicksNeeded)); 730 uint32_t available = endTicksAvailable - offset; 731 ticksWritten += available; 732 toWrite -= available; 733 offset = endTicksAvailable; 734 } 735 output.AppendNullData(toWrite); 736 LOG(LogLevel::Verbose, 737 ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to " 738 "%f (samples %" PRId64 " to %" PRId64 ")", 739 this, track, toWrite, MediaTimeToSeconds(t), 740 MediaTimeToSeconds(end), offset, endTicksNeeded)); 741 ticksWritten += toWrite; 742 } 743 output.ApplyVolume(mGlobalVolume * aOutput.mVolume); 744 } 745 t = end; 746 747 output.Mix(mMixer, aOutputChannelCount, mSampleRate); 748 } 749 return ticksWritten; 750 } 751 752 DeviceInputTrack* MediaTrackGraph::GetDeviceInputTrackMainThread( 753 CubebUtils::AudioDeviceID aID) { 754 MOZ_ASSERT(NS_IsMainThread()); 755 auto* impl = static_cast<MediaTrackGraphImpl*>(this); 756 return impl->mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID); 757 } 758 759 NativeInputTrack* MediaTrackGraph::GetNativeInputTrackMainThread() { 760 MOZ_ASSERT(NS_IsMainThread()); 761 auto* impl = static_cast<MediaTrackGraphImpl*>(this); 762 return impl->mDeviceInputTrackManagerMainThread.GetNativeInputTrack(); 763 } 764 765 void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) { 766 MOZ_ASSERT(OnGraphThread()); 767 LOG(LogLevel::Debug, 768 ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId)); 769 770 mDeviceInputTrackManagerGraphThread.Add(aTrack); 771 772 if (aTrack->AsNativeInputTrack()) { 773 // CheckDriver() will check for changes, after all control messages are 774 // processed. 775 return; 776 } 777 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack(); 778 MOZ_ASSERT(nonNative); 779 // Start non-native input right away. 780 nonNative->StartAudio(MakeRefPtr<AudioInputSource>( 781 MakeRefPtr<AudioInputSourceListener>(nonNative), 782 nonNative->GenerateSourceId(), nonNative->mDeviceId, 783 AudioInputChannelCount(nonNative->mDeviceId), 784 AudioInputDevicePreference(nonNative->mDeviceId) == AudioInputType::Voice, 785 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate())); 786 } 787 788 void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) { 789 MOZ_ASSERT(NS_IsMainThread()); 790 MOZ_ASSERT(aTrack); 791 792 LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this, 793 aTrack, aTrack->mDeviceId)); 794 795 class Message : public ControlMessage { 796 public: 797 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack) 798 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {} 799 void Run() override { 800 TRACE("MTG::OpenAudioInputImpl ControlMessage"); 801 mGraph->OpenAudioInputImpl(mInputTrack); 802 } 803 MediaTrackGraphImpl* mGraph; 804 DeviceInputTrack* mInputTrack; 805 }; 806 807 mDeviceInputTrackManagerMainThread.Add(aTrack); 808 UpdateEnumeratorDefaultDeviceTracking(); 809 810 this->AppendMessage(MakeUnique<Message>(this, aTrack)); 811 } 812 813 void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) { 814 MOZ_ASSERT(OnGraphThread()); 815 816 LOG(LogLevel::Debug, 817 ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId)); 818 819 if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) { 820 nonNative->StopAudio(); 821 mDeviceInputTrackManagerGraphThread.Remove(aTrack); 822 return; 823 } 824 825 MOZ_ASSERT(aTrack->AsNativeInputTrack()); 826 827 mDeviceInputTrackManagerGraphThread.Remove(aTrack); 828 // CheckDriver() will check for changes, after all control messages are 829 // processed. 830 } 831 832 void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) { 833 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 834 mOutputDevices.RemoveElementsBy([&](OutputDeviceEntry& aDeviceRef) { 835 aDeviceRef.mTrackOutputs.RemoveElement(aTrack); 836 // mReceiver is null for the primary output device, which is retained for 837 // AudioCallbackDriver output even when no tracks have audio outputs. 838 return aDeviceRef.mTrackOutputs.IsEmpty() && aDeviceRef.mReceiver; 839 }); 840 } 841 842 void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) { 843 MOZ_ASSERT(NS_IsMainThread()); 844 MOZ_ASSERT(aTrack); 845 846 LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p", 847 this, aTrack, aTrack->mDeviceId)); 848 849 class Message : public ControlMessage { 850 public: 851 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack) 852 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {} 853 void Run() override { 854 TRACE("MTG::CloseAudioInputImpl ControlMessage"); 855 mGraph->CloseAudioInputImpl(mInputTrack); 856 } 857 MediaTrackGraphImpl* mGraph; 858 DeviceInputTrack* mInputTrack; 859 }; 860 861 // DeviceInputTrack is still alive (in mTracks) even we remove it here, since 862 // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio 863 // for more details. 864 mDeviceInputTrackManagerMainThread.Remove(aTrack); 865 UpdateEnumeratorDefaultDeviceTracking(); 866 867 this->AppendMessage(MakeUnique<Message>(this, aTrack)); 868 869 if (aTrack->AsNativeInputTrack()) { 870 LOG(LogLevel::Debug, 871 ("%p Native input device %p is closed!", this, aTrack->mDeviceId)); 872 SetNewNativeInput(); 873 } 874 } 875 876 // All AudioInput listeners get the same speaker data (at least for now). 877 void MediaTrackGraphImpl::NotifyOutputData(const AudioChunk& aChunk) { 878 if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) { 879 return; 880 } 881 882 #if defined(MOZ_WEBRTC) 883 for (const auto& track : mTracks) { 884 if (const auto& t = track->AsAudioProcessingTrack()) { 885 t->NotifyOutputData(this, aChunk); 886 } 887 } 888 #endif 889 } 890 891 void MediaTrackGraphImpl::NotifyInputStopped() { 892 NativeInputTrack* native = 893 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); 894 if (!native) { 895 return; 896 } 897 native->NotifyInputStopped(this); 898 } 899 900 void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer, 901 size_t aFrames, TrackRate aRate, 902 uint32_t aChannels, 903 uint32_t aAlreadyBuffered) { 904 // Either we have an audio input device, or we just removed the audio input 905 // this iteration, and we're switching back to an output-only driver next 906 // iteration. 907 NativeInputTrack* native = 908 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); 909 MOZ_ASSERT(native || Switching()); 910 if (!native) { 911 return; 912 } 913 native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels, 914 aAlreadyBuffered); 915 } 916 917 void MediaTrackGraphImpl::NotifySetRequestedInputProcessingParamsResult( 918 AudioCallbackDriver* aDriver, int aGeneration, 919 Result<cubeb_input_processing_params, int>&& aResult) { 920 MOZ_ASSERT(NS_IsMainThread()); 921 NativeInputTrack* native = 922 mDeviceInputTrackManagerMainThread.GetNativeInputTrack(); 923 if (!native) { 924 return; 925 } 926 QueueControlMessageWithNoShutdown([this, self = RefPtr(this), 927 driver = RefPtr(aDriver), aGeneration, 928 result = std::move(aResult)]() mutable { 929 NativeInputTrack* native = 930 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); 931 if (!native) { 932 return; 933 } 934 if (driver != mDriver) { 935 return; 936 } 937 native->NotifySetRequestedProcessingParamsResult(this, aGeneration, result); 938 }); 939 } 940 941 void MediaTrackGraphImpl::DeviceChangedImpl() { 942 MOZ_ASSERT(OnGraphThread()); 943 NativeInputTrack* native = 944 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); 945 if (!native) { 946 return; 947 } 948 native->DeviceChanged(this); 949 } 950 951 void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) { 952 MOZ_ASSERT(OnGraphThread()); 953 mMaxOutputChannelCount = aMaxChannelCount; 954 } 955 956 void MediaTrackGraphImpl::DeviceChanged() { 957 // This is safe to be called from any thread: this message comes from an 958 // underlying platform API, and we don't have much guarantees. If it is not 959 // called from the main thread (and it probably will rarely be), it will post 960 // itself to the main thread, and the actual device change message will be ran 961 // and acted upon on the graph thread. 962 if (!NS_IsMainThread()) { 963 RefPtr<nsIRunnable> runnable = WrapRunnable( 964 RefPtr<MediaTrackGraphImpl>(this), &MediaTrackGraphImpl::DeviceChanged); 965 mMainThread->Dispatch(runnable.forget()); 966 return; 967 } 968 969 class Message : public ControlMessage { 970 public: 971 explicit Message(MediaTrackGraph* aGraph) 972 : ControlMessage(nullptr), 973 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)) {} 974 void Run() override { 975 TRACE("MTG::DeviceChangeImpl ControlMessage"); 976 mGraphImpl->DeviceChangedImpl(); 977 } 978 // We know that this is valid, because the graph can't shutdown if it has 979 // messages. 980 MediaTrackGraphImpl* mGraphImpl; 981 }; 982 983 if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) { 984 // This is a special case where the origin of this event cannot control the 985 // lifetime of the graph, because the graph is controling the lifetime of 986 // the AudioCallbackDriver where the event originated. 987 // We know the graph is soon going away, so there's no need to notify about 988 // this device change. 989 return; 990 } 991 992 // Reset the latency, it will get fetched again next time it's queried. 993 MOZ_ASSERT(NS_IsMainThread()); 994 mAudioOutputLatency = 0.0; 995 996 // Dispatch to the bg thread to do the (potentially expensive) query of the 997 // maximum channel count, and then dispatch back to the main thread, then to 998 // the graph, with the new info. The "special case" above is to be handled 999 // back on the main thread as well for the same reasons. 1000 RefPtr<MediaTrackGraphImpl> self = this; 1001 NS_DispatchBackgroundTask(NS_NewRunnableFunction( 1002 "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() { 1003 uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels(); 1004 self->Dispatch(NS_NewRunnableFunction( 1005 "MaxChannelCountUpdateToMainThread", 1006 [self{self}, maxChannelCount]() { 1007 class MessageToGraph : public ControlMessage { 1008 public: 1009 explicit MessageToGraph(MediaTrackGraph* aGraph, 1010 uint32_t aMaxChannelCount) 1011 : ControlMessage(nullptr), 1012 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)), 1013 mMaxChannelCount(aMaxChannelCount) {} 1014 void Run() override { 1015 TRACE("MTG::SetMaxOutputChannelCount ControlMessage") 1016 mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount); 1017 } 1018 MediaTrackGraphImpl* mGraphImpl; 1019 uint32_t mMaxChannelCount; 1020 }; 1021 1022 if (self->mMainThreadTrackCount == 0 && 1023 self->mMainThreadPortCount == 0) { 1024 // See comments above. 1025 return; 1026 } 1027 1028 self->AppendMessage( 1029 MakeUnique<MessageToGraph>(self, maxChannelCount)); 1030 })); 1031 })); 1032 1033 AppendMessage(MakeUnique<Message>(this)); 1034 } 1035 1036 static const char* GetAudioInputTypeString(const AudioInputType& aType) { 1037 return aType == AudioInputType::Voice ? "Voice" : "Unknown"; 1038 } 1039 1040 void MediaTrackGraph::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) { 1041 MOZ_ASSERT(OnGraphThread()); 1042 auto* impl = static_cast<MediaTrackGraphImpl*>(this); 1043 impl->ReevaluateInputDevice(aID); 1044 } 1045 1046 void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) { 1047 MOZ_ASSERT(OnGraphThread()); 1048 LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID)); 1049 1050 DeviceInputTrack* track = 1051 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID); 1052 if (!track) { 1053 LOG(LogLevel::Debug, 1054 ("%p: No DeviceInputTrack for this device. Ignore", this)); 1055 return; 1056 } 1057 if (track->AsNativeInputTrack()) { 1058 // CheckDriver() will check for changes, after all control messages are 1059 // processed. 1060 return; 1061 } 1062 bool needToSwitch = false; 1063 1064 NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack(); 1065 MOZ_ASSERT(nonNative); 1066 if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) { 1067 LOG(LogLevel::Debug, 1068 ("%p: %u-channel non-native input device %p (track %p) is " 1069 "re-configured to %d-channel", 1070 this, nonNative->NumberOfChannels(), aID, track, 1071 AudioInputChannelCount(aID))); 1072 needToSwitch = true; 1073 } 1074 if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) { 1075 LOG(LogLevel::Debug, 1076 ("%p: %s-type non-native input device %p (track %p) is re-configured " 1077 "to %s-type", 1078 this, GetAudioInputTypeString(nonNative->DevicePreference()), aID, 1079 track, GetAudioInputTypeString(AudioInputDevicePreference(aID)))); 1080 needToSwitch = true; 1081 } 1082 1083 if (needToSwitch) { 1084 nonNative->StopAudio(); 1085 nonNative->StartAudio(MakeRefPtr<AudioInputSource>( 1086 MakeRefPtr<AudioInputSourceListener>(nonNative), 1087 nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID), 1088 AudioInputDevicePreference(aID) == AudioInputType::Voice, 1089 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate())); 1090 } 1091 } 1092 1093 bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const { 1094 // either we're on the right thread (and calling CurrentDriver() is safe), 1095 // or we're going to fail the assert anyway, so don't cross-check 1096 // via CurrentDriver(). 1097 return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread(); 1098 } 1099 1100 bool MediaTrackGraphImpl::OnGraphThread() const { 1101 // we're on the right thread (and calling mDriver is safe), 1102 MOZ_ASSERT(mDriver); 1103 if (mGraphRunner && mGraphRunner->OnThread()) { 1104 return true; 1105 } 1106 return mDriver->OnThread(); 1107 } 1108 1109 bool MediaTrackGraphImpl::Destroyed() const { 1110 MOZ_ASSERT(NS_IsMainThread()); 1111 return !mSelfRef; 1112 } 1113 1114 bool MediaTrackGraphImpl::ShouldUpdateMainThread() { 1115 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 1116 if (mRealtime) { 1117 return true; 1118 } 1119 1120 TimeStamp now = TimeStamp::Now(); 1121 // For offline graphs, update now if it has been long enough since the last 1122 // update, or if it has reached the end. 1123 if (now - mLastMainThreadUpdate > CurrentDriver()->IterationDuration() || 1124 mStateComputedTime >= mEndTime) { 1125 mLastMainThreadUpdate = now; 1126 return true; 1127 } 1128 return false; 1129 } 1130 1131 void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) { 1132 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 1133 mMonitor.AssertCurrentThreadOwns(); 1134 1135 // We don't want to frequently update the main thread about timing update 1136 // when we are not running in realtime. 1137 if (aFinalUpdate || ShouldUpdateMainThread()) { 1138 // Strip updates that will be obsoleted below, so as to keep the length of 1139 // mTrackUpdates sane. 1140 size_t keptUpdateCount = 0; 1141 for (size_t i = 0; i < mTrackUpdates.Length(); ++i) { 1142 MediaTrack* track = mTrackUpdates[i].mTrack; 1143 // RemoveTrackGraphThread() clears mTrack in updates for 1144 // tracks that are removed from the graph. 1145 MOZ_ASSERT(!track || track->GraphImpl() == this); 1146 if (!track || track->MainThreadNeedsUpdates()) { 1147 // Discard this update as it has either been cleared when the track 1148 // was destroyed or there will be a newer update below. 1149 continue; 1150 } 1151 if (keptUpdateCount != i) { 1152 mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]); 1153 MOZ_ASSERT(!mTrackUpdates[i].mTrack); 1154 } 1155 ++keptUpdateCount; 1156 } 1157 mTrackUpdates.TruncateLength(keptUpdateCount); 1158 1159 mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() + 1160 mSuspendedTracks.Length()); 1161 for (MediaTrack* track : AllTracks()) { 1162 if (!track->MainThreadNeedsUpdates()) { 1163 continue; 1164 } 1165 TrackUpdate* update = mTrackUpdates.AppendElement(); 1166 update->mTrack = track; 1167 // No blocking to worry about here, since we've passed 1168 // UpdateCurrentTimeForTracks. 1169 update->mNextMainThreadCurrentTime = 1170 track->GraphTimeToTrackTime(mProcessedTime); 1171 update->mNextMainThreadEnded = track->mNotifiedEnded; 1172 } 1173 mNextMainThreadGraphTime = mProcessedTime; 1174 if (!mPendingUpdateRunnables.IsEmpty()) { 1175 mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables)); 1176 } 1177 } 1178 1179 // If this is the final update, then a stable state event will soon be 1180 // posted just before this thread finishes, and so there is no need to also 1181 // post here. 1182 if (!aFinalUpdate && 1183 // Don't send the message to the main thread if it's not going to have 1184 // any work to do. 1185 !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) { 1186 EnsureStableStateEventPosted(); 1187 } 1188 } 1189 1190 GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) { 1191 if (aTime % WEBAUDIO_BLOCK_SIZE == 0) { 1192 return aTime; 1193 } 1194 return RoundUpToNextAudioBlock(aTime); 1195 } 1196 1197 GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) { 1198 uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS; 1199 uint64_t nextBlock = block + 1; 1200 GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS; 1201 return nextTime; 1202 } 1203 1204 void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock( 1205 uint32_t aTrackIndex, TrackRate aSampleRate) { 1206 MOZ_ASSERT(OnGraphThread()); 1207 MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker, 1208 "Cycle breaker is not AudioNodeTrack?"); 1209 1210 while (mProcessedTime < mStateComputedTime) { 1211 // Microtask checkpoints are in between render quanta. 1212 nsAutoMicroTask mt; 1213 1214 GraphTime next = RoundUpToNextAudioBlock(mProcessedTime); 1215 for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) { 1216 auto nt = static_cast<AudioNodeTrack*>(mTracks[i]); 1217 MOZ_ASSERT(nt->AsAudioNodeTrack()); 1218 nt->ProduceOutputBeforeInput(mProcessedTime); 1219 } 1220 for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) { 1221 ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack(); 1222 if (pt) { 1223 pt->ProcessInput( 1224 mProcessedTime, next, 1225 (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0); 1226 } 1227 } 1228 mProcessedTime = next; 1229 } 1230 MOZ_ASSERT(mProcessedTime == mStateComputedTime, 1231 "Something went wrong with rounding to block boundaries"); 1232 } 1233 1234 void MediaTrackGraphImpl::RunMessageAfterProcessing( 1235 UniquePtr<ControlMessageInterface> aMessage) { 1236 MOZ_ASSERT(OnGraphThread()); 1237 1238 if (mFrontMessageQueue.IsEmpty()) { 1239 mFrontMessageQueue.AppendElement(); 1240 } 1241 1242 // Only one block is used for messages from the graph thread. 1243 MOZ_ASSERT(mFrontMessageQueue.Length() == 1); 1244 mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage)); 1245 } 1246 1247 void MediaTrackGraphImpl::RunMessagesInQueue() { 1248 TRACE("MTG::RunMessagesInQueue"); 1249 MOZ_ASSERT(OnGraphThread()); 1250 // Calculate independent action times for each batch of messages (each 1251 // batch corresponding to an event loop task). This isolates the performance 1252 // of different scripts to some extent. 1253 for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) { 1254 nsTArray<UniquePtr<ControlMessageInterface>>& messages = 1255 mFrontMessageQueue[i].mMessages; 1256 1257 for (uint32_t j = 0; j < messages.Length(); ++j) { 1258 TRACE("ControlMessage::Run"); 1259 messages[j]->Run(); 1260 } 1261 } 1262 mFrontMessageQueue.Clear(); 1263 } 1264 1265 void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) { 1266 TRACE("MTG::UpdateGraph"); 1267 MOZ_ASSERT(OnGraphThread()); 1268 MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime); 1269 // The next state computed time can be the same as the previous: it 1270 // means the driver would have been blocking indefinitly, but the graph has 1271 // been woken up right after having been to sleep. 1272 MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime); 1273 1274 CheckDriver(); 1275 UpdateTrackOrder(); 1276 1277 // Always do another iteration if there are tracks waiting to resume. 1278 bool ensureNextIteration = !mPendingResumeOperations.IsEmpty(); 1279 1280 for (MediaTrack* track : mTracks) { 1281 if (SourceMediaTrack* is = track->AsSourceTrack()) { 1282 ensureNextIteration |= is->PullNewData(aEndBlockingDecisions); 1283 is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions); 1284 } 1285 if (track->mEnded) { 1286 // The track's not suspended, and since it's ended, underruns won't 1287 // stop it playing out. So there's no blocking other than what we impose 1288 // here. 1289 GraphTime endTime = track->GetEnd() + track->mStartTime; 1290 if (endTime <= mStateComputedTime) { 1291 LOG(LogLevel::Verbose, 1292 ("%p: MediaTrack %p is blocked due to being ended", this, track)); 1293 track->mStartBlocking = mStateComputedTime; 1294 } else { 1295 LOG(LogLevel::Verbose, 1296 ("%p: MediaTrack %p has ended, but is not blocked yet (current " 1297 "time %f, end at %f)", 1298 this, track, MediaTimeToSeconds(mStateComputedTime), 1299 MediaTimeToSeconds(endTime))); 1300 // Data can't be added to a ended track, so underruns are irrelevant. 1301 MOZ_ASSERT(endTime <= aEndBlockingDecisions); 1302 track->mStartBlocking = endTime; 1303 } 1304 } else { 1305 track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions); 1306 1307 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED 1308 if (SourceMediaTrack* s = track->AsSourceTrack()) { 1309 if (s->Ended()) { 1310 continue; 1311 } 1312 { 1313 MutexAutoLock lock(s->mMutex); 1314 if (!s->mUpdateTrack->mPullingEnabled) { 1315 // The invariant that data must be provided is only enforced when 1316 // pulling. 1317 continue; 1318 } 1319 } 1320 if (track->GetEnd() < 1321 track->GraphTimeToTrackTime(aEndBlockingDecisions)) { 1322 LOG(LogLevel::Error, 1323 ("%p: SourceMediaTrack %p (%s) is live and pulled, " 1324 "but wasn't fed " 1325 "enough data. TrackListeners=%zu. Track-end=%f, " 1326 "Iteration-end=%f", 1327 this, track, 1328 (track->mType == MediaSegment::AUDIO ? "audio" : "video"), 1329 track->mTrackListeners.Length(), 1330 MediaTimeToSeconds(track->GetEnd()), 1331 MediaTimeToSeconds( 1332 track->GraphTimeToTrackTime(aEndBlockingDecisions)))); 1333 MOZ_DIAGNOSTIC_CRASH( 1334 "A non-ended SourceMediaTrack wasn't fed " 1335 "enough data by NotifyPull"); 1336 } 1337 } 1338 #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */ 1339 } 1340 } 1341 1342 for (MediaTrack* track : mSuspendedTracks) { 1343 track->mStartBlocking = mStateComputedTime; 1344 } 1345 1346 // If the loop is woken up so soon that mStateComputedTime does not advance 1347 // or if an offline graph is not currently rendering, we end up having 1348 // aEndBlockingDecisions == mStateComputedTime. 1349 // Since the process interval [mStateComputedTime, aEndBlockingDecision) is 1350 // empty, Process() will not find any unblocked track and so will not 1351 // ensure another iteration. If the graph should be rendering, then ensure 1352 // another iteration to render. 1353 if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime && 1354 mStateComputedTime < mEndTime)) { 1355 EnsureNextIteration(); 1356 } 1357 } 1358 1359 void MediaTrackGraphImpl::SelectOutputDeviceForAEC() { 1360 MOZ_ASSERT(OnGraphThread()); 1361 size_t currentDeviceIndex = mOutputDevices.IndexOf(mOutputDeviceForAEC); 1362 if (currentDeviceIndex == mOutputDevices.NoIndex) { 1363 // Outputs for this device have been removed. 1364 // Fall back to the primary output device. 1365 LOG(LogLevel::Info, ("%p: No remaining outputs to device %p. " 1366 "Switch to primary output device %p for AEC", 1367 this, mOutputDeviceForAEC, PrimaryOutputDeviceID())); 1368 mOutputDeviceForAEC = PrimaryOutputDeviceID(); 1369 currentDeviceIndex = 0; 1370 MOZ_ASSERT(mOutputDevices[0].mDeviceID == mOutputDeviceForAEC); 1371 } 1372 if (mOutputDevices.Length() == 1) { 1373 // No other output devices so there is no choice. 1374 return; 1375 } 1376 1377 // The output is considered silent intentionally only if the whole duration 1378 // (often more than just this processing interval) of audio data in the 1379 // MediaSegment is null so as to reduce switching between output devices 1380 // should there be short durations of silence. 1381 auto HasNonNullAudio = [](const TrackAndVolume& aTV) { 1382 return aTV.mVolume != 0 && !aTV.mTrack->IsSuspended() && 1383 !aTV.mTrack->GetData()->IsNull(); 1384 }; 1385 // Keep using the same output device stream if it has non-null data, 1386 // so as to stay with a stream having ongoing audio. If the output stream 1387 // is switched, the echo cancellation algorithm can take some time to adjust 1388 // to the change in delay, so there is less value in switching back and 1389 // forth between output devices for very short sounds. 1390 for (const auto& output : mOutputDevices[currentDeviceIndex].mTrackOutputs) { 1391 if (HasNonNullAudio(output)) { 1392 return; 1393 } 1394 } 1395 // The current output device is silent. Use another if it has non-null data. 1396 for (const auto& outputDeviceEntry : mOutputDevices) { 1397 for (const auto& output : outputDeviceEntry.mTrackOutputs) { 1398 if (HasNonNullAudio(output)) { 1399 // Switch to this device. 1400 LOG(LogLevel::Info, 1401 ("%p: Switch output device for AEC from silent %p to non-null %p", 1402 this, mOutputDeviceForAEC, outputDeviceEntry.mDeviceID)); 1403 mOutputDeviceForAEC = outputDeviceEntry.mDeviceID; 1404 return; 1405 } 1406 } 1407 } 1408 // Null data for all outputs. Keep using the same device. 1409 } 1410 1411 void MediaTrackGraphImpl::Process(MixerCallbackReceiver* aMixerReceiver) { 1412 TRACE("MTG::Process"); 1413 MOZ_ASSERT(OnGraphThread()); 1414 if (mStateComputedTime == mProcessedTime) { // No frames to render. 1415 return; 1416 } 1417 1418 // Play track contents. 1419 bool allBlockedForever = true; 1420 // True when we've done ProcessInput for all processed tracks. 1421 bool doneAllProducing = false; 1422 const GraphTime oldProcessedTime = mProcessedTime; 1423 1424 // Figure out what each track wants to do 1425 for (uint32_t i = 0; i < mTracks.Length(); ++i) { 1426 MediaTrack* track = mTracks[i]; 1427 if (!doneAllProducing) { 1428 ProcessedMediaTrack* pt = track->AsProcessedTrack(); 1429 if (pt) { 1430 AudioNodeTrack* n = track->AsAudioNodeTrack(); 1431 if (n) { 1432 #ifdef DEBUG 1433 // Verify that the sampling rate for all of the following tracks is 1434 // the same 1435 for (uint32_t j = i + 1; j < mTracks.Length(); ++j) { 1436 AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack(); 1437 if (nextTrack) { 1438 MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate, 1439 "All AudioNodeTracks in the graph must have the same " 1440 "sampling rate"); 1441 } 1442 } 1443 #endif 1444 // Since an AudioNodeTrack is present, go ahead and 1445 // produce audio block by block for all the rest of the tracks. 1446 ProduceDataForTracksBlockByBlock(i, n->mSampleRate); 1447 doneAllProducing = true; 1448 } else { 1449 pt->ProcessInput(mProcessedTime, mStateComputedTime, 1450 ProcessedMediaTrack::ALLOW_END); 1451 // Assert that a live track produced enough data 1452 MOZ_ASSERT_IF(!track->mEnded, 1453 track->GetEnd() >= GraphTimeToTrackTimeWithBlocking( 1454 track, mStateComputedTime)); 1455 } 1456 } 1457 } 1458 if (track->mStartBlocking > oldProcessedTime) { 1459 allBlockedForever = false; 1460 } 1461 } 1462 mProcessedTime = mStateComputedTime; 1463 1464 SelectOutputDeviceForAEC(); 1465 for (const auto& outputDeviceEntry : mOutputDevices) { 1466 uint32_t outputChannelCount; 1467 if (!outputDeviceEntry.mReceiver) { // primary output 1468 if (!aMixerReceiver) { 1469 // Running off a system clock driver. No need to mix output. 1470 continue; 1471 } 1472 MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver(), 1473 "Driver must be AudioCallbackDriver if aMixerReceiver"); 1474 // Use the number of channel the driver expects: this is the number of 1475 // channel that can be output by the underlying system level audio stream. 1476 outputChannelCount = 1477 CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount(); 1478 } else { 1479 outputChannelCount = AudioOutputChannelCount(outputDeviceEntry); 1480 } 1481 MOZ_ASSERT(mRealtime, 1482 "If there's an output device, this graph must be realtime"); 1483 mMixer.StartMixing(); 1484 // This is the number of frames that are written to the output buffer, for 1485 // this iteration. 1486 TrackTime ticksPlayed = 0; 1487 for (const auto& t : outputDeviceEntry.mTrackOutputs) { 1488 TrackTime ticksPlayedForThisTrack = 1489 PlayAudio(t, oldProcessedTime, outputChannelCount); 1490 if (ticksPlayed == 0) { 1491 ticksPlayed = ticksPlayedForThisTrack; 1492 } else { 1493 MOZ_ASSERT( 1494 !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed, 1495 "Each track should have the same number of frames."); 1496 } 1497 } 1498 1499 if (ticksPlayed == 0) { 1500 // Nothing was played, so the mixer doesn't know how many frames were 1501 // processed. We still tell it so AudioCallbackDriver knows how much has 1502 // been processed. (bug 1406027) 1503 mMixer.Mix(nullptr, outputChannelCount, 1504 mStateComputedTime - oldProcessedTime, mSampleRate); 1505 } 1506 AudioChunk* outputChunk = mMixer.MixedChunk(); 1507 if (outputDeviceEntry.mDeviceID == mOutputDeviceForAEC) { 1508 // Callback any observers for the AEC speaker data. Note that one 1509 // (maybe) of these will be full-duplex, the others will get their input 1510 // data off separate cubeb callbacks. 1511 NotifyOutputData(*outputChunk); 1512 } 1513 if (!outputDeviceEntry.mReceiver) { // primary output 1514 aMixerReceiver->MixerCallback(outputChunk, mSampleRate); 1515 } else { 1516 outputDeviceEntry.mReceiver->EnqueueAudio(*outputChunk); 1517 } 1518 } 1519 1520 if (!allBlockedForever) { 1521 EnsureNextIteration(); 1522 } 1523 } 1524 1525 bool MediaTrackGraphImpl::UpdateMainThreadState() { 1526 MOZ_ASSERT(OnGraphThread()); 1527 if (mForceShutDownReceived) { 1528 for (MediaTrack* track : AllTracks()) { 1529 track->OnGraphThreadDone(); 1530 } 1531 } 1532 { 1533 MonitorAutoLock lock(mMonitor); 1534 bool finalUpdate = 1535 mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty()); 1536 PrepareUpdatesToMainThreadState(finalUpdate); 1537 if (!finalUpdate) { 1538 return true; 1539 } 1540 // The JSContext will not be used again. 1541 // Clear main thread access while under monitor. 1542 mJSContext = nullptr; 1543 } 1544 dom::WorkletThread::DeleteCycleCollectedJSContext(); 1545 // Enter shutdown mode when this iteration is completed. 1546 // No need to Destroy tracks here. The main-thread owner of each 1547 // track is responsible for calling Destroy on them. 1548 return false; 1549 } 1550 1551 auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime, 1552 MixerCallbackReceiver* aMixerReceiver) 1553 -> IterationResult { 1554 if (mGraphRunner) { 1555 return mGraphRunner->OneIteration(aStateTime, aMixerReceiver); 1556 } 1557 1558 return OneIterationImpl(aStateTime, aMixerReceiver); 1559 } 1560 1561 auto MediaTrackGraphImpl::OneIterationImpl( 1562 GraphTime aStateTime, MixerCallbackReceiver* aMixerReceiver) 1563 -> IterationResult { 1564 TRACE("MTG::OneIterationImpl"); 1565 TRACE_AUDIO_CALLBACK_FRAME_COUNT( 1566 "MTG graph advance", aStateTime - mStateComputedTime, mSampleRate); 1567 1568 if (SoftRealTimeLimitReached()) { 1569 TRACE("MTG::Demoting real-time thread!"); 1570 DemoteThreadFromRealTime(); 1571 } 1572 1573 // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph 1574 // thread, and so the monitor need not be held to check mLifecycleState. 1575 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline 1576 // graphs that have not started. 1577 1578 // While changes occur on mainthread, this assert confirms that 1579 // this code shouldn't run if mainthread might be changing the state (to 1580 // > LIFECYCLE_RUNNING) 1581 1582 // Ignore mutex warning: static during execution of the graph 1583 MOZ_PUSH_IGNORE_THREAD_SAFETY 1584 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING); 1585 MOZ_POP_THREAD_SAFETY 1586 1587 MOZ_ASSERT(OnGraphThread()); 1588 1589 WebCore::DenormalDisabler disabler; 1590 1591 // Process graph message from the main thread for this iteration. 1592 SwapMessageQueues(); 1593 RunMessagesInQueue(); 1594 1595 // Process MessagePort events. 1596 // These require a single thread, which has an nsThread with an event queue. 1597 if (mGraphRunner || !mRealtime) { 1598 TRACE("MTG::MessagePort events"); 1599 NS_ProcessPendingEvents(nullptr); 1600 } 1601 1602 UpdateGraph(aStateTime); 1603 1604 mStateComputedTime = aStateTime; 1605 1606 GraphTime oldProcessedTime = mProcessedTime; 1607 Process(aMixerReceiver); 1608 MOZ_ASSERT(mProcessedTime == aStateTime); 1609 1610 UpdateCurrentTimeForTracks(oldProcessedTime); 1611 1612 ProcessChunkMetadata(oldProcessedTime); 1613 1614 // Process graph messages queued from RunMessageAfterProcessing() on this 1615 // thread during the iteration. 1616 RunMessagesInQueue(); 1617 1618 if (!UpdateMainThreadState()) { 1619 if (Switching()) { 1620 // We'll never get to do this switch. Clear mNextDriver to break the 1621 // ref-cycle graph->nextDriver->currentDriver->graph. 1622 SwitchAtNextIteration(nullptr); 1623 } 1624 return IterationResult::CreateStop( 1625 NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this, 1626 &MediaTrackGraphImpl::SignalMainThreadCleanup)); 1627 } 1628 1629 if (Switching()) { 1630 RefPtr<GraphDriver> nextDriver = std::move(mNextDriver); 1631 return IterationResult::CreateSwitchDriver( 1632 nextDriver, NewRunnableMethod<StoreRefPtrPassByPtr<GraphDriver>>( 1633 "MediaTrackGraphImpl::SetCurrentDriver", this, 1634 &MediaTrackGraphImpl::SetCurrentDriver, nextDriver)); 1635 } 1636 1637 return IterationResult::CreateStillProcessing(); 1638 } 1639 1640 void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) { 1641 MOZ_ASSERT(NS_IsMainThread()); 1642 mMonitor.AssertCurrentThreadOwns(); 1643 1644 MediaTrack* track = aUpdate->mTrack; 1645 if (!track) return; 1646 track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime; 1647 track->mMainThreadEnded = aUpdate->mNextMainThreadEnded; 1648 1649 if (track->ShouldNotifyTrackEnded()) { 1650 track->NotifyMainThreadListeners(); 1651 } 1652 } 1653 1654 void MediaTrackGraphImpl::ForceShutDown() { 1655 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread"); 1656 LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this)); 1657 1658 if (mShutdownBlocker) { 1659 // Avoid waiting forever for a graph to shut down 1660 // synchronously. Reports are that some 3rd-party audio drivers 1661 // occasionally hang in shutdown (both for us and Chrome). 1662 NS_NewTimerWithCallback( 1663 getter_AddRefs(mShutdownTimer), this, 1664 MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT, 1665 nsITimer::TYPE_ONE_SHOT); 1666 } 1667 1668 class Message final : public ControlMessage { 1669 public: 1670 explicit Message(MediaTrackGraphImpl* aGraph) 1671 : ControlMessage(nullptr), mGraph(aGraph) {} 1672 void Run() override { 1673 TRACE("MTG::ForceShutdown ControlMessage"); 1674 mGraph->mForceShutDownReceived = true; 1675 } 1676 // The graph owns this message. 1677 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph; 1678 }; 1679 1680 if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) { 1681 // If both the track and port counts are zero, the regular shutdown 1682 // sequence will progress shortly to shutdown threads and destroy the graph. 1683 AppendMessage(MakeUnique<Message>(this)); 1684 InterruptJS(); 1685 } 1686 } 1687 1688 NS_IMETHODIMP 1689 MediaTrackGraphImpl::Notify(nsITimer* aTimer) { 1690 MOZ_ASSERT(NS_IsMainThread()); 1691 MOZ_ASSERT(!mShutdownBlocker, "MediaTrackGraph took too long to shut down!"); 1692 // Sigh, graph took too long to shut down. Stop blocking system 1693 // shutdown and hope all is well. 1694 RemoveShutdownBlocker(); 1695 mOutputDevicesChangedListener.DisconnectIfExists(); 1696 return NS_OK; 1697 } 1698 1699 static nsCString GetDocumentTitle(uint64_t aWindowID) { 1700 MOZ_ASSERT(NS_IsMainThread()); 1701 nsCString title; 1702 auto* win = nsGlobalWindowInner::GetInnerWindowWithId(aWindowID); 1703 if (!win) { 1704 return title; 1705 } 1706 Document* doc = win->GetExtantDoc(); 1707 if (!doc) { 1708 return title; 1709 } 1710 nsAutoString titleUTF16; 1711 doc->GetTitle(titleUTF16); 1712 CopyUTF16toUTF8(titleUTF16, title); 1713 return title; 1714 } 1715 1716 NS_IMETHODIMP 1717 MediaTrackGraphImpl::Observe(nsISupports* aSubject, const char* aTopic, 1718 const char16_t* aData) { 1719 MOZ_ASSERT(NS_IsMainThread()); 1720 MOZ_ASSERT(strcmp(aTopic, "document-title-changed") == 0); 1721 nsCString streamName = GetDocumentTitle(mWindowID); 1722 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get())); 1723 if (streamName.IsEmpty()) { 1724 return NS_OK; 1725 } 1726 QueueControlMessageWithNoShutdown( 1727 [self = RefPtr{this}, this, streamName = std::move(streamName)] { 1728 CurrentDriver()->SetStreamName(streamName); 1729 }); 1730 return NS_OK; 1731 } 1732 1733 bool MediaTrackGraphImpl::AddShutdownBlocker() { 1734 MOZ_ASSERT(NS_IsMainThread()); 1735 MOZ_ASSERT(!mShutdownBlocker); 1736 1737 class Blocker : public media::ShutdownBlocker { 1738 const RefPtr<MediaTrackGraphImpl> mGraph; 1739 1740 public: 1741 Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName) 1742 : media::ShutdownBlocker(aName), mGraph(aGraph) {} 1743 1744 NS_IMETHOD 1745 BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override { 1746 mGraph->ForceShutDown(); 1747 return NS_OK; 1748 } 1749 }; 1750 1751 nsCOMPtr<nsIAsyncShutdownClient> barrier = media::GetShutdownBarrier(); 1752 if (!barrier) { 1753 // We're already shutting down, we won't be able to add a blocker, bail. 1754 LOG(LogLevel::Error, 1755 ("%p: Couldn't get shutdown barrier, won't add shutdown blocker", 1756 this)); 1757 return false; 1758 } 1759 1760 // Blocker names must be distinct. 1761 nsString blockerName; 1762 blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this); 1763 mShutdownBlocker = MakeAndAddRef<Blocker>(this, blockerName); 1764 nsresult rv = barrier->AddBlocker(mShutdownBlocker, 1765 NS_LITERAL_STRING_FROM_CSTRING(__FILE__), 1766 __LINE__, u"MediaTrackGraph shutdown"_ns); 1767 MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); 1768 return true; 1769 } 1770 1771 void MediaTrackGraphImpl::RemoveShutdownBlocker() { 1772 if (!mShutdownBlocker) { 1773 return; 1774 } 1775 media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker); 1776 mShutdownBlocker = nullptr; 1777 } 1778 1779 NS_IMETHODIMP 1780 MediaTrackGraphImpl::GetName(nsACString& aName) { 1781 aName.AssignLiteral("MediaTrackGraphImpl"); 1782 return NS_OK; 1783 } 1784 1785 namespace { 1786 1787 class MediaTrackGraphShutDownRunnable : public Runnable { 1788 public: 1789 explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph) 1790 : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {} 1791 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. 1792 // See bug 1535398. 1793 MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override { 1794 TRACE("MTG::MediaTrackGraphShutDownRunnable runnable"); 1795 MOZ_ASSERT(NS_IsMainThread()); 1796 MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver, 1797 "We should know the graph thread control loop isn't running!"); 1798 1799 LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get())); 1800 1801 // We've asserted the graph isn't running. Use mDriver instead of 1802 // CurrentDriver to avoid thread-safety checks 1803 #if 0 // AudioCallbackDrivers are released asynchronously anyways 1804 // XXX a better test would be have setting mGraphDriverRunning make sure 1805 // any current callback has finished and block future ones -- or just 1806 // handle it all in Shutdown()! 1807 if (mGraph->mDriver->AsAudioCallbackDriver()) { 1808 MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback()); 1809 } 1810 #endif 1811 1812 for (MediaTrackGraphImpl::PendingResumeOperation& op : 1813 mGraph->mPendingResumeOperations) { 1814 op.Abort(); 1815 } 1816 1817 if (mGraph->mGraphRunner) { 1818 RefPtr<GraphRunner>(mGraph->mGraphRunner)->Shutdown(); 1819 } 1820 1821 RefPtr<GraphDriver>(mGraph->mDriver)->Shutdown(); 1822 1823 // Release the driver now so that an AudioCallbackDriver will release its 1824 // SharedThreadPool reference. Each SharedThreadPool reference must be 1825 // released before SharedThreadPool::SpinUntilEmpty() runs on 1826 // xpcom-shutdown-threads. Don't wait for GC/CC to release references to 1827 // objects owning tracks, or for expiration of mGraph->mShutdownTimer, 1828 // which won't otherwise release its reference on the graph until 1829 // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads. 1830 mGraph->SetCurrentDriver(nullptr); 1831 1832 // Safe to access these without the monitor since the graph isn't running. 1833 // We may be one of several graphs. Drop ticket to eventually unblock 1834 // shutdown. 1835 if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) { 1836 MOZ_ASSERT( 1837 false, 1838 "AudioCallbackDriver took too long to shut down and we let shutdown" 1839 " continue - freezing and leaking"); 1840 1841 // The timer fired, so we may be deeper in shutdown now. Block any 1842 // further teardown and just leak, for safety. 1843 return NS_OK; 1844 } 1845 1846 // mGraph's thread is not running so it's OK to do whatever here 1847 for (MediaTrack* track : mGraph->AllTracks()) { 1848 // Clean up all MediaSegments since we cannot release Images too 1849 // late during shutdown. Also notify listeners that they were removed 1850 // so they can clean up any gfx resources. 1851 track->RemoveAllResourcesAndListenersImpl(); 1852 } 1853 1854 #ifdef DEBUG 1855 { 1856 MonitorAutoLock lock(mGraph->mMonitor); 1857 MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty()); 1858 } 1859 #endif 1860 mGraph->mPendingUpdateRunnables.Clear(); 1861 1862 mGraph->RemoveShutdownBlocker(); 1863 1864 // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION 1865 // stage, since completion of that stage requires all tracks to be freed, 1866 // which requires shutdown to proceed. 1867 1868 if (mGraph->IsEmpty()) { 1869 // mGraph is no longer needed, so delete it. 1870 mGraph->Destroy(); 1871 } else { 1872 // The graph is not empty. We must be in a forced shutdown. 1873 // Some later AppendMessage will detect that the graph has 1874 // been emptied, and delete it. 1875 NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?"); 1876 mGraph->LifecycleStateRef() = 1877 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION; 1878 } 1879 return NS_OK; 1880 } 1881 1882 private: 1883 RefPtr<MediaTrackGraphImpl> mGraph; 1884 }; 1885 1886 class MediaTrackGraphStableStateRunnable : public Runnable { 1887 public: 1888 explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph, 1889 bool aSourceIsMTG) 1890 : Runnable("MediaTrackGraphStableStateRunnable"), 1891 mGraph(aGraph), 1892 mSourceIsMTG(aSourceIsMTG) {} 1893 NS_IMETHOD Run() override { 1894 TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage"); 1895 if (mGraph) { 1896 mGraph->RunInStableState(mSourceIsMTG); 1897 } 1898 return NS_OK; 1899 } 1900 1901 private: 1902 RefPtr<MediaTrackGraphImpl> mGraph; 1903 bool mSourceIsMTG; 1904 }; 1905 1906 /* 1907 * Control messages forwarded from main thread to graph manager thread 1908 */ 1909 class CreateMessage : public ControlMessage { 1910 public: 1911 explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {} 1912 void Run() override { 1913 TRACE("MTG::AddTrackGraphThread ControlMessage"); 1914 mTrack->GraphImpl()->AddTrackGraphThread(mTrack); 1915 } 1916 void RunDuringShutdown() override { 1917 // Make sure to run this message during shutdown too, to make sure 1918 // that we balance the number of tracks registered with the graph 1919 // as they're destroyed during shutdown. 1920 Run(); 1921 } 1922 }; 1923 1924 } // namespace 1925 1926 void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) { 1927 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread"); 1928 1929 nsTArray<nsCOMPtr<nsIRunnable>> runnables; 1930 // When we're doing a forced shutdown, pending control messages may be 1931 // run on the main thread via RunDuringShutdown. Those messages must 1932 // run without the graph monitor being held. So, we collect them here. 1933 nsTArray<UniquePtr<ControlMessageInterface>> 1934 controlMessagesToRunDuringShutdown; 1935 1936 { 1937 MonitorAutoLock lock(mMonitor); 1938 if (aSourceIsMTG) { 1939 MOZ_ASSERT(mPostedRunInStableStateEvent); 1940 mPostedRunInStableStateEvent = false; 1941 } 1942 1943 // This should be kept in sync with the LifecycleState enum in 1944 // MediaTrackGraphImpl.h 1945 const char* LifecycleState_str[] = { 1946 "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING", 1947 "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP", 1948 "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN", 1949 "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"}; 1950 1951 if (LifecycleStateRef() != LIFECYCLE_RUNNING) { 1952 LOG(LogLevel::Debug, 1953 ("%p: Running stable state callback. Current state: %s", this, 1954 LifecycleState_str[LifecycleStateRef()])); 1955 } 1956 1957 runnables = std::move(mUpdateRunnables); 1958 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) { 1959 TrackUpdate* update = &mTrackUpdates[i]; 1960 if (update->mTrack) { 1961 ApplyTrackUpdate(update); 1962 } 1963 } 1964 mTrackUpdates.Clear(); 1965 1966 mMainThreadGraphTime = mNextMainThreadGraphTime; 1967 1968 if (mCurrentTaskMessageQueue.IsEmpty()) { 1969 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP && 1970 IsEmpty()) { 1971 // Complete shutdown. First, ensure that this graph is no longer used. 1972 // A new graph graph will be created if one is needed. 1973 // Asynchronously clean up old graph. We don't want to do this 1974 // synchronously because it spins the event loop waiting for threads 1975 // to shut down, and we don't want to do that in a stable state handler. 1976 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; 1977 LOG(LogLevel::Debug, 1978 ("%p: Sending MediaTrackGraphShutDownRunnable", this)); 1979 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this); 1980 mMainThread->Dispatch(event.forget()); 1981 } 1982 } else { 1983 if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) { 1984 MessageBlock* block = mBackMessageQueue.AppendElement(); 1985 block->mMessages = std::move(mCurrentTaskMessageQueue); 1986 EnsureNextIteration(); 1987 } 1988 1989 // If this MediaTrackGraph has entered regular (non-forced) shutdown it 1990 // is not able to process any more messages. Those messages being added to 1991 // the graph in the first place is an error. 1992 MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() < 1993 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP || 1994 mForceShutDownReceived); 1995 } 1996 1997 if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) { 1998 // Start the driver now. We couldn't start it earlier because the graph 1999 // might exit immediately on finding it has no tracks. The first message 2000 // for a new graph must create a track. 2001 MOZ_ASSERT(MessagesQueued()); 2002 2003 LOG(LogLevel::Debug, 2004 ("%p: Starting a graph with a %s", this, 2005 CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver" 2006 : "SystemClockDriver")); 2007 LifecycleStateRef() = LIFECYCLE_RUNNING; 2008 mGraphDriverRunning = true; 2009 RefPtr<GraphDriver> driver = CurrentDriver(); 2010 driver->Start(); 2011 // It's not safe to Shutdown() a thread from StableState, and 2012 // releasing this may shutdown a SystemClockDriver thread. 2013 // Proxy the release to outside of StableState. 2014 NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver", 2015 driver.forget(), 2016 true); // always proxy 2017 } 2018 2019 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP && 2020 mForceShutDownReceived) { 2021 // Defer calls to RunDuringShutdown() to happen while mMonitor is not 2022 // held. 2023 for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) { 2024 MessageBlock& mb = mBackMessageQueue[i]; 2025 controlMessagesToRunDuringShutdown.AppendElements( 2026 std::move(mb.mMessages)); 2027 } 2028 mBackMessageQueue.Clear(); 2029 MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty()); 2030 // Stop MediaTrackGraph threads. 2031 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; 2032 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this); 2033 mMainThread->Dispatch(event.forget()); 2034 } 2035 2036 mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING; 2037 } 2038 2039 // Make sure we get a new current time in the next event loop task 2040 if (!aSourceIsMTG) { 2041 MOZ_ASSERT(mPostedRunInStableState); 2042 mPostedRunInStableState = false; 2043 } 2044 2045 for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) { 2046 controlMessagesToRunDuringShutdown[i]->RunDuringShutdown(); 2047 } 2048 2049 #ifdef DEBUG 2050 mCanRunMessagesSynchronously = 2051 !mGraphDriverRunning && 2052 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; 2053 #endif 2054 2055 for (uint32_t i = 0; i < runnables.Length(); ++i) { 2056 runnables[i]->Run(); 2057 } 2058 } 2059 2060 void MediaTrackGraphImpl::EnsureRunInStableState() { 2061 MOZ_ASSERT(NS_IsMainThread(), "main thread only"); 2062 2063 if (mPostedRunInStableState) return; 2064 mPostedRunInStableState = true; 2065 nsCOMPtr<nsIRunnable> event = 2066 new MediaTrackGraphStableStateRunnable(this, false); 2067 nsContentUtils::RunInStableState(event.forget()); 2068 } 2069 2070 void MediaTrackGraphImpl::EnsureStableStateEventPosted() { 2071 MOZ_ASSERT(OnGraphThread()); 2072 mMonitor.AssertCurrentThreadOwns(); 2073 2074 if (mPostedRunInStableStateEvent) return; 2075 mPostedRunInStableStateEvent = true; 2076 nsCOMPtr<nsIRunnable> event = 2077 new MediaTrackGraphStableStateRunnable(this, true); 2078 mMainThread->Dispatch(event.forget()); 2079 } 2080 2081 void MediaTrackGraphImpl::SignalMainThreadCleanup() { 2082 MOZ_ASSERT(mDriver->OnThread()); 2083 2084 MonitorAutoLock lock(mMonitor); 2085 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline 2086 // graphs that have not started. 2087 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING); 2088 LOG(LogLevel::Debug, 2089 ("%p: MediaTrackGraph waiting for main thread cleanup", this)); 2090 LifecycleStateRef() = 2091 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP; 2092 EnsureStableStateEventPosted(); 2093 } 2094 2095 void MediaTrackGraphImpl::AppendMessage( 2096 UniquePtr<ControlMessageInterface> aMessage) { 2097 MOZ_ASSERT(NS_IsMainThread(), "main thread only"); 2098 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0); 2099 2100 if (!mGraphDriverRunning && 2101 LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) { 2102 // The graph control loop is not running and main thread cleanup has 2103 // happened. From now on we can't append messages to 2104 // mCurrentTaskMessageQueue, because that will never be processed again, so 2105 // just RunDuringShutdown this message. This should only happen during 2106 // forced shutdown, or after a non-realtime graph has finished processing. 2107 #ifdef DEBUG 2108 MOZ_ASSERT(mCanRunMessagesSynchronously); 2109 mCanRunMessagesSynchronously = false; 2110 #endif 2111 aMessage->RunDuringShutdown(); 2112 #ifdef DEBUG 2113 mCanRunMessagesSynchronously = true; 2114 #endif 2115 if (IsEmpty() && 2116 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) { 2117 Destroy(); 2118 } 2119 return; 2120 } 2121 2122 mCurrentTaskMessageQueue.AppendElement(std::move(aMessage)); 2123 EnsureRunInStableState(); 2124 } 2125 2126 void MediaTrackGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable) { 2127 mMainThread->Dispatch(std::move(aRunnable)); 2128 } 2129 2130 MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType, 2131 MediaSegment* aSegment) 2132 : mSampleRate(aSampleRate), 2133 mType(aType), 2134 mSegment(aSegment), 2135 mStartTime(0), 2136 mForgottenTime(0), 2137 mEnded(false), 2138 mNotifiedEnded(false), 2139 mDisabledMode(DisabledTrackMode::ENABLED), 2140 mStartBlocking(GRAPH_TIME_MAX), 2141 mSuspendedCount(0), 2142 mMainThreadCurrentTime(0), 2143 mMainThreadEnded(false), 2144 mEndedNotificationSent(false), 2145 mMainThreadDestroyed(false), 2146 mGraph(nullptr) { 2147 MOZ_COUNT_CTOR(MediaTrack); 2148 MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType); 2149 } 2150 2151 MediaTrack::~MediaTrack() { 2152 MOZ_COUNT_DTOR(MediaTrack); 2153 NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already"); 2154 NS_ASSERTION(mMainThreadListeners.IsEmpty(), 2155 "All main thread listeners should have been removed"); 2156 } 2157 2158 size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { 2159 size_t amount = 0; 2160 2161 // Not owned: 2162 // - mGraph - Not reported here 2163 // - mConsumers - elements 2164 // Future: 2165 // - mLastPlayedVideoFrame 2166 // - mTrackListeners - elements 2167 2168 amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf); 2169 amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf); 2170 amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf); 2171 2172 return amount; 2173 } 2174 2175 size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { 2176 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); 2177 } 2178 2179 void MediaTrack::IncrementSuspendCount() { 2180 ++mSuspendedCount; 2181 if (mSuspendedCount != 1 || !mGraph) { 2182 MOZ_ASSERT(mGraph || mConsumers.IsEmpty()); 2183 return; 2184 } 2185 AssertOnGraphThreadOrNotRunning(); 2186 auto* graph = GraphImpl(); 2187 for (uint32_t i = 0; i < mConsumers.Length(); ++i) { 2188 mConsumers[i]->Suspended(); 2189 } 2190 MOZ_ASSERT(graph->mTracks.Contains(this)); 2191 graph->mTracks.RemoveElement(this); 2192 graph->mSuspendedTracks.AppendElement(this); 2193 graph->SetTrackOrderDirty(); 2194 } 2195 2196 void MediaTrack::DecrementSuspendCount() { 2197 MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun"); 2198 --mSuspendedCount; 2199 if (mSuspendedCount != 0 || !mGraph) { 2200 MOZ_ASSERT(mGraph || mConsumers.IsEmpty()); 2201 return; 2202 } 2203 AssertOnGraphThreadOrNotRunning(); 2204 auto* graph = GraphImpl(); 2205 for (uint32_t i = 0; i < mConsumers.Length(); ++i) { 2206 mConsumers[i]->Resumed(); 2207 } 2208 MOZ_ASSERT(graph->mSuspendedTracks.Contains(this)); 2209 graph->mSuspendedTracks.RemoveElement(this); 2210 graph->mTracks.AppendElement(this); 2211 graph->SetTrackOrderDirty(); 2212 } 2213 2214 void ProcessedMediaTrack::DecrementSuspendCount() { 2215 mCycleMarker = NOT_VISITED; 2216 MediaTrack::DecrementSuspendCount(); 2217 } 2218 2219 MediaTrackGraphImpl* MediaTrack::GraphImpl() { 2220 return static_cast<MediaTrackGraphImpl*>(mGraph); 2221 } 2222 2223 const MediaTrackGraphImpl* MediaTrack::GraphImpl() const { 2224 return static_cast<MediaTrackGraphImpl*>(mGraph); 2225 } 2226 2227 void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) { 2228 MOZ_ASSERT(!mGraph, "Should only be called once"); 2229 MOZ_ASSERT(mSampleRate == aGraph->GraphRate()); 2230 mGraph = aGraph; 2231 } 2232 2233 void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) { 2234 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(aGraph); 2235 SetGraphImpl(graph); 2236 } 2237 2238 TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const { 2239 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime || 2240 aTime <= mStartBlocking, 2241 "Incorrectly ignoring blocking!"); 2242 return aTime - mStartTime; 2243 } 2244 2245 GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const { 2246 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime || 2247 aTime + mStartTime <= mStartBlocking, 2248 "Incorrectly ignoring blocking!"); 2249 return aTime + mStartTime; 2250 } 2251 2252 TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const { 2253 return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime); 2254 } 2255 2256 void MediaTrack::RemoveAllResourcesAndListenersImpl() { 2257 GraphImpl()->AssertOnGraphThreadOrNotRunning(); 2258 2259 for (auto& l : mTrackListeners.Clone()) { 2260 l->NotifyRemoved(Graph()); 2261 } 2262 mTrackListeners.Clear(); 2263 2264 RemoveAllDirectListenersImpl(); 2265 2266 if (mSegment) { 2267 mSegment->Clear(); 2268 } 2269 } 2270 2271 void MediaTrack::DestroyImpl() { 2272 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) { 2273 mConsumers[i]->Disconnect(); 2274 } 2275 if (mSegment) { 2276 mSegment->Clear(); 2277 } 2278 mGraph = nullptr; 2279 } 2280 2281 void MediaTrack::Destroy() { 2282 // Keep this track alive until we leave this method 2283 RefPtr<MediaTrack> kungFuDeathGrip = this; 2284 // Keep a reference to the graph, since Message might RunDuringShutdown() 2285 // synchronously and make GraphImpl() invalid. 2286 RefPtr<MediaTrackGraphImpl> graph = GraphImpl(); 2287 2288 QueueControlOrShutdownMessage( 2289 [self = RefPtr{this}, this](IsInShutdown aInShutdown) { 2290 if (aInShutdown == IsInShutdown::No) { 2291 OnGraphThreadDone(); 2292 } 2293 TRACE("MediaTrack::Destroy ControlMessage"); 2294 RemoveAllResourcesAndListenersImpl(); 2295 auto* graph = GraphImpl(); 2296 DestroyImpl(); 2297 graph->RemoveTrackGraphThread(this); 2298 }); 2299 graph->RemoveTrack(this); 2300 // Message::RunDuringShutdown may have removed this track from the graph, 2301 // but our kungFuDeathGrip above will have kept this track alive if 2302 // necessary. 2303 mMainThreadDestroyed = true; 2304 } 2305 2306 uint64_t MediaTrack::GetWindowId() const { return GraphImpl()->mWindowID; } 2307 2308 TrackTime MediaTrack::GetEnd() const { 2309 return mSegment ? mSegment->GetDuration() : 0; 2310 } 2311 2312 void MediaTrack::AddAudioOutput(void* aKey, const AudioDeviceInfo* aSink) { 2313 MOZ_ASSERT(NS_IsMainThread()); 2314 AudioDeviceID deviceID = nullptr; 2315 TrackRate preferredSampleRate = 0; 2316 if (aSink) { 2317 deviceID = aSink->DeviceID(); 2318 preferredSampleRate = static_cast<TrackRate>(aSink->DefaultRate()); 2319 } 2320 AddAudioOutput(aKey, deviceID, preferredSampleRate); 2321 } 2322 2323 void MediaTrack::AddAudioOutput(void* aKey, CubebUtils::AudioDeviceID aDeviceID, 2324 TrackRate aPreferredSampleRate) { 2325 MOZ_ASSERT(NS_IsMainThread()); 2326 if (mMainThreadDestroyed) { 2327 return; 2328 } 2329 LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this)); 2330 GraphImpl()->RegisterAudioOutput(this, aKey, aDeviceID, aPreferredSampleRate); 2331 } 2332 2333 void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey, 2334 float aVolume) { 2335 MOZ_ASSERT(NS_IsMainThread()); 2336 for (auto& params : mAudioOutputParams) { 2337 if (params.mKey == aKey && aTrack == params.mTrack) { 2338 params.mVolume = aVolume; 2339 UpdateAudioOutput(aTrack, params.mDeviceID); 2340 return; 2341 } 2342 } 2343 MOZ_CRASH("Audio output key not found when setting the volume."); 2344 } 2345 2346 void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) { 2347 if (mMainThreadDestroyed) { 2348 return; 2349 } 2350 GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume); 2351 } 2352 2353 void MediaTrack::RemoveAudioOutput(void* aKey) { 2354 MOZ_ASSERT(NS_IsMainThread()); 2355 if (mMainThreadDestroyed) { 2356 return; 2357 } 2358 LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this)); 2359 GraphImpl()->UnregisterAudioOutput(this, aKey); 2360 } 2361 2362 void MediaTrackGraphImpl::RegisterAudioOutput( 2363 MediaTrack* aTrack, void* aKey, CubebUtils::AudioDeviceID aDeviceID, 2364 TrackRate aPreferredSampleRate) { 2365 MOZ_ASSERT(NS_IsMainThread()); 2366 MOZ_ASSERT(!mAudioOutputParams.Contains(TrackAndKey{aTrack, aKey})); 2367 2368 IncrementOutputDeviceRefCnt(aDeviceID, aPreferredSampleRate); 2369 2370 mAudioOutputParams.EmplaceBack( 2371 TrackKeyDeviceAndVolume{aTrack, aKey, aDeviceID, 1.f}); 2372 2373 UpdateAudioOutput(aTrack, aDeviceID); 2374 } 2375 2376 void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack, 2377 void* aKey) { 2378 MOZ_ASSERT(NS_IsMainThread()); 2379 2380 size_t index = mAudioOutputParams.IndexOf(TrackAndKey{aTrack, aKey}); 2381 MOZ_ASSERT(index != mAudioOutputParams.NoIndex); 2382 AudioDeviceID deviceID = mAudioOutputParams[index].mDeviceID; 2383 mAudioOutputParams.UnorderedRemoveElementAt(index); 2384 2385 UpdateAudioOutput(aTrack, deviceID); 2386 2387 DecrementOutputDeviceRefCnt(deviceID); 2388 } 2389 2390 void MediaTrackGraphImpl::UpdateAudioOutput(MediaTrack* aTrack, 2391 AudioDeviceID aDeviceID) { 2392 MOZ_ASSERT(NS_IsMainThread()); 2393 MOZ_ASSERT(!aTrack->IsDestroyed()); 2394 2395 float volume = 0.f; 2396 bool found = false; 2397 for (const auto& params : mAudioOutputParams) { 2398 if (params.mTrack == aTrack && params.mDeviceID == aDeviceID) { 2399 volume += params.mVolume; 2400 found = true; 2401 } 2402 } 2403 2404 QueueControlMessageWithNoShutdown( 2405 // track has a strong reference to this. 2406 [track = RefPtr{aTrack}, aDeviceID, volume, found] { 2407 TRACE("MediaTrack::UpdateAudioOutput ControlMessage"); 2408 MediaTrackGraphImpl* graph = track->GraphImpl(); 2409 auto& outputDevicesRef = graph->mOutputDevices; 2410 size_t deviceIndex = outputDevicesRef.IndexOf(aDeviceID); 2411 MOZ_ASSERT(deviceIndex != outputDevicesRef.NoIndex); 2412 auto& deviceOutputsRef = outputDevicesRef[deviceIndex].mTrackOutputs; 2413 if (found) { 2414 for (auto& outputRef : deviceOutputsRef) { 2415 if (outputRef.mTrack == track) { 2416 outputRef.mVolume = volume; 2417 return; 2418 } 2419 } 2420 deviceOutputsRef.EmplaceBack(TrackAndVolume{track, volume}); 2421 } else { 2422 DebugOnly<bool> removed = deviceOutputsRef.RemoveElement(track); 2423 MOZ_ASSERT(removed); 2424 // mOutputDevices[0] is retained for AudioCallbackDriver output even 2425 // when no tracks have audio outputs. 2426 if (deviceIndex != 0 && deviceOutputsRef.IsEmpty()) { 2427 // The device is no longer in use. 2428 outputDevicesRef.UnorderedRemoveElementAt(deviceIndex); 2429 } 2430 } 2431 }); 2432 } 2433 2434 void MediaTrackGraphImpl::IncrementOutputDeviceRefCnt( 2435 AudioDeviceID aDeviceID, TrackRate aPreferredSampleRate) { 2436 MOZ_ASSERT(NS_IsMainThread()); 2437 2438 for (auto& elementRef : mOutputDeviceRefCnts) { 2439 if (elementRef.mDeviceID == aDeviceID) { 2440 ++elementRef.mRefCnt; 2441 return; 2442 } 2443 } 2444 MOZ_ASSERT(aDeviceID != mPrimaryOutputDeviceID, 2445 "mOutputDeviceRefCnts should always have the primary device"); 2446 // Need to add an output device. 2447 // Output via another graph for this device. 2448 // This sample rate is not exposed to content. 2449 TrackRate sampleRate = 2450 aPreferredSampleRate != 0 2451 ? aPreferredSampleRate 2452 : static_cast<TrackRate>(CubebUtils::PreferredSampleRate( 2453 /*aShouldResistFingerprinting*/ false)); 2454 MediaTrackGraph* newGraph = MediaTrackGraphImpl::GetInstance( 2455 MediaTrackGraph::AUDIO_THREAD_DRIVER, mWindowID, sampleRate, aDeviceID, 2456 GetMainThreadSerialEventTarget()); 2457 // CreateCrossGraphReceiver wants the sample rate of this graph. 2458 RefPtr receiver = newGraph->CreateCrossGraphReceiver(mSampleRate); 2459 receiver->AddAudioOutput(nullptr, aDeviceID, sampleRate); 2460 mOutputDeviceRefCnts.EmplaceBack( 2461 DeviceReceiverAndCount{aDeviceID, receiver, 1}); 2462 2463 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aDeviceID, 2464 receiver = std::move(receiver)]() mutable { 2465 TRACE("MediaTrackGraph add output device ControlMessage"); 2466 MOZ_ASSERT(!mOutputDevices.Contains(aDeviceID)); 2467 mOutputDevices.EmplaceBack( 2468 OutputDeviceEntry{aDeviceID, std::move(receiver)}); 2469 }); 2470 } 2471 2472 void MediaTrackGraphImpl::DecrementOutputDeviceRefCnt(AudioDeviceID aDeviceID) { 2473 MOZ_ASSERT(NS_IsMainThread()); 2474 2475 size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID); 2476 MOZ_ASSERT(index != mOutputDeviceRefCnts.NoIndex); 2477 // mOutputDeviceRefCnts[0] is retained for consistency with 2478 // mOutputDevices[0], which is retained for AudioCallbackDriver output even 2479 // when no tracks have audio outputs. 2480 if (--mOutputDeviceRefCnts[index].mRefCnt == 0 && index != 0) { 2481 mOutputDeviceRefCnts[index].mReceiver->Destroy(); 2482 mOutputDeviceRefCnts.UnorderedRemoveElementAt(index); 2483 } 2484 } 2485 2486 void MediaTrack::Suspend() { 2487 // This can happen if this method has been called asynchronously, and the 2488 // track has been destroyed since then. 2489 if (mMainThreadDestroyed) { 2490 return; 2491 } 2492 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] { 2493 TRACE("MediaTrack::IncrementSuspendCount ControlMessage"); 2494 IncrementSuspendCount(); 2495 }); 2496 } 2497 2498 void MediaTrack::Resume() { 2499 // This can happen if this method has been called asynchronously, and the 2500 // track has been destroyed since then. 2501 if (mMainThreadDestroyed) { 2502 return; 2503 } 2504 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] { 2505 TRACE("MediaTrack::DecrementSuspendCount ControlMessage"); 2506 DecrementSuspendCount(); 2507 }); 2508 } 2509 2510 void MediaTrack::AddListenerImpl( 2511 already_AddRefed<MediaTrackListener> aListener) { 2512 RefPtr<MediaTrackListener> l(aListener); 2513 mTrackListeners.AppendElement(std::move(l)); 2514 2515 PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle(); 2516 mTrackListeners.LastElement()->NotifyPrincipalHandleChanged( 2517 Graph(), lastPrincipalHandle); 2518 if (mNotifiedEnded) { 2519 mTrackListeners.LastElement()->NotifyEnded(Graph()); 2520 } 2521 if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) { 2522 mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false); 2523 } 2524 } 2525 2526 void MediaTrack::AddListener(MediaTrackListener* aListener) { 2527 MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners"); 2528 if (mMainThreadDestroyed) { 2529 return; 2530 } 2531 QueueControlMessageWithNoShutdown( 2532 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable { 2533 TRACE("MediaTrack::AddListenerImpl ControlMessage"); 2534 AddListenerImpl(listener.forget()); 2535 }); 2536 } 2537 2538 void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) { 2539 for (size_t i = 0; i < mTrackListeners.Length(); ++i) { 2540 if (mTrackListeners[i] == aListener) { 2541 mTrackListeners[i]->NotifyRemoved(Graph()); 2542 mTrackListeners.RemoveElementAt(i); 2543 return; 2544 } 2545 } 2546 } 2547 2548 RefPtr<GenericPromise> MediaTrack::RemoveListener( 2549 MediaTrackListener* aListener) { 2550 MozPromiseHolder<GenericPromise> promiseHolder; 2551 RefPtr<GenericPromise> p = promiseHolder.Ensure(__func__); 2552 if (mMainThreadDestroyed) { 2553 promiseHolder.Reject(NS_ERROR_FAILURE, __func__); 2554 return p; 2555 } 2556 QueueControlOrShutdownMessage( 2557 [self = RefPtr{this}, this, listener = RefPtr{aListener}, 2558 promiseHolder = std::move(promiseHolder)](IsInShutdown) mutable { 2559 TRACE("MediaTrack::RemoveListenerImpl ControlMessage"); 2560 // During shutdown we still want the listener's NotifyRemoved to be 2561 // called, since not doing that might block shutdown of other modules. 2562 RemoveListenerImpl(listener); 2563 promiseHolder.Resolve(true, __func__); 2564 }); 2565 return p; 2566 } 2567 2568 void MediaTrack::AddDirectListenerImpl( 2569 already_AddRefed<DirectMediaTrackListener> aListener) { 2570 AssertOnGraphThread(); 2571 // Base implementation, for tracks that don't support direct track listeners. 2572 RefPtr<DirectMediaTrackListener> listener = aListener; 2573 listener->NotifyDirectListenerInstalled( 2574 DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED); 2575 } 2576 2577 void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) { 2578 if (mMainThreadDestroyed) { 2579 return; 2580 } 2581 QueueControlMessageWithNoShutdown( 2582 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable { 2583 TRACE("MediaTrack::AddDirectListenerImpl ControlMessage"); 2584 AddDirectListenerImpl(listener.forget()); 2585 }); 2586 } 2587 2588 void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) { 2589 // Base implementation, the listener was never added so nothing to do. 2590 } 2591 2592 void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) { 2593 if (mMainThreadDestroyed) { 2594 return; 2595 } 2596 QueueControlOrShutdownMessage( 2597 [self = RefPtr{this}, this, listener = RefPtr{aListener}](IsInShutdown) { 2598 TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage"); 2599 // During shutdown we still want the listener's 2600 // NotifyDirectListenerUninstalled to be called, since not doing that 2601 // might block shutdown of other modules. 2602 RemoveDirectListenerImpl(listener); 2603 }); 2604 } 2605 2606 void MediaTrack::RunAfterPendingUpdates( 2607 already_AddRefed<nsIRunnable> aRunnable) { 2608 MOZ_ASSERT(NS_IsMainThread()); 2609 if (mMainThreadDestroyed) { 2610 return; 2611 } 2612 QueueControlOrShutdownMessage( 2613 [self = RefPtr{this}, this, 2614 runnable = nsCOMPtr{aRunnable}](IsInShutdown aInShutdown) mutable { 2615 TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage"); 2616 if (aInShutdown == IsInShutdown::No) { 2617 Graph()->DispatchToMainThreadStableState(runnable.forget()); 2618 } else { 2619 // Don't run mRunnable now as it may call AppendMessage() which would 2620 // assume that there are no remaining 2621 // controlMessagesToRunDuringShutdown. 2622 MOZ_ASSERT(NS_IsMainThread()); 2623 GraphImpl()->Dispatch(runnable.forget()); 2624 } 2625 }); 2626 } 2627 2628 void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) { 2629 AssertOnGraphThread(); 2630 MOZ_DIAGNOSTIC_ASSERT( 2631 aMode == DisabledTrackMode::ENABLED || 2632 mDisabledMode == DisabledTrackMode::ENABLED, 2633 "Changing disabled track mode for a track is not allowed"); 2634 DisabledTrackMode oldMode = CombinedDisabledMode(); 2635 mDisabledMode = aMode; 2636 NotifyIfDisabledModeChangedFrom(oldMode); 2637 } 2638 2639 void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) { 2640 if (mMainThreadDestroyed) { 2641 return; 2642 } 2643 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aMode]() { 2644 TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage"); 2645 SetDisabledTrackModeImpl(aMode); 2646 }); 2647 } 2648 2649 void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment, 2650 MediaSegment* aRawSegment) { 2651 AssertOnGraphThread(); 2652 mozilla::ApplyTrackDisabling(mDisabledMode, aSegment, aRawSegment); 2653 } 2654 2655 void MediaTrack::AddMainThreadListener( 2656 MainThreadMediaTrackListener* aListener) { 2657 MOZ_ASSERT(NS_IsMainThread()); 2658 MOZ_ASSERT(aListener); 2659 MOZ_ASSERT(!mMainThreadListeners.Contains(aListener)); 2660 2661 mMainThreadListeners.AppendElement(aListener); 2662 2663 // If it is not yet time to send the notification, then exit here. 2664 if (!mEndedNotificationSent) { 2665 return; 2666 } 2667 2668 class NotifyRunnable final : public Runnable { 2669 public: 2670 explicit NotifyRunnable(MediaTrack* aTrack) 2671 : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {} 2672 2673 NS_IMETHOD Run() override { 2674 TRACE("MediaTrack::NotifyMainThreadListeners Runnable"); 2675 MOZ_ASSERT(NS_IsMainThread()); 2676 mTrack->NotifyMainThreadListeners(); 2677 return NS_OK; 2678 } 2679 2680 private: 2681 ~NotifyRunnable() = default; 2682 2683 RefPtr<MediaTrack> mTrack; 2684 }; 2685 2686 nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this); 2687 GraphImpl()->Dispatch(runnable.forget()); 2688 } 2689 2690 void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, 2691 GraphTime aBlockedTime) { 2692 mStartTime += aBlockedTime; 2693 2694 if (!mSegment) { 2695 // No data to be forgotten. 2696 return; 2697 } 2698 2699 TrackTime time = aCurrentTime - mStartTime; 2700 // Only prune if there is a reasonable chunk (50ms) to forget, so we don't 2701 // spend too much time pruning segments. 2702 const TrackTime minChunkSize = mSampleRate * 50 / 1000; 2703 if (time < mForgottenTime + minChunkSize) { 2704 return; 2705 } 2706 2707 mForgottenTime = std::min(GetEnd() - 1, time); 2708 mSegment->ForgetUpTo(mForgottenTime); 2709 } 2710 2711 void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) { 2712 DisabledTrackMode mode = CombinedDisabledMode(); 2713 if (aOldMode == mode) { 2714 return; 2715 } 2716 2717 for (const auto& listener : mTrackListeners) { 2718 listener->NotifyEnabledStateChanged( 2719 Graph(), mode != DisabledTrackMode::SILENCE_BLACK); 2720 } 2721 2722 for (const auto& c : mConsumers) { 2723 if (c->GetDestination()) { 2724 c->GetDestination()->OnInputDisabledModeChanged(mode); 2725 } 2726 } 2727 } 2728 2729 void MediaTrack::QueueMessage(UniquePtr<ControlMessageInterface> aMessage) { 2730 MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); 2731 MOZ_RELEASE_ASSERT(!IsDestroyed()); 2732 GraphImpl()->AppendMessage(std::move(aMessage)); 2733 } 2734 2735 void MediaTrack::RunMessageAfterProcessing( 2736 UniquePtr<ControlMessageInterface> aMessage) { 2737 AssertOnGraphThread(); 2738 GraphImpl()->RunMessageAfterProcessing(std::move(aMessage)); 2739 } 2740 2741 SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType, 2742 TrackRate aSampleRate) 2743 : MediaTrack(aSampleRate, aType, 2744 aType == MediaSegment::AUDIO 2745 ? static_cast<MediaSegment*>(new AudioSegment()) 2746 : static_cast<MediaSegment*>(new VideoSegment())), 2747 mMutex("mozilla::media::SourceMediaTrack") { 2748 mUpdateTrack = MakeUnique<TrackData>(); 2749 mUpdateTrack->mInputRate = aSampleRate; 2750 mUpdateTrack->mResamplerChannelCount = 0; 2751 mUpdateTrack->mData = UniquePtr<MediaSegment>(mSegment->CreateEmptyClone()); 2752 mUpdateTrack->mEnded = false; 2753 mUpdateTrack->mPullingEnabled = false; 2754 mUpdateTrack->mGraphThreadDone = false; 2755 } 2756 2757 void SourceMediaTrack::DestroyImpl() { 2758 GraphImpl()->AssertOnGraphThreadOrNotRunning(); 2759 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) { 2760 // Disconnect before we come under mMutex's lock since it can call back 2761 // through RemoveDirectListenerImpl() and deadlock. 2762 mConsumers[i]->Disconnect(); 2763 } 2764 2765 // Hold mMutex while mGraph is reset so that other threads holding mMutex 2766 // can null-check know that the graph will not destroyed. 2767 MutexAutoLock lock(mMutex); 2768 mUpdateTrack = nullptr; 2769 MediaTrack::DestroyImpl(); 2770 } 2771 2772 void SourceMediaTrack::SetPullingEnabled(bool aEnabled) { 2773 class Message : public ControlMessage { 2774 public: 2775 Message(SourceMediaTrack* aTrack, bool aEnabled) 2776 : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {} 2777 void Run() override { 2778 TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage"); 2779 MutexAutoLock lock(mTrack->mMutex); 2780 if (!mTrack->mUpdateTrack) { 2781 // We can't enable pulling for a track that has ended. We ignore 2782 // this if we're disabling pulling, since shutdown sequences are 2783 // complex. If there's truly an issue we'll have issues enabling anyway. 2784 MOZ_ASSERT_IF(mEnabled, mTrack->mEnded); 2785 return; 2786 } 2787 MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO, 2788 "Pulling is not allowed for video"); 2789 mTrack->mUpdateTrack->mPullingEnabled = mEnabled; 2790 } 2791 SourceMediaTrack* mTrack; 2792 bool mEnabled; 2793 }; 2794 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled)); 2795 } 2796 2797 bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) { 2798 TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this); 2799 TrackTime t; 2800 TrackTime current; 2801 { 2802 if (mEnded) { 2803 return false; 2804 } 2805 MutexAutoLock lock(mMutex); 2806 if (mUpdateTrack->mEnded) { 2807 return false; 2808 } 2809 if (!mUpdateTrack->mPullingEnabled) { 2810 return false; 2811 } 2812 // Compute how much track time we'll need assuming we don't block 2813 // the track at all. 2814 t = GraphTimeToTrackTime(aDesiredUpToTime); 2815 current = GetEnd() + mUpdateTrack->mData->GetDuration(); 2816 } 2817 if (t <= current) { 2818 return false; 2819 } 2820 LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f", 2821 GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t), 2822 GraphImpl()->MediaTimeToSeconds(current))); 2823 for (auto& l : mTrackListeners) { 2824 l->NotifyPull(Graph(), current, t); 2825 } 2826 return true; 2827 } 2828 2829 /** 2830 * This moves chunks from aIn to aOut. For audio this is simple. For video 2831 * we carry durations over if present, or extend up to aDesiredUpToTime if not. 2832 * 2833 * We also handle "resetters" from captured media elements. This type of source 2834 * pushes future frames into the track, and should it need to remove some, e.g., 2835 * because of a seek or pause, it tells us by letting time go backwards. Without 2836 * this, tracks would be live for too long after a seek or pause. 2837 */ 2838 static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn, 2839 MediaSegment* aOut, TrackTime aCurrentTime, 2840 TrackTime aDesiredUpToTime) 2841 MOZ_REQUIRES(aTrack->GetMutex()) { 2842 MOZ_ASSERT(aIn->GetType() == aOut->GetType()); 2843 MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime); 2844 MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime); 2845 if (aIn->GetType() == MediaSegment::AUDIO) { 2846 AudioSegment* in = static_cast<AudioSegment*>(aIn); 2847 AudioSegment* out = static_cast<AudioSegment*>(aOut); 2848 TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime; 2849 TrackTime end = std::min(in->GetDuration(), desiredDurationToMove); 2850 2851 out->AppendSlice(*in, 0, end); 2852 in->RemoveLeading(end); 2853 2854 aTrack->GetMutex().AssertCurrentThreadOwns(); 2855 out->ApplyVolume(aTrack->GetVolumeLocked()); 2856 } else { 2857 VideoSegment* in = static_cast<VideoSegment*>(aIn); 2858 VideoSegment* out = static_cast<VideoSegment*>(aOut); 2859 for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) { 2860 MOZ_ASSERT(!c->mTimeStamp.IsNull()); 2861 VideoChunk* last = out->GetLastChunk(); 2862 if (!last || last->mTimeStamp.IsNull()) { 2863 // This is the first frame, or the last frame pushed to `out` has been 2864 // all consumed. Just append and we deal with its duration later. 2865 out->AppendFrame(*c); 2866 if (c->GetDuration() > 0) { 2867 out->ExtendLastFrameBy(c->GetDuration()); 2868 } 2869 continue; 2870 } 2871 2872 // We now know when this frame starts, aka when the last frame ends. 2873 2874 if (c->mTimeStamp < last->mTimeStamp) { 2875 // Time is going backwards. This is a resetting frame from 2876 // DecodedStream. Clear everything up to currentTime. 2877 out->Clear(); 2878 out->AppendNullData(aCurrentTime); 2879 } 2880 2881 // Append the current frame (will have duration 0). 2882 out->AppendFrame(*c); 2883 if (c->GetDuration() > 0) { 2884 out->ExtendLastFrameBy(c->GetDuration()); 2885 } 2886 } 2887 if (out->GetDuration() < aDesiredUpToTime) { 2888 out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration()); 2889 } 2890 in->Clear(); 2891 MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed"); 2892 } 2893 } 2894 2895 void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime, 2896 GraphTime aDesiredUpToTime) { 2897 MutexAutoLock lock(mMutex); 2898 2899 if (!mUpdateTrack) { 2900 MOZ_ASSERT(mEnded); 2901 return; 2902 } 2903 2904 TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime); 2905 2906 ApplyTrackDisabling(mUpdateTrack->mData.get()); 2907 2908 if (!mUpdateTrack->mData->IsEmpty()) { 2909 for (const auto& l : mTrackListeners) { 2910 l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData); 2911 } 2912 } 2913 TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime); 2914 TrackTime endTime = trackDesiredUpToTime; 2915 if (mUpdateTrack->mEnded) { 2916 endTime = std::min(trackDesiredUpToTime, 2917 GetEnd() + mUpdateTrack->mData->GetDuration()); 2918 } 2919 LOG(LogLevel::Verbose, 2920 ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64, 2921 GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime))); 2922 MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(), 2923 trackCurrentTime, endTime); 2924 if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) { 2925 mEnded = true; 2926 mUpdateTrack = nullptr; 2927 } 2928 } 2929 2930 void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) { 2931 mMutex.AssertCurrentThreadOwns(); 2932 if (aSegment->GetType() != MediaSegment::AUDIO || 2933 mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) { 2934 return; 2935 } 2936 AudioSegment* segment = static_cast<AudioSegment*>(aSegment); 2937 segment->ResampleChunks(mUpdateTrack->mResampler, 2938 &mUpdateTrack->mResamplerChannelCount, 2939 mUpdateTrack->mInputRate, GraphImpl()->GraphRate()); 2940 } 2941 2942 void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime( 2943 GraphTime aCurrentTime, GraphTime aBlockedTime) { 2944 MutexAutoLock lock(mMutex); 2945 MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime); 2946 } 2947 2948 void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) { 2949 MutexAutoLock lock(mMutex); 2950 if (!mUpdateTrack) { 2951 return; 2952 } 2953 MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO); 2954 // Set the new input rate and reset the resampler. 2955 mUpdateTrack->mInputRate = aRate; 2956 mUpdateTrack->mResampler.own(nullptr); 2957 mUpdateTrack->mResamplerChannelCount = 0; 2958 } 2959 2960 TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment, 2961 MediaSegment* aRawSegment) { 2962 MutexAutoLock lock(mMutex); 2963 MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType); 2964 TrackTime appended = 0; 2965 if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) { 2966 aSegment->Clear(); 2967 return appended; 2968 } 2969 2970 // Data goes into mData, and on the next iteration of the MTG moves 2971 // into the track's segment after NotifyQueuedTrackChanges(). This adds 2972 // 0-10ms of delay before data gets to direct listeners. 2973 // Indirect listeners (via subsequent TrackUnion nodes) are synced to 2974 // playout time, and so can be delayed by buffering. 2975 2976 // Apply track disabling before notifying any consumers directly 2977 // or inserting into the graph 2978 mozilla::ApplyTrackDisabling(mDirectDisabledMode, aSegment, aRawSegment); 2979 2980 ResampleAudioToGraphSampleRate(aSegment); 2981 2982 // Must notify first, since AppendFrom() will empty out aSegment 2983 NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment); 2984 appended = aSegment->GetDuration(); 2985 mUpdateTrack->mData->AppendFrom(aSegment); // note: aSegment is now dead 2986 { 2987 auto graph = GraphImpl(); 2988 MonitorAutoLock lock(graph->GetMonitor()); 2989 if (graph->CurrentDriver()) { // graph has not completed forced shutdown 2990 graph->EnsureNextIteration(); 2991 } 2992 } 2993 2994 return appended; 2995 } 2996 2997 TrackTime SourceMediaTrack::ClearFutureData() { 2998 MutexAutoLock lock(mMutex); 2999 auto graph = GraphImpl(); 3000 if (!mUpdateTrack || !graph) { 3001 return 0; 3002 } 3003 3004 TrackTime duration = mUpdateTrack->mData->GetDuration(); 3005 mUpdateTrack->mData->Clear(); 3006 return duration; 3007 } 3008 3009 void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) { 3010 mMutex.AssertCurrentThreadOwns(); 3011 3012 for (const auto& l : mDirectTrackListeners) { 3013 TrackTime offset = 0; // FIX! need a separate TrackTime.... or the end of 3014 // the internal buffer 3015 l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset, 3016 *aSegment); 3017 } 3018 } 3019 3020 void SourceMediaTrack::AddDirectListenerImpl( 3021 already_AddRefed<DirectMediaTrackListener> aListener) { 3022 AssertOnGraphThread(); 3023 MutexAutoLock lock(mMutex); 3024 3025 RefPtr<DirectMediaTrackListener> listener = aListener; 3026 LOG(LogLevel::Debug, 3027 ("%p: Adding direct track listener %p to source track %p", GraphImpl(), 3028 listener.get(), this)); 3029 3030 MOZ_ASSERT(mType == MediaSegment::VIDEO); 3031 for (const auto& l : mDirectTrackListeners) { 3032 if (l == listener) { 3033 listener->NotifyDirectListenerInstalled( 3034 DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS); 3035 return; 3036 } 3037 } 3038 3039 mDirectTrackListeners.AppendElement(listener); 3040 3041 LOG(LogLevel::Debug, 3042 ("%p: Added direct track listener %p", GraphImpl(), listener.get())); 3043 listener->NotifyDirectListenerInstalled( 3044 DirectMediaTrackListener::InstallationResult::SUCCESS); 3045 3046 if (mDisabledMode != DisabledTrackMode::ENABLED) { 3047 listener->IncreaseDisabled(mDisabledMode); 3048 } 3049 3050 if (mEnded) { 3051 return; 3052 } 3053 3054 // Pass buffered data to the listener 3055 VideoSegment bufferedData; 3056 size_t videoFrames = 0; 3057 VideoSegment& segment = *GetData<VideoSegment>(); 3058 for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded(); 3059 iter.Next()) { 3060 if (iter->mTimeStamp.IsNull()) { 3061 // No timestamp means this is only for the graph's internal book-keeping, 3062 // denoting a late start of the track. 3063 continue; 3064 } 3065 ++videoFrames; 3066 bufferedData.AppendFrame(*iter); 3067 } 3068 3069 VideoSegment& video = static_cast<VideoSegment&>(*mUpdateTrack->mData); 3070 for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded(); 3071 iter.Next()) { 3072 ++videoFrames; 3073 MOZ_ASSERT(!iter->mTimeStamp.IsNull()); 3074 bufferedData.AppendFrame(*iter); 3075 } 3076 3077 LOG(LogLevel::Info, 3078 ("%p: Notifying direct listener %p of %zu video frames and duration " 3079 "%" PRId64, 3080 GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration())); 3081 listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData); 3082 } 3083 3084 void SourceMediaTrack::RemoveDirectListenerImpl( 3085 DirectMediaTrackListener* aListener) { 3086 mGraph->AssertOnGraphThreadOrNotRunning(); 3087 MutexAutoLock lock(mMutex); 3088 for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) { 3089 const RefPtr<DirectMediaTrackListener>& l = mDirectTrackListeners[i]; 3090 if (l == aListener) { 3091 if (mDisabledMode != DisabledTrackMode::ENABLED) { 3092 aListener->DecreaseDisabled(mDisabledMode); 3093 } 3094 aListener->NotifyDirectListenerUninstalled(); 3095 mDirectTrackListeners.RemoveElementAt(i); 3096 } 3097 } 3098 } 3099 3100 void SourceMediaTrack::End() { 3101 MutexAutoLock lock(mMutex); 3102 if (!mUpdateTrack) { 3103 // Already ended 3104 return; 3105 } 3106 mUpdateTrack->mEnded = true; 3107 if (auto graph = GraphImpl()) { 3108 MonitorAutoLock lock(graph->GetMonitor()); 3109 if (graph->CurrentDriver()) { // graph has not completed forced shutdown 3110 graph->EnsureNextIteration(); 3111 } 3112 } 3113 } 3114 3115 void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) { 3116 AssertOnGraphThread(); 3117 { 3118 MutexAutoLock lock(mMutex); 3119 const DisabledTrackMode oldMode = mDirectDisabledMode; 3120 const bool oldEnabled = oldMode == DisabledTrackMode::ENABLED; 3121 const bool enabled = aMode == DisabledTrackMode::ENABLED; 3122 mDirectDisabledMode = aMode; 3123 for (const auto& l : mDirectTrackListeners) { 3124 if (!oldEnabled && enabled) { 3125 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting " 3126 "direct listener enabled", 3127 GraphImpl(), this)); 3128 l->DecreaseDisabled(oldMode); 3129 } else if (oldEnabled && !enabled) { 3130 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting " 3131 "direct listener disabled", 3132 GraphImpl(), this)); 3133 l->IncreaseDisabled(aMode); 3134 } 3135 } 3136 } 3137 MediaTrack::SetDisabledTrackModeImpl(aMode); 3138 } 3139 3140 uint32_t SourceMediaTrack::NumberOfChannels() const { 3141 AudioSegment* audio = GetData<AudioSegment>(); 3142 MOZ_DIAGNOSTIC_ASSERT(audio); 3143 if (!audio) { 3144 return 0; 3145 } 3146 return audio->MaxChannelCount(); 3147 } 3148 3149 void SourceMediaTrack::RemoveAllDirectListenersImpl() { 3150 GraphImpl()->AssertOnGraphThreadOrNotRunning(); 3151 MutexAutoLock lock(mMutex); 3152 3153 for (auto& l : mDirectTrackListeners.Clone()) { 3154 l->NotifyDirectListenerUninstalled(); 3155 } 3156 mDirectTrackListeners.Clear(); 3157 } 3158 3159 void SourceMediaTrack::SetVolume(float aVolume) { 3160 MutexAutoLock lock(mMutex); 3161 mVolume = aVolume; 3162 } 3163 3164 float SourceMediaTrack::GetVolumeLocked() { 3165 mMutex.AssertCurrentThreadOwns(); 3166 return mVolume; 3167 } 3168 3169 SourceMediaTrack::~SourceMediaTrack() = default; 3170 3171 void MediaInputPort::Init() { 3172 mGraph->AssertOnGraphThreadOrNotRunning(); 3173 LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)", mGraph, 3174 this, mSource, mDest)); 3175 // Only connect the port if it wasn't disconnected on allocation. 3176 if (mSource) { 3177 mSource->AddConsumer(this); 3178 mDest->AddInput(this); 3179 } 3180 // mPortCount decremented via MediaInputPort::Destroy's message 3181 ++mGraph->mPortCount; 3182 } 3183 3184 void MediaInputPort::Disconnect() { 3185 mGraph->AssertOnGraphThreadOrNotRunning(); 3186 NS_ASSERTION(!mSource == !mDest, 3187 "mSource and mDest must either both be null or both non-null"); 3188 3189 if (!mSource) { 3190 return; 3191 } 3192 3193 mSource->RemoveConsumer(this); 3194 mDest->RemoveInput(this); 3195 mSource = nullptr; 3196 mDest = nullptr; 3197 3198 mGraph->SetTrackOrderDirty(); 3199 } 3200 3201 MediaTrack* MediaInputPort::GetSource() const { 3202 mGraph->AssertOnGraphThreadOrNotRunning(); 3203 return mSource; 3204 } 3205 3206 ProcessedMediaTrack* MediaInputPort::GetDestination() const { 3207 mGraph->AssertOnGraphThreadOrNotRunning(); 3208 return mDest; 3209 } 3210 3211 MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval( 3212 MediaInputPort const* aPort, GraphTime aTime) { 3213 InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false}; 3214 if (!aPort) { 3215 result.mStart = aTime; 3216 result.mInputIsBlocked = true; 3217 return result; 3218 } 3219 aPort->mGraph->AssertOnGraphThreadOrNotRunning(); 3220 if (aTime >= aPort->mDest->mStartBlocking) { 3221 return result; 3222 } 3223 result.mStart = aTime; 3224 result.mEnd = aPort->mDest->mStartBlocking; 3225 result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking; 3226 if (!result.mInputIsBlocked) { 3227 result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking); 3228 } 3229 return result; 3230 } 3231 3232 void MediaInputPort::Suspended() { 3233 mGraph->AssertOnGraphThreadOrNotRunning(); 3234 mDest->InputSuspended(this); 3235 } 3236 3237 void MediaInputPort::Resumed() { 3238 mGraph->AssertOnGraphThreadOrNotRunning(); 3239 mDest->InputResumed(this); 3240 } 3241 3242 void MediaInputPort::Destroy() { 3243 class Message : public ControlMessage { 3244 public: 3245 explicit Message(MediaInputPort* aPort) 3246 : ControlMessage(nullptr), mPort(aPort) {} 3247 void Run() override { 3248 TRACE("MediaInputPort::Destroy ControlMessage"); 3249 mPort->Disconnect(); 3250 --mPort->GraphImpl()->mPortCount; 3251 mPort->SetGraphImpl(nullptr); 3252 NS_RELEASE(mPort); 3253 } 3254 void RunDuringShutdown() override { Run(); } 3255 MediaInputPort* mPort; 3256 }; 3257 // Keep a reference to the graph, since Message might RunDuringShutdown() 3258 // synchronously and make GraphImpl() invalid. 3259 RefPtr<MediaTrackGraphImpl> graph = mGraph; 3260 graph->AppendMessage(MakeUnique<Message>(this)); 3261 --graph->mMainThreadPortCount; 3262 } 3263 3264 MediaTrackGraphImpl* MediaInputPort::GraphImpl() const { 3265 mGraph->AssertOnGraphThreadOrNotRunning(); 3266 return mGraph; 3267 } 3268 3269 MediaTrackGraph* MediaInputPort::Graph() const { return mGraph; } 3270 3271 void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) { 3272 MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once"); 3273 DebugOnly<MediaTrackGraphImpl*> graph = mGraph ? mGraph : aGraph; 3274 MOZ_ASSERT(graph->OnGraphThreadOrNotRunning()); 3275 mGraph = aGraph; 3276 } 3277 3278 already_AddRefed<MediaInputPort> ProcessedMediaTrack::AllocateInputPort( 3279 MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) { 3280 // This method creates two references to the MediaInputPort: one for 3281 // the main thread, and one for the MediaTrackGraph. 3282 class Message : public ControlMessage { 3283 public: 3284 explicit Message(MediaInputPort* aPort) 3285 : ControlMessage(aPort->mDest), mPort(aPort) {} 3286 void Run() override { 3287 TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage"); 3288 mPort->Init(); 3289 // The graph holds its reference implicitly 3290 mPort->GraphImpl()->SetTrackOrderDirty(); 3291 NS_ADDREF(mPort.get()); 3292 } 3293 void RunDuringShutdown() override { Run(); } 3294 RefPtr<MediaInputPort> mPort; 3295 }; 3296 3297 MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType); 3298 RefPtr<MediaInputPort> port; 3299 if (aTrack->IsDestroyed()) { 3300 // Create a port that's disconnected, which is what it'd be after its source 3301 // track is Destroy()ed normally. Disconnect() is idempotent so destroying 3302 // this later is fine. 3303 port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber, 3304 aOutputNumber); 3305 } else { 3306 MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl()); 3307 port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber, 3308 aOutputNumber); 3309 } 3310 ++GraphImpl()->mMainThreadPortCount; 3311 GraphImpl()->AppendMessage(MakeUnique<Message>(port)); 3312 return port.forget(); 3313 } 3314 3315 void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) { 3316 class Message : public ControlMessage { 3317 public: 3318 Message(ProcessedMediaTrack* aTrack, bool aAutoend) 3319 : ControlMessage(aTrack), mAutoend(aAutoend) {} 3320 void Run() override { 3321 TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage"); 3322 static_cast<ProcessedMediaTrack*>(mTrack)->SetAutoendImpl(mAutoend); 3323 } 3324 bool mAutoend; 3325 }; 3326 if (mMainThreadDestroyed) { 3327 return; 3328 } 3329 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutoend)); 3330 } 3331 3332 void ProcessedMediaTrack::DestroyImpl() { 3333 for (int32_t i = mInputs.Length() - 1; i >= 0; --i) { 3334 mInputs[i]->Disconnect(); 3335 } 3336 3337 for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) { 3338 mSuspendedInputs[i]->Disconnect(); 3339 } 3340 3341 MediaTrack::DestroyImpl(); 3342 // The track order is only important if there are connections, in which 3343 // case MediaInputPort::Disconnect() called SetTrackOrderDirty(). 3344 // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call 3345 // SetTrackOrderDirty(), for other reasons. 3346 } 3347 3348 MediaTrackGraphImpl::MediaTrackGraphImpl(uint64_t aWindowID, 3349 TrackRate aSampleRate, 3350 AudioDeviceID aPrimaryOutputDeviceID, 3351 nsISerialEventTarget* aMainThread) 3352 : MediaTrackGraph(aSampleRate, aPrimaryOutputDeviceID), 3353 mWindowID(aWindowID), 3354 mFirstCycleBreaker(0) 3355 // An offline graph is not initially processing. 3356 , 3357 mPortCount(0), 3358 mMonitor("MediaTrackGraphImpl"), 3359 mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED), 3360 mPostedRunInStableStateEvent(false), 3361 mGraphDriverRunning(false), 3362 mPostedRunInStableState(false), 3363 mTrackOrderDirty(false), 3364 mMainThread(aMainThread), 3365 mGlobalVolume(CubebUtils::GetVolumeScale()) 3366 #ifdef DEBUG 3367 , 3368 mCanRunMessagesSynchronously(false) 3369 #endif 3370 , 3371 mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"), 3372 mAudioOutputLatency(0.0), 3373 mMaxOutputChannelCount(CubebUtils::MaxNumberOfChannels()) { 3374 } 3375 3376 void MediaTrackGraphImpl::Init(GraphDriverType aDriverRequested, 3377 GraphRunType aRunTypeRequested, 3378 uint32_t aChannelCount) { 3379 mSelfRef = this; 3380 mEndTime = aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX; 3381 mRealtime = aDriverRequested != OFFLINE_THREAD_DRIVER; 3382 // The primary output device always exists because an AudioCallbackDriver 3383 // may exist, and want to be fed data, even when no tracks have audio 3384 // outputs. 3385 mOutputDeviceRefCnts.EmplaceBack( 3386 DeviceReceiverAndCount{mPrimaryOutputDeviceID, nullptr, 0}); 3387 mOutputDevices.EmplaceBack(OutputDeviceEntry{mPrimaryOutputDeviceID}); 3388 3389 bool failedToGetShutdownBlocker = false; 3390 if (!IsNonRealtime()) { 3391 failedToGetShutdownBlocker = !AddShutdownBlocker(); 3392 } 3393 3394 mGraphRunner = aRunTypeRequested == SINGLE_THREAD 3395 ? GraphRunner::Create(this) 3396 : already_AddRefed<GraphRunner>(nullptr); 3397 3398 if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) || 3399 failedToGetShutdownBlocker) { 3400 MonitorAutoLock lock(mMonitor); 3401 // At least one of the following happened 3402 // - Failed to create thread. 3403 // - Failed to install a shutdown blocker when one is needed. 3404 // Because we have a fail state, jump to last phase of the lifecycle. 3405 mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION; 3406 RemoveShutdownBlocker(); // No-op if blocker wasn't added. 3407 #ifdef DEBUG 3408 mCanRunMessagesSynchronously = true; 3409 #endif 3410 return; 3411 } 3412 if (mRealtime) { 3413 if (aDriverRequested == AUDIO_THREAD_DRIVER) { 3414 // Always start with zero input channels, and no particular preferences 3415 // for the input channel. 3416 mDriver = new AudioCallbackDriver( 3417 this, nullptr, mSampleRate, aChannelCount, 0, PrimaryOutputDeviceID(), 3418 nullptr, AudioInputType::Unknown, Nothing()); 3419 } else { 3420 mDriver = new SystemClockDriver(this, nullptr, mSampleRate); 3421 } 3422 nsCString streamName = GetDocumentTitle(mWindowID); 3423 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get())); 3424 mDriver->SetStreamName(streamName); 3425 } else { 3426 mDriver = new OfflineClockDriver(this, mSampleRate); 3427 } 3428 3429 mLastMainThreadUpdate = TimeStamp::Now(); 3430 3431 RegisterWeakAsyncMemoryReporter(this); 3432 } 3433 3434 #ifdef DEBUG 3435 bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const { 3436 return aDriver->OnThread() || 3437 (mGraphRunner && mGraphRunner->InDriverIteration(aDriver)); 3438 } 3439 #endif 3440 3441 void MediaTrackGraphImpl::Destroy() { 3442 // First unregister from memory reporting. 3443 UnregisterWeakMemoryReporter(this); 3444 3445 // Clear the self reference which will destroy this instance if all 3446 // associated GraphDrivers are destroyed. 3447 mSelfRef = nullptr; 3448 } 3449 3450 // Internal method has a Window ID parameter so that TestAudioTrackGraph 3451 // GTests can create a graph without a window. 3452 /* static */ 3453 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists( 3454 uint64_t aWindowID, TrackRate aSampleRate, 3455 AudioDeviceID aPrimaryOutputDeviceID) { 3456 MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); 3457 MOZ_ASSERT(aSampleRate > 0); 3458 3459 GraphHashSet::Ptr p = 3460 Graphs()->lookup({aWindowID, aSampleRate, aPrimaryOutputDeviceID}); 3461 return p ? *p : nullptr; 3462 } 3463 3464 // Public method has an nsPIDOMWindowInner* parameter to ensure that the 3465 // window is a real inner Window, not a WindowProxy. 3466 /* static */ 3467 MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists( 3468 nsPIDOMWindowInner* aWindow, TrackRate aSampleRate, 3469 AudioDeviceID aPrimaryOutputDeviceID) { 3470 TrackRate sampleRate = 3471 aSampleRate ? aSampleRate 3472 : CubebUtils::PreferredSampleRate( 3473 aWindow->AsGlobal()->ShouldResistFingerprinting( 3474 RFPTarget::AudioSampleRate)); 3475 return MediaTrackGraphImpl::GetInstanceIfExists( 3476 aWindow->WindowID(), sampleRate, aPrimaryOutputDeviceID); 3477 } 3478 3479 /* static */ 3480 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance( 3481 GraphDriverType aGraphDriverRequested, uint64_t aWindowID, 3482 TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID, 3483 nsISerialEventTarget* aMainThread) { 3484 MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); 3485 MOZ_ASSERT(aSampleRate > 0); 3486 MOZ_ASSERT(aGraphDriverRequested != OFFLINE_THREAD_DRIVER, 3487 "Use CreateNonRealtimeInstance() for offline graphs"); 3488 3489 MediaTrackGraphImpl* graph = 3490 GetInstanceIfExists(aWindowID, aSampleRate, aPrimaryOutputDeviceID); 3491 if (graph) { // graph already exists 3492 return graph; 3493 } 3494 3495 GraphRunType runType = DIRECT_DRIVER; 3496 if (Preferences::GetBool("media.audiograph.single_thread.enabled", true)) { 3497 runType = SINGLE_THREAD; 3498 } 3499 3500 // In a real time graph, the number of output channels is determined by 3501 // the underlying number of channel of the default audio output device. 3502 uint32_t channelCount = CubebUtils::MaxNumberOfChannels(); 3503 graph = new MediaTrackGraphImpl(aWindowID, aSampleRate, 3504 aPrimaryOutputDeviceID, aMainThread); 3505 graph->Init(aGraphDriverRequested, runType, channelCount); 3506 MOZ_ALWAYS_TRUE(Graphs()->putNew( 3507 {aWindowID, aSampleRate, aPrimaryOutputDeviceID}, graph)); 3508 3509 LOG(LogLevel::Debug, ("Starting up MediaTrackGraph %p for window 0x%" PRIx64, 3510 graph, aWindowID)); 3511 3512 return graph; 3513 } 3514 3515 /* static */ 3516 MediaTrackGraph* MediaTrackGraph::GetInstance( 3517 GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow, 3518 TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID) { 3519 TrackRate sampleRate = 3520 aSampleRate ? aSampleRate 3521 : CubebUtils::PreferredSampleRate( 3522 aWindow->AsGlobal()->ShouldResistFingerprinting( 3523 RFPTarget::AudioSampleRate)); 3524 return MediaTrackGraphImpl::GetInstance( 3525 aGraphDriverRequested, aWindow->WindowID(), sampleRate, 3526 aPrimaryOutputDeviceID, GetMainThreadSerialEventTarget()); 3527 } 3528 3529 MediaTrackGraph* MediaTrackGraphImpl::CreateNonRealtimeInstance( 3530 TrackRate aSampleRate) { 3531 MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); 3532 3533 nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget(); 3534 // Offline graphs have 0 output channel count: they write the output to a 3535 // buffer, not an audio output track. 3536 MediaTrackGraphImpl* graph = new MediaTrackGraphImpl( 3537 0, aSampleRate, DEFAULT_OUTPUT_DEVICE, mainThread); 3538 graph->Init(OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, 0); 3539 3540 LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph)); 3541 3542 return graph; 3543 } 3544 3545 MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance( 3546 TrackRate aSampleRate) { 3547 return MediaTrackGraphImpl::CreateNonRealtimeInstance(aSampleRate); 3548 } 3549 3550 void MediaTrackGraph::ForceShutDown() { 3551 MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); 3552 3553 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this); 3554 3555 graph->ForceShutDown(); 3556 } 3557 3558 NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIObserver, 3559 nsIThreadObserver, nsITimerCallback, nsINamed) 3560 3561 NS_IMETHODIMP 3562 MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport, 3563 nsISupports* aData, bool aAnonymize) { 3564 MOZ_ASSERT(NS_IsMainThread()); 3565 if (mMainThreadTrackCount == 0) { 3566 // No tracks to report. 3567 FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>()); 3568 return NS_OK; 3569 } 3570 3571 class Message final : public ControlMessage { 3572 public: 3573 Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport, 3574 nsISupports* aHandlerData) 3575 : ControlMessage(nullptr), 3576 mGraph(aGraph), 3577 mHandleReport(aHandleReport), 3578 mHandlerData(aHandlerData) {} 3579 void Run() override { 3580 TRACE("MTG::CollectSizesForMemoryReport ControlMessage"); 3581 mGraph->CollectSizesForMemoryReport(mHandleReport.forget(), 3582 mHandlerData.forget()); 3583 } 3584 void RunDuringShutdown() override { 3585 // Run this message during shutdown too, so that endReports is called. 3586 Run(); 3587 } 3588 MediaTrackGraphImpl* mGraph; 3589 // nsMemoryReporterManager keeps the callback and data alive only if it 3590 // does not time out. 3591 nsCOMPtr<nsIHandleReportCallback> mHandleReport; 3592 nsCOMPtr<nsISupports> mHandlerData; 3593 }; 3594 3595 AppendMessage(MakeUnique<Message>(this, aHandleReport, aData)); 3596 3597 return NS_OK; 3598 } 3599 3600 void MediaTrackGraphImpl::CollectSizesForMemoryReport( 3601 already_AddRefed<nsIHandleReportCallback> aHandleReport, 3602 already_AddRefed<nsISupports> aHandlerData) { 3603 class FinishCollectRunnable final : public Runnable { 3604 public: 3605 explicit FinishCollectRunnable( 3606 already_AddRefed<nsIHandleReportCallback> aHandleReport, 3607 already_AddRefed<nsISupports> aHandlerData) 3608 : mozilla::Runnable("FinishCollectRunnable"), 3609 mHandleReport(aHandleReport), 3610 mHandlerData(aHandlerData) {} 3611 3612 NS_IMETHOD Run() override { 3613 TRACE("MTG::FinishCollectReports ControlMessage"); 3614 MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData, 3615 std::move(mAudioTrackSizes)); 3616 return NS_OK; 3617 } 3618 3619 nsTArray<AudioNodeSizes> mAudioTrackSizes; 3620 3621 private: 3622 ~FinishCollectRunnable() = default; 3623 3624 // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its 3625 // constructor modifies the ref-count, which cannot be done off main 3626 // thread. 3627 RefPtr<nsIHandleReportCallback> mHandleReport; 3628 RefPtr<nsISupports> mHandlerData; 3629 }; 3630 3631 RefPtr<FinishCollectRunnable> runnable = new FinishCollectRunnable( 3632 std::move(aHandleReport), std::move(aHandlerData)); 3633 3634 auto audioTrackSizes = &runnable->mAudioTrackSizes; 3635 3636 for (MediaTrack* t : AllTracks()) { 3637 AudioNodeTrack* track = t->AsAudioNodeTrack(); 3638 if (track) { 3639 AudioNodeSizes* usage = audioTrackSizes->AppendElement(); 3640 track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage); 3641 } 3642 } 3643 3644 mMainThread->Dispatch(runnable.forget()); 3645 } 3646 3647 void MediaTrackGraphImpl::FinishCollectReports( 3648 nsIHandleReportCallback* aHandleReport, nsISupports* aData, 3649 const nsTArray<AudioNodeSizes>& aAudioTrackSizes) { 3650 MOZ_ASSERT(NS_IsMainThread()); 3651 3652 nsCOMPtr<nsIMemoryReporterManager> manager = 3653 do_GetService("@mozilla.org/memory-reporter-manager;1"); 3654 3655 if (!manager) return; 3656 3657 #define REPORT(_path, _amount, _desc) \ 3658 aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \ 3659 nsLiteralCString(_desc), aData); 3660 3661 for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) { 3662 const AudioNodeSizes& usage = aAudioTrackSizes[i]; 3663 const char* const nodeType = 3664 usage.mNodeType ? usage.mNodeType : "<unknown>"; 3665 3666 nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects", 3667 nodeType); 3668 REPORT(enginePath, usage.mEngine, 3669 "Memory used by AudioNode engine objects (Web Audio)."); 3670 3671 nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects", 3672 nodeType); 3673 REPORT(trackPath, usage.mTrack, 3674 "Memory used by AudioNode track objects (Web Audio)."); 3675 } 3676 3677 size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf); 3678 if (hrtfLoaders) { 3679 REPORT(nsLiteralCString( 3680 "explicit/webaudio/audio-node/PannerNode/hrtf-databases"), 3681 hrtfLoaders, "Memory used by PannerNode databases (Web Audio)."); 3682 } 3683 3684 #undef REPORT 3685 3686 manager->EndReport(); 3687 } 3688 3689 SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) { 3690 SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate()); 3691 AddTrack(track); 3692 return track; 3693 } 3694 3695 ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack( 3696 MediaSegment::Type aType) { 3697 ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType); 3698 AddTrack(track); 3699 return track; 3700 } 3701 3702 AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() { 3703 AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate()); 3704 AddTrack(track); 3705 return track; 3706 } 3707 3708 CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter( 3709 CrossGraphReceiver* aReceiver) { 3710 CrossGraphTransmitter* track = 3711 new CrossGraphTransmitter(GraphRate(), aReceiver); 3712 AddTrack(track); 3713 return track; 3714 } 3715 3716 CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver( 3717 TrackRate aTransmitterRate) { 3718 CrossGraphReceiver* track = 3719 new CrossGraphReceiver(GraphRate(), aTransmitterRate); 3720 AddTrack(track); 3721 return track; 3722 } 3723 3724 void MediaTrackGraph::AddTrack(MediaTrack* aTrack) { 3725 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this); 3726 MOZ_ASSERT(NS_IsMainThread()); 3727 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED 3728 if (graph->mRealtime) { 3729 GraphHashSet::Ptr p = Graphs()->lookup(*graph); 3730 MOZ_DIAGNOSTIC_ASSERT(p, "Graph must not be shutting down"); 3731 } 3732 #endif 3733 if (graph->mMainThreadTrackCount == 0) { 3734 nsCOMPtr<nsIObserverService> observerService = 3735 mozilla::services::GetObserverService(); 3736 if (observerService) { 3737 observerService->AddObserver(graph, "document-title-changed", false); 3738 } 3739 } 3740 3741 NS_ADDREF(aTrack); 3742 aTrack->SetGraphImpl(graph); 3743 ++graph->mMainThreadTrackCount; 3744 graph->AppendMessage(MakeUnique<CreateMessage>(aTrack)); 3745 } 3746 3747 void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) { 3748 MOZ_ASSERT(NS_IsMainThread()); 3749 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0); 3750 3751 mAudioOutputParams.RemoveElementsBy( 3752 [&](const TrackKeyDeviceAndVolume& aElement) { 3753 if (aElement.mTrack != aTrack) { 3754 return false; 3755 }; 3756 DecrementOutputDeviceRefCnt(aElement.mDeviceID); 3757 return true; 3758 }); 3759 3760 if (--mMainThreadTrackCount == 0) { 3761 LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from " 3762 "main thread. Graph will shut down.", 3763 this, aTrack)); 3764 if (mRealtime) { 3765 // Find the graph in the hash table and remove it. 3766 GraphHashSet* graphs = Graphs(); 3767 GraphHashSet::Ptr p = graphs->lookup(*this); 3768 MOZ_ASSERT(*p == this); 3769 graphs->remove(p); 3770 3771 nsCOMPtr<nsIObserverService> observerService = 3772 mozilla::services::GetObserverService(); 3773 if (observerService) { 3774 observerService->RemoveObserver(this, "document-title-changed"); 3775 } 3776 } 3777 // The graph thread will shut itself down soon, but won't be able to do 3778 // that if JS continues to run. 3779 InterruptJS(); 3780 } 3781 } 3782 3783 auto MediaTrackGraphImpl::NotifyWhenDeviceStarted(AudioDeviceID aDeviceID) 3784 -> RefPtr<GraphStartedPromise> { 3785 MOZ_ASSERT(NS_IsMainThread()); 3786 3787 size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID); 3788 if (index == decltype(mOutputDeviceRefCnts)::NoIndex) { 3789 return GraphStartedPromise::CreateAndReject(NS_ERROR_INVALID_ARG, __func__); 3790 } 3791 3792 MozPromiseHolder<GraphStartedPromise> h; 3793 RefPtr<GraphStartedPromise> p = h.Ensure(__func__); 3794 3795 if (CrossGraphReceiver* receiver = mOutputDeviceRefCnts[index].mReceiver) { 3796 receiver->GraphImpl()->NotifyWhenPrimaryDeviceStarted(std::move(h)); 3797 return p; 3798 } 3799 3800 // aSink corresponds to the primary audio output device of this graph. 3801 NotifyWhenPrimaryDeviceStarted(std::move(h)); 3802 return p; 3803 } 3804 3805 void MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted( 3806 MozPromiseHolder<GraphStartedPromise>&& aHolder) { 3807 MOZ_ASSERT(NS_IsMainThread()); 3808 if (mOutputDeviceRefCnts[0].mRefCnt == 0) { 3809 // There are no track outputs that require the device, so the creator of 3810 // this promise no longer needs to know when the graph is running. Don't 3811 // keep the graph alive with another message. 3812 aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__); 3813 return; 3814 } 3815 3816 QueueControlOrShutdownMessage( 3817 [self = RefPtr{this}, this, 3818 holder = std::move(aHolder)](IsInShutdown aInShutdown) mutable { 3819 if (aInShutdown == IsInShutdown::Yes) { 3820 holder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__); 3821 return; 3822 } 3823 3824 TRACE("MTG::NotifyWhenPrimaryDeviceStarted ControlMessage"); 3825 // This runs on the graph thread, so when this runs, and the current 3826 // driver is an AudioCallbackDriver, we know the audio hardware is 3827 // started. If not, we are going to switch soon, keep reposting this 3828 // ControlMessage. 3829 if (CurrentDriver()->AsAudioCallbackDriver() && 3830 CurrentDriver()->ThreadRunning() && 3831 !CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) { 3832 // Avoid Resolve's locking on the graph thread by doing it on main. 3833 Dispatch(NS_NewRunnableFunction( 3834 "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted::Resolver", 3835 [holder = std::move(holder)]() mutable { 3836 holder.Resolve(true, __func__); 3837 })); 3838 } else { 3839 DispatchToMainThreadStableState( 3840 NewRunnableMethod< 3841 StoreCopyPassByRRef<MozPromiseHolder<GraphStartedPromise>>>( 3842 "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted", this, 3843 &MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted, 3844 std::move(holder))); 3845 } 3846 }); 3847 } 3848 3849 class AudioContextOperationControlMessage : public ControlMessage { 3850 using AudioContextOperationPromise = 3851 MediaTrackGraph::AudioContextOperationPromise; 3852 3853 public: 3854 AudioContextOperationControlMessage( 3855 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks, 3856 AudioContextOperation aOperation, 3857 MozPromiseHolder<AudioContextOperationPromise>&& aHolder) 3858 : ControlMessage(aDestinationTrack), 3859 mTracks(std::move(aTracks)), 3860 mAudioContextOperation(aOperation), 3861 mHolder(std::move(aHolder)) {} 3862 void Run() override { 3863 TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage", 3864 kAudioContextOptionsStrings[static_cast<uint8_t>( 3865 mAudioContextOperation)]); 3866 mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this); 3867 } 3868 void RunDuringShutdown() override { 3869 MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close, 3870 "We should be reviving the graph?"); 3871 mHolder.Reject(false, __func__); 3872 } 3873 3874 nsTArray<RefPtr<MediaTrack>> mTracks; 3875 AudioContextOperation mAudioContextOperation; 3876 MozPromiseHolder<AudioContextOperationPromise> mHolder; 3877 }; 3878 3879 void MediaTrackGraphImpl::ApplyAudioContextOperationImpl( 3880 AudioContextOperationControlMessage* aMessage) { 3881 MOZ_ASSERT(OnGraphThread()); 3882 // Initialize state to zero. This silences a GCC warning about uninitialized 3883 // values, because although the switch below initializes state for all valid 3884 // enum values, the actual value could be any integer that fits in the enum. 3885 AudioContextState state{0}; 3886 switch (aMessage->mAudioContextOperation) { 3887 // Suspend and Close operations may be performed immediately because no 3888 // specific kind of GraphDriver is required. CheckDriver() will schedule 3889 // a change to a SystemCallbackDriver if all tracks are suspended. 3890 case AudioContextOperation::Suspend: 3891 state = AudioContextState::Suspended; 3892 break; 3893 case AudioContextOperation::Close: 3894 state = AudioContextState::Closed; 3895 break; 3896 case AudioContextOperation::Resume: 3897 // Resume operations require an AudioCallbackDriver. CheckDriver() will 3898 // schedule an AudioCallbackDriver if necessary and process pending 3899 // operations if and when an AudioCallbackDriver is running. 3900 mPendingResumeOperations.EmplaceBack(aMessage); 3901 return; 3902 } 3903 // First resolve any pending Resume promises for the same AudioContext so as 3904 // to resolve its associated promises in the same order as they were 3905 // created. These Resume operations are considered complete and immediately 3906 // canceled by the Suspend or Close. 3907 MediaTrack* destinationTrack = aMessage->GetTrack(); 3908 bool shrinking = false; 3909 auto moveDest = mPendingResumeOperations.begin(); 3910 for (PendingResumeOperation& op : mPendingResumeOperations) { 3911 if (op.DestinationTrack() == destinationTrack) { 3912 op.Apply(this); 3913 shrinking = true; 3914 continue; 3915 } 3916 if (shrinking) { // Fill-in gaps in the array. 3917 *moveDest = std::move(op); 3918 } 3919 ++moveDest; 3920 } 3921 mPendingResumeOperations.TruncateLength(moveDest - 3922 mPendingResumeOperations.begin()); 3923 3924 for (MediaTrack* track : aMessage->mTracks) { 3925 track->IncrementSuspendCount(); 3926 } 3927 // Resolve after main thread state is up to date with completed processing. 3928 DispatchToMainThreadStableState(NS_NewRunnableFunction( 3929 "MediaTrackGraphImpl::ApplyAudioContextOperationImpl", 3930 [holder = std::move(aMessage->mHolder), state]() mutable { 3931 holder.Resolve(state, __func__); 3932 })); 3933 } 3934 3935 MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation( 3936 AudioContextOperationControlMessage* aMessage) 3937 : mDestinationTrack(aMessage->GetTrack()), 3938 mTracks(std::move(aMessage->mTracks)), 3939 mHolder(std::move(aMessage->mHolder)) { 3940 MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume); 3941 } 3942 3943 void MediaTrackGraphImpl::PendingResumeOperation::Apply( 3944 MediaTrackGraphImpl* aGraph) { 3945 MOZ_ASSERT(aGraph->OnGraphThread()); 3946 for (MediaTrack* track : mTracks) { 3947 track->DecrementSuspendCount(); 3948 } 3949 // The graph is provided through the parameter so that it is available even 3950 // when the track is destroyed. 3951 aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction( 3952 "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable { 3953 holder.Resolve(AudioContextState::Running, __func__); 3954 })); 3955 } 3956 3957 void MediaTrackGraphImpl::PendingResumeOperation::Abort() { 3958 // The graph is shutting down before the operation completed. 3959 MOZ_ASSERT(!mDestinationTrack->GraphImpl() || 3960 mDestinationTrack->GraphImpl()->LifecycleStateRef() == 3961 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN); 3962 mHolder.Reject(false, __func__); 3963 } 3964 3965 auto MediaTrackGraph::ApplyAudioContextOperation( 3966 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks, 3967 AudioContextOperation aOperation) -> RefPtr<AudioContextOperationPromise> { 3968 MozPromiseHolder<AudioContextOperationPromise> holder; 3969 RefPtr<AudioContextOperationPromise> p = holder.Ensure(__func__); 3970 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this); 3971 graphImpl->AppendMessage(MakeUnique<AudioContextOperationControlMessage>( 3972 aDestinationTrack, std::move(aTracks), aOperation, std::move(holder))); 3973 return p; 3974 } 3975 3976 uint32_t MediaTrackGraphImpl::PrimaryOutputChannelCount() const { 3977 MOZ_ASSERT(!mOutputDevices[0].mReceiver); 3978 return AudioOutputChannelCount(mOutputDevices[0]); 3979 } 3980 3981 uint32_t MediaTrackGraphImpl::AudioOutputChannelCount( 3982 const OutputDeviceEntry& aDevice) const { 3983 MOZ_ASSERT(OnGraphThread()); 3984 // The audio output channel count for a graph is the maximum of the output 3985 // channel count of all the tracks with outputs to this device, or the max 3986 // audio output channel count the machine can do, whichever is smaller. 3987 uint32_t channelCount = 0; 3988 for (const auto& output : aDevice.mTrackOutputs) { 3989 channelCount = std::max(channelCount, output.mTrack->NumberOfChannels()); 3990 } 3991 channelCount = std::min(channelCount, mMaxOutputChannelCount); 3992 if (channelCount) { 3993 return channelCount; 3994 } else { 3995 // null aDevice.mReceiver indicates the primary graph output device. 3996 if (!aDevice.mReceiver && CurrentDriver()->AsAudioCallbackDriver()) { 3997 return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount(); 3998 } 3999 return 2; 4000 } 4001 } 4002 4003 double MediaTrackGraph::AudioOutputLatency() { 4004 return static_cast<MediaTrackGraphImpl*>(this)->AudioOutputLatency(); 4005 } 4006 4007 double MediaTrackGraphImpl::AudioOutputLatency() { 4008 MOZ_ASSERT(NS_IsMainThread()); 4009 if (mAudioOutputLatency != 0.0) { 4010 return mAudioOutputLatency; 4011 } 4012 MonitorAutoLock lock(mMonitor); 4013 if (CurrentDriver()->AsAudioCallbackDriver()) { 4014 mAudioOutputLatency = CurrentDriver() 4015 ->AsAudioCallbackDriver() 4016 ->AudioOutputLatency() 4017 .ToSeconds(); 4018 } else { 4019 // Failure mode: return 0.0 if running on a normal thread. 4020 mAudioOutputLatency = 0.0; 4021 } 4022 4023 return mAudioOutputLatency; 4024 } 4025 4026 bool MediaTrackGraph::OutputForAECMightDrift() { 4027 return static_cast<MediaTrackGraphImpl*>(this)->OutputForAECMightDrift(); 4028 } 4029 bool MediaTrackGraph::OutputForAECIsPrimary() { 4030 return static_cast<MediaTrackGraphImpl*>(this)->OutputForAECIsPrimary(); 4031 } 4032 bool MediaTrackGraph::IsNonRealtime() const { 4033 return !static_cast<const MediaTrackGraphImpl*>(this)->mRealtime; 4034 } 4035 4036 void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) { 4037 MOZ_ASSERT(NS_IsMainThread(), "main thread only"); 4038 4039 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this); 4040 NS_ASSERTION(!graph->mRealtime, "non-realtime only"); 4041 4042 graph->QueueControlMessageWithNoShutdown([graph = RefPtr{graph}, 4043 aTicksToProcess]() { 4044 TRACE("MTG::StartNonRealtimeProcessing ControlMessage"); 4045 MOZ_ASSERT(graph->mStateComputedTime == 0); 4046 MOZ_ASSERT(graph->mEndTime == 0, 4047 "StartNonRealtimeProcessing should be called only once"); 4048 graph->mEndTime = aTicksToProcess; 4049 OfflineClockDriver* driver = graph->CurrentDriver()->AsOfflineClockDriver(); 4050 MOZ_ASSERT(driver); 4051 driver->SetTickCountToRender(aTicksToProcess); 4052 }); 4053 } 4054 4055 void MediaTrackGraphImpl::InterruptJS() { 4056 MonitorAutoLock lock(mMonitor); 4057 mInterruptJSCalled = true; 4058 if (mJSContext) { 4059 JS_RequestInterruptCallback(mJSContext); 4060 } 4061 } 4062 4063 static bool InterruptCallback(JSContext* aCx) { 4064 // Interrupt future calls also. 4065 JS_RequestInterruptCallback(aCx); 4066 // Stop execution. 4067 return false; 4068 } 4069 4070 void MediaTrackGraph::NotifyJSContext(JSContext* aCx) { 4071 MOZ_ASSERT(OnGraphThread()); 4072 MOZ_ASSERT(aCx); 4073 4074 auto* impl = static_cast<MediaTrackGraphImpl*>(this); 4075 MonitorAutoLock lock(impl->mMonitor); 4076 if (impl->mJSContext) { 4077 MOZ_ASSERT(impl->mJSContext == aCx); 4078 return; 4079 } 4080 JS_AddInterruptCallback(aCx, InterruptCallback); 4081 impl->mJSContext = aCx; 4082 if (impl->mInterruptJSCalled) { 4083 JS_RequestInterruptCallback(aCx); 4084 } 4085 } 4086 4087 void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) { 4088 MediaTrack* t = aPort->GetSource(); 4089 if (!t->IsSuspended()) { 4090 mInputs.AppendElement(aPort); 4091 } else { 4092 mSuspendedInputs.AppendElement(aPort); 4093 } 4094 GraphImpl()->SetTrackOrderDirty(); 4095 } 4096 4097 void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) { 4098 GraphImpl()->AssertOnGraphThreadOrNotRunning(); 4099 mInputs.RemoveElement(aPort); 4100 mSuspendedInputs.AppendElement(aPort); 4101 GraphImpl()->SetTrackOrderDirty(); 4102 } 4103 4104 void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) { 4105 GraphImpl()->AssertOnGraphThreadOrNotRunning(); 4106 mSuspendedInputs.RemoveElement(aPort); 4107 mInputs.AppendElement(aPort); 4108 GraphImpl()->SetTrackOrderDirty(); 4109 } 4110 4111 void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) { 4112 MOZ_ASSERT(OnGraphThread()); 4113 LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver)); 4114 if (GraphDriver* nextDriver = NextDriver()) { 4115 if (nextDriver != CurrentDriver()) { 4116 LOG(LogLevel::Debug, 4117 ("%p: Discarding previous next driver: %p", this, nextDriver)); 4118 } 4119 } 4120 mNextDriver = aNextDriver; 4121 } 4122 4123 void MediaTrackGraph::RegisterCaptureTrackForWindow( 4124 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) { 4125 MOZ_ASSERT(NS_IsMainThread()); 4126 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this); 4127 graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack); 4128 } 4129 4130 void MediaTrackGraphImpl::RegisterCaptureTrackForWindow( 4131 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) { 4132 MOZ_ASSERT(NS_IsMainThread()); 4133 WindowAndTrack winAndTrack; 4134 winAndTrack.mWindowId = aWindowId; 4135 winAndTrack.mCaptureTrackSink = aCaptureTrack; 4136 mWindowCaptureTracks.AppendElement(winAndTrack); 4137 } 4138 4139 void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) { 4140 MOZ_ASSERT(NS_IsMainThread()); 4141 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this); 4142 graphImpl->UnregisterCaptureTrackForWindow(aWindowId); 4143 } 4144 4145 void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) { 4146 MOZ_ASSERT(NS_IsMainThread()); 4147 mWindowCaptureTracks.RemoveElementsBy( 4148 [aWindowId](const auto& track) { return track.mWindowId == aWindowId; }); 4149 } 4150 4151 already_AddRefed<MediaInputPort> MediaTrackGraph::ConnectToCaptureTrack( 4152 uint64_t aWindowId, MediaTrack* aMediaTrack) { 4153 return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId, 4154 aMediaTrack); 4155 } 4156 4157 already_AddRefed<MediaInputPort> MediaTrackGraphImpl::ConnectToCaptureTrack( 4158 uint64_t aWindowId, MediaTrack* aMediaTrack) { 4159 MOZ_ASSERT(NS_IsMainThread()); 4160 for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) { 4161 if (mWindowCaptureTracks[i].mWindowId == aWindowId) { 4162 ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink; 4163 return sink->AllocateInputPort(aMediaTrack); 4164 } 4165 } 4166 return nullptr; 4167 } 4168 4169 void MediaTrackGraph::DispatchToMainThreadStableState( 4170 already_AddRefed<nsIRunnable> aRunnable) { 4171 AssertOnGraphThreadOrNotRunning(); 4172 static_cast<MediaTrackGraphImpl*>(this) 4173 ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable)); 4174 } 4175 4176 Watchable<mozilla::GraphTime>& MediaTrackGraphImpl::CurrentTime() { 4177 MOZ_ASSERT(NS_IsMainThread()); 4178 return mMainThreadGraphTime; 4179 } 4180 4181 GraphTime MediaTrackGraph::ProcessedTime() const { 4182 AssertOnGraphThreadOrNotRunning(); 4183 return static_cast<const MediaTrackGraphImpl*>(this)->mProcessedTime; 4184 } 4185 4186 void* MediaTrackGraph::CurrentDriver() const { 4187 AssertOnGraphThreadOrNotRunning(); 4188 return static_cast<const MediaTrackGraphImpl*>(this)->mDriver; 4189 } 4190 4191 uint32_t MediaTrackGraphImpl::AudioInputChannelCount( 4192 CubebUtils::AudioDeviceID aID) { 4193 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 4194 DeviceInputTrack* t = 4195 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID); 4196 return t ? t->MaxRequestedInputChannels() : 0; 4197 } 4198 4199 AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference( 4200 CubebUtils::AudioDeviceID aID) { 4201 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 4202 DeviceInputTrack* t = 4203 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID); 4204 return t && t->HasVoiceInput() ? AudioInputType::Voice 4205 : AudioInputType::Unknown; 4206 } 4207 4208 void MediaTrackGraphImpl::SetNewNativeInput() { 4209 MOZ_ASSERT(NS_IsMainThread()); 4210 MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack()); 4211 4212 LOG(LogLevel::Debug, ("%p SetNewNativeInput", this)); 4213 4214 NonNativeInputTrack* track = 4215 mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack(); 4216 if (!track) { 4217 LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this)); 4218 return; 4219 } 4220 4221 const CubebUtils::AudioDeviceID deviceId = track->mDeviceId; 4222 const PrincipalHandle principal = track->mPrincipalHandle; 4223 4224 LOG(LogLevel::Debug, 4225 ("%p Select device %p as the new native input device", this, deviceId)); 4226 4227 struct TrackListener { 4228 DeviceInputConsumerTrack* track; 4229 // Keep its reference so it won't be dropped when after 4230 // DisconnectDeviceInput(). 4231 RefPtr<AudioDataListener> listener; 4232 }; 4233 nsTArray<TrackListener> pairs; 4234 4235 for (const auto& t : track->GetConsumerTracks()) { 4236 pairs.AppendElement( 4237 TrackListener{t.get(), t->GetAudioDataListener().get()}); 4238 } 4239 4240 for (TrackListener& pair : pairs) { 4241 pair.track->DisconnectDeviceInput(); 4242 } 4243 4244 for (TrackListener& pair : pairs) { 4245 pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal); 4246 LOG(LogLevel::Debug, 4247 ("%p: Reinitialize AudioProcessingTrack %p for device %p", this, 4248 pair.track, deviceId)); 4249 } 4250 4251 LOG(LogLevel::Debug, 4252 ("%p Native input device is set to device %p now", this, deviceId)); 4253 4254 MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack()); 4255 } 4256 4257 void MediaTrackGraphImpl::UpdateEnumeratorDefaultDeviceTracking() { 4258 MOZ_ASSERT(NS_IsMainThread()); 4259 auto onExit = MakeScopeExit([&] { UpdateDefaultDevice(); }); 4260 4261 if (!mDeviceInputTrackManagerMainThread.GetNativeInputTrack()) { 4262 mEnumeratorMainThread = nullptr; 4263 mOutputDevicesChangedListener.DisconnectIfExists(); 4264 LOG(LogLevel::Debug, 4265 ("%p No longer tracking system default output device", this)); 4266 return; 4267 } 4268 4269 if (mEnumeratorMainThread) { 4270 onExit.release(); 4271 return; 4272 } 4273 4274 mEnumeratorMainThread = CubebDeviceEnumerator::GetInstance(); 4275 mOutputDevicesChangedListener = 4276 mEnumeratorMainThread->OnAudioOutputDeviceListChange().Connect( 4277 GetCurrentSerialEventTarget(), this, 4278 &MediaTrackGraphImpl::UpdateDefaultDevice); 4279 LOG(LogLevel::Debug, ("%p Now tracking system default output device", this)); 4280 } 4281 4282 void MediaTrackGraphImpl::UpdateDefaultDevice() { 4283 MOZ_ASSERT(NS_IsMainThread()); 4284 CubebUtils::AudioDeviceID id = nullptr; 4285 auto onExit = MakeScopeExit([&] { 4286 mDefaultOutputDeviceID.store(id, std::memory_order_relaxed); 4287 LOG(LogLevel::Debug, 4288 ("%p Tracked system default output device ID is now %p", this, id)); 4289 }); 4290 4291 if (!mEnumeratorMainThread) { 4292 return; 4293 } 4294 4295 auto dev = 4296 mEnumeratorMainThread->DefaultDevice(CubebDeviceEnumerator::Side::OUTPUT); 4297 if (!dev) { 4298 return; 4299 } 4300 4301 id = dev->DeviceID(); 4302 } 4303 4304 // nsIThreadObserver methods 4305 4306 NS_IMETHODIMP 4307 MediaTrackGraphImpl::OnDispatchedEvent() { 4308 MonitorAutoLock lock(mMonitor); 4309 GraphDriver* driver = CurrentDriver(); 4310 if (!driver) { 4311 // The ipc::BackgroundChild started by `UniqueMessagePortId()` destruction 4312 // can queue messages on the thread used for the graph after graph 4313 // shutdown. See bug 1955768. 4314 // 4315 // Other threads may have already taken a reference to this graph as the 4316 // observer, so clearing the thread's observer (on the graph thread) would 4317 // not be effective to prevent this callback from being invoked. 4318 // https://searchfox.org/mozilla-central/rev/8b52ebe3cddf0e63bb42e8b51618bc3ab8dcb12d/xpcom/threads/ThreadEventQueue.cpp#113,135,139 4319 return NS_OK; 4320 } 4321 driver->EnsureNextIteration(); 4322 return NS_OK; 4323 } 4324 4325 NS_IMETHODIMP 4326 MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) { 4327 return NS_OK; 4328 } 4329 4330 NS_IMETHODIMP 4331 MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) { 4332 return NS_OK; 4333 } 4334 } // namespace mozilla 4335 4336 #undef LOG