AudioSinkWrapper.cpp (20299B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "AudioSinkWrapper.h" 8 9 #include "AudioDeviceInfo.h" 10 #include "AudioSink.h" 11 #include "VideoUtils.h" 12 #include "mozilla/Logging.h" 13 #include "mozilla/StaticPrefs_media.h" 14 #include "nsPrintfCString.h" 15 #include "nsThreadManager.h" 16 17 mozilla::LazyLogModule gAudioSinkWrapperLog("AudioSinkWrapper"); 18 #define LOG(...) \ 19 MOZ_LOG(gAudioSinkWrapperLog, mozilla::LogLevel::Debug, (__VA_ARGS__)); 20 #define LOGV(...) \ 21 MOZ_LOG(gAudioSinkWrapperLog, mozilla::LogLevel::Verbose, (__VA_ARGS__)); 22 23 namespace mozilla { 24 25 using media::TimeUnit; 26 27 AudioSinkWrapper::~AudioSinkWrapper() = default; 28 29 void AudioSinkWrapper::Shutdown() { 30 AssertOwnerThread(); 31 MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped."); 32 mSinkCreator = nullptr; 33 } 34 35 /* static */ 36 already_AddRefed<TaskQueue> AudioSinkWrapper::CreateAsyncInitTaskQueue() { 37 return nsThreadManager::get().CreateBackgroundTaskQueue("AsyncAudioSinkInit"); 38 } 39 40 RefPtr<MediaSink::EndedPromise> AudioSinkWrapper::OnEnded(TrackType aType) { 41 AssertOwnerThread(); 42 MOZ_ASSERT(mIsStarted, "Must be called after playback starts."); 43 if (aType == TrackInfo::kAudioTrack) { 44 return mEndedPromise; 45 } 46 return nullptr; 47 } 48 49 TimeUnit AudioSinkWrapper::GetEndTime(TrackType aType) const { 50 AssertOwnerThread(); 51 MOZ_ASSERT(mIsStarted, "Must be called after playback starts."); 52 if (aType != TrackInfo::kAudioTrack) { 53 return TimeUnit::Zero(); 54 } 55 56 if (mAudioSink && mAudioSink->AudioStreamCallbackStarted()) { 57 auto time = mAudioSink->GetEndTime(); 58 LOGV("%p: GetEndTime return %lf from sink", this, time.ToSeconds()); 59 return time; 60 } 61 62 RefPtr<const AudioData> audio = mAudioQueue.PeekBack(); 63 if (audio) { 64 LOGV("%p: GetEndTime return %lf from queue", this, 65 audio->GetEndTime().ToSeconds()); 66 return audio->GetEndTime(); 67 } 68 69 LOGV("%p: GetEndTime return %lf from last packet", this, 70 mLastPacketEndTime.ToSeconds()); 71 return mLastPacketEndTime; 72 } 73 74 TimeUnit AudioSinkWrapper::GetSystemClockPosition(TimeStamp aNow) const { 75 AssertOwnerThread(); 76 MOZ_ASSERT(!mClockStartTime.IsNull()); 77 // Time elapsed since we started playing. 78 double delta = (aNow - mClockStartTime).ToSeconds(); 79 // Take playback rate into account. 80 return mPositionAtClockStart + 81 TimeUnit::FromSeconds(delta * mParams.mPlaybackRate); 82 } 83 84 bool AudioSinkWrapper::IsMuted() const { 85 AssertOwnerThread(); 86 return mParams.mVolume == 0.0; 87 } 88 89 TimeUnit AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) { 90 AssertOwnerThread(); 91 MOZ_ASSERT(mIsStarted, "Must be called after playback starts."); 92 93 TimeUnit pos; 94 TimeStamp t = TimeStamp::Now(); 95 96 if (mAudioSink) { 97 if (mLastClockSource == ClockSource::SystemClock) { 98 TimeUnit switchTime = GetSystemClockPosition(t); 99 // Update the _actual_ start time of the audio stream now that it has 100 // started, preventing any clock discontinuity. 101 mAudioSink->UpdateStartTime(switchTime); 102 LOGV("%p: switching to audio clock at media time %lf", this, 103 switchTime.ToSeconds()); 104 } 105 // Rely on the audio sink to report playback position when it is not ended. 106 pos = mAudioSink->GetPosition(); 107 LOGV("%p: Getting position from the Audio Sink %lf", this, pos.ToSeconds()); 108 mLastClockSource = ClockSource::AudioStream; 109 } else if (!mClockStartTime.IsNull()) { 110 // Calculate playback position using system clock if we are still playing, 111 // but not rendering the audio, because this audio sink is muted. 112 pos = GetSystemClockPosition(t); 113 LOGV("%p: Getting position from the system clock %lf", this, 114 pos.ToSeconds()); 115 if (mAudioQueue.GetSize() > 0) { 116 // Audio track, but it won't be dequeued. Discard packets 117 // that are behind the current media time, to keep the queue size under 118 // control. 119 DropAudioPacketsIfNeeded(pos); 120 } 121 // Without an AudioSink, it's necessary to manually check if the audio has 122 // "ended", meaning that all the audio packets have been consumed, 123 // to resolve the ended promise. 124 if (CheckIfEnded()) { 125 MOZ_ASSERT(!mAudioSink); 126 mEndedPromiseHolder.ResolveIfExists(true, __func__); 127 } 128 mLastClockSource = ClockSource::SystemClock; 129 130 if (!mAudioSink && mAsyncCreateCount == 0 && NeedAudioSink() && 131 t > mRetrySinkTime) { 132 MaybeAsyncCreateAudioSink(mAudioDevice); 133 } 134 } else { 135 // Return how long we've played if we are not playing. 136 pos = mPositionAtClockStart; 137 LOGV("%p: Getting static position, not playing %lf", this, pos.ToSeconds()); 138 mLastClockSource = ClockSource::Paused; 139 } 140 141 if (aTimeStamp) { 142 *aTimeStamp = t; 143 } 144 145 return pos; 146 } 147 148 bool AudioSinkWrapper::CheckIfEnded() const { 149 return mAudioQueue.IsFinished() && mAudioQueue.GetSize() == 0u; 150 } 151 152 bool AudioSinkWrapper::HasUnplayedFrames(TrackType aType) const { 153 AssertOwnerThread(); 154 return mAudioSink ? mAudioSink->HasUnplayedFrames() : false; 155 } 156 157 media::TimeUnit AudioSinkWrapper::UnplayedDuration(TrackType aType) const { 158 AssertOwnerThread(); 159 return mAudioSink ? mAudioSink->UnplayedDuration() : media::TimeUnit::Zero(); 160 } 161 162 void AudioSinkWrapper::DropAudioPacketsIfNeeded( 163 const TimeUnit& aMediaPosition) { 164 RefPtr<AudioData> audio = mAudioQueue.PeekFront(); 165 uint32_t dropped = 0; 166 while (audio && audio->GetEndTime() < aMediaPosition) { 167 // drop this packet, try the next one 168 audio = mAudioQueue.PopFront(); 169 dropped++; 170 if (audio) { 171 mLastPacketEndTime = audio->GetEndTime(); 172 LOGV( 173 "Dropping audio packets: media position: %lf, " 174 "packet dropped: [%lf, %lf] (%u so far).\n", 175 aMediaPosition.ToSeconds(), audio->mTime.ToSeconds(), 176 (audio->GetEndTime()).ToSeconds(), dropped); 177 } 178 audio = mAudioQueue.PeekFront(); 179 } 180 } 181 182 void AudioSinkWrapper::OnMuted(bool aMuted) { 183 AssertOwnerThread(); 184 LOG("%p: AudioSinkWrapper::OnMuted(%s)", this, aMuted ? "true" : "false"); 185 // Nothing to do 186 if (mAudioEnded) { 187 LOG("%p: AudioSinkWrapper::OnMuted, but no audio track", this); 188 return; 189 } 190 if (aMuted) { 191 if (mAudioSink) { 192 LOG("AudioSinkWrapper muted, shutting down AudioStream."); 193 ShutDownAudioSink(); 194 } 195 } else { 196 LOG("%p: AudioSinkWrapper unmuted, maybe re-creating an AudioStream.", 197 this); 198 MaybeAsyncCreateAudioSink(mAudioDevice); 199 } 200 } 201 202 void AudioSinkWrapper::SetVolume(double aVolume) { 203 AssertOwnerThread(); 204 205 bool wasMuted = mParams.mVolume == 0; 206 bool nowMuted = aVolume == 0.; 207 mParams.mVolume = aVolume; 208 209 if (!wasMuted && nowMuted) { 210 OnMuted(true); 211 } else if (wasMuted && !nowMuted) { 212 OnMuted(false); 213 } 214 215 if (mAudioSink) { 216 mAudioSink->SetVolume(aVolume); 217 } 218 } 219 220 void AudioSinkWrapper::SetStreamName(const nsAString& aStreamName) { 221 AssertOwnerThread(); 222 if (mAudioSink) { 223 mAudioSink->SetStreamName(aStreamName); 224 } 225 } 226 227 void AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate) { 228 AssertOwnerThread(); 229 if (mAudioSink) { 230 // Pass the playback rate to the audio sink. The underlying AudioStream 231 // will handle playback rate changes and report correct audio position. 232 mAudioSink->SetPlaybackRate(aPlaybackRate); 233 } else if (!mClockStartTime.IsNull()) { 234 // Adjust playback duration and start time when we are still playing. 235 TimeStamp now = TimeStamp::Now(); 236 mPositionAtClockStart = GetSystemClockPosition(now); 237 mClockStartTime = now; 238 } 239 // mParams.mPlaybackRate affects GetSystemClockPosition(). It should be 240 // updated after the calls to GetSystemClockPosition(); 241 mParams.mPlaybackRate = aPlaybackRate; 242 243 // Do nothing when not playing. Changes in playback rate will be taken into 244 // account by GetSystemClockPosition(). 245 } 246 247 void AudioSinkWrapper::SetPreservesPitch(bool aPreservesPitch) { 248 AssertOwnerThread(); 249 mParams.mPreservesPitch = aPreservesPitch; 250 if (mAudioSink) { 251 mAudioSink->SetPreservesPitch(aPreservesPitch); 252 } 253 } 254 255 void AudioSinkWrapper::SetPlaying(bool aPlaying) { 256 AssertOwnerThread(); 257 LOG("%p: AudioSinkWrapper::SetPlaying %s", this, aPlaying ? "true" : "false"); 258 259 // Resume/pause matters only when playback started. 260 if (!mIsStarted) { 261 return; 262 } 263 264 if (mAudioSink) { 265 mAudioSink->SetPlaying(aPlaying); 266 } 267 268 if (aPlaying) { 269 MOZ_ASSERT(mClockStartTime.IsNull()); 270 TimeUnit switchTime = GetPosition(); 271 mClockStartTime = TimeStamp::Now(); 272 if (!mAudioSink && NeedAudioSink()) { 273 LOG("%p: AudioSinkWrapper::SetPlaying : starting an AudioSink", this); 274 DropAudioPacketsIfNeeded(switchTime); 275 SyncCreateAudioSink(switchTime); 276 } 277 } else { 278 // Remember how long we've played. 279 mPositionAtClockStart = GetPosition(); 280 // mClockStartTime must be updated later since GetPosition() 281 // depends on the value of mClockStartTime. 282 mClockStartTime = TimeStamp(); 283 } 284 } 285 286 RefPtr<GenericPromise> AudioSinkWrapper::SetAudioDevice( 287 RefPtr<AudioDeviceInfo> aDevice) { 288 return MaybeAsyncCreateAudioSink(std::move(aDevice)); 289 } 290 291 double AudioSinkWrapper::PlaybackRate() const { 292 AssertOwnerThread(); 293 return mParams.mPlaybackRate; 294 } 295 296 nsresult AudioSinkWrapper::Start(const TimeUnit& aStartTime, 297 const MediaInfo& aInfo) { 298 LOG("%p AudioSinkWrapper::Start", this); 299 AssertOwnerThread(); 300 MOZ_ASSERT(!mIsStarted, "playback already started."); 301 302 mIsStarted = true; 303 mPositionAtClockStart = aStartTime; 304 mClockStartTime = TimeStamp::Now(); 305 mAudioEnded = IsAudioSourceEnded(aInfo); 306 mLastPacketEndTime = TimeUnit::Zero(); 307 308 if (mAudioEnded) { 309 // Resolve promise if we start playback at the end position of the audio. 310 mEndedPromise = 311 aInfo.HasAudio() 312 ? MediaSink::EndedPromise::CreateAndResolve(true, __func__) 313 : nullptr; 314 return NS_OK; 315 } 316 317 mEndedPromise = mEndedPromiseHolder.Ensure(__func__); 318 if (!NeedAudioSink()) { 319 return NS_OK; 320 } 321 return SyncCreateAudioSink(aStartTime); 322 } 323 324 bool AudioSinkWrapper::NeedAudioSink() { 325 // An AudioSink is needed if unmuted, playing, and not ended. The not-ended 326 // check also avoids creating an AudioSink when there is no audio track. 327 return !IsMuted() && IsPlaying() && !mEndedPromiseHolder.IsEmpty(); 328 } 329 330 void AudioSinkWrapper::StartAudioSink(UniquePtr<AudioSink> aAudioSink, 331 const TimeUnit& aStartTime) { 332 AssertOwnerThread(); 333 MOZ_ASSERT(!mAudioSink); 334 mAudioSink = std::move(aAudioSink); 335 mAudioSink->Start(mParams, aStartTime) 336 ->Then(mOwnerThread.GetEventTarget(), __func__, this, 337 &AudioSinkWrapper::OnAudioEnded) 338 ->Track(mAudioSinkEndedRequest); 339 } 340 341 void AudioSinkWrapper::ShutDownAudioSink() { 342 AssertOwnerThread(); 343 mAudioSinkEndedRequest.DisconnectIfExists(); 344 if (IsPlaying()) { 345 mPositionAtClockStart = mAudioSink->GetPosition(); 346 mClockStartTime = TimeStamp::Now(); 347 } 348 mAudioSink->ShutDown(); 349 mLastPacketEndTime = mAudioSink->GetEndTime(); 350 mAudioSink = nullptr; 351 } 352 353 RefPtr<GenericPromise> AudioSinkWrapper::MaybeAsyncCreateAudioSink( 354 RefPtr<AudioDeviceInfo> aDevice) { 355 AssertOwnerThread(); 356 UniquePtr<AudioSink> audioSink; 357 if (NeedAudioSink() && (!mAudioSink || aDevice != mAudioDevice)) { 358 LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: AudioSink needed", 359 this); 360 if (mAudioSink) { 361 ShutDownAudioSink(); 362 } 363 audioSink = mSinkCreator(); 364 } else { 365 LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: no AudioSink change", 366 this); 367 // Bounce off the background thread to keep promise resolution in order. 368 } 369 mAudioDevice = std::move(aDevice); 370 ++mAsyncCreateCount; 371 using Promise = 372 MozPromise<UniquePtr<AudioSink>, nsresult, /* IsExclusive = */ true>; 373 return InvokeAsync( 374 mAsyncInitTaskQueue, 375 "MaybeAsyncCreateAudioSink (Async part: initialization)", 376 [self = RefPtr<AudioSinkWrapper>(this), 377 audioSink{std::move(audioSink)}, audioDevice = mAudioDevice, 378 this]() mutable { 379 if (!audioSink || !mAsyncInitTaskQueue->IsEmpty()) { 380 // Either an AudioSink is not required or there's a 381 // pending task to init an AudioSink with a possibly 382 // different device. 383 return Promise::CreateAndResolve(nullptr, __func__); 384 } 385 386 LOG("AudioSink initialization on background thread"); 387 // This can take about 200ms, e.g. on Windows, we don't 388 // want to do it on the MDSM thread, because it would 389 // make the clock not update for that amount of time, and 390 // the video would therefore not update. The Start() call 391 // is very cheap on the other hand, we can do it from the 392 // MDSM thread. 393 nsresult rv = audioSink->InitializeAudioStream( 394 audioDevice, AudioSink::InitializationType::UNMUTING); 395 if (NS_FAILED(rv)) { 396 LOG("Async AudioSink initialization failed"); 397 return Promise::CreateAndReject(rv, __func__); 398 } 399 return Promise::CreateAndResolve(std::move(audioSink), __func__); 400 }) 401 ->Then( 402 mOwnerThread.GetEventTarget(), 403 "MaybeAsyncCreateAudioSink (Async part: start from MDSM thread)", 404 [self = RefPtr<AudioSinkWrapper>(this), audioDevice = mAudioDevice, 405 this](Promise::ResolveOrRejectValue&& aValue) mutable { 406 LOG("AudioSink async init done, back on MDSM thread"); 407 --mAsyncCreateCount; 408 UniquePtr<AudioSink> audioSink; 409 if (aValue.IsResolve()) { 410 audioSink = std::move(aValue.ResolveValue()); 411 } 412 // It's possible that the newly created AudioSink isn't needed at 413 // this point, in some cases: 414 // 1. An AudioSink was created synchronously while this 415 // AudioSink was initialized asynchronously, bail out here. This 416 // happens when seeking (which does a synchronous initialization) 417 // right after unmuting. mEndedPromiseHolder is managed by the 418 // other AudioSink, so don't touch it here. 419 // 2. The media element was muted while the async initialization 420 // was happening. 421 // 3. The AudioSinkWrapper was paused or stopped during 422 // asynchronous initialization. 423 // 4. The audio has ended during asynchronous initialization. 424 // 5. A change to a potentially different sink device is pending. 425 if (mAudioSink || !NeedAudioSink() || audioDevice != mAudioDevice) { 426 LOG("AudioSink async initialization isn't needed."); 427 if (audioSink) { 428 LOG("Shutting down unneeded AudioSink."); 429 audioSink->ShutDown(); 430 } 431 return GenericPromise::CreateAndResolve(true, __func__); 432 } 433 434 if (aValue.IsReject()) { 435 if (audioDevice) { 436 // Device will be started when available again. 437 ScheduleRetrySink(); 438 } else { 439 // Default device not available. Report error. 440 MOZ_ASSERT(!mAudioSink); 441 mEndedPromiseHolder.RejectIfExists(aValue.RejectValue(), 442 __func__); 443 } 444 return GenericPromise::CreateAndResolve(true, __func__); 445 } 446 447 if (!audioSink) { 448 // No-op because either an existing AudioSink was suitable or no 449 // AudioSink was needed when MaybeAsyncCreateAudioSink() set up 450 // this task. We now need a new AudioSink, but that will be 451 // handled by another task, either already pending or a delayed 452 // retry task yet to be created by GetPosition(). 453 return GenericPromise::CreateAndResolve(true, __func__); 454 } 455 456 MOZ_ASSERT(!mAudioSink); 457 // Avoiding the side effects of GetPosition() creating another 458 // sink another AudioSink and resolving mEndedPromiseHolder, which 459 // the new audioSink will now manage. 460 TimeUnit switchTime = GetSystemClockPosition(TimeStamp::Now()); 461 DropAudioPacketsIfNeeded(switchTime); 462 mLastClockSource = ClockSource::SystemClock; 463 464 LOG("AudioSink async, start"); 465 StartAudioSink(std::move(audioSink), switchTime); 466 return GenericPromise::CreateAndResolve(true, __func__); 467 }); 468 } 469 470 nsresult AudioSinkWrapper::SyncCreateAudioSink(const TimeUnit& aStartTime) { 471 AssertOwnerThread(); 472 MOZ_ASSERT(!mAudioSink); 473 MOZ_ASSERT(!mAudioSinkEndedRequest.Exists()); 474 475 LOG("%p: AudioSinkWrapper::SyncCreateAudioSink(%lf)", this, 476 aStartTime.ToSeconds()); 477 478 UniquePtr<AudioSink> audioSink = mSinkCreator(); 479 nsresult rv = audioSink->InitializeAudioStream( 480 mAudioDevice, AudioSink::InitializationType::INITIAL); 481 if (NS_FAILED(rv)) { 482 LOG("Sync AudioSinkWrapper initialization failed"); 483 // If a specific device has been specified through setSinkId() 484 // the sink is started after the device becomes available again. 485 if (mAudioDevice) { 486 ScheduleRetrySink(); 487 return NS_OK; 488 } 489 // If a default output device is not available, the system may not support 490 // audio output. Report an error so that playback can be aborted if there 491 // is no video. 492 mEndedPromiseHolder.RejectIfExists(rv, __func__); 493 return rv; 494 } 495 StartAudioSink(std::move(audioSink), aStartTime); 496 497 return NS_OK; 498 } 499 500 void AudioSinkWrapper::ScheduleRetrySink() { 501 mRetrySinkTime = 502 TimeStamp::Now() + TimeDuration::FromMilliseconds( 503 StaticPrefs::media_audio_device_retry_ms()); 504 } 505 506 bool AudioSinkWrapper::IsAudioSourceEnded(const MediaInfo& aInfo) const { 507 // no audio or empty audio queue which won't get data anymore is equivalent to 508 // audio ended 509 return !aInfo.HasAudio() || 510 (mAudioQueue.IsFinished() && mAudioQueue.GetSize() == 0u); 511 } 512 513 void AudioSinkWrapper::Stop() { 514 AssertOwnerThread(); 515 MOZ_ASSERT(mIsStarted, "playback not started."); 516 517 LOG("%p: AudioSinkWrapper::Stop", this); 518 519 mIsStarted = false; 520 mClockStartTime = TimeStamp(); 521 mPositionAtClockStart = TimeUnit::Invalid(); 522 mAudioEnded = true; 523 if (mAudioSink) { 524 ShutDownAudioSink(); 525 } 526 527 mEndedPromiseHolder.ResolveIfExists(true, __func__); 528 mEndedPromise = nullptr; 529 } 530 531 bool AudioSinkWrapper::IsStarted() const { 532 AssertOwnerThread(); 533 return mIsStarted; 534 } 535 536 bool AudioSinkWrapper::IsPlaying() const { 537 AssertOwnerThread(); 538 MOZ_ASSERT(mClockStartTime.IsNull() || IsStarted()); 539 return !mClockStartTime.IsNull(); 540 } 541 542 void AudioSinkWrapper::OnAudioEnded( 543 const EndedPromise::ResolveOrRejectValue& aValue) { 544 AssertOwnerThread(); 545 // This callback on mAudioSinkEndedRequest should have been disconnected if 546 // mEndedPromiseHolder has been settled. 547 MOZ_ASSERT(!mEndedPromiseHolder.IsEmpty()); 548 LOG("%p: AudioSinkWrapper::OnAudioEnded %i", this, aValue.IsResolve()); 549 mAudioSinkEndedRequest.Complete(); 550 ShutDownAudioSink(); 551 // System time is now used for the clock as video may not have ended. 552 if (aValue.IsResolve()) { 553 mAudioEnded = true; 554 mEndedPromiseHolder.Resolve(aValue.ResolveValue(), __func__); 555 return; 556 } 557 if (mAudioDevice) { 558 ScheduleRetrySink(); // Device will be restarted when available again. 559 return; 560 } 561 // Default device not available. Report error. 562 mEndedPromiseHolder.Reject(aValue.RejectValue(), __func__); 563 } 564 565 void AudioSinkWrapper::GetDebugInfo(dom::MediaSinkDebugInfo& aInfo) { 566 AssertOwnerThread(); 567 aInfo.mAudioSinkWrapper.mIsPlaying = IsPlaying(); 568 aInfo.mAudioSinkWrapper.mIsStarted = IsStarted(); 569 aInfo.mAudioSinkWrapper.mAudioEnded = mAudioEnded; 570 if (mAudioSink) { 571 mAudioSink->GetDebugInfo(aInfo); 572 } 573 } 574 575 } // namespace mozilla 576 577 #undef LOG 578 #undef LOGV