TestAudioTrackGraph.cpp (143128B)
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=2 et sw=2 tw=80: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 5 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "CrossGraphPort.h" 8 #include "DeviceInputTrack.h" 9 #include "MediaTrackGraphImpl.h" 10 #include "gmock/gmock.h" 11 #include "gtest/gtest-printers.h" 12 #include "gtest/gtest.h" 13 #ifdef MOZ_WEBRTC 14 # include "MediaEngineWebRTCAudio.h" 15 #endif // MOZ_WEBRTC 16 #include "MockCubeb.h" 17 #include "WavDumper.h" 18 #include "mozilla/Preferences.h" 19 #include "mozilla/SpinEventLoopUntil.h" 20 #include "mozilla/gtest/WaitFor.h" 21 22 using namespace mozilla; 23 using testing::AtLeast; 24 using testing::Eq; 25 using testing::InSequence; 26 using testing::MockFunction; 27 using testing::Return; 28 using testing::StrEq; 29 using testing::StrictMock; 30 31 // Short-hand for InvokeAsync on the current thread. 32 #define InvokeAsync(f) InvokeAsync(GetCurrentSerialEventTarget(), __func__, f) 33 34 // Short-hand for DispatchToCurrentThread with a function. 35 #define DispatchFunction(f) \ 36 NS_DispatchToCurrentThread(NS_NewRunnableFunction(__func__, f)) 37 38 // Short-hand for DispatchToCurrentThread with a method with arguments 39 #define DispatchMethod(t, m, args...) \ 40 NS_DispatchToCurrentThread(NewRunnableMethod(__func__, t, m, ##args)) 41 42 // Short-hand for draining the current threads event queue, i.e. processing 43 // those runnables dispatched per above. 44 #define ProcessEventQueue() \ 45 while (NS_ProcessNextEvent(nullptr, false)) { \ 46 } 47 48 namespace { 49 #ifdef MOZ_WEBRTC 50 /* 51 * Common ControlMessages 52 */ 53 struct StartInputProcessing : public ControlMessage { 54 const RefPtr<AudioProcessingTrack> mProcessingTrack; 55 const RefPtr<AudioInputProcessing> mInputProcessing; 56 57 StartInputProcessing(AudioProcessingTrack* aTrack, 58 AudioInputProcessing* aInputProcessing) 59 : ControlMessage(aTrack), 60 mProcessingTrack(aTrack), 61 mInputProcessing(aInputProcessing) {} 62 void Run() override { mInputProcessing->Start(mTrack->Graph()); } 63 }; 64 65 struct StopInputProcessing : public ControlMessage { 66 const RefPtr<AudioInputProcessing> mInputProcessing; 67 68 explicit StopInputProcessing(AudioProcessingTrack* aTrack, 69 AudioInputProcessing* aInputProcessing) 70 : ControlMessage(aTrack), mInputProcessing(aInputProcessing) {} 71 void Run() override { mInputProcessing->Stop(mTrack->Graph()); } 72 }; 73 74 void QueueApplySettings(AudioProcessingTrack* aTrack, 75 AudioInputProcessing* aInputProcessing, 76 const MediaEnginePrefs& aSettings) { 77 aTrack->QueueControlMessageWithNoShutdown( 78 [inputProcessing = RefPtr{aInputProcessing}, aSettings, 79 // If the track is not connected to a device then the particular 80 // AudioDeviceID (nullptr) passed to ReevaluateInputDevice() is not 81 // important. 82 deviceId = aTrack->DeviceId().valueOr(nullptr), 83 graph = aTrack->Graph()] { 84 inputProcessing->ApplySettings(graph, deviceId, aSettings); 85 }); 86 } 87 88 void QueueExpectIsPassThrough(AudioProcessingTrack* aTrack, 89 AudioInputProcessing* aInputProcessing) { 90 aTrack->QueueControlMessageWithNoShutdown( 91 [inputProcessing = RefPtr{aInputProcessing}, graph = aTrack->Graph()] { 92 EXPECT_EQ(inputProcessing->IsPassThrough(graph), true); 93 }); 94 } 95 #endif // MOZ_WEBRTC 96 97 class GoFaster : public ControlMessage { 98 MockCubeb* mCubeb; 99 100 public: 101 explicit GoFaster(MockCubeb* aCubeb) 102 : ControlMessage(nullptr), mCubeb(aCubeb) {} 103 void Run() override { mCubeb->GoFaster(); } 104 }; 105 106 struct StartNonNativeInput : public ControlMessage { 107 const RefPtr<NonNativeInputTrack> mInputTrack; 108 RefPtr<AudioInputSource> mInputSource; 109 110 StartNonNativeInput(NonNativeInputTrack* aInputTrack, 111 RefPtr<AudioInputSource>&& aInputSource) 112 : ControlMessage(aInputTrack), 113 mInputTrack(aInputTrack), 114 mInputSource(std::move(aInputSource)) {} 115 void Run() override { mInputTrack->StartAudio(std::move(mInputSource)); } 116 }; 117 118 struct StopNonNativeInput : public ControlMessage { 119 const RefPtr<NonNativeInputTrack> mInputTrack; 120 121 explicit StopNonNativeInput(NonNativeInputTrack* aInputTrack) 122 : ControlMessage(aInputTrack), mInputTrack(aInputTrack) {} 123 void Run() override { mInputTrack->StopAudio(); } 124 }; 125 126 // Helper for detecting when fallback driver has been switched away, for use 127 // with RunningMode::Manual. 128 class OnFallbackListener : public MediaTrackListener { 129 const RefPtr<MediaTrack> mTrack; 130 Atomic<bool> mOnFallback{true}; 131 132 public: 133 explicit OnFallbackListener(MediaTrack* aTrack) : mTrack(aTrack) {} 134 135 void Reset() { mOnFallback = true; } 136 bool OnFallback() { return mOnFallback; } 137 138 void NotifyOutput(MediaTrackGraph*, TrackTime) override { 139 if (auto* ad = 140 mTrack->GraphImpl()->CurrentDriver()->AsAudioCallbackDriver()) { 141 mOnFallback = ad->OnFallback(); 142 } 143 } 144 }; 145 146 class MockAudioDataListener : public AudioDataListener { 147 protected: 148 ~MockAudioDataListener() = default; 149 150 public: 151 MockAudioDataListener() = default; 152 153 MOCK_METHOD(uint32_t, RequestedInputChannelCount, (MediaTrackGraph*), 154 (const)); 155 MOCK_METHOD(cubeb_input_processing_params, RequestedInputProcessingParams, 156 (MediaTrackGraph*), (const)); 157 MOCK_METHOD(bool, IsVoiceInput, (MediaTrackGraph*), (const)); 158 MOCK_METHOD(void, DeviceChanged, (MediaTrackGraph*)); 159 MOCK_METHOD(void, Disconnect, (MediaTrackGraph*)); 160 MOCK_METHOD(void, NotifySetRequestedInputProcessingParams, 161 (MediaTrackGraph*, int, cubeb_input_processing_params)); 162 MOCK_METHOD(void, NotifySetRequestedInputProcessingParamsResult, 163 (MediaTrackGraph*, int, 164 (const Result<cubeb_input_processing_params, int>&))); 165 }; 166 } // namespace 167 168 /* 169 * The set of tests here are a bit special. In part because they're async and 170 * depends on the graph thread to function. In part because they depend on main 171 * thread stable state to send messages to the graph. 172 * 173 * Any message sent from the main thread to the graph through the graph's 174 * various APIs are scheduled to run in stable state. Stable state occurs after 175 * a task in the main thread eventloop has run to completion. 176 * 177 * Since gtests are generally sync and on main thread, calling into the graph 178 * may schedule a stable state runnable but with no task in the eventloop to 179 * trigger stable state. Therefore care must be taken to always call into the 180 * graph from a task, typically via InvokeAsync or a dispatch to main thread. 181 */ 182 183 TEST(TestAudioTrackGraph, DifferentDeviceIDs) 184 { 185 MockCubeb* cubeb = new MockCubeb(); 186 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 187 188 MediaTrackGraph* g1 = MediaTrackGraphImpl::GetInstance( 189 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 190 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 191 /*OutputDeviceID*/ nullptr, GetMainThreadSerialEventTarget()); 192 193 MediaTrackGraph* g2 = MediaTrackGraphImpl::GetInstance( 194 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 195 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 196 /*OutputDeviceID*/ reinterpret_cast<cubeb_devid>(1), 197 GetMainThreadSerialEventTarget()); 198 199 MediaTrackGraph* g1_2 = MediaTrackGraphImpl::GetInstance( 200 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 201 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 202 /*OutputDeviceID*/ nullptr, GetMainThreadSerialEventTarget()); 203 204 MediaTrackGraph* g2_2 = MediaTrackGraphImpl::GetInstance( 205 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 206 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 207 /*OutputDeviceID*/ reinterpret_cast<cubeb_devid>(1), 208 GetMainThreadSerialEventTarget()); 209 210 EXPECT_NE(g1, g2) << "Different graphs due to different device ids"; 211 EXPECT_EQ(g1, g1_2) << "Same graphs for same device ids"; 212 EXPECT_EQ(g2, g2_2) << "Same graphs for same device ids"; 213 214 for (MediaTrackGraph* g : {g1, g2}) { 215 // Dummy track to make graph rolling. Add it and remove it to remove the 216 // graph from the global hash table and let it shutdown. 217 218 using SourceTrackPromise = MozPromise<SourceMediaTrack*, nsresult, true>; 219 auto p = InvokeAsync([g] { 220 return SourceTrackPromise::CreateAndResolve( 221 g->CreateSourceTrack(MediaSegment::AUDIO), __func__); 222 }); 223 224 WaitFor(cubeb->StreamInitEvent()); 225 RefPtr<SourceMediaTrack> dummySource = WaitFor(p).unwrap(); 226 227 DispatchMethod(dummySource, &SourceMediaTrack::Destroy); 228 229 WaitFor(cubeb->StreamDestroyEvent()); 230 } 231 } 232 233 TEST(TestAudioTrackGraph, SetOutputDeviceID) 234 { 235 MockCubeb* cubeb = new MockCubeb(); 236 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 237 238 // Set the output device id in GetInstance method confirm that it is the one 239 // used in cubeb_stream_init. 240 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 241 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 242 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 243 /*OutputDeviceID*/ reinterpret_cast<cubeb_devid>(2), 244 GetMainThreadSerialEventTarget()); 245 246 // Dummy track to make graph rolling. Add it and remove it to remove the 247 // graph from the global hash table and let it shutdown. 248 RefPtr<SourceMediaTrack> dummySource; 249 DispatchFunction( 250 [&] { dummySource = graph->CreateSourceTrack(MediaSegment::AUDIO); }); 251 252 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 253 254 EXPECT_EQ(stream->GetOutputDeviceID(), reinterpret_cast<cubeb_devid>(2)) 255 << "After init confirm the expected output device id"; 256 257 // Test has finished, destroy the track to shutdown the MTG. 258 DispatchMethod(dummySource, &SourceMediaTrack::Destroy); 259 WaitFor(cubeb->StreamDestroyEvent()); 260 } 261 262 TEST(TestAudioTrackGraph, StreamName) 263 { 264 MockCubeb* cubeb = new MockCubeb(); 265 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 266 267 // Initialize a graph with a system thread driver to check that the stream 268 // name survives the driver switch. 269 MediaTrackGraphImpl* graph = MediaTrackGraphImpl::GetInstance( 270 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 271 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 272 /*OutputDeviceID*/ reinterpret_cast<cubeb_devid>(1), 273 GetMainThreadSerialEventTarget()); 274 nsLiteralCString name1("name1"); 275 graph->CurrentDriver()->SetStreamName(name1); 276 277 // Dummy track to start the graph rolling and switch to an 278 // AudioCallbackDriver. 279 RefPtr<SourceMediaTrack> dummySource; 280 DispatchFunction( 281 [&] { dummySource = graph->CreateSourceTrack(MediaSegment::AUDIO); }); 282 283 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 284 EXPECT_STREQ(stream->StreamName(), name1.get()); 285 286 // Test a name change on an existing stream. 287 nsLiteralCString name2("name2"); 288 DispatchFunction([&] { 289 graph->QueueControlMessageWithNoShutdown( 290 [&] { graph->CurrentDriver()->SetStreamName(name2); }); 291 }); 292 nsCString name = WaitFor(stream->NameSetEvent()); 293 EXPECT_EQ(name, name2); 294 295 // Test has finished. Destroy the track to shutdown the MTG. 296 DispatchMethod(dummySource, &SourceMediaTrack::Destroy); 297 WaitFor(cubeb->StreamDestroyEvent()); 298 } 299 300 TEST(TestAudioTrackGraph, NotifyDeviceStarted) 301 { 302 MockCubeb* cubeb = new MockCubeb(); 303 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 304 305 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 306 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 307 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 308 nullptr, GetMainThreadSerialEventTarget()); 309 310 RefPtr<SourceMediaTrack> dummySource; 311 (void)WaitForResolve(InvokeAsync([&] { 312 // Dummy track to make graph rolling. Add it and remove it to remove the 313 // graph from the global hash table and let it shutdown. 314 dummySource = graph->CreateSourceTrack(MediaSegment::AUDIO); 315 dummySource->AddAudioOutput(reinterpret_cast<void*>(1), nullptr); 316 return graph->NotifyWhenDeviceStarted(nullptr); 317 })); 318 319 { 320 MediaTrackGraphImpl* graph = dummySource->GraphImpl(); 321 MonitorAutoLock lock(graph->GetMonitor()); 322 EXPECT_TRUE(graph->CurrentDriver()->AsAudioCallbackDriver()); 323 EXPECT_TRUE(graph->CurrentDriver()->ThreadRunning()); 324 } 325 326 // Test has finished, destroy the track to shutdown the MTG. 327 DispatchMethod(dummySource, &SourceMediaTrack::Destroy); 328 WaitFor(cubeb->StreamDestroyEvent()); 329 } 330 331 TEST(TestAudioTrackGraph, NonNativeInputTrackStartAndStop) 332 { 333 MockCubeb* cubeb = new MockCubeb(); 334 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 335 336 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 337 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 338 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 339 nullptr, GetMainThreadSerialEventTarget()); 340 341 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 342 343 // Add a NonNativeInputTrack to graph, making graph create an output-only 344 // AudioCallbackDriver since NonNativeInputTrack is an audio-type MediaTrack. 345 RefPtr<NonNativeInputTrack> track; 346 DispatchFunction([&] { 347 track = new NonNativeInputTrack(graph->GraphRate(), deviceId, 348 PRINCIPAL_HANDLE_NONE); 349 graph->AddTrack(track); 350 }); 351 352 RefPtr<SmartMockCubebStream> driverStream = WaitFor(cubeb->StreamInitEvent()); 353 EXPECT_FALSE(driverStream->mHasInput); 354 EXPECT_TRUE(driverStream->mHasOutput); 355 356 // Main test below: 357 { 358 const AudioInputSource::Id sourceId = 1; 359 const uint32_t channels = 2; 360 const TrackRate rate = 48000; 361 362 // Start and stop the audio in NonNativeInputTrack. 363 { 364 struct DeviceInfo { 365 uint32_t mChannelCount; 366 AudioInputType mType; 367 }; 368 using DeviceQueryPromise = 369 MozPromise<DeviceInfo, nsresult, /* IsExclusive = */ true>; 370 371 struct DeviceQueryMessage : public ControlMessage { 372 const NonNativeInputTrack* mInputTrack; 373 MozPromiseHolder<DeviceQueryPromise> mHolder; 374 375 DeviceQueryMessage(NonNativeInputTrack* aInputTrack, 376 MozPromiseHolder<DeviceQueryPromise>&& aHolder) 377 : ControlMessage(aInputTrack), 378 mInputTrack(aInputTrack), 379 mHolder(std::move(aHolder)) {} 380 void Run() override { 381 DeviceInfo info = {mInputTrack->NumberOfChannels(), 382 mInputTrack->DevicePreference()}; 383 // mHolder.Resolve(info, __func__); 384 mTrack->GraphImpl()->Dispatch(NS_NewRunnableFunction( 385 "TestAudioTrackGraph::DeviceQueryMessage", 386 [holder = std::move(mHolder), devInfo = info]() mutable { 387 holder.Resolve(devInfo, __func__); 388 })); 389 } 390 }; 391 392 // No input channels and device preference before start. 393 { 394 MozPromiseHolder<DeviceQueryPromise> h; 395 RefPtr<DeviceQueryPromise> p = h.Ensure(__func__); 396 DispatchFunction([&] { 397 track->GraphImpl()->AppendMessage( 398 MakeUnique<DeviceQueryMessage>(track.get(), std::move(h))); 399 }); 400 Result<DeviceInfo, nsresult> r = WaitFor(p); 401 ASSERT_TRUE(r.isOk()); 402 DeviceInfo info = r.unwrap(); 403 404 EXPECT_EQ(info.mChannelCount, 0U); 405 EXPECT_EQ(info.mType, AudioInputType::Unknown); 406 } 407 408 DispatchFunction([&] { 409 track->GraphImpl()->AppendMessage(MakeUnique<StartNonNativeInput>( 410 track.get(), MakeRefPtr<AudioInputSource>( 411 MakeRefPtr<AudioInputSourceListener>(track.get()), 412 sourceId, deviceId, channels, true /* voice */, 413 PRINCIPAL_HANDLE_NONE, rate, graph->GraphRate()))); 414 }); 415 RefPtr<SmartMockCubebStream> nonNativeStream = 416 WaitFor(cubeb->StreamInitEvent()); 417 EXPECT_TRUE(nonNativeStream->mHasInput); 418 EXPECT_FALSE(nonNativeStream->mHasOutput); 419 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), deviceId); 420 EXPECT_EQ(nonNativeStream->InputChannels(), channels); 421 EXPECT_EQ(nonNativeStream->SampleRate(), static_cast<uint32_t>(rate)); 422 423 // Input channels and device preference should be set after start. 424 { 425 MozPromiseHolder<DeviceQueryPromise> h; 426 RefPtr<DeviceQueryPromise> p = h.Ensure(__func__); 427 DispatchFunction([&] { 428 track->GraphImpl()->AppendMessage( 429 MakeUnique<DeviceQueryMessage>(track.get(), std::move(h))); 430 }); 431 Result<DeviceInfo, nsresult> r = WaitFor(p); 432 ASSERT_TRUE(r.isOk()); 433 DeviceInfo info = r.unwrap(); 434 435 EXPECT_EQ(info.mChannelCount, channels); 436 EXPECT_EQ(info.mType, AudioInputType::Voice); 437 } 438 439 (void)WaitFor(nonNativeStream->FramesProcessedEvent()); 440 441 DispatchFunction([&] { 442 track->GraphImpl()->AppendMessage( 443 MakeUnique<StopNonNativeInput>(track.get())); 444 }); 445 RefPtr<SmartMockCubebStream> destroyedStream = 446 WaitFor(cubeb->StreamDestroyEvent()); 447 EXPECT_EQ(destroyedStream.get(), nonNativeStream.get()); 448 449 // No input channels and device preference after stop. 450 { 451 MozPromiseHolder<DeviceQueryPromise> h; 452 RefPtr<DeviceQueryPromise> p = h.Ensure(__func__); 453 DispatchFunction([&] { 454 track->GraphImpl()->AppendMessage( 455 MakeUnique<DeviceQueryMessage>(track.get(), std::move(h))); 456 }); 457 Result<DeviceInfo, nsresult> r = WaitFor(p); 458 ASSERT_TRUE(r.isOk()); 459 DeviceInfo info = r.unwrap(); 460 461 EXPECT_EQ(info.mChannelCount, 0U); 462 EXPECT_EQ(info.mType, AudioInputType::Unknown); 463 } 464 } 465 466 // Make sure the NonNativeInputTrack can restart and stop its audio. 467 { 468 DispatchFunction([&] { 469 track->GraphImpl()->AppendMessage(MakeUnique<StartNonNativeInput>( 470 track.get(), MakeRefPtr<AudioInputSource>( 471 MakeRefPtr<AudioInputSourceListener>(track.get()), 472 sourceId, deviceId, channels, true, 473 PRINCIPAL_HANDLE_NONE, rate, graph->GraphRate()))); 474 }); 475 RefPtr<SmartMockCubebStream> nonNativeStream = 476 WaitFor(cubeb->StreamInitEvent()); 477 EXPECT_TRUE(nonNativeStream->mHasInput); 478 EXPECT_FALSE(nonNativeStream->mHasOutput); 479 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), deviceId); 480 EXPECT_EQ(nonNativeStream->InputChannels(), channels); 481 EXPECT_EQ(nonNativeStream->SampleRate(), static_cast<uint32_t>(rate)); 482 483 (void)WaitFor(nonNativeStream->FramesProcessedEvent()); 484 485 DispatchFunction([&] { 486 track->GraphImpl()->AppendMessage( 487 MakeUnique<StopNonNativeInput>(track.get())); 488 }); 489 RefPtr<SmartMockCubebStream> destroyedStream = 490 WaitFor(cubeb->StreamDestroyEvent()); 491 EXPECT_EQ(destroyedStream.get(), nonNativeStream.get()); 492 } 493 } 494 495 // Clean up. 496 DispatchFunction([&] { track->Destroy(); }); 497 RefPtr<SmartMockCubebStream> destroyedStream = 498 WaitFor(cubeb->StreamDestroyEvent()); 499 EXPECT_EQ(destroyedStream.get(), driverStream.get()); 500 } 501 502 TEST(TestAudioTrackGraph, NonNativeInputTrackErrorCallback) 503 { 504 MockCubeb* cubeb = new MockCubeb(); 505 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 506 507 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 508 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 509 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 510 nullptr, GetMainThreadSerialEventTarget()); 511 512 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 513 514 // Add a NonNativeInputTrack to graph, making graph create an output-only 515 // AudioCallbackDriver since NonNativeInputTrack is an audio-type MediaTrack. 516 RefPtr<NonNativeInputTrack> track; 517 DispatchFunction([&] { 518 track = new NonNativeInputTrack(graph->GraphRate(), deviceId, 519 PRINCIPAL_HANDLE_NONE); 520 graph->AddTrack(track); 521 }); 522 523 RefPtr<SmartMockCubebStream> driverStream = WaitFor(cubeb->StreamInitEvent()); 524 EXPECT_FALSE(driverStream->mHasInput); 525 EXPECT_TRUE(driverStream->mHasOutput); 526 527 // Main test below: 528 { 529 const AudioInputSource::Id sourceId = 1; 530 const uint32_t channels = 2; 531 const TrackRate rate = 48000; 532 533 // Launch and start the non-native audio stream. 534 DispatchFunction([&] { 535 track->GraphImpl()->AppendMessage(MakeUnique<StartNonNativeInput>( 536 track.get(), MakeRefPtr<AudioInputSource>( 537 MakeRefPtr<AudioInputSourceListener>(track.get()), 538 sourceId, deviceId, channels, true, 539 PRINCIPAL_HANDLE_NONE, rate, graph->GraphRate()))); 540 }); 541 RefPtr<SmartMockCubebStream> nonNativeStream = 542 WaitFor(cubeb->StreamInitEvent()); 543 EXPECT_TRUE(nonNativeStream->mHasInput); 544 EXPECT_FALSE(nonNativeStream->mHasOutput); 545 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), deviceId); 546 EXPECT_EQ(nonNativeStream->InputChannels(), channels); 547 EXPECT_EQ(nonNativeStream->SampleRate(), static_cast<uint32_t>(rate)); 548 549 // Make sure the audio stream is running. 550 (void)WaitFor(nonNativeStream->FramesProcessedEvent()); 551 552 // Force an error. This results in the audio stream destroying. 553 DispatchFunction([&] { nonNativeStream->ForceError(); }); 554 WaitFor(nonNativeStream->ErrorForcedEvent()); 555 556 RefPtr<SmartMockCubebStream> destroyedStream = 557 WaitFor(cubeb->StreamDestroyEvent()); 558 EXPECT_EQ(destroyedStream.get(), nonNativeStream.get()); 559 } 560 561 // Make sure it's ok to call audio stop again. 562 DispatchFunction([&] { 563 track->GraphImpl()->AppendMessage( 564 MakeUnique<StopNonNativeInput>(track.get())); 565 }); 566 567 // Clean up. 568 DispatchFunction([&] { track->Destroy(); }); 569 RefPtr<SmartMockCubebStream> destroyedStream = 570 WaitFor(cubeb->StreamDestroyEvent()); 571 EXPECT_EQ(destroyedStream.get(), driverStream.get()); 572 } 573 574 class TestDeviceInputConsumerTrack : public DeviceInputConsumerTrack { 575 public: 576 static TestDeviceInputConsumerTrack* Create(MediaTrackGraph* aGraph) { 577 MOZ_RELEASE_ASSERT(NS_IsMainThread()); 578 TestDeviceInputConsumerTrack* track = 579 new TestDeviceInputConsumerTrack(aGraph->GraphRate()); 580 aGraph->AddTrack(track); 581 return track; 582 } 583 584 void Destroy() { 585 MOZ_RELEASE_ASSERT(NS_IsMainThread()); 586 DisconnectDeviceInput(); 587 DeviceInputConsumerTrack::Destroy(); 588 } 589 590 void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override { 591 MOZ_RELEASE_ASSERT(aFrom < aTo); 592 593 if (mInputs.IsEmpty()) { 594 GetData<AudioSegment>()->AppendNullData(aTo - aFrom); 595 } else { 596 MOZ_RELEASE_ASSERT(mInputs.Length() == 1); 597 AudioSegment data; 598 DeviceInputConsumerTrack::GetInputSourceData(data, aFrom, aTo); 599 GetData<AudioSegment>()->AppendFrom(&data); 600 } 601 }; 602 603 uint32_t NumberOfChannels() const override { 604 if (mInputs.IsEmpty()) { 605 return 0; 606 } 607 DeviceInputTrack* t = mInputs[0]->GetSource()->AsDeviceInputTrack(); 608 MOZ_RELEASE_ASSERT(t); 609 return t->NumberOfChannels(); 610 } 611 612 private: 613 explicit TestDeviceInputConsumerTrack(TrackRate aSampleRate) 614 : DeviceInputConsumerTrack(aSampleRate) {} 615 }; 616 617 TEST(TestAudioTrackGraph, DeviceChangedCallback) 618 { 619 MockCubeb* cubeb = new MockCubeb(); 620 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 621 622 MediaTrackGraph* graphImpl = MediaTrackGraphImpl::GetInstance( 623 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 624 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 625 nullptr, GetMainThreadSerialEventTarget()); 626 627 class TestAudioDataListener : public StrictMock<MockAudioDataListener> { 628 public: 629 TestAudioDataListener(uint32_t aChannelCount, bool aIsVoice) { 630 EXPECT_CALL(*this, RequestedInputChannelCount) 631 .WillRepeatedly(Return(aChannelCount)); 632 EXPECT_CALL(*this, RequestedInputProcessingParams) 633 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NONE)); 634 EXPECT_CALL(*this, IsVoiceInput).WillRepeatedly(Return(aIsVoice)); 635 { 636 InSequence s; 637 EXPECT_CALL(*this, DeviceChanged); 638 EXPECT_CALL(*this, Disconnect); 639 } 640 } 641 642 private: 643 ~TestAudioDataListener() = default; 644 }; 645 646 // Create a full-duplex AudioCallbackDriver by creating a NativeInputTrack. 647 const CubebUtils::AudioDeviceID device1 = (CubebUtils::AudioDeviceID)1; 648 RefPtr<TestAudioDataListener> listener1 = new TestAudioDataListener(1, false); 649 RefPtr<TestDeviceInputConsumerTrack> track1 = 650 TestDeviceInputConsumerTrack::Create(graphImpl); 651 track1->ConnectDeviceInput(device1, listener1.get(), PRINCIPAL_HANDLE_NONE); 652 653 EXPECT_TRUE(track1->ConnectedToNativeDevice()); 654 EXPECT_FALSE(track1->ConnectedToNonNativeDevice()); 655 auto started = 656 InvokeAsync([&] { return graphImpl->NotifyWhenDeviceStarted(nullptr); }); 657 RefPtr<SmartMockCubebStream> stream1 = WaitFor(cubeb->StreamInitEvent()); 658 EXPECT_TRUE(stream1->mHasInput); 659 EXPECT_TRUE(stream1->mHasOutput); 660 EXPECT_EQ(stream1->GetInputDeviceID(), device1); 661 (void)WaitFor(started); 662 663 // Create a NonNativeInputTrack, and make sure its DeviceChangeCallback works. 664 const CubebUtils::AudioDeviceID device2 = (CubebUtils::AudioDeviceID)2; 665 RefPtr<TestAudioDataListener> listener2 = new TestAudioDataListener(2, true); 666 RefPtr<TestDeviceInputConsumerTrack> track2 = 667 TestDeviceInputConsumerTrack::Create(graphImpl); 668 track2->ConnectDeviceInput(device2, listener2.get(), PRINCIPAL_HANDLE_NONE); 669 670 EXPECT_FALSE(track2->ConnectedToNativeDevice()); 671 EXPECT_TRUE(track2->ConnectedToNonNativeDevice()); 672 RefPtr<SmartMockCubebStream> stream2 = WaitFor(cubeb->StreamInitEvent()); 673 EXPECT_TRUE(stream2->mHasInput); 674 EXPECT_FALSE(stream2->mHasOutput); 675 EXPECT_EQ(stream2->GetInputDeviceID(), device2); 676 677 // Produce a device-changed event for the NonNativeInputTrack. 678 DispatchFunction([&] { stream2->ForceDeviceChanged(); }); 679 WaitFor(stream2->DeviceChangeForcedEvent()); 680 681 // Produce a device-changed event for the NativeInputTrack. 682 DispatchFunction([&] { stream1->ForceDeviceChanged(); }); 683 WaitFor(stream1->DeviceChangeForcedEvent()); 684 685 // Destroy the NonNativeInputTrack. 686 DispatchFunction([&] { 687 track2->DisconnectDeviceInput(); 688 track2->Destroy(); 689 }); 690 RefPtr<SmartMockCubebStream> destroyedStream = 691 WaitFor(cubeb->StreamDestroyEvent()); 692 EXPECT_EQ(destroyedStream.get(), stream2.get()); 693 694 // Destroy the NativeInputTrack. 695 DispatchFunction([&] { 696 track1->DisconnectDeviceInput(); 697 track1->Destroy(); 698 }); 699 destroyedStream = WaitFor(cubeb->StreamDestroyEvent()); 700 EXPECT_EQ(destroyedStream.get(), stream1.get()); 701 } 702 703 // The native audio stream (a.k.a. GraphDriver) and the non-native audio stream 704 // should always be the same as the max requested input channel of its paired 705 // DeviceInputTracks. This test checks if the audio stream paired with the 706 // DeviceInputTrack will follow the max requested input channel or not. 707 // 708 // The main focus for this test is to make sure DeviceInputTrack::OpenAudio and 709 // ::CloseAudio works as what we expect. Besides, This test also confirms 710 // MediaTrackGraph::ReevaluateInputDevice works correctly by using a 711 // test-only AudioDataListener. 712 // 713 // This test is pretty similar to RestartAudioIfProcessingMaxChannelCountChanged 714 // below, which tests the same thing but using AudioProcessingTrack. 715 // AudioProcessingTrack is the consumer of the DeviceInputTrack used in wild. 716 // It has its own customized AudioDataListener. However, it only tests when 717 // MOZ_WEBRTC is defined. 718 TEST(TestAudioTrackGraph, RestartAudioIfMaxChannelCountChanged) 719 { 720 MockCubeb* cubeb = new MockCubeb(); 721 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 722 auto unforcer = WaitFor(cubeb->ForceAudioThread()).unwrap(); 723 (void)unforcer; 724 725 MediaTrackGraph* graphImpl = MediaTrackGraphImpl::GetInstance( 726 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 727 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 728 nullptr, GetMainThreadSerialEventTarget()); 729 730 // A test-only AudioDataListener that simulates AudioInputProcessing's setter 731 // and getter for the input channel count. 732 class TestAudioDataListener : public StrictMock<MockAudioDataListener> { 733 public: 734 TestAudioDataListener(uint32_t aChannelCount, bool aIsVoice) { 735 EXPECT_CALL(*this, RequestedInputChannelCount) 736 .WillRepeatedly(Return(aChannelCount)); 737 EXPECT_CALL(*this, RequestedInputProcessingParams) 738 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NONE)); 739 EXPECT_CALL(*this, IsVoiceInput).WillRepeatedly(Return(aIsVoice)); 740 EXPECT_CALL(*this, Disconnect); 741 } 742 // Main thread API 743 void SetInputChannelCount(MediaTrackGraph* aGraph, 744 CubebUtils::AudioDeviceID aDevice, 745 uint32_t aChannelCount) { 746 MOZ_RELEASE_ASSERT(NS_IsMainThread()); 747 static_cast<MediaTrackGraphImpl*>(aGraph) 748 ->QueueControlMessageWithNoShutdown( 749 [this, self = RefPtr(this), aGraph, aDevice, aChannelCount] { 750 EXPECT_CALL(*this, RequestedInputChannelCount) 751 .WillRepeatedly(Return(aChannelCount)); 752 aGraph->ReevaluateInputDevice(aDevice); 753 }); 754 } 755 756 private: 757 ~TestAudioDataListener() = default; 758 }; 759 760 // Request a new input channel count and expect to have a new stream. 761 auto setNewChannelCount = [&](const RefPtr<TestAudioDataListener>& aListener, 762 RefPtr<SmartMockCubebStream>& aStream, 763 uint32_t aChannelCount) { 764 ASSERT_TRUE(!!aListener); 765 ASSERT_TRUE(!!aStream); 766 ASSERT_TRUE(aStream->mHasInput); 767 ASSERT_NE(aChannelCount, 0U); 768 769 const CubebUtils::AudioDeviceID device = aStream->GetInputDeviceID(); 770 771 bool destroyed = false; 772 MediaEventListener destroyListener = cubeb->StreamDestroyEvent().Connect( 773 AbstractThread::GetCurrent(), 774 [&](const RefPtr<SmartMockCubebStream>& aDestroyed) { 775 destroyed = aDestroyed.get() == aStream.get(); 776 }); 777 778 RefPtr<SmartMockCubebStream> newStream; 779 MediaEventListener restartListener = cubeb->StreamInitEvent().Connect( 780 AbstractThread::GetCurrent(), 781 [&](const RefPtr<SmartMockCubebStream>& aCreated) { 782 newStream = aCreated; 783 }); 784 785 DispatchFunction([&] { 786 aListener->SetInputChannelCount(graphImpl, device, aChannelCount); 787 }); 788 789 SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( 790 "TEST(TestAudioTrackGraph, RestartAudioIfMaxChannelCountChanged) #1"_ns, 791 [&] { return destroyed && newStream; }); 792 793 destroyListener.Disconnect(); 794 restartListener.Disconnect(); 795 796 aStream = newStream; 797 }; 798 799 // Open a new track and expect to have a new stream. 800 auto openTrack = [&](RefPtr<SmartMockCubebStream>& aCurrentStream, 801 RefPtr<TestDeviceInputConsumerTrack>& aTrack, 802 const RefPtr<TestAudioDataListener>& aListener, 803 CubebUtils::AudioDeviceID aDevice) { 804 ASSERT_TRUE(!!aCurrentStream); 805 ASSERT_TRUE(aCurrentStream->mHasInput); 806 ASSERT_TRUE(!aTrack); 807 ASSERT_TRUE(!!aListener); 808 809 bool destroyed = false; 810 MediaEventListener destroyListener = cubeb->StreamDestroyEvent().Connect( 811 AbstractThread::GetCurrent(), 812 [&](const RefPtr<SmartMockCubebStream>& aDestroyed) { 813 destroyed = aDestroyed.get() == aCurrentStream.get(); 814 }); 815 816 RefPtr<SmartMockCubebStream> newStream; 817 MediaEventListener restartListener = cubeb->StreamInitEvent().Connect( 818 AbstractThread::GetCurrent(), 819 [&](const RefPtr<SmartMockCubebStream>& aCreated) { 820 newStream = aCreated; 821 }); 822 823 aTrack = TestDeviceInputConsumerTrack::Create(graphImpl); 824 aTrack->ConnectDeviceInput(aDevice, aListener.get(), PRINCIPAL_HANDLE_NONE); 825 826 SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( 827 "TEST(TestAudioTrackGraph, RestartAudioIfMaxChannelCountChanged) #2"_ns, 828 [&] { return destroyed && newStream; }); 829 830 destroyListener.Disconnect(); 831 restartListener.Disconnect(); 832 833 aCurrentStream = newStream; 834 }; 835 836 // Test for the native input device first then non-native device. The 837 // non-native device will be destroyed before the native device in case of 838 // causing a driver switching. 839 840 // Test for the native device. 841 const CubebUtils::AudioDeviceID nativeDevice = (CubebUtils::AudioDeviceID)1; 842 RefPtr<TestDeviceInputConsumerTrack> track1; 843 RefPtr<TestAudioDataListener> listener1; 844 RefPtr<SmartMockCubebStream> nativeStream; 845 RefPtr<TestDeviceInputConsumerTrack> track2; 846 RefPtr<TestAudioDataListener> listener2; 847 { 848 // Open a 1-channel NativeInputTrack. 849 listener1 = new TestAudioDataListener(1, false); 850 track1 = TestDeviceInputConsumerTrack::Create(graphImpl); 851 track1->ConnectDeviceInput(nativeDevice, listener1.get(), 852 PRINCIPAL_HANDLE_NONE); 853 854 EXPECT_TRUE(track1->ConnectedToNativeDevice()); 855 EXPECT_FALSE(track1->ConnectedToNonNativeDevice()); 856 auto started = InvokeAsync( 857 [&] { return graphImpl->NotifyWhenDeviceStarted(nullptr); }); 858 nativeStream = WaitFor(cubeb->StreamInitEvent()); 859 EXPECT_TRUE(nativeStream->mHasInput); 860 EXPECT_TRUE(nativeStream->mHasOutput); 861 EXPECT_EQ(nativeStream->GetInputDeviceID(), nativeDevice); 862 (void)WaitFor(started); 863 864 // Open a 2-channel NativeInputTrack and wait for a new driver since the 865 // max-channel for the native device becomes 2 now. 866 listener2 = new TestAudioDataListener(2, false); 867 openTrack(nativeStream, track2, listener2, nativeDevice); 868 EXPECT_EQ(nativeStream->InputChannels(), 2U); 869 870 // Set the second NativeInputTrack to 1-channel and wait for a new driver 871 // since the max-channel for the native device becomes 1 now. 872 setNewChannelCount(listener2, nativeStream, 1); 873 EXPECT_EQ(nativeStream->InputChannels(), 1U); 874 875 // Set the first NativeInputTrack to 2-channel and wait for a new driver 876 // since the max input channel for the native device becomes 2 now. 877 setNewChannelCount(listener1, nativeStream, 2); 878 EXPECT_EQ(nativeStream->InputChannels(), 2U); 879 } 880 881 // Test for the non-native device. 882 { 883 const CubebUtils::AudioDeviceID nonNativeDevice = 884 (CubebUtils::AudioDeviceID)2; 885 886 // Open a 1-channel NonNativeInputTrack. 887 RefPtr<TestAudioDataListener> listener3 = 888 new TestAudioDataListener(1, false); 889 RefPtr<TestDeviceInputConsumerTrack> track3 = 890 TestDeviceInputConsumerTrack::Create(graphImpl); 891 track3->ConnectDeviceInput(nonNativeDevice, listener3.get(), 892 PRINCIPAL_HANDLE_NONE); 893 EXPECT_FALSE(track3->ConnectedToNativeDevice()); 894 EXPECT_TRUE(track3->ConnectedToNonNativeDevice()); 895 896 RefPtr<SmartMockCubebStream> nonNativeStream = 897 WaitFor(cubeb->StreamInitEvent()); 898 EXPECT_TRUE(nonNativeStream->mHasInput); 899 EXPECT_FALSE(nonNativeStream->mHasOutput); 900 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), nonNativeDevice); 901 EXPECT_EQ(nonNativeStream->InputChannels(), 1U); 902 903 // Open a 2-channel NonNativeInputTrack and wait for a new stream since 904 // the max-channel for the non-native device becomes 2 now. 905 RefPtr<TestAudioDataListener> listener4 = 906 new TestAudioDataListener(2, false); 907 RefPtr<TestDeviceInputConsumerTrack> track4; 908 openTrack(nonNativeStream, track4, listener4, nonNativeDevice); 909 EXPECT_EQ(nonNativeStream->InputChannels(), 2U); 910 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), nonNativeDevice); 911 912 // Set the second NonNativeInputTrack to 1-channel and wait for a new 913 // driver since the max-channel for the non-native device becomes 1 now. 914 setNewChannelCount(listener4, nonNativeStream, 1); 915 EXPECT_EQ(nonNativeStream->InputChannels(), 1U); 916 917 // Set the first NonNativeInputTrack to 2-channel and wait for a new 918 // driver since the max input channel for the non-native device becomes 2 919 // now. 920 setNewChannelCount(listener3, nonNativeStream, 2); 921 EXPECT_EQ(nonNativeStream->InputChannels(), 2U); 922 923 // Close the second NonNativeInputTrack (1-channel) then the first one 924 // (2-channel) so we won't result in another stream creation. 925 DispatchFunction([&] { 926 track4->DisconnectDeviceInput(); 927 track4->Destroy(); 928 }); 929 DispatchFunction([&] { 930 track3->DisconnectDeviceInput(); 931 track3->Destroy(); 932 }); 933 RefPtr<SmartMockCubebStream> destroyedStream = 934 WaitFor(cubeb->StreamDestroyEvent()); 935 EXPECT_EQ(destroyedStream.get(), nonNativeStream.get()); 936 } 937 938 // Tear down for the native device. 939 { 940 // Close the second NativeInputTrack (1-channel) then the first one 941 // (2-channel) so we won't have driver switching. 942 DispatchFunction([&] { 943 track2->DisconnectDeviceInput(); 944 track2->Destroy(); 945 }); 946 DispatchFunction([&] { 947 track1->DisconnectDeviceInput(); 948 track1->Destroy(); 949 }); 950 RefPtr<SmartMockCubebStream> destroyedStream = 951 WaitFor(cubeb->StreamDestroyEvent()); 952 EXPECT_EQ(destroyedStream.get(), nativeStream.get()); 953 } 954 } 955 956 // This test is pretty similar to SwitchNativeAudioProcessingTrack below, which 957 // tests the same thing but using AudioProcessingTrack. AudioProcessingTrack is 958 // the consumer of the DeviceInputTrack used in wild. It has its own customized 959 // AudioDataListener. However, it only tests when MOZ_WEBRTC is defined. 960 TEST(TestAudioTrackGraph, SwitchNativeInputDevice) 961 { 962 class TestAudioDataListener : public StrictMock<MockAudioDataListener> { 963 public: 964 TestAudioDataListener(uint32_t aChannelCount, bool aIsVoice) { 965 EXPECT_CALL(*this, RequestedInputChannelCount) 966 .WillRepeatedly(Return(aChannelCount)); 967 EXPECT_CALL(*this, RequestedInputProcessingParams) 968 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NONE)); 969 EXPECT_CALL(*this, IsVoiceInput).WillRepeatedly(Return(aIsVoice)); 970 } 971 972 private: 973 ~TestAudioDataListener() = default; 974 }; 975 976 MockCubeb* cubeb = new MockCubeb(); 977 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 978 979 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 980 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 981 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 982 nullptr, GetMainThreadSerialEventTarget()); 983 984 auto switchNativeDevice = 985 [&](RefPtr<SmartMockCubebStream>&& aCurrentNativeStream, 986 RefPtr<TestDeviceInputConsumerTrack>& aCurrentNativeTrack, 987 RefPtr<SmartMockCubebStream>& aNextNativeStream, 988 RefPtr<TestDeviceInputConsumerTrack>& aNextNativeTrack) { 989 ASSERT_TRUE(aCurrentNativeStream->mHasInput); 990 ASSERT_TRUE(aCurrentNativeStream->mHasOutput); 991 ASSERT_TRUE(aNextNativeStream->mHasInput); 992 ASSERT_FALSE(aNextNativeStream->mHasOutput); 993 994 std::cerr << "Switching native input from device " 995 << aCurrentNativeStream->GetInputDeviceID() << " to " 996 << aNextNativeStream->GetInputDeviceID() << std::endl; 997 998 uint32_t destroyed = 0; 999 MediaEventListener destroyListener = 1000 cubeb->StreamDestroyEvent().Connect( 1001 AbstractThread::GetCurrent(), 1002 [&](const RefPtr<SmartMockCubebStream>& aDestroyed) { 1003 if (aDestroyed.get() == aCurrentNativeStream.get() || 1004 aDestroyed.get() == aNextNativeStream.get()) { 1005 std::cerr << "cubeb stream " << aDestroyed.get() 1006 << " (device " << aDestroyed->GetInputDeviceID() 1007 << ") has been destroyed" << std::endl; 1008 destroyed += 1; 1009 } 1010 }); 1011 1012 RefPtr<SmartMockCubebStream> newStream; 1013 MediaEventListener restartListener = cubeb->StreamInitEvent().Connect( 1014 AbstractThread::GetCurrent(), 1015 [&](const RefPtr<SmartMockCubebStream>& aCreated) { 1016 // Make sure new stream has input, to prevent from getting a 1017 // temporary output-only AudioCallbackDriver after closing current 1018 // native device but before setting a new native input. 1019 if (aCreated->mHasInput) { 1020 ASSERT_TRUE(aCreated->mHasOutput); 1021 newStream = aCreated; 1022 } 1023 }); 1024 1025 std::cerr << "Close device " << aCurrentNativeStream->GetInputDeviceID() 1026 << std::endl; 1027 DispatchFunction([&] { 1028 aCurrentNativeTrack->DisconnectDeviceInput(); 1029 aCurrentNativeTrack->Destroy(); 1030 }); 1031 1032 std::cerr << "Wait for the switching" << std::endl; 1033 SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( 1034 "TEST(TestAudioTrackGraph, SwitchNativeInputDevice)"_ns, 1035 [&] { return destroyed >= 2 && newStream; }); 1036 1037 destroyListener.Disconnect(); 1038 restartListener.Disconnect(); 1039 1040 aCurrentNativeStream = nullptr; 1041 aNextNativeStream = newStream; 1042 1043 std::cerr << "Now the native input is device " 1044 << aNextNativeStream->GetInputDeviceID() << std::endl; 1045 }; 1046 1047 // Open a DeviceInputConsumerTrack for device 1. 1048 const CubebUtils::AudioDeviceID device1 = (CubebUtils::AudioDeviceID)1; 1049 RefPtr<TestDeviceInputConsumerTrack> track1 = 1050 TestDeviceInputConsumerTrack::Create(graph); 1051 RefPtr<TestAudioDataListener> listener1 = new TestAudioDataListener(1, false); 1052 EXPECT_CALL(*listener1, Disconnect); 1053 track1->ConnectDeviceInput(device1, listener1, PRINCIPAL_HANDLE_NONE); 1054 EXPECT_EQ(track1->DeviceId().value(), device1); 1055 1056 auto started = 1057 InvokeAsync([&] { return graph->NotifyWhenDeviceStarted(nullptr); }); 1058 1059 RefPtr<SmartMockCubebStream> stream1 = WaitFor(cubeb->StreamInitEvent()); 1060 EXPECT_TRUE(stream1->mHasInput); 1061 EXPECT_TRUE(stream1->mHasOutput); 1062 EXPECT_EQ(stream1->InputChannels(), 1U); 1063 EXPECT_EQ(stream1->GetInputDeviceID(), device1); 1064 (void)WaitFor(started); 1065 std::cerr << "Device " << device1 << " is opened (stream " << stream1.get() 1066 << ")" << std::endl; 1067 1068 // Open a DeviceInputConsumerTrack for device 2. 1069 const CubebUtils::AudioDeviceID device2 = (CubebUtils::AudioDeviceID)2; 1070 RefPtr<TestDeviceInputConsumerTrack> track2 = 1071 TestDeviceInputConsumerTrack::Create(graph); 1072 RefPtr<TestAudioDataListener> listener2 = new TestAudioDataListener(2, false); 1073 EXPECT_CALL(*listener2, Disconnect).Times(2); 1074 track2->ConnectDeviceInput(device2, listener2, PRINCIPAL_HANDLE_NONE); 1075 EXPECT_EQ(track2->DeviceId().value(), device2); 1076 1077 RefPtr<SmartMockCubebStream> stream2 = WaitFor(cubeb->StreamInitEvent()); 1078 EXPECT_TRUE(stream2->mHasInput); 1079 EXPECT_FALSE(stream2->mHasOutput); 1080 EXPECT_EQ(stream2->InputChannels(), 2U); 1081 EXPECT_EQ(stream2->GetInputDeviceID(), device2); 1082 std::cerr << "Device " << device2 << " is opened (stream " << stream2.get() 1083 << ")" << std::endl; 1084 1085 // Open a DeviceInputConsumerTrack for device 3. 1086 const CubebUtils::AudioDeviceID device3 = (CubebUtils::AudioDeviceID)3; 1087 RefPtr<TestDeviceInputConsumerTrack> track3 = 1088 TestDeviceInputConsumerTrack::Create(graph); 1089 RefPtr<TestAudioDataListener> listener3 = new TestAudioDataListener(1, false); 1090 EXPECT_CALL(*listener3, Disconnect).Times(2); 1091 track3->ConnectDeviceInput(device3, listener3, PRINCIPAL_HANDLE_NONE); 1092 EXPECT_EQ(track3->DeviceId().value(), device3); 1093 1094 RefPtr<SmartMockCubebStream> stream3 = WaitFor(cubeb->StreamInitEvent()); 1095 EXPECT_TRUE(stream3->mHasInput); 1096 EXPECT_FALSE(stream3->mHasOutput); 1097 EXPECT_EQ(stream3->InputChannels(), 1U); 1098 EXPECT_EQ(stream3->GetInputDeviceID(), device3); 1099 std::cerr << "Device " << device3 << " is opened (stream " << stream3.get() 1100 << ")" << std::endl; 1101 1102 // Close device 1, so the native input device is switched from device 1 to 1103 // device 2. 1104 switchNativeDevice(std::move(stream1), track1, stream2, track2); 1105 EXPECT_TRUE(stream2->mHasInput); 1106 EXPECT_TRUE(stream2->mHasOutput); 1107 EXPECT_EQ(stream2->InputChannels(), 2U); 1108 EXPECT_EQ(stream2->GetInputDeviceID(), device2); 1109 { 1110 NativeInputTrack* native = track2->Graph()->GetNativeInputTrackMainThread(); 1111 ASSERT_TRUE(!!native); 1112 EXPECT_EQ(native->mDeviceId, device2); 1113 } 1114 1115 // Close device 2, so the native input device is switched from device 2 to 1116 // device 3. 1117 switchNativeDevice(std::move(stream2), track2, stream3, track3); 1118 EXPECT_TRUE(stream3->mHasInput); 1119 EXPECT_TRUE(stream3->mHasOutput); 1120 EXPECT_EQ(stream3->InputChannels(), 1U); 1121 EXPECT_EQ(stream3->GetInputDeviceID(), device3); 1122 { 1123 NativeInputTrack* native = track3->Graph()->GetNativeInputTrackMainThread(); 1124 ASSERT_TRUE(!!native); 1125 EXPECT_EQ(native->mDeviceId, device3); 1126 } 1127 1128 // Clean up. 1129 std::cerr << "Close device " << device3 << std::endl; 1130 DispatchFunction([&] { 1131 track3->DisconnectDeviceInput(); 1132 track3->Destroy(); 1133 }); 1134 RefPtr<SmartMockCubebStream> destroyedStream = 1135 WaitFor(cubeb->StreamDestroyEvent()); 1136 EXPECT_EQ(destroyedStream.get(), stream3.get()); 1137 { 1138 NativeInputTrack* native = graph->GetNativeInputTrackMainThread(); 1139 ASSERT_TRUE(!native); 1140 } 1141 std::cerr << "No native input now" << std::endl; 1142 } 1143 1144 #ifdef MOZ_WEBRTC 1145 TEST(TestAudioTrackGraph, ErrorCallback) 1146 { 1147 MockCubeb* cubeb = new MockCubeb(); 1148 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 1149 1150 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 1151 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 1152 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 1153 nullptr, GetMainThreadSerialEventTarget()); 1154 1155 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 1156 1157 // Dummy track to make graph rolling. Add it and remove it to remove the 1158 // graph from the global hash table and let it shutdown. 1159 // 1160 // We open an input through this track so that there's something triggering 1161 // EnsureNextIteration on the fallback driver after the callback driver has 1162 // gotten the error, and to check that a replacement cubeb_stream receives 1163 // output from the graph. 1164 RefPtr<AudioProcessingTrack> processingTrack; 1165 RefPtr<AudioInputProcessing> listener; 1166 auto started = InvokeAsync([&] { 1167 processingTrack = AudioProcessingTrack::Create(graph); 1168 listener = new AudioInputProcessing(2); 1169 QueueExpectIsPassThrough(processingTrack, listener); 1170 processingTrack->SetInputProcessing(listener); 1171 processingTrack->GraphImpl()->AppendMessage( 1172 MakeUnique<StartInputProcessing>(processingTrack, listener)); 1173 processingTrack->ConnectDeviceInput(deviceId, listener, 1174 PRINCIPAL_HANDLE_NONE); 1175 EXPECT_EQ(processingTrack->DeviceId().value(), deviceId); 1176 processingTrack->AddAudioOutput(reinterpret_cast<void*>(1), nullptr); 1177 return graph->NotifyWhenDeviceStarted(nullptr); 1178 }); 1179 1180 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 1181 Result<bool, nsresult> rv = WaitFor(started); 1182 EXPECT_TRUE(rv.unwrapOr(false)); 1183 1184 // Force a cubeb state_callback error and see that we don't crash. 1185 DispatchFunction([&] { stream->ForceError(); }); 1186 1187 // Wait for the error to take effect, and the driver to restart and receive 1188 // output. 1189 bool errored = false; 1190 MediaEventListener errorListener = stream->ErrorForcedEvent().Connect( 1191 AbstractThread::GetCurrent(), [&] { errored = true; }); 1192 stream = WaitFor(cubeb->StreamInitEvent()); 1193 WaitFor(stream->FramesVerifiedEvent()); 1194 // The error event is notified after CUBEB_STATE_ERROR triggers other 1195 // threads to init a new cubeb_stream, so there is a theoretical chance that 1196 // `errored` might not be set when `stream` is set. 1197 errorListener.Disconnect(); 1198 EXPECT_TRUE(errored); 1199 1200 // Clean up. 1201 DispatchFunction([&] { 1202 processingTrack->GraphImpl()->AppendMessage( 1203 MakeUnique<StopInputProcessing>(processingTrack, listener)); 1204 processingTrack->DisconnectDeviceInput(); 1205 processingTrack->Destroy(); 1206 }); 1207 WaitFor(cubeb->StreamDestroyEvent()); 1208 } 1209 1210 TEST(TestAudioTrackGraph, AudioProcessingTrack) 1211 { 1212 MockCubeb* cubeb = new MockCubeb(); 1213 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 1214 auto unforcer = WaitFor(cubeb->ForceAudioThread()).unwrap(); 1215 (void)unforcer; 1216 1217 // Start on a system clock driver, then switch to full-duplex in one go. If we 1218 // did output-then-full-duplex we'd risk a second NotifyWhenDeviceStarted 1219 // resolving early after checking the first audio driver only. 1220 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 1221 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 1222 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 1223 nullptr, GetMainThreadSerialEventTarget()); 1224 1225 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 1226 1227 RefPtr<AudioProcessingTrack> processingTrack; 1228 RefPtr<ProcessedMediaTrack> outputTrack; 1229 RefPtr<MediaInputPort> port; 1230 RefPtr<AudioInputProcessing> listener; 1231 auto p = InvokeAsync([&] { 1232 processingTrack = AudioProcessingTrack::Create(graph); 1233 outputTrack = graph->CreateForwardedInputTrack(MediaSegment::AUDIO); 1234 outputTrack->QueueSetAutoend(false); 1235 outputTrack->AddAudioOutput(reinterpret_cast<void*>(1), nullptr); 1236 port = outputTrack->AllocateInputPort(processingTrack); 1237 /* Primary graph: Open Audio Input through SourceMediaTrack */ 1238 listener = new AudioInputProcessing(2); 1239 QueueExpectIsPassThrough(processingTrack, listener); 1240 processingTrack->SetInputProcessing(listener); 1241 processingTrack->GraphImpl()->AppendMessage( 1242 MakeUnique<StartInputProcessing>(processingTrack, listener)); 1243 // Device id does not matter. Ignore. 1244 processingTrack->ConnectDeviceInput(deviceId, listener, 1245 PRINCIPAL_HANDLE_NONE); 1246 return graph->NotifyWhenDeviceStarted(nullptr); 1247 }); 1248 1249 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 1250 EXPECT_TRUE(stream->mHasInput); 1251 (void)WaitFor(p); 1252 1253 // Wait for a second worth of audio data. GoFaster is dispatched through a 1254 // ControlMessage so that it is called in the first audio driver iteration. 1255 // Otherwise the audio driver might be going very fast while the fallback 1256 // system clock driver is still in an iteration. 1257 DispatchFunction([&] { 1258 processingTrack->GraphImpl()->AppendMessage(MakeUnique<GoFaster>(cubeb)); 1259 }); 1260 uint32_t totalFrames = 0; 1261 WaitUntil(stream->FramesVerifiedEvent(), [&](uint32_t aFrames) { 1262 totalFrames += aFrames; 1263 return totalFrames > static_cast<uint32_t>(graph->GraphRate()); 1264 }); 1265 cubeb->DontGoFaster(); 1266 1267 // Clean up. 1268 DispatchFunction([&] { 1269 outputTrack->RemoveAudioOutput((void*)1); 1270 outputTrack->Destroy(); 1271 port->Destroy(); 1272 processingTrack->GraphImpl()->AppendMessage( 1273 MakeUnique<StopInputProcessing>(processingTrack, listener)); 1274 processingTrack->DisconnectDeviceInput(); 1275 processingTrack->Destroy(); 1276 }); 1277 1278 uint32_t inputRate = stream->SampleRate(); 1279 uint32_t inputFrequency = stream->InputFrequency(); 1280 uint64_t preSilenceSamples; 1281 uint32_t estimatedFreq; 1282 uint32_t nrDiscontinuities; 1283 std::tie(preSilenceSamples, estimatedFreq, nrDiscontinuities) = 1284 WaitFor(stream->OutputVerificationEvent()); 1285 1286 EXPECT_EQ(estimatedFreq, inputFrequency); 1287 std::cerr << "PreSilence: " << preSilenceSamples << std::endl; 1288 // We buffer 128 frames. See NativeInputTrack::NotifyInputData. 1289 EXPECT_GE(preSilenceSamples, 128U); 1290 // If the fallback system clock driver is doing a graph iteration before the 1291 // first audio driver iteration comes in, that iteration is ignored and 1292 // results in zeros. It takes one fallback driver iteration *after* the audio 1293 // driver has started to complete the switch, *usually* resulting two 1294 // 10ms-iterations of silence; sometimes only one. 1295 EXPECT_LE(preSilenceSamples, 128U + 2 * inputRate / 100 /* 2*10ms */); 1296 // The waveform from AudioGenerator starts at 0, but we don't control its 1297 // ending, so we expect a discontinuity there. 1298 EXPECT_LE(nrDiscontinuities, 1U); 1299 } 1300 1301 TEST(TestAudioTrackGraph, ReConnectDeviceInput) 1302 { 1303 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); 1304 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 1305 1306 // 48k is a native processing rate, and avoids a resampling pass compared 1307 // to 44.1k. The resampler may add take a few frames to stabilize, which show 1308 // as unexected discontinuities in the test. 1309 const TrackRate rate = 48000; 1310 // Use a drift factor so that we don't dont produce perfect 10ms-chunks. 1311 // This will exercise whatever buffers are in the audio processing pipeline, 1312 // and the bookkeeping surrounding them. 1313 const long step = 10 * rate * 1111 / 1000 / PR_MSEC_PER_SEC; 1314 1315 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 1316 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, rate, nullptr, 1317 GetMainThreadSerialEventTarget()); 1318 1319 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 1320 1321 RefPtr<AudioProcessingTrack> processingTrack; 1322 RefPtr<ProcessedMediaTrack> outputTrack; 1323 RefPtr<MediaInputPort> port; 1324 RefPtr<AudioInputProcessing> listener; 1325 RefPtr<OnFallbackListener> fallbackListener; 1326 DispatchFunction([&] { 1327 processingTrack = AudioProcessingTrack::Create(graph); 1328 outputTrack = graph->CreateForwardedInputTrack(MediaSegment::AUDIO); 1329 outputTrack->QueueSetAutoend(false); 1330 outputTrack->AddAudioOutput(reinterpret_cast<void*>(1), nullptr); 1331 port = outputTrack->AllocateInputPort(processingTrack); 1332 1333 const int32_t channelCount = 2; 1334 listener = new AudioInputProcessing(channelCount); 1335 processingTrack->SetInputProcessing(listener); 1336 processingTrack->GraphImpl()->AppendMessage( 1337 MakeUnique<StartInputProcessing>(processingTrack, listener)); 1338 processingTrack->ConnectDeviceInput(deviceId, listener, 1339 PRINCIPAL_HANDLE_NONE); 1340 MediaEnginePrefs settings; 1341 settings.mChannels = channelCount; 1342 settings.mAgcOn = true; // Turn off pass-through. 1343 // AGC1 Mode 0 interferes with AudioVerifier's frequency estimation 1344 // through zero-crossing counts. 1345 settings.mAgc2Forced = true; 1346 QueueApplySettings(processingTrack, listener, settings); 1347 1348 fallbackListener = new OnFallbackListener(processingTrack); 1349 processingTrack->AddListener(fallbackListener); 1350 }); 1351 1352 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 1353 EXPECT_TRUE(stream->mHasInput); 1354 1355 while ( 1356 stream->State() 1357 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 1358 .valueOr(true)) { 1359 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1360 } 1361 1362 // Wait for the AudioCallbackDriver to come into effect. 1363 while (fallbackListener->OnFallback()) { 1364 EXPECT_EQ(stream->ManualDataCallback(0), 1365 MockCubebStream::KeepProcessing::Yes); 1366 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1367 } 1368 1369 // Iterate for a second worth of audio data. 1370 for (long frames = 0; frames < graph->GraphRate(); frames += step) { 1371 stream->ManualDataCallback(step); 1372 } 1373 1374 // Close the input to see that no asserts go off due to bad state. 1375 DispatchFunction([&] { processingTrack->DisconnectDeviceInput(); }); 1376 1377 // Dispatch the disconnect message. 1378 ProcessEventQueue(); 1379 // Run the disconnect message and switch driver. 1380 auto initPromise = TakeN(cubeb->StreamInitEvent(), 1); 1381 EXPECT_EQ(stream->ManualDataCallback(0), MockCubebStream::KeepProcessing::No); 1382 std::tie(stream) = WaitFor(initPromise).unwrap()[0]; 1383 EXPECT_FALSE(stream->mHasInput); 1384 1385 while ( 1386 stream->State() 1387 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 1388 .valueOr(true)) { 1389 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1390 } 1391 1392 // Wait for the new AudioCallbackDriver to come into effect. 1393 fallbackListener->Reset(); 1394 while (fallbackListener->OnFallback()) { 1395 EXPECT_EQ(stream->ManualDataCallback(0), 1396 MockCubebStream::KeepProcessing::Yes); 1397 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1398 } 1399 1400 // Output-only. Iterate for another second before unmuting. 1401 for (long frames = 0; frames < graph->GraphRate(); frames += step) { 1402 stream->ManualDataCallback(step); 1403 } 1404 1405 // Re-open the input to again see that no asserts go off due to bad state. 1406 DispatchFunction([&] { 1407 // Device id does not matter. Ignore. 1408 processingTrack->ConnectDeviceInput(deviceId, listener, 1409 PRINCIPAL_HANDLE_NONE); 1410 }); 1411 // Dispatch the connect message. 1412 ProcessEventQueue(); 1413 // Run the connect message and switch driver. 1414 initPromise = TakeN(cubeb->StreamInitEvent(), 1); 1415 EXPECT_EQ(stream->ManualDataCallback(0), MockCubebStream::KeepProcessing::No); 1416 std::tie(stream) = WaitFor(initPromise).unwrap()[0]; 1417 EXPECT_TRUE(stream->mHasInput); 1418 1419 while ( 1420 stream->State() 1421 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 1422 .valueOr(true)) { 1423 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1424 } 1425 1426 // Wait for the new AudioCallbackDriver to come into effect. 1427 fallbackListener->Reset(); 1428 while (fallbackListener->OnFallback()) { 1429 EXPECT_EQ(stream->ManualDataCallback(0), 1430 MockCubebStream::KeepProcessing::Yes); 1431 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1432 } 1433 1434 // Full-duplex. Iterate for another second before finishing. 1435 for (long frames = 0; frames < graph->GraphRate(); frames += step) { 1436 stream->ManualDataCallback(step); 1437 } 1438 1439 // Clean up. 1440 DispatchFunction([&] { 1441 processingTrack->RemoveListener(fallbackListener); 1442 outputTrack->RemoveAudioOutput((void*)1); 1443 outputTrack->Destroy(); 1444 port->Destroy(); 1445 processingTrack->GraphImpl()->AppendMessage( 1446 MakeUnique<StopInputProcessing>(processingTrack, listener)); 1447 processingTrack->DisconnectDeviceInput(); 1448 processingTrack->Destroy(); 1449 }); 1450 1451 // Dispatch the clean-up messages. 1452 ProcessEventQueue(); 1453 // Run the clean-up messages and shut down driver. 1454 EXPECT_EQ(stream->ManualDataCallback(0), MockCubebStream::KeepProcessing::No); 1455 1456 uint32_t inputFrequency = stream->InputFrequency(); 1457 uint64_t preSilenceSamples; 1458 uint32_t estimatedFreq; 1459 uint32_t nrDiscontinuities; 1460 std::tie(preSilenceSamples, estimatedFreq, nrDiscontinuities) = 1461 WaitFor(stream->OutputVerificationEvent()); 1462 1463 EXPECT_EQ(estimatedFreq, inputFrequency); 1464 std::cerr << "PreSilence: " << preSilenceSamples << "\n"; 1465 // We buffer 128 frames. See NativeInputTrack::NotifyInputData. 1466 // When not in passthrough the AudioInputProcessing packetizer also buffers 1467 // 10ms of silence, pulled in from NativeInputTrack when being run by the 1468 // fallback SystemClockDriver. 1469 EXPECT_EQ(preSilenceSamples, WEBAUDIO_BLOCK_SIZE + rate / 100); 1470 // The waveform from AudioGenerator starts at 0, but we don't control its 1471 // ending, so we expect a discontinuity there. Note that this check is only 1472 // for the waveform on the stream *after* re-opening the input. 1473 EXPECT_LE(nrDiscontinuities, 1U); 1474 } 1475 1476 // Sum the signal to mono and compute the root mean square, in float32, 1477 // regardless of the input format. 1478 float rmsf32(AudioDataValue* aSamples, uint32_t aChannels, uint32_t aFrames) { 1479 float downmixed; 1480 float rms = 0.; 1481 uint32_t readIdx = 0; 1482 for (uint32_t i = 0; i < aFrames; i++) { 1483 downmixed = 0.; 1484 for (uint32_t j = 0; j < aChannels; j++) { 1485 downmixed += ConvertAudioSample<float>(aSamples[readIdx++]); 1486 } 1487 rms += downmixed * downmixed; 1488 } 1489 rms = rms / aFrames; 1490 return sqrt(rms); 1491 } 1492 1493 TEST(TestAudioTrackGraph, AudioProcessingTrackDisabling) 1494 { 1495 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); 1496 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 1497 1498 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 1499 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 1500 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 1501 nullptr, GetMainThreadSerialEventTarget()); 1502 1503 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 1504 1505 RefPtr<AudioProcessingTrack> processingTrack; 1506 RefPtr<ProcessedMediaTrack> outputTrack; 1507 RefPtr<MediaInputPort> port; 1508 RefPtr<AudioInputProcessing> listener; 1509 RefPtr<OnFallbackListener> fallbackListener; 1510 DispatchFunction([&] { 1511 processingTrack = AudioProcessingTrack::Create(graph); 1512 outputTrack = graph->CreateForwardedInputTrack(MediaSegment::AUDIO); 1513 outputTrack->QueueSetAutoend(false); 1514 outputTrack->AddAudioOutput(reinterpret_cast<void*>(1), nullptr); 1515 port = outputTrack->AllocateInputPort(processingTrack); 1516 /* Primary graph: Open Audio Input through SourceMediaTrack */ 1517 listener = new AudioInputProcessing(2); 1518 QueueExpectIsPassThrough(processingTrack, listener); 1519 processingTrack->SetInputProcessing(listener); 1520 processingTrack->ConnectDeviceInput(deviceId, listener, 1521 PRINCIPAL_HANDLE_NONE); 1522 processingTrack->GraphImpl()->AppendMessage( 1523 MakeUnique<StartInputProcessing>(processingTrack, listener)); 1524 fallbackListener = new OnFallbackListener(processingTrack); 1525 processingTrack->AddListener(fallbackListener); 1526 }); 1527 1528 ProcessEventQueue(); 1529 1530 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 1531 EXPECT_TRUE(stream->mHasInput); 1532 1533 while ( 1534 stream->State() 1535 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 1536 .valueOr(true)) { 1537 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1538 } 1539 1540 // Wait for the AudioCallbackDriver to come into effect. 1541 while (fallbackListener->OnFallback()) { 1542 EXPECT_EQ(stream->ManualDataCallback(0), 1543 MockCubebStream::KeepProcessing::Yes); 1544 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 1545 } 1546 1547 stream->SetOutputRecordingEnabled(true); 1548 1549 // Wait for a second worth of audio data. 1550 const long step = graph->GraphRate() / 100; // 10ms 1551 for (long frames = 0; frames < graph->GraphRate(); frames += step) { 1552 stream->ManualDataCallback(step); 1553 } 1554 1555 const uint32_t ITERATION_COUNT = 5; 1556 uint32_t iterations = ITERATION_COUNT; 1557 DisabledTrackMode nextMode = DisabledTrackMode::SILENCE_BLACK; 1558 while (iterations--) { 1559 // toggle the track enabled mode, wait a second, do this ITERATION_COUNT 1560 // times 1561 DispatchFunction([&] { 1562 processingTrack->SetDisabledTrackMode(nextMode); 1563 if (nextMode == DisabledTrackMode::SILENCE_BLACK) { 1564 nextMode = DisabledTrackMode::ENABLED; 1565 } else { 1566 nextMode = DisabledTrackMode::SILENCE_BLACK; 1567 } 1568 }); 1569 1570 ProcessEventQueue(); 1571 1572 for (long frames = 0; frames < graph->GraphRate(); frames += step) { 1573 stream->ManualDataCallback(step); 1574 } 1575 } 1576 1577 // Clean up. 1578 DispatchFunction([&] { 1579 outputTrack->RemoveAudioOutput((void*)1); 1580 outputTrack->Destroy(); 1581 port->Destroy(); 1582 processingTrack->GraphImpl()->AppendMessage( 1583 MakeUnique<StopInputProcessing>(processingTrack, listener)); 1584 processingTrack->RemoveListener(fallbackListener); 1585 processingTrack->DisconnectDeviceInput(); 1586 processingTrack->Destroy(); 1587 }); 1588 1589 ProcessEventQueue(); 1590 1591 // Close the input and switch driver. 1592 while (stream->ManualDataCallback(0) != MockCubebStream::KeepProcessing::No) { 1593 std::cerr << "Waiting for switch...\n"; 1594 } 1595 1596 uint64_t preSilenceSamples; 1597 uint32_t estimatedFreq; 1598 uint32_t nrDiscontinuities; 1599 std::tie(preSilenceSamples, estimatedFreq, nrDiscontinuities) = 1600 WaitFor(stream->OutputVerificationEvent()); 1601 1602 auto data = stream->TakeRecordedOutput(); 1603 1604 // check that there is non-silence and silence at the expected time in the 1605 // stereo recording, while allowing for a bit of scheduling uncertainty, by 1606 // checking half a second after the theoretical muting/unmuting. 1607 // non-silence starts around: 0s, 2s, 4s 1608 // silence start around: 1s, 3s, 5s 1609 // To detect silence or non-silence, we compute the RMS of the signal for 1610 // 100ms. 1611 float noisyTime_s[] = {0.5, 2.5, 4.5}; 1612 float silenceTime_s[] = {1.5, 3.5, 5.5}; 1613 1614 uint32_t rate = graph->GraphRate(); 1615 for (float& time : noisyTime_s) { 1616 uint32_t startIdx = time * rate * 2 /* stereo */; 1617 EXPECT_NE(rmsf32(&(data[startIdx]), 2, rate / 10), 0.0); 1618 } 1619 1620 for (float& time : silenceTime_s) { 1621 uint32_t startIdx = time * rate * 2 /* stereo */; 1622 EXPECT_EQ(rmsf32(&(data[startIdx]), 2, rate / 10), 0.0); 1623 } 1624 } 1625 1626 TEST(TestAudioTrackGraph, SetRequestedInputChannelCount) 1627 { 1628 MockCubeb* cubeb = new MockCubeb(); 1629 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 1630 1631 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 1632 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 1633 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 1634 nullptr, GetMainThreadSerialEventTarget()); 1635 1636 // Open a 2-channel native input stream. 1637 const CubebUtils::AudioDeviceID device1 = (CubebUtils::AudioDeviceID)1; 1638 RefPtr<AudioProcessingTrack> track1 = AudioProcessingTrack::Create(graph); 1639 RefPtr<AudioInputProcessing> listener1 = new AudioInputProcessing(2); 1640 track1->SetInputProcessing(listener1); 1641 QueueExpectIsPassThrough(track1, listener1); 1642 track1->GraphImpl()->AppendMessage( 1643 MakeUnique<StartInputProcessing>(track1, listener1)); 1644 track1->ConnectDeviceInput(device1, listener1, PRINCIPAL_HANDLE_NONE); 1645 EXPECT_EQ(track1->DeviceId().value(), device1); 1646 1647 auto started = 1648 InvokeAsync([&] { return graph->NotifyWhenDeviceStarted(nullptr); }); 1649 1650 RefPtr<SmartMockCubebStream> stream1 = WaitFor(cubeb->StreamInitEvent()); 1651 EXPECT_TRUE(stream1->mHasInput); 1652 EXPECT_TRUE(stream1->mHasOutput); 1653 EXPECT_EQ(stream1->InputChannels(), 2U); 1654 EXPECT_EQ(stream1->GetInputDeviceID(), device1); 1655 (void)WaitFor(started); 1656 1657 // Open a 1-channel non-native input stream. 1658 const CubebUtils::AudioDeviceID device2 = (CubebUtils::AudioDeviceID)2; 1659 RefPtr<AudioProcessingTrack> track2 = AudioProcessingTrack::Create(graph); 1660 RefPtr<AudioInputProcessing> listener2 = new AudioInputProcessing(1); 1661 track2->SetInputProcessing(listener2); 1662 QueueExpectIsPassThrough(track2, listener2); 1663 track2->GraphImpl()->AppendMessage( 1664 MakeUnique<StartInputProcessing>(track2, listener2)); 1665 track2->ConnectDeviceInput(device2, listener2, PRINCIPAL_HANDLE_NONE); 1666 EXPECT_EQ(track2->DeviceId().value(), device2); 1667 1668 RefPtr<SmartMockCubebStream> stream2 = WaitFor(cubeb->StreamInitEvent()); 1669 EXPECT_TRUE(stream2->mHasInput); 1670 EXPECT_FALSE(stream2->mHasOutput); 1671 EXPECT_EQ(stream2->InputChannels(), 1U); 1672 EXPECT_EQ(stream2->GetInputDeviceID(), device2); 1673 1674 // Request a new input channel count. This should re-create new input stream 1675 // accordingly. 1676 auto setNewChannelCount = [&](const RefPtr<AudioProcessingTrack> aTrack, 1677 const RefPtr<AudioInputProcessing>& aListener, 1678 RefPtr<SmartMockCubebStream>& aStream, 1679 int32_t aChannelCount) { 1680 bool destroyed = false; 1681 MediaEventListener destroyListener = cubeb->StreamDestroyEvent().Connect( 1682 AbstractThread::GetCurrent(), 1683 [&](const RefPtr<SmartMockCubebStream>& aDestroyed) { 1684 destroyed = aDestroyed.get() == aStream.get(); 1685 }); 1686 1687 RefPtr<SmartMockCubebStream> newStream; 1688 MediaEventListener restartListener = cubeb->StreamInitEvent().Connect( 1689 AbstractThread::GetCurrent(), 1690 [&](const RefPtr<SmartMockCubebStream>& aCreated) { 1691 newStream = aCreated; 1692 }); 1693 1694 MediaEnginePrefs settings; 1695 settings.mChannels = aChannelCount; 1696 QueueApplySettings(aTrack, aListener, settings); 1697 1698 SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( 1699 "TEST(TestAudioTrackGraph, SetRequestedInputChannelCount)"_ns, 1700 [&] { return destroyed && newStream; }); 1701 1702 destroyListener.Disconnect(); 1703 restartListener.Disconnect(); 1704 1705 aStream = newStream; 1706 }; 1707 1708 // Set the native input stream's input channel count to 1. 1709 setNewChannelCount(track1, listener1, stream1, 1); 1710 EXPECT_TRUE(stream1->mHasInput); 1711 EXPECT_TRUE(stream1->mHasOutput); 1712 EXPECT_EQ(stream1->InputChannels(), 1U); 1713 EXPECT_EQ(stream1->GetInputDeviceID(), device1); 1714 1715 // Set the non-native input stream's input channel count to 2. 1716 setNewChannelCount(track2, listener2, stream2, 2); 1717 EXPECT_TRUE(stream2->mHasInput); 1718 EXPECT_FALSE(stream2->mHasOutput); 1719 EXPECT_EQ(stream2->InputChannels(), 2U); 1720 EXPECT_EQ(stream2->GetInputDeviceID(), device2); 1721 1722 // Close the non-native input stream. 1723 DispatchFunction([&] { 1724 track2->GraphImpl()->AppendMessage( 1725 MakeUnique<StopInputProcessing>(track2, listener2)); 1726 track2->DisconnectDeviceInput(); 1727 track2->Destroy(); 1728 }); 1729 RefPtr<SmartMockCubebStream> destroyed = WaitFor(cubeb->StreamDestroyEvent()); 1730 EXPECT_EQ(destroyed.get(), stream2.get()); 1731 1732 // Close the native input stream. 1733 DispatchFunction([&] { 1734 track1->GraphImpl()->AppendMessage( 1735 MakeUnique<StopInputProcessing>(track1, listener1)); 1736 track1->DisconnectDeviceInput(); 1737 track1->Destroy(); 1738 }); 1739 destroyed = WaitFor(cubeb->StreamDestroyEvent()); 1740 EXPECT_EQ(destroyed.get(), stream1.get()); 1741 } 1742 1743 // The native audio stream (a.k.a. GraphDriver) and the non-native audio stream 1744 // should always be the same as the max requested input channel of its paired 1745 // AudioProcessingTracks. This test checks if the audio stream paired with the 1746 // AudioProcessingTrack will follow the max requested input channel or not. 1747 // 1748 // This test is pretty similar to RestartAudioIfMaxChannelCountChanged above, 1749 // which makes sure the related DeviceInputTrack operations for the test here 1750 // works correctly. Instead of using a test-only AudioDataListener, we use 1751 // AudioInputProcessing here to simulate the real world use case. 1752 TEST(TestAudioTrackGraph, RestartAudioIfProcessingMaxChannelCountChanged) 1753 { 1754 MockCubeb* cubeb = new MockCubeb(); 1755 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 1756 auto unforcer = WaitFor(cubeb->ForceAudioThread()).unwrap(); 1757 (void)unforcer; 1758 1759 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 1760 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 1761 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 1762 nullptr, GetMainThreadSerialEventTarget()); 1763 1764 // Request a new input channel count and expect to have a new stream. 1765 auto setNewChannelCount = [&](const RefPtr<AudioProcessingTrack>& aTrack, 1766 const RefPtr<AudioInputProcessing>& aListener, 1767 RefPtr<SmartMockCubebStream>& aStream, 1768 int32_t aChannelCount) { 1769 ASSERT_TRUE(!!aTrack); 1770 ASSERT_TRUE(!!aListener); 1771 ASSERT_TRUE(!!aStream); 1772 ASSERT_TRUE(aStream->mHasInput); 1773 ASSERT_NE(aChannelCount, 0); 1774 1775 bool destroyed = false; 1776 MediaEventListener destroyListener = cubeb->StreamDestroyEvent().Connect( 1777 AbstractThread::GetCurrent(), 1778 [&](const RefPtr<SmartMockCubebStream>& aDestroyed) { 1779 destroyed = aDestroyed.get() == aStream.get(); 1780 }); 1781 1782 RefPtr<SmartMockCubebStream> newStream; 1783 MediaEventListener restartListener = cubeb->StreamInitEvent().Connect( 1784 AbstractThread::GetCurrent(), 1785 [&](const RefPtr<SmartMockCubebStream>& aCreated) { 1786 newStream = aCreated; 1787 }); 1788 1789 MediaEnginePrefs settings; 1790 settings.mChannels = aChannelCount; 1791 QueueApplySettings(aTrack, aListener, settings); 1792 1793 SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( 1794 "TEST(TestAudioTrackGraph, RestartAudioIfProcessingMaxChannelCountChanged) #1"_ns, 1795 [&] { return destroyed && newStream; }); 1796 1797 destroyListener.Disconnect(); 1798 restartListener.Disconnect(); 1799 1800 aStream = newStream; 1801 }; 1802 1803 // Open a new track and expect to have a new stream. 1804 auto openTrack = [&](RefPtr<SmartMockCubebStream>& aCurrentStream, 1805 RefPtr<AudioProcessingTrack>& aTrack, 1806 RefPtr<AudioInputProcessing>& aListener, 1807 CubebUtils::AudioDeviceID aDevice, 1808 uint32_t aChannelCount) { 1809 ASSERT_TRUE(!!aCurrentStream); 1810 ASSERT_TRUE(aCurrentStream->mHasInput); 1811 ASSERT_TRUE(aChannelCount > aCurrentStream->InputChannels()); 1812 ASSERT_TRUE(!aTrack); 1813 ASSERT_TRUE(!aListener); 1814 1815 bool destroyed = false; 1816 MediaEventListener destroyListener = cubeb->StreamDestroyEvent().Connect( 1817 AbstractThread::GetCurrent(), 1818 [&](const RefPtr<SmartMockCubebStream>& aDestroyed) { 1819 destroyed = aDestroyed.get() == aCurrentStream.get(); 1820 }); 1821 1822 RefPtr<SmartMockCubebStream> newStream; 1823 MediaEventListener restartListener = cubeb->StreamInitEvent().Connect( 1824 AbstractThread::GetCurrent(), 1825 [&](const RefPtr<SmartMockCubebStream>& aCreated) { 1826 newStream = aCreated; 1827 }); 1828 1829 aTrack = AudioProcessingTrack::Create(graph); 1830 aListener = new AudioInputProcessing(aChannelCount); 1831 aTrack->SetInputProcessing(aListener); 1832 QueueExpectIsPassThrough(aTrack, aListener); 1833 aTrack->GraphImpl()->AppendMessage( 1834 MakeUnique<StartInputProcessing>(aTrack, aListener)); 1835 1836 DispatchFunction([&] { 1837 aTrack->ConnectDeviceInput(aDevice, aListener, PRINCIPAL_HANDLE_NONE); 1838 }); 1839 1840 SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( 1841 "TEST(TestAudioTrackGraph, RestartAudioIfProcessingMaxChannelCountChanged) #2"_ns, 1842 [&] { return destroyed && newStream; }); 1843 1844 destroyListener.Disconnect(); 1845 restartListener.Disconnect(); 1846 1847 aCurrentStream = newStream; 1848 }; 1849 1850 // Test for the native input device first then non-native device. The 1851 // non-native device will be destroyed before the native device in case of 1852 // causing a native-device-switching. 1853 1854 // Test for the native device. 1855 const CubebUtils::AudioDeviceID nativeDevice = (CubebUtils::AudioDeviceID)1; 1856 RefPtr<AudioProcessingTrack> track1; 1857 RefPtr<AudioInputProcessing> listener1; 1858 RefPtr<SmartMockCubebStream> nativeStream; 1859 RefPtr<AudioProcessingTrack> track2; 1860 RefPtr<AudioInputProcessing> listener2; 1861 { 1862 // Open a 1-channel AudioProcessingTrack for the native device. 1863 track1 = AudioProcessingTrack::Create(graph); 1864 listener1 = new AudioInputProcessing(1); 1865 track1->SetInputProcessing(listener1); 1866 QueueExpectIsPassThrough(track1, listener1); 1867 track1->GraphImpl()->AppendMessage( 1868 MakeUnique<StartInputProcessing>(track1, listener1)); 1869 track1->ConnectDeviceInput(nativeDevice, listener1, PRINCIPAL_HANDLE_NONE); 1870 EXPECT_EQ(track1->DeviceId().value(), nativeDevice); 1871 1872 auto started = 1873 InvokeAsync([&] { return graph->NotifyWhenDeviceStarted(nullptr); }); 1874 1875 nativeStream = WaitFor(cubeb->StreamInitEvent()); 1876 EXPECT_TRUE(nativeStream->mHasInput); 1877 EXPECT_TRUE(nativeStream->mHasOutput); 1878 EXPECT_EQ(nativeStream->InputChannels(), 1U); 1879 EXPECT_EQ(nativeStream->GetInputDeviceID(), nativeDevice); 1880 (void)WaitFor(started); 1881 1882 // Open a 2-channel AudioProcessingTrack for the native device and wait for 1883 // a new driver since the max-channel for the native device becomes 2 now. 1884 openTrack(nativeStream, track2, listener2, nativeDevice, 2); 1885 EXPECT_EQ(nativeStream->InputChannels(), 2U); 1886 1887 // Set the second AudioProcessingTrack for the native device to 1-channel 1888 // and wait for a new driver since the max-channel for the native device 1889 // becomes 1 now. 1890 setNewChannelCount(track2, listener2, nativeStream, 1); 1891 EXPECT_EQ(nativeStream->InputChannels(), 1U); 1892 1893 // Set the first AudioProcessingTrack for the native device to 2-channel and 1894 // wait for a new driver since the max input channel for the native device 1895 // becomes 2 now. 1896 setNewChannelCount(track1, listener1, nativeStream, 2); 1897 EXPECT_EQ(nativeStream->InputChannels(), 2U); 1898 } 1899 1900 // Test for the non-native device. 1901 { 1902 const CubebUtils::AudioDeviceID nonNativeDevice = 1903 (CubebUtils::AudioDeviceID)2; 1904 1905 // Open a 1-channel AudioProcessingTrack for the non-native device. 1906 RefPtr<AudioProcessingTrack> track3 = AudioProcessingTrack::Create(graph); 1907 RefPtr<AudioInputProcessing> listener3 = new AudioInputProcessing(1); 1908 track3->SetInputProcessing(listener3); 1909 QueueExpectIsPassThrough(track3, listener3); 1910 track3->GraphImpl()->AppendMessage( 1911 MakeUnique<StartInputProcessing>(track3, listener3)); 1912 track3->ConnectDeviceInput(nonNativeDevice, listener3, 1913 PRINCIPAL_HANDLE_NONE); 1914 EXPECT_EQ(track3->DeviceId().value(), nonNativeDevice); 1915 1916 RefPtr<SmartMockCubebStream> nonNativeStream = 1917 WaitFor(cubeb->StreamInitEvent()); 1918 EXPECT_TRUE(nonNativeStream->mHasInput); 1919 EXPECT_FALSE(nonNativeStream->mHasOutput); 1920 EXPECT_EQ(nonNativeStream->InputChannels(), 1U); 1921 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), nonNativeDevice); 1922 1923 // Open a 2-channel AudioProcessingTrack for the non-native device and wait 1924 // for a new stream since the max-channel for the non-native device becomes 1925 // 2 now. 1926 RefPtr<AudioProcessingTrack> track4; 1927 RefPtr<AudioInputProcessing> listener4; 1928 openTrack(nonNativeStream, track4, listener4, nonNativeDevice, 2); 1929 EXPECT_EQ(nonNativeStream->InputChannels(), 2U); 1930 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), nonNativeDevice); 1931 1932 // Set the second AudioProcessingTrack for the non-native to 1-channel and 1933 // wait for a new driver since the max-channel for the non-native device 1934 // becomes 1 now. 1935 setNewChannelCount(track4, listener4, nonNativeStream, 1); 1936 EXPECT_EQ(nonNativeStream->InputChannels(), 1U); 1937 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), nonNativeDevice); 1938 1939 // Set the first AudioProcessingTrack for the non-native device to 2-channel 1940 // and wait for a new driver since the max input channel for the non-native 1941 // device becomes 2 now. 1942 setNewChannelCount(track3, listener3, nonNativeStream, 2); 1943 EXPECT_EQ(nonNativeStream->InputChannels(), 2U); 1944 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), nonNativeDevice); 1945 1946 // Close the second AudioProcessingTrack (1-channel) for the non-native 1947 // device then the first one (2-channel) so we won't result in another 1948 // stream creation. 1949 DispatchFunction([&] { 1950 track4->GraphImpl()->AppendMessage( 1951 MakeUnique<StopInputProcessing>(track4, listener4)); 1952 track4->DisconnectDeviceInput(); 1953 track4->Destroy(); 1954 }); 1955 DispatchFunction([&] { 1956 track3->GraphImpl()->AppendMessage( 1957 MakeUnique<StopInputProcessing>(track3, listener3)); 1958 track3->DisconnectDeviceInput(); 1959 track3->Destroy(); 1960 }); 1961 RefPtr<SmartMockCubebStream> destroyedStream = 1962 WaitFor(cubeb->StreamDestroyEvent()); 1963 EXPECT_EQ(destroyedStream.get(), nonNativeStream.get()); 1964 } 1965 1966 // Tear down for the native device. 1967 { 1968 // Close the second AudioProcessingTrack (1-channel) for the native device 1969 // then the first one (2-channel) so we won't have driver switching. 1970 DispatchFunction([&] { 1971 track2->GraphImpl()->AppendMessage( 1972 MakeUnique<StopInputProcessing>(track2, listener2)); 1973 track2->DisconnectDeviceInput(); 1974 track2->Destroy(); 1975 }); 1976 DispatchFunction([&] { 1977 track1->GraphImpl()->AppendMessage( 1978 MakeUnique<StopInputProcessing>(track1, listener1)); 1979 track1->DisconnectDeviceInput(); 1980 track1->Destroy(); 1981 }); 1982 RefPtr<SmartMockCubebStream> destroyedStream = 1983 WaitFor(cubeb->StreamDestroyEvent()); 1984 EXPECT_EQ(destroyedStream.get(), nativeStream.get()); 1985 } 1986 } 1987 1988 TEST(TestAudioTrackGraph, SetInputChannelCountBeforeAudioCallbackDriver) 1989 { 1990 MockCubeb* cubeb = new MockCubeb(); 1991 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 1992 1993 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 1994 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 1995 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 1996 nullptr, GetMainThreadSerialEventTarget()); 1997 1998 // Set the input channel count of AudioInputProcessing, which will force 1999 // MediaTrackGraph to re-evaluate input device, when the MediaTrackGraph is 2000 // driven by the SystemClockDriver. 2001 2002 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 2003 RefPtr<AudioProcessingTrack> track; 2004 RefPtr<AudioInputProcessing> listener; 2005 DispatchFunction([&] { 2006 track = AudioProcessingTrack::Create(graph); 2007 listener = new AudioInputProcessing(2); 2008 QueueExpectIsPassThrough(track, listener); 2009 track->SetInputProcessing(listener); 2010 2011 MediaEnginePrefs settings; 2012 settings.mChannels = 1; 2013 QueueApplySettings(track, listener, settings); 2014 }); 2015 2016 // Wait for AudioCallbackDriver to init output-only stream. 2017 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 2018 EXPECT_FALSE(stream->mHasInput); 2019 EXPECT_TRUE(stream->mHasOutput); 2020 2021 // Open a full-duplex AudioCallbackDriver. 2022 RefPtr<MediaInputPort> port; 2023 DispatchFunction([&] { 2024 track->GraphImpl()->AppendMessage( 2025 MakeUnique<StartInputProcessing>(track, listener)); 2026 track->ConnectDeviceInput(deviceId, listener, PRINCIPAL_HANDLE_NONE); 2027 }); 2028 2029 stream = WaitFor(cubeb->StreamInitEvent()); 2030 EXPECT_TRUE(stream->mHasInput); 2031 EXPECT_TRUE(stream->mHasOutput); 2032 EXPECT_EQ(stream->InputChannels(), 1U); 2033 2034 (void)WaitFor( 2035 InvokeAsync([&] { return graph->NotifyWhenDeviceStarted(nullptr); })); 2036 2037 // Clean up. 2038 DispatchFunction([&] { 2039 track->GraphImpl()->AppendMessage( 2040 MakeUnique<StopInputProcessing>(track, listener)); 2041 track->DisconnectDeviceInput(); 2042 track->Destroy(); 2043 }); 2044 (void)WaitFor(cubeb->StreamDestroyEvent()); 2045 } 2046 2047 TEST(TestAudioTrackGraph, StartAudioDeviceBeforeStartingAudioProcessing) 2048 { 2049 MockCubeb* cubeb = new MockCubeb(); 2050 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 2051 2052 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 2053 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 2054 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 2055 nullptr, GetMainThreadSerialEventTarget()); 2056 2057 // Create a duplex AudioCallbackDriver 2058 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 2059 RefPtr<AudioProcessingTrack> track; 2060 RefPtr<AudioInputProcessing> listener; 2061 DispatchFunction([&] { 2062 track = AudioProcessingTrack::Create(graph); 2063 listener = new AudioInputProcessing(2); 2064 QueueExpectIsPassThrough(track, listener); 2065 track->SetInputProcessing(listener); 2066 // Start audio device without starting audio processing. 2067 track->ConnectDeviceInput(deviceId, listener, PRINCIPAL_HANDLE_NONE); 2068 }); 2069 2070 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 2071 EXPECT_TRUE(stream->mHasInput); 2072 EXPECT_TRUE(stream->mHasOutput); 2073 2074 // Wait for a second to make sure audio output callback has been fired. 2075 DispatchFunction( 2076 [&] { track->GraphImpl()->AppendMessage(MakeUnique<GoFaster>(cubeb)); }); 2077 { 2078 uint32_t totalFrames = 0; 2079 WaitUntil(stream->FramesProcessedEvent(), [&](uint32_t aFrames) { 2080 totalFrames += aFrames; 2081 return totalFrames > static_cast<uint32_t>(graph->GraphRate()); 2082 }); 2083 } 2084 cubeb->DontGoFaster(); 2085 2086 // Start the audio processing. 2087 DispatchFunction([&] { 2088 track->GraphImpl()->AppendMessage( 2089 MakeUnique<StartInputProcessing>(track, listener)); 2090 }); 2091 2092 // Wait for a second to make sure audio output callback has been fired. 2093 DispatchFunction( 2094 [&] { track->GraphImpl()->AppendMessage(MakeUnique<GoFaster>(cubeb)); }); 2095 { 2096 uint32_t totalFrames = 0; 2097 WaitUntil(stream->FramesProcessedEvent(), [&](uint32_t aFrames) { 2098 totalFrames += aFrames; 2099 return totalFrames > static_cast<uint32_t>(graph->GraphRate()); 2100 }); 2101 } 2102 cubeb->DontGoFaster(); 2103 2104 // Clean up. 2105 DispatchFunction([&] { 2106 track->DisconnectDeviceInput(); 2107 track->Destroy(); 2108 }); 2109 (void)WaitFor(cubeb->StreamDestroyEvent()); 2110 } 2111 2112 TEST(TestAudioTrackGraph, StopAudioProcessingBeforeStoppingAudioDevice) 2113 { 2114 MockCubeb* cubeb = new MockCubeb(); 2115 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 2116 2117 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 2118 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 2119 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 2120 nullptr, GetMainThreadSerialEventTarget()); 2121 2122 // Create a duplex AudioCallbackDriver 2123 const CubebUtils::AudioDeviceID deviceId = (CubebUtils::AudioDeviceID)1; 2124 RefPtr<AudioProcessingTrack> track; 2125 RefPtr<AudioInputProcessing> listener; 2126 DispatchFunction([&] { 2127 track = AudioProcessingTrack::Create(graph); 2128 listener = new AudioInputProcessing(2); 2129 QueueExpectIsPassThrough(track, listener); 2130 track->SetInputProcessing(listener); 2131 track->GraphImpl()->AppendMessage( 2132 MakeUnique<StartInputProcessing>(track, listener)); 2133 track->ConnectDeviceInput(deviceId, listener, PRINCIPAL_HANDLE_NONE); 2134 }); 2135 2136 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 2137 EXPECT_TRUE(stream->mHasInput); 2138 EXPECT_TRUE(stream->mHasOutput); 2139 2140 // Wait for a second to make sure audio output callback has been fired. 2141 DispatchFunction( 2142 [&] { track->GraphImpl()->AppendMessage(MakeUnique<GoFaster>(cubeb)); }); 2143 { 2144 uint32_t totalFrames = 0; 2145 WaitUntil(stream->FramesProcessedEvent(), [&](uint32_t aFrames) { 2146 totalFrames += aFrames; 2147 return totalFrames > static_cast<uint32_t>(graph->GraphRate()); 2148 }); 2149 } 2150 cubeb->DontGoFaster(); 2151 2152 // Stop the audio processing 2153 DispatchFunction([&] { 2154 track->GraphImpl()->AppendMessage( 2155 MakeUnique<StopInputProcessing>(track, listener)); 2156 }); 2157 2158 // Wait for a second to make sure audio output callback has been fired. 2159 DispatchFunction( 2160 [&] { track->GraphImpl()->AppendMessage(MakeUnique<GoFaster>(cubeb)); }); 2161 { 2162 uint32_t totalFrames = 0; 2163 WaitUntil(stream->FramesProcessedEvent(), [&](uint32_t aFrames) { 2164 totalFrames += aFrames; 2165 return totalFrames > static_cast<uint32_t>(graph->GraphRate()); 2166 }); 2167 } 2168 cubeb->DontGoFaster(); 2169 2170 // Clean up. 2171 DispatchFunction([&] { 2172 track->DisconnectDeviceInput(); 2173 track->Destroy(); 2174 }); 2175 (void)WaitFor(cubeb->StreamDestroyEvent()); 2176 } 2177 2178 // This test is pretty similar to SwitchNativeInputDevice above, which makes 2179 // sure the related DeviceInputTrack operations for the test here works 2180 // correctly. Instead of using a test-only DeviceInputTrack consumer, we use 2181 // AudioProcessingTrack here to simulate the real world use case. 2182 TEST(TestAudioTrackGraph, SwitchNativeAudioProcessingTrack) 2183 { 2184 MockCubeb* cubeb = new MockCubeb(); 2185 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 2186 2187 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 2188 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 2189 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 2190 nullptr, GetMainThreadSerialEventTarget()); 2191 2192 auto switchNativeDevice = 2193 [&](RefPtr<SmartMockCubebStream>&& aCurrentNativeStream, 2194 RefPtr<AudioProcessingTrack>& aCurrentNativeTrack, 2195 RefPtr<AudioInputProcessing>& aCurrentNativeListener, 2196 RefPtr<SmartMockCubebStream>& aNextNativeStream, 2197 RefPtr<AudioProcessingTrack>& aNextNativeTrack) { 2198 ASSERT_TRUE(aCurrentNativeStream->mHasInput); 2199 ASSERT_TRUE(aCurrentNativeStream->mHasOutput); 2200 ASSERT_TRUE(aNextNativeStream->mHasInput); 2201 ASSERT_FALSE(aNextNativeStream->mHasOutput); 2202 2203 std::cerr << "Switching native input from device " 2204 << aCurrentNativeStream->GetInputDeviceID() << " to " 2205 << aNextNativeStream->GetInputDeviceID() << std::endl; 2206 2207 uint32_t destroyed = 0; 2208 MediaEventListener destroyListener = 2209 cubeb->StreamDestroyEvent().Connect( 2210 AbstractThread::GetCurrent(), 2211 [&](const RefPtr<SmartMockCubebStream>& aDestroyed) { 2212 if (aDestroyed.get() == aCurrentNativeStream.get() || 2213 aDestroyed.get() == aNextNativeStream.get()) { 2214 std::cerr << "cubeb stream " << aDestroyed.get() 2215 << " (device " << aDestroyed->GetInputDeviceID() 2216 << ") has been destroyed" << std::endl; 2217 destroyed += 1; 2218 } 2219 }); 2220 2221 RefPtr<SmartMockCubebStream> newStream; 2222 MediaEventListener restartListener = cubeb->StreamInitEvent().Connect( 2223 AbstractThread::GetCurrent(), 2224 [&](const RefPtr<SmartMockCubebStream>& aCreated) { 2225 // Make sure new stream has input, to prevent from getting a 2226 // temporary output-only AudioCallbackDriver after closing current 2227 // native device but before setting a new native input. 2228 if (aCreated->mHasInput) { 2229 ASSERT_TRUE(aCreated->mHasOutput); 2230 newStream = aCreated; 2231 } 2232 }); 2233 2234 std::cerr << "Close device " << aCurrentNativeStream->GetInputDeviceID() 2235 << std::endl; 2236 DispatchFunction([&] { 2237 aCurrentNativeTrack->GraphImpl()->AppendMessage( 2238 MakeUnique<StopInputProcessing>(aCurrentNativeTrack, 2239 aCurrentNativeListener)); 2240 aCurrentNativeTrack->DisconnectDeviceInput(); 2241 aCurrentNativeTrack->Destroy(); 2242 }); 2243 2244 std::cerr << "Wait for the switching" << std::endl; 2245 SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( 2246 "TEST(TestAudioTrackGraph, SwitchNativeAudioProcessingTrack)"_ns, 2247 [&] { return destroyed >= 2 && newStream; }); 2248 2249 destroyListener.Disconnect(); 2250 restartListener.Disconnect(); 2251 2252 aCurrentNativeStream = nullptr; 2253 aNextNativeStream = newStream; 2254 2255 std::cerr << "Now the native input is device " 2256 << aNextNativeStream->GetInputDeviceID() << std::endl; 2257 }; 2258 2259 // Open a AudioProcessingTrack for device 1. 2260 const CubebUtils::AudioDeviceID device1 = (CubebUtils::AudioDeviceID)1; 2261 RefPtr<AudioProcessingTrack> track1 = AudioProcessingTrack::Create(graph); 2262 RefPtr<AudioInputProcessing> listener1 = new AudioInputProcessing(1); 2263 track1->SetInputProcessing(listener1); 2264 QueueExpectIsPassThrough(track1, listener1); 2265 track1->GraphImpl()->AppendMessage( 2266 MakeUnique<StartInputProcessing>(track1, listener1)); 2267 track1->ConnectDeviceInput(device1, listener1, PRINCIPAL_HANDLE_NONE); 2268 EXPECT_EQ(track1->DeviceId().value(), device1); 2269 2270 auto started = 2271 InvokeAsync([&] { return graph->NotifyWhenDeviceStarted(nullptr); }); 2272 2273 RefPtr<SmartMockCubebStream> stream1 = WaitFor(cubeb->StreamInitEvent()); 2274 EXPECT_TRUE(stream1->mHasInput); 2275 EXPECT_TRUE(stream1->mHasOutput); 2276 EXPECT_EQ(stream1->InputChannels(), 1U); 2277 EXPECT_EQ(stream1->GetInputDeviceID(), device1); 2278 (void)WaitFor(started); 2279 std::cerr << "Device " << device1 << " is opened (stream " << stream1.get() 2280 << ")" << std::endl; 2281 2282 // Open a AudioProcessingTrack for device 2. 2283 const CubebUtils::AudioDeviceID device2 = (CubebUtils::AudioDeviceID)2; 2284 RefPtr<AudioProcessingTrack> track2 = AudioProcessingTrack::Create(graph); 2285 RefPtr<AudioInputProcessing> listener2 = new AudioInputProcessing(2); 2286 track2->SetInputProcessing(listener2); 2287 QueueExpectIsPassThrough(track2, listener2); 2288 track2->GraphImpl()->AppendMessage( 2289 MakeUnique<StartInputProcessing>(track2, listener2)); 2290 track2->ConnectDeviceInput(device2, listener2, PRINCIPAL_HANDLE_NONE); 2291 EXPECT_EQ(track2->DeviceId().value(), device2); 2292 2293 RefPtr<SmartMockCubebStream> stream2 = WaitFor(cubeb->StreamInitEvent()); 2294 EXPECT_TRUE(stream2->mHasInput); 2295 EXPECT_FALSE(stream2->mHasOutput); 2296 EXPECT_EQ(stream2->InputChannels(), 2U); 2297 EXPECT_EQ(stream2->GetInputDeviceID(), device2); 2298 std::cerr << "Device " << device2 << " is opened (stream " << stream2.get() 2299 << ")" << std::endl; 2300 2301 // Open a AudioProcessingTrack for device 3. 2302 const CubebUtils::AudioDeviceID device3 = (CubebUtils::AudioDeviceID)3; 2303 RefPtr<AudioProcessingTrack> track3 = AudioProcessingTrack::Create(graph); 2304 RefPtr<AudioInputProcessing> listener3 = new AudioInputProcessing(1); 2305 track3->SetInputProcessing(listener3); 2306 QueueExpectIsPassThrough(track3, listener3); 2307 track3->GraphImpl()->AppendMessage( 2308 MakeUnique<StartInputProcessing>(track3, listener3)); 2309 track3->ConnectDeviceInput(device3, listener3, PRINCIPAL_HANDLE_NONE); 2310 EXPECT_EQ(track3->DeviceId().value(), device3); 2311 2312 RefPtr<SmartMockCubebStream> stream3 = WaitFor(cubeb->StreamInitEvent()); 2313 EXPECT_TRUE(stream3->mHasInput); 2314 EXPECT_FALSE(stream3->mHasOutput); 2315 EXPECT_EQ(stream3->InputChannels(), 1U); 2316 EXPECT_EQ(stream3->GetInputDeviceID(), device3); 2317 std::cerr << "Device " << device3 << " is opened (stream " << stream3.get() 2318 << ")" << std::endl; 2319 2320 // Close device 1, so the native input device is switched from device 1 to 2321 // device 2. 2322 switchNativeDevice(std::move(stream1), track1, listener1, stream2, track2); 2323 EXPECT_TRUE(stream2->mHasInput); 2324 EXPECT_TRUE(stream2->mHasOutput); 2325 EXPECT_EQ(stream2->InputChannels(), 2U); 2326 EXPECT_EQ(stream2->GetInputDeviceID(), device2); 2327 { 2328 NativeInputTrack* native = track2->Graph()->GetNativeInputTrackMainThread(); 2329 ASSERT_TRUE(!!native); 2330 EXPECT_EQ(native->mDeviceId, device2); 2331 } 2332 2333 // Close device 2, so the native input device is switched from device 2 to 2334 // device 3. 2335 switchNativeDevice(std::move(stream2), track2, listener2, stream3, track3); 2336 EXPECT_TRUE(stream3->mHasInput); 2337 EXPECT_TRUE(stream3->mHasOutput); 2338 EXPECT_EQ(stream3->InputChannels(), 1U); 2339 EXPECT_EQ(stream3->GetInputDeviceID(), device3); 2340 { 2341 NativeInputTrack* native = track3->Graph()->GetNativeInputTrackMainThread(); 2342 ASSERT_TRUE(!!native); 2343 EXPECT_EQ(native->mDeviceId, device3); 2344 } 2345 2346 // Clean up. 2347 std::cerr << "Close device " << device3 << std::endl; 2348 DispatchFunction([&] { 2349 track3->GraphImpl()->AppendMessage( 2350 MakeUnique<StopInputProcessing>(track3, listener3)); 2351 track3->DisconnectDeviceInput(); 2352 track3->Destroy(); 2353 }); 2354 RefPtr<SmartMockCubebStream> destroyedStream = 2355 WaitFor(cubeb->StreamDestroyEvent()); 2356 EXPECT_EQ(destroyedStream.get(), stream3.get()); 2357 { 2358 NativeInputTrack* native = graph->GetNativeInputTrackMainThread(); 2359 ASSERT_TRUE(!native); 2360 } 2361 std::cerr << "No native input now" << std::endl; 2362 } 2363 2364 void TestCrossGraphPort(uint32_t aInputRate, uint32_t aOutputRate, 2365 float aDriftFactor, uint32_t aRunTimeSeconds = 10, 2366 uint32_t aNumExpectedUnderruns = 0) { 2367 std::cerr << "TestCrossGraphPort input: " << aInputRate 2368 << ", output: " << aOutputRate << ", driftFactor: " << aDriftFactor 2369 << std::endl; 2370 2371 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); 2372 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 2373 2374 /* Primary graph: Create the graph. */ 2375 MediaTrackGraph* primary = MediaTrackGraphImpl::GetInstance( 2376 MediaTrackGraph::SYSTEM_THREAD_DRIVER, 2377 /*Window ID*/ 1, aInputRate, nullptr, GetMainThreadSerialEventTarget()); 2378 2379 /* Partner graph: Create the graph. */ 2380 MediaTrackGraph* partner = MediaTrackGraphImpl::GetInstance( 2381 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, aOutputRate, 2382 /*OutputDeviceID*/ reinterpret_cast<cubeb_devid>(1), 2383 GetMainThreadSerialEventTarget()); 2384 2385 const CubebUtils::AudioDeviceID inputDeviceId = (CubebUtils::AudioDeviceID)1; 2386 2387 RefPtr<AudioProcessingTrack> processingTrack; 2388 RefPtr<AudioInputProcessing> listener; 2389 RefPtr<OnFallbackListener> primaryFallbackListener; 2390 DispatchFunction([&] { 2391 /* Primary graph: Create input track and open it */ 2392 processingTrack = AudioProcessingTrack::Create(primary); 2393 listener = new AudioInputProcessing(2); 2394 QueueExpectIsPassThrough(processingTrack, listener); 2395 processingTrack->SetInputProcessing(listener); 2396 processingTrack->GraphImpl()->AppendMessage( 2397 MakeUnique<StartInputProcessing>(processingTrack, listener)); 2398 processingTrack->ConnectDeviceInput(inputDeviceId, listener, 2399 PRINCIPAL_HANDLE_NONE); 2400 primaryFallbackListener = new OnFallbackListener(processingTrack); 2401 processingTrack->AddListener(primaryFallbackListener); 2402 }); 2403 2404 RefPtr<SmartMockCubebStream> inputStream = WaitFor(cubeb->StreamInitEvent()); 2405 2406 while ( 2407 inputStream->State() 2408 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 2409 .valueOr(true)) { 2410 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2411 } 2412 2413 // Wait for the primary AudioCallbackDriver to come into effect. 2414 while (primaryFallbackListener->OnFallback()) { 2415 EXPECT_EQ(inputStream->ManualDataCallback(0), 2416 MockCubebStream::KeepProcessing::Yes); 2417 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2418 } 2419 2420 RefPtr<CrossGraphTransmitter> transmitter; 2421 RefPtr<MediaInputPort> port; 2422 RefPtr<CrossGraphReceiver> receiver; 2423 RefPtr<OnFallbackListener> partnerFallbackListener; 2424 DispatchFunction([&] { 2425 processingTrack->RemoveListener(primaryFallbackListener); 2426 2427 /* Partner graph: Create CrossGraphReceiver */ 2428 receiver = partner->CreateCrossGraphReceiver(primary->GraphRate()); 2429 2430 /* Primary graph: Create CrossGraphTransmitter */ 2431 transmitter = primary->CreateCrossGraphTransmitter(receiver); 2432 2433 /* How the input track connects to another ProcessedMediaTrack. 2434 * Check in MediaManager how it is connected to AudioStreamTrack. */ 2435 port = transmitter->AllocateInputPort(processingTrack); 2436 receiver->AddAudioOutput((void*)1, partner->PrimaryOutputDeviceID(), 0); 2437 2438 partnerFallbackListener = new OnFallbackListener(receiver); 2439 receiver->AddListener(partnerFallbackListener); 2440 }); 2441 2442 RefPtr<SmartMockCubebStream> partnerStream = 2443 WaitFor(cubeb->StreamInitEvent()); 2444 2445 while ( 2446 partnerStream->State() 2447 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 2448 .valueOr(true)) { 2449 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2450 } 2451 2452 // Process the CrossGraphTransmitter on the primary graph. 2453 EXPECT_EQ(inputStream->ManualDataCallback(0), 2454 MockCubebStream::KeepProcessing::Yes); 2455 2456 // Wait for the partner AudioCallbackDriver to come into effect. 2457 while (partnerFallbackListener->OnFallback()) { 2458 EXPECT_EQ(partnerStream->ManualDataCallback(0), 2459 MockCubebStream::KeepProcessing::Yes); 2460 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2461 } 2462 2463 DispatchFunction([&] { receiver->RemoveListener(partnerFallbackListener); }); 2464 ProcessEventQueue(); 2465 2466 nsIThread* currentThread = NS_GetCurrentThread(); 2467 cubeb_state inputState = CUBEB_STATE_STARTED; 2468 MediaEventListener inputStateListener = inputStream->StateEvent().Connect( 2469 currentThread, [&](cubeb_state aState) { inputState = aState; }); 2470 cubeb_state partnerState = CUBEB_STATE_STARTED; 2471 MediaEventListener partnerStateListener = partnerStream->StateEvent().Connect( 2472 currentThread, [&](cubeb_state aState) { partnerState = aState; }); 2473 2474 const media::TimeUnit runtime = media::TimeUnit::FromSeconds(aRunTimeSeconds); 2475 // 10ms per iteration. 2476 const media::TimeUnit step = media::TimeUnit::FromSeconds(0.01); 2477 { 2478 media::TimeUnit pos = media::TimeUnit::Zero(); 2479 long inputFrames = 0; 2480 long outputFrames = 0; 2481 while (pos < runtime) { 2482 pos += step; 2483 const long newInputFrames = pos.ToTicksAtRate(aInputRate); 2484 const long newOutputFrames = 2485 (pos.MultDouble(aDriftFactor)).ToTicksAtRate(aOutputRate); 2486 EXPECT_EQ(inputStream->ManualDataCallback(newInputFrames - inputFrames), 2487 MockCubebStream::KeepProcessing::Yes); 2488 EXPECT_EQ( 2489 partnerStream->ManualDataCallback(newOutputFrames - outputFrames), 2490 MockCubebStream::KeepProcessing::Yes); 2491 2492 inputFrames = newInputFrames; 2493 outputFrames = newOutputFrames; 2494 } 2495 } 2496 2497 DispatchFunction([&] { 2498 // Clean up on MainThread 2499 receiver->RemoveAudioOutput((void*)1); 2500 receiver->Destroy(); 2501 transmitter->Destroy(); 2502 port->Destroy(); 2503 processingTrack->GraphImpl()->AppendMessage( 2504 MakeUnique<StopInputProcessing>(processingTrack, listener)); 2505 processingTrack->DisconnectDeviceInput(); 2506 processingTrack->Destroy(); 2507 }); 2508 2509 ProcessEventQueue(); 2510 2511 EXPECT_EQ(inputStream->ManualDataCallback(128), 2512 MockCubebStream::KeepProcessing::No); 2513 EXPECT_EQ(partnerStream->ManualDataCallback(128), 2514 MockCubebStream::KeepProcessing::No); 2515 2516 uint32_t inputFrequency = inputStream->InputFrequency(); 2517 2518 uint64_t preSilenceSamples; 2519 float estimatedFreq; 2520 uint32_t nrDiscontinuities; 2521 std::tie(preSilenceSamples, estimatedFreq, nrDiscontinuities) = 2522 WaitFor(partnerStream->OutputVerificationEvent()); 2523 2524 EXPECT_NEAR(estimatedFreq, inputFrequency / aDriftFactor, 5); 2525 // Note that pre-silence is in the output rate. The buffering is on the input 2526 // side. There is one block buffered in NativeInputTrack. Then 2527 // AudioDriftCorrection sets its pre-buffering so that *after* the first 2528 // resample of real input data, the buffer contains enough data to match the 2529 // desired level, which is initially 50ms. I.e. silence = buffering - 2530 // inputStep + outputStep. Note that the steps here are rounded up to block 2531 // size. 2532 const media::TimeUnit inputBuffering(WEBAUDIO_BLOCK_SIZE, aInputRate); 2533 const media::TimeUnit buffering = 2534 media::TimeUnit::FromSeconds(0.05).ToBase(aInputRate); 2535 const media::TimeUnit inputStepSize( 2536 MediaTrackGraphImpl::RoundUpToEndOfAudioBlock( 2537 step.ToTicksAtRate(aInputRate)), 2538 aInputRate); 2539 const media::TimeUnit outputStepSize = 2540 media::TimeUnit(MediaTrackGraphImpl::RoundUpToEndOfAudioBlock( 2541 step.ToBase(aOutputRate) 2542 .MultDouble(aDriftFactor) 2543 .ToTicksAtRate(aOutputRate)), 2544 aOutputRate) 2545 .ToBase(aInputRate); 2546 const uint32_t expectedPreSilence = 2547 (outputStepSize + inputBuffering + buffering - inputStepSize) 2548 .ToBase(aInputRate) 2549 .ToBase<media::TimeUnit::CeilingPolicy>(aOutputRate) 2550 .ToTicksAtRate(aOutputRate); 2551 // Use a margin of 0.1% of the expected pre-silence, since the resampler is 2552 // adapting to drift and will process the pre-silence frames. Because of 2553 // rounding errors, we don't use a margin lower than 1. 2554 const uint32_t margin = std::max(1U, expectedPreSilence / 1000); 2555 EXPECT_NEAR(preSilenceSamples, expectedPreSilence, margin); 2556 // The waveform from AudioGenerator starts at 0, but we don't control its 2557 // ending, so we expect a discontinuity there. For each expected underrun 2558 // there could be an additional 2 discontinuities (start and end of the silent 2559 // period). 2560 EXPECT_LE(nrDiscontinuities, 1U + 2 * aNumExpectedUnderruns); 2561 2562 SpinEventLoopUntil("streams have stopped"_ns, [&] { 2563 return inputState == CUBEB_STATE_STOPPED && 2564 partnerState == CUBEB_STATE_STOPPED; 2565 }); 2566 inputStateListener.Disconnect(); 2567 partnerStateListener.Disconnect(); 2568 } 2569 2570 TEST(TestAudioTrackGraph, CrossGraphPort) 2571 { 2572 TestCrossGraphPort(44100, 44100, 1); 2573 TestCrossGraphPort(44100, 44100, 1.006); 2574 TestCrossGraphPort(44100, 44100, 0.994); 2575 2576 TestCrossGraphPort(48000, 44100, 1); 2577 TestCrossGraphPort(48000, 44100, 1.006); 2578 TestCrossGraphPort(48000, 44100, 0.994); 2579 2580 TestCrossGraphPort(44100, 48000, 1); 2581 TestCrossGraphPort(44100, 48000, 1.006); 2582 TestCrossGraphPort(44100, 48000, 0.994); 2583 2584 TestCrossGraphPort(52110, 17781, 1); 2585 TestCrossGraphPort(52110, 17781, 1.006); 2586 TestCrossGraphPort(52110, 17781, 0.994); 2587 } 2588 2589 TEST(TestAudioTrackGraph, CrossGraphPortUnderrun) 2590 { 2591 TestCrossGraphPort(44100, 44100, 1.01, 30, 1); 2592 TestCrossGraphPort(44100, 44100, 1.03, 40, 3); 2593 2594 TestCrossGraphPort(48000, 44100, 1.01, 30, 1); 2595 TestCrossGraphPort(48000, 44100, 1.03, 40, 3); 2596 2597 TestCrossGraphPort(44100, 48000, 1.01, 30, 1); 2598 TestCrossGraphPort(44100, 48000, 1.03, 40, 3); 2599 2600 TestCrossGraphPort(52110, 17781, 1.01, 30, 1); 2601 TestCrossGraphPort(52110, 17781, 1.03, 40, 3); 2602 } 2603 2604 TEST(TestAudioTrackGraph, SecondaryOutputDevice) 2605 { 2606 MockCubeb* cubeb = new MockCubeb(); 2607 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 2608 2609 const TrackRate primaryRate = 48000; 2610 const TrackRate secondaryRate = 44100; // for secondary output device 2611 2612 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 2613 MediaTrackGraph::SYSTEM_THREAD_DRIVER, 2614 /*Window ID*/ 1, primaryRate, nullptr, GetMainThreadSerialEventTarget()); 2615 2616 RefPtr<AudioProcessingTrack> processingTrack; 2617 RefPtr<AudioInputProcessing> listener; 2618 DispatchFunction([&] { 2619 /* Create an input track and connect it to a device */ 2620 processingTrack = AudioProcessingTrack::Create(graph); 2621 listener = new AudioInputProcessing(2); 2622 QueueExpectIsPassThrough(processingTrack, listener); 2623 processingTrack->SetInputProcessing(listener); 2624 processingTrack->GraphImpl()->AppendMessage( 2625 MakeUnique<StartInputProcessing>(processingTrack, listener)); 2626 processingTrack->ConnectDeviceInput(nullptr, listener, 2627 PRINCIPAL_HANDLE_NONE); 2628 }); 2629 RefPtr<SmartMockCubebStream> primaryStream = 2630 WaitFor(cubeb->StreamInitEvent()); 2631 2632 const void* secondaryDeviceID = CubebUtils::AudioDeviceID(2); 2633 DispatchFunction([&] { 2634 processingTrack->AddAudioOutput(nullptr, secondaryDeviceID, secondaryRate); 2635 processingTrack->SetAudioOutputVolume(nullptr, 0.f); 2636 }); 2637 RefPtr<SmartMockCubebStream> secondaryStream = 2638 WaitFor(cubeb->StreamInitEvent()); 2639 EXPECT_EQ(secondaryStream->GetOutputDeviceID(), secondaryDeviceID); 2640 EXPECT_EQ(static_cast<TrackRate>(secondaryStream->SampleRate()), 2641 secondaryRate); 2642 2643 nsIThread* currentThread = NS_GetCurrentThread(); 2644 uint32_t audioFrames = 0; // excludes pre-silence 2645 MediaEventListener audioListener = 2646 secondaryStream->FramesVerifiedEvent().Connect( 2647 currentThread, [&](uint32_t aFrames) { audioFrames += aFrames; }); 2648 2649 // Wait for 100ms of pre-silence to verify that SetAudioOutputVolume() is 2650 // effective. 2651 uint32_t processedFrames = 0; 2652 WaitUntil(secondaryStream->FramesProcessedEvent(), [&](uint32_t aFrames) { 2653 processedFrames += aFrames; 2654 return processedFrames > static_cast<uint32_t>(secondaryRate / 10); 2655 }); 2656 EXPECT_EQ(audioFrames, 0U) << "audio frames at zero volume"; 2657 2658 secondaryStream->SetOutputRecordingEnabled(true); 2659 DispatchFunction( 2660 [&] { processingTrack->SetAudioOutputVolume(nullptr, 1.f); }); 2661 2662 // Wait for enough audio after initial silence to check the frequency. 2663 SpinEventLoopUntil("200ms of audio"_ns, [&] { 2664 return audioFrames > static_cast<uint32_t>(secondaryRate / 5); 2665 }); 2666 audioListener.Disconnect(); 2667 2668 // Stop recording now so as not to record the discontinuity when the 2669 // CrossGraphReceiver is removed from the secondary graph before its 2670 // AudioCallbackDriver is stopped. 2671 secondaryStream->SetOutputRecordingEnabled(false); 2672 2673 DispatchFunction([&] { processingTrack->RemoveAudioOutput(nullptr); }); 2674 WaitFor(secondaryStream->OutputVerificationEvent()); 2675 // The frequency from OutputVerificationEvent() is estimated by 2676 // AudioVerifier from a zero-crossing count. When the discontinuity from 2677 // the volume change is resampled, the discontinuity presents as 2678 // oscillations, which increase the zero-crossing count and corrupt the 2679 // frequency estimate. Trim off sufficient leading from the output to 2680 // remove this discontinuity. 2681 uint32_t channelCount = secondaryStream->OutputChannels(); 2682 nsTArray<AudioDataValue> output = secondaryStream->TakeRecordedOutput(); 2683 size_t leadingIndex = 0; 2684 for (; leadingIndex < output.Length() && output[leadingIndex] == 0.f; 2685 leadingIndex += channelCount) { 2686 }; 2687 leadingIndex += 10 * channelCount; // skip discontinuity oscillations 2688 EXPECT_LT(leadingIndex, output.Length()); 2689 auto trimmed = Span(output).From(std::min(leadingIndex, output.Length())); 2690 size_t frameCount = trimmed.Length() / channelCount; 2691 uint32_t inputFrequency = primaryStream->InputFrequency(); 2692 AudioVerifier<AudioDataValue> verifier(secondaryRate, inputFrequency); 2693 verifier.AppendDataInterleaved(trimmed.Elements(), frameCount, channelCount); 2694 EXPECT_EQ(verifier.EstimatedFreq(), inputFrequency); 2695 // AudioVerifier considers the previous value before the initial sample to 2696 // be zero and so considers any initial sample >> 0 to be a discontinuity. 2697 EXPECT_EQ(verifier.CountDiscontinuities(), 1U); 2698 2699 DispatchFunction([&] { 2700 // Clean up 2701 processingTrack->GraphImpl()->AppendMessage( 2702 MakeUnique<StopInputProcessing>(processingTrack, listener)); 2703 processingTrack->DisconnectDeviceInput(); 2704 processingTrack->Destroy(); 2705 }); 2706 WaitFor(primaryStream->OutputVerificationEvent()); 2707 } 2708 2709 // Test when AudioInputProcessing expects clock drift 2710 TEST(TestAudioTrackGraph, ClockDriftExpectation) 2711 { 2712 MockCubeb* cubeb = new MockCubeb(); 2713 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 2714 2715 const TrackRate rate = 44100; 2716 2717 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 2718 MediaTrackGraph::SYSTEM_THREAD_DRIVER, 2719 /*Window ID*/ 1, rate, nullptr, GetMainThreadSerialEventTarget()); 2720 2721 auto createInputProcessing = 2722 [&](CubebUtils::AudioDeviceID aDeviceID, 2723 RefPtr<AudioProcessingTrack>* aProcessingTrack, 2724 RefPtr<AudioInputProcessing>* aInputProcessing) { 2725 /* Create an input track and connect it to a device */ 2726 const int32_t channelCount = 2; 2727 RefPtr processingTrack = AudioProcessingTrack::Create(graph); 2728 RefPtr inputProcessing = new AudioInputProcessing(channelCount); 2729 processingTrack->SetInputProcessing(inputProcessing); 2730 MediaEnginePrefs settings; 2731 settings.mChannels = channelCount; 2732 settings.mAecOn = true; 2733 QueueApplySettings(processingTrack, inputProcessing, settings); 2734 processingTrack->GraphImpl()->AppendMessage( 2735 MakeUnique<StartInputProcessing>(processingTrack, inputProcessing)); 2736 processingTrack->ConnectDeviceInput(aDeviceID, inputProcessing, 2737 PRINCIPAL_HANDLE_NONE); 2738 aProcessingTrack->swap(processingTrack); 2739 aInputProcessing->swap(inputProcessing); 2740 }; 2741 2742 // Native input, which uses a duplex stream 2743 RefPtr<AudioProcessingTrack> processingTrack1; 2744 RefPtr<AudioInputProcessing> inputProcessing1; 2745 DispatchFunction([&] { 2746 createInputProcessing(nullptr, &processingTrack1, &inputProcessing1); 2747 }); 2748 RefPtr<SmartMockCubebStream> primaryStream = 2749 WaitFor(cubeb->StreamInitEvent()); 2750 EXPECT_GT(primaryStream->OutputChannels(), 0U); 2751 // Non-native input 2752 const auto* nonNativeInputDeviceID = CubebUtils::AudioDeviceID(1); 2753 RefPtr<AudioProcessingTrack> processingTrack2; 2754 RefPtr<AudioInputProcessing> inputProcessing2; 2755 DispatchFunction([&] { 2756 createInputProcessing(nonNativeInputDeviceID, &processingTrack2, 2757 &inputProcessing2); 2758 processingTrack2->AddAudioOutput(nullptr, nullptr, rate); 2759 }); 2760 2761 RefPtr<SmartMockCubebStream> nonNativeInputStream = 2762 WaitFor(cubeb->StreamInitEvent()); 2763 EXPECT_EQ(nonNativeInputStream->GetInputDeviceID(), nonNativeInputDeviceID); 2764 EXPECT_EQ(nonNativeInputStream->OutputChannels(), 0U); 2765 2766 // Wait until non-native input signal reaches the output, when input 2767 // processing has run and so has been configured. 2768 WaitFor(primaryStream->FramesVerifiedEvent()); 2769 2770 const void* secondaryOutputDeviceID = CubebUtils::AudioDeviceID(2); 2771 DispatchFunction([&] { 2772 // Check input processing config with output to primary device. 2773 processingTrack1->QueueControlMessageWithNoShutdown([&] { 2774 EXPECT_FALSE(inputProcessing1->HadAECAndDrift()); 2775 EXPECT_TRUE(inputProcessing2->HadAECAndDrift()); 2776 }); 2777 2778 // Switch output to a secondary device. 2779 processingTrack2->RemoveAudioOutput(nullptr); 2780 processingTrack2->AddAudioOutput(nullptr, secondaryOutputDeviceID, rate); 2781 }); 2782 2783 RefPtr<SmartMockCubebStream> secondaryOutputStream = 2784 WaitFor(cubeb->StreamInitEvent()); 2785 EXPECT_EQ(secondaryOutputStream->GetOutputDeviceID(), 2786 secondaryOutputDeviceID); 2787 2788 WaitFor(secondaryOutputStream->FramesVerifiedEvent()); 2789 DispatchFunction([&] { 2790 // Check input processing config with output to secondary device. 2791 processingTrack1->QueueControlMessageWithNoShutdown([&] { 2792 EXPECT_TRUE(inputProcessing1->HadAECAndDrift()); 2793 EXPECT_TRUE(inputProcessing2->HadAECAndDrift()); 2794 }); 2795 }); 2796 2797 auto destroyInputProcessing = [&](AudioProcessingTrack* aProcessingTrack, 2798 AudioInputProcessing* aInputProcessing) { 2799 aProcessingTrack->GraphImpl()->AppendMessage( 2800 MakeUnique<StopInputProcessing>(aProcessingTrack, aInputProcessing)); 2801 aProcessingTrack->DisconnectDeviceInput(); 2802 aProcessingTrack->Destroy(); 2803 }; 2804 2805 DispatchFunction([&] { 2806 // Clean up 2807 destroyInputProcessing(processingTrack1, inputProcessing1); 2808 destroyInputProcessing(processingTrack2, inputProcessing2); 2809 }); 2810 // Wait for stream stop to ensure that expectations have been checked. 2811 WaitFor(nonNativeInputStream->OutputVerificationEvent()); 2812 } 2813 #endif // MOZ_WEBRTC 2814 2815 TEST(TestAudioTrackGraph, PlatformProcessing) 2816 { 2817 constexpr cubeb_input_processing_params allParams = 2818 CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION | 2819 CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION | 2820 CUBEB_INPUT_PROCESSING_PARAM_AUTOMATIC_GAIN_CONTROL | 2821 CUBEB_INPUT_PROCESSING_PARAM_VOICE_ISOLATION; 2822 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); 2823 cubeb->SetSupportedInputProcessingParams(allParams, CUBEB_OK); 2824 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 2825 2826 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 2827 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 2828 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 2829 nullptr, GetMainThreadSerialEventTarget()); 2830 2831 const CubebUtils::AudioDeviceID device = (CubebUtils::AudioDeviceID)1; 2832 2833 // Set up mock listener. 2834 RefPtr<MockAudioDataListener> listener = MakeRefPtr<MockAudioDataListener>(); 2835 EXPECT_CALL(*listener, IsVoiceInput).WillRepeatedly(Return(true)); 2836 EXPECT_CALL(*listener, RequestedInputChannelCount).WillRepeatedly(Return(1)); 2837 EXPECT_CALL(*listener, RequestedInputProcessingParams) 2838 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 2839 EXPECT_CALL(*listener, Disconnect); 2840 2841 // Expectations. 2842 const Result<cubeb_input_processing_params, int> notSupportedResult( 2843 Err(CUBEB_ERROR_NOT_SUPPORTED)); 2844 const Result<cubeb_input_processing_params, int> errorResult( 2845 Err(CUBEB_ERROR)); 2846 const Result<cubeb_input_processing_params, int> noneResult( 2847 CUBEB_INPUT_PROCESSING_PARAM_NONE); 2848 const Result<cubeb_input_processing_params, int> echoResult( 2849 CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION); 2850 const Result<cubeb_input_processing_params, int> noiseResult( 2851 CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION); 2852 Atomic<int> numProcessingParamsResults(0); 2853 { 2854 InSequence s; 2855 // On first driver start. 2856 EXPECT_CALL(*listener, 2857 NotifySetRequestedInputProcessingParams( 2858 graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 2859 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult( 2860 graph, 1, Eq(std::ref(echoResult)))) 2861 .WillOnce([&] { ++numProcessingParamsResults; }); 2862 // After requesting something else. 2863 EXPECT_CALL(*listener, 2864 NotifySetRequestedInputProcessingParams( 2865 graph, 2, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 2866 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult( 2867 graph, 2, Eq(std::ref(noiseResult)))) 2868 .WillOnce([&] { ++numProcessingParamsResults; }); 2869 // After error request. 2870 EXPECT_CALL(*listener, 2871 NotifySetRequestedInputProcessingParams( 2872 graph, 3, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 2873 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult( 2874 graph, 3, Eq(std::ref(errorResult)))) 2875 .WillOnce([&] { ++numProcessingParamsResults; }); 2876 // After requesting None, because of the previous error. 2877 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParams( 2878 graph, 4, CUBEB_INPUT_PROCESSING_PARAM_NONE)); 2879 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult( 2880 graph, 4, Eq(std::ref(noneResult)))) 2881 .WillOnce([&] { ++numProcessingParamsResults; }); 2882 // After driver switch. 2883 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult( 2884 graph, 4, Eq(std::ref(noneResult)))) 2885 .WillOnce([&] { ++numProcessingParamsResults; }); 2886 // After requesting something not supported. 2887 EXPECT_CALL(*listener, 2888 NotifySetRequestedInputProcessingParams( 2889 graph, 5, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 2890 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult( 2891 graph, 5, Eq(std::ref(noneResult)))) 2892 .WillOnce([&] { ++numProcessingParamsResults; }); 2893 // After requesting something with backend not supporting processing params. 2894 EXPECT_CALL(*listener, 2895 NotifySetRequestedInputProcessingParams( 2896 graph, 6, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 2897 EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult( 2898 graph, 6, Eq(std::ref(notSupportedResult)))) 2899 .WillOnce([&] { ++numProcessingParamsResults; }); 2900 } 2901 2902 // Open a device. 2903 RefPtr<TestDeviceInputConsumerTrack> track; 2904 RefPtr<OnFallbackListener> fallbackListener; 2905 DispatchFunction([&] { 2906 track = TestDeviceInputConsumerTrack::Create(graph); 2907 track->ConnectDeviceInput(device, listener, PRINCIPAL_HANDLE_NONE); 2908 fallbackListener = new OnFallbackListener(track); 2909 track->AddListener(fallbackListener); 2910 }); 2911 2912 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 2913 EXPECT_TRUE(stream->mHasInput); 2914 EXPECT_TRUE(stream->mHasOutput); 2915 EXPECT_EQ(stream->InputChannels(), 1U); 2916 EXPECT_EQ(stream->GetInputDeviceID(), device); 2917 2918 while ( 2919 stream->State() 2920 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 2921 .valueOr(true)) { 2922 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2923 } 2924 2925 const auto waitForResult = [&](int aNumResult) { 2926 while (numProcessingParamsResults < aNumResult) { 2927 EXPECT_EQ(stream->ManualDataCallback(0), 2928 MockCubebStream::KeepProcessing::Yes); 2929 NS_ProcessNextEvent(); 2930 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2931 } 2932 }; 2933 2934 // Wait for the AudioCallbackDriver to come into effect. 2935 while (fallbackListener->OnFallback()) { 2936 EXPECT_EQ(stream->ManualDataCallback(0), 2937 MockCubebStream::KeepProcessing::Yes); 2938 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2939 } 2940 2941 // Wait for the first result after driver creation. 2942 waitForResult(1); 2943 2944 // Request new processing params. 2945 EXPECT_CALL(*listener, RequestedInputProcessingParams) 2946 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 2947 waitForResult(2); 2948 2949 // Test with returning error on new request. 2950 cubeb->SetInputProcessingApplyRv(CUBEB_ERROR); 2951 EXPECT_CALL(*listener, RequestedInputProcessingParams) 2952 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 2953 waitForResult(3); 2954 2955 // Test unsetting all params. 2956 cubeb->SetInputProcessingApplyRv(CUBEB_OK); 2957 EXPECT_CALL(*listener, RequestedInputProcessingParams) 2958 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NONE)); 2959 waitForResult(4); 2960 2961 // Switch driver. 2962 EXPECT_CALL(*listener, RequestedInputChannelCount).WillRepeatedly(Return(2)); 2963 // ReevaluateInputDevice() is not required for the native input, but is 2964 // called anyway. 2965 DispatchFunction([&] { 2966 track->QueueControlMessageWithNoShutdown( 2967 [&] { graph->ReevaluateInputDevice(device); }); 2968 }); 2969 ProcessEventQueue(); 2970 // Process the reevaluation message and perform the switch. 2971 auto initPromise = TakeN(cubeb->StreamInitEvent(), 1); 2972 EXPECT_EQ(stream->ManualDataCallback(0), MockCubebStream::KeepProcessing::No); 2973 std::tie(stream) = WaitFor(initPromise).unwrap()[0]; 2974 EXPECT_TRUE(stream->mHasInput); 2975 EXPECT_TRUE(stream->mHasOutput); 2976 EXPECT_EQ(stream->InputChannels(), 2U); 2977 EXPECT_EQ(stream->GetInputDeviceID(), device); 2978 2979 while ( 2980 stream->State() 2981 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 2982 .valueOr(true)) { 2983 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2984 } 2985 2986 // Wait for the new AudioCallbackDriver to come into effect. 2987 fallbackListener->Reset(); 2988 while (fallbackListener->OnFallback()) { 2989 EXPECT_EQ(stream->ManualDataCallback(0), 2990 MockCubebStream::KeepProcessing::Yes); 2991 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 2992 } 2993 2994 // Wait for the first result after driver creation. 2995 waitForResult(5); 2996 2997 // Test requesting something not supported. 2998 cubeb->SetSupportedInputProcessingParams( 2999 CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION | 3000 CUBEB_INPUT_PROCESSING_PARAM_AUTOMATIC_GAIN_CONTROL | 3001 CUBEB_INPUT_PROCESSING_PARAM_VOICE_ISOLATION, 3002 CUBEB_OK); 3003 EXPECT_CALL(*listener, RequestedInputProcessingParams) 3004 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 3005 waitForResult(6); 3006 3007 // Test requesting something when unsupported by backend. 3008 cubeb->SetSupportedInputProcessingParams(CUBEB_INPUT_PROCESSING_PARAM_NONE, 3009 CUBEB_ERROR_NOT_SUPPORTED); 3010 EXPECT_CALL(*listener, RequestedInputProcessingParams) 3011 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 3012 waitForResult(7); 3013 3014 // Clean up. 3015 DispatchFunction([&] { 3016 track->RemoveListener(fallbackListener); 3017 track->Destroy(); 3018 }); 3019 ProcessEventQueue(); 3020 // Process the destroy message and shut down. 3021 EXPECT_EQ(stream->ManualDataCallback(0), MockCubebStream::KeepProcessing::No); 3022 RefPtr<SmartMockCubebStream> destroyedStream = 3023 WaitFor(cubeb->StreamDestroyEvent()); 3024 EXPECT_EQ(destroyedStream.get(), stream.get()); 3025 { 3026 NativeInputTrack* native = graph->GetNativeInputTrackMainThread(); 3027 ASSERT_TRUE(!native); 3028 } 3029 } 3030 3031 TEST(TestAudioTrackGraph, PlatformProcessingNonNativeToNativeSwitch) 3032 { 3033 constexpr cubeb_input_processing_params allParams = 3034 CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION | 3035 CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION | 3036 CUBEB_INPUT_PROCESSING_PARAM_AUTOMATIC_GAIN_CONTROL | 3037 CUBEB_INPUT_PROCESSING_PARAM_VOICE_ISOLATION; 3038 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); 3039 cubeb->SetSupportedInputProcessingParams(allParams, CUBEB_OK); 3040 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 3041 3042 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 3043 MediaTrackGraph::SYSTEM_THREAD_DRIVER, /*Window ID*/ 1, 3044 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 3045 nullptr, GetMainThreadSerialEventTarget()); 3046 3047 const CubebUtils::AudioDeviceID firstDevice = (CubebUtils::AudioDeviceID)1; 3048 const CubebUtils::AudioDeviceID secondDevice = (CubebUtils::AudioDeviceID)2; 3049 3050 // Set up mock listeners. 3051 RefPtr<MockAudioDataListener> firstListener = 3052 MakeRefPtr<MockAudioDataListener>(); 3053 EXPECT_CALL(*firstListener, IsVoiceInput).WillRepeatedly(Return(true)); 3054 EXPECT_CALL(*firstListener, RequestedInputChannelCount) 3055 .WillRepeatedly(Return(1)); 3056 EXPECT_CALL(*firstListener, RequestedInputProcessingParams) 3057 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NONE)); 3058 3059 RefPtr<MockAudioDataListener> secondListener = 3060 MakeRefPtr<MockAudioDataListener>(); 3061 EXPECT_CALL(*secondListener, IsVoiceInput).WillRepeatedly(Return(true)); 3062 EXPECT_CALL(*secondListener, RequestedInputChannelCount) 3063 .WillRepeatedly(Return(1)); 3064 EXPECT_CALL(*secondListener, RequestedInputProcessingParams) 3065 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 3066 3067 // Expectations. 3068 const Result<cubeb_input_processing_params, int> noneResult( 3069 CUBEB_INPUT_PROCESSING_PARAM_NONE); 3070 const Result<cubeb_input_processing_params, int> echoResult( 3071 CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION); 3072 const Result<cubeb_input_processing_params, int> noiseResult( 3073 CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION); 3074 Atomic<int> numProcessingParamsResults(0); 3075 { 3076 InSequence s; 3077 // On first driver start. 3078 EXPECT_CALL(*firstListener, NotifySetRequestedInputProcessingParamsResult( 3079 graph, 0, Eq(std::ref(noneResult)))) 3080 .WillOnce([&] { ++numProcessingParamsResults; }); 3081 // After param update. 3082 EXPECT_CALL(*firstListener, 3083 NotifySetRequestedInputProcessingParams( 3084 graph, 1, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 3085 EXPECT_CALL(*firstListener, NotifySetRequestedInputProcessingParamsResult( 3086 graph, 1, Eq(std::ref(noiseResult)))) 3087 .WillOnce([&] { ++numProcessingParamsResults; }); 3088 // After second param update. 3089 EXPECT_CALL(*firstListener, 3090 NotifySetRequestedInputProcessingParams( 3091 graph, 2, CUBEB_INPUT_PROCESSING_PARAM_NONE)); 3092 EXPECT_CALL(*firstListener, NotifySetRequestedInputProcessingParamsResult( 3093 graph, 2, Eq(std::ref(noneResult)))) 3094 .WillOnce([&] { ++numProcessingParamsResults; }); 3095 // On non-native device's start. 3096 EXPECT_CALL(*secondListener, 3097 NotifySetRequestedInputProcessingParams( 3098 graph, 3, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 3099 EXPECT_CALL(*secondListener, NotifySetRequestedInputProcessingParamsResult( 3100 graph, 3, Eq(std::ref(echoResult)))) 3101 .WillOnce([&] { ++numProcessingParamsResults; }); 3102 // After switch to native device for second device. 3103 EXPECT_CALL(*firstListener, Disconnect); 3104 EXPECT_CALL(*secondListener, Disconnect); 3105 EXPECT_CALL(*secondListener, 3106 NotifySetRequestedInputProcessingParams( 3107 graph, 4, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION)); 3108 EXPECT_CALL(*secondListener, NotifySetRequestedInputProcessingParamsResult( 3109 graph, 4, Eq(std::ref(echoResult)))) 3110 .WillOnce([&] { ++numProcessingParamsResults; }); 3111 // After param update. 3112 EXPECT_CALL(*secondListener, 3113 NotifySetRequestedInputProcessingParams( 3114 graph, 5, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 3115 EXPECT_CALL(*secondListener, NotifySetRequestedInputProcessingParamsResult( 3116 graph, 5, Eq(std::ref(noiseResult)))) 3117 .WillOnce([&] { ++numProcessingParamsResults; }); 3118 EXPECT_CALL(*secondListener, Disconnect); 3119 } 3120 3121 // Open the first input device. It will be the native device of the graph. 3122 RefPtr<TestDeviceInputConsumerTrack> firstTrack; 3123 RefPtr<OnFallbackListener> fallbackListener; 3124 DispatchFunction([&] { 3125 firstTrack = TestDeviceInputConsumerTrack::Create(graph); 3126 firstTrack->ConnectDeviceInput(firstDevice, firstListener, 3127 PRINCIPAL_HANDLE_NONE); 3128 fallbackListener = new OnFallbackListener(firstTrack); 3129 firstTrack->AddListener(fallbackListener); 3130 }); 3131 3132 RefPtr<SmartMockCubebStream> nativeStream = WaitFor(cubeb->StreamInitEvent()); 3133 RefPtr<SmartMockCubebStream> nonNativeStream; 3134 EXPECT_TRUE(nativeStream->mHasInput); 3135 EXPECT_TRUE(nativeStream->mHasOutput); 3136 EXPECT_EQ(nativeStream->InputChannels(), 1U); 3137 EXPECT_EQ(nativeStream->GetInputDeviceID(), firstDevice); 3138 3139 while ( 3140 nativeStream->State() 3141 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 3142 .valueOr(true)) { 3143 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3144 } 3145 3146 // Wait for the AudioCallbackDriver to come into effect. 3147 while (fallbackListener->OnFallback()) { 3148 EXPECT_EQ(nativeStream->ManualDataCallback(0), 3149 MockCubebStream::KeepProcessing::Yes); 3150 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3151 } 3152 3153 const auto waitForResult = [&](int aNumResult) { 3154 while (numProcessingParamsResults < aNumResult) { 3155 EXPECT_EQ(nativeStream->ManualDataCallback(0), 3156 MockCubebStream::KeepProcessing::Yes); 3157 if (nonNativeStream) { 3158 EXPECT_EQ(nonNativeStream->ManualDataCallback(0), 3159 MockCubebStream::KeepProcessing::Yes); 3160 } 3161 NS_ProcessNextEvent(); 3162 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3163 } 3164 }; 3165 3166 // Wait for the first result after driver creation. 3167 waitForResult(1); 3168 3169 // Request new processing params. 3170 EXPECT_CALL(*firstListener, RequestedInputProcessingParams) 3171 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 3172 waitForResult(2); 3173 3174 // Again request new processing params, to move the processing params 3175 // generation of the first device higher so as to avoid ambiguity with the 3176 // processing params generation of the second device. 3177 EXPECT_CALL(*firstListener, RequestedInputProcessingParams) 3178 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NONE)); 3179 waitForResult(3); 3180 3181 // Open the second input device. It will be a non-native device of the graph. 3182 RefPtr<TestDeviceInputConsumerTrack> secondTrack; 3183 DispatchFunction([&] { 3184 secondTrack = TestDeviceInputConsumerTrack::Create(graph); 3185 secondTrack->ConnectDeviceInput(secondDevice, secondListener, 3186 PRINCIPAL_HANDLE_NONE); 3187 secondTrack->AddListener(fallbackListener); 3188 }); 3189 3190 auto initPromise = TakeN(cubeb->StreamInitEvent(), 1); 3191 DispatchFunction([&] { 3192 // TakeN dispatches a task that runs SpinEventLoopUntil while blocking the 3193 // caller. Make sure any event loop spinning that needs to be intertwined 3194 // with driving the graph runs in a nested task, so as to not block the test 3195 // flow. I.e. if the caller that gets blocked on the TakeN task is WaitFor 3196 // below, we will deadlock, because the graph must be iterated to unblock 3197 // the TakeN task. 3198 EXPECT_EQ(nativeStream->ManualDataCallback(0), 3199 MockCubebStream::KeepProcessing::Yes); 3200 ProcessEventQueue(); 3201 EXPECT_EQ(nativeStream->ManualDataCallback(0), 3202 MockCubebStream::KeepProcessing::Yes); 3203 }); 3204 3205 std::tie(nonNativeStream) = WaitFor(initPromise).unwrap()[0]; 3206 EXPECT_TRUE(nonNativeStream->mHasInput); 3207 EXPECT_FALSE(nonNativeStream->mHasOutput); 3208 EXPECT_EQ(nonNativeStream->InputChannels(), 1U); 3209 EXPECT_EQ(nonNativeStream->GetInputDeviceID(), secondDevice); 3210 3211 while ( 3212 nonNativeStream->State() 3213 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 3214 .valueOr(true)) { 3215 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3216 } 3217 3218 // Wait for the first result after non-native device input creation. 3219 waitForResult(4); 3220 3221 // Disconnect native input. First non-native input should be promoted to 3222 // native, and processing param generation should increase monotonically 3223 // through the switch. 3224 DispatchFunction([&] { 3225 firstTrack->RemoveListener(fallbackListener); 3226 secondTrack->AddListener(fallbackListener); 3227 firstTrack->DisconnectDeviceInput(); 3228 }); 3229 ProcessEventQueue(); 3230 initPromise = TakeN(cubeb->StreamInitEvent(), 1); 3231 // Process the disconnect message, perform the switch, 3232 // and check that the second device is now used with the new graph driver. 3233 EXPECT_EQ(nativeStream->ManualDataCallback(0), 3234 MockCubebStream::KeepProcessing::No); 3235 std::tie(nativeStream) = WaitFor(initPromise).unwrap()[0]; 3236 EXPECT_TRUE(nativeStream->mHasInput); 3237 EXPECT_TRUE(nativeStream->mHasOutput); 3238 EXPECT_EQ(nativeStream->InputChannels(), 1U); 3239 EXPECT_EQ(nativeStream->GetInputDeviceID(), secondDevice); 3240 3241 while ( 3242 nativeStream->State() 3243 .map([](cubeb_state aState) { return aState != CUBEB_STATE_STARTED; }) 3244 .valueOr(true)) { 3245 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3246 } 3247 3248 // Wait for the new AudioCallbackDriver to come into effect. 3249 fallbackListener->Reset(); 3250 while (fallbackListener->OnFallback()) { 3251 EXPECT_EQ(nativeStream->ManualDataCallback(0), 3252 MockCubebStream::KeepProcessing::Yes); 3253 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3254 } 3255 3256 // So that waitForResult doesn't make callbacks to it anymore. 3257 nonNativeStream = nullptr; 3258 3259 // Wait for the first result after driver creation. 3260 waitForResult(5); 3261 3262 // Request new processing params. 3263 EXPECT_CALL(*secondListener, RequestedInputProcessingParams) 3264 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION)); 3265 waitForResult(6); 3266 3267 // Clean up. 3268 DispatchFunction([&] { 3269 firstTrack->Destroy(); 3270 secondTrack->RemoveListener(fallbackListener); 3271 secondTrack->Destroy(); 3272 }); 3273 auto destroyPromise = TakeN(cubeb->StreamDestroyEvent(), 1); 3274 DispatchFunction([&] { 3275 ProcessEventQueue(); 3276 // Process the destroy message and shut down native. 3277 EXPECT_EQ(nativeStream->ManualDataCallback(0), 3278 MockCubebStream::KeepProcessing::No); 3279 }); 3280 RefPtr<SmartMockCubebStream> destroyedStream; 3281 std::tie(destroyedStream) = WaitFor(destroyPromise).unwrap()[0]; 3282 EXPECT_EQ(destroyedStream, nativeStream); 3283 { 3284 NativeInputTrack* native = graph->GetNativeInputTrackMainThread(); 3285 ASSERT_TRUE(!native); 3286 } 3287 } 3288 3289 class MockProcessedMediaTrack : public ProcessedMediaTrack { 3290 public: 3291 explicit MockProcessedMediaTrack(TrackRate aRate) 3292 : ProcessedMediaTrack(aRate, MediaSegment::AUDIO, new AudioSegment()) { 3293 ON_CALL(*this, ProcessInput) 3294 .WillByDefault([segment = GetData<AudioSegment>()]( 3295 GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { 3296 segment->AppendNullData(aTo - aFrom); 3297 }); 3298 } 3299 3300 MOCK_METHOD(void, ProcessInput, 3301 (GraphTime aFrom, GraphTime aTo, uint32_t aFlags), (override)); 3302 3303 uint32_t NumberOfChannels() const override { return 2; }; 3304 }; 3305 3306 TEST(TestAudioTrackGraph, EmptyProcessingInterval) 3307 { 3308 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); 3309 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 3310 3311 MediaTrackGraph* graph = MediaTrackGraphImpl::GetInstance( 3312 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 3313 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 3314 nullptr, GetMainThreadSerialEventTarget()); 3315 3316 RefPtr processedTrack = new MockProcessedMediaTrack(graph->GraphRate()); 3317 RefPtr fallbackListener = new OnFallbackListener(processedTrack); 3318 MockFunction<void(const char* name)> checkpoint; 3319 { 3320 InSequence s; 3321 EXPECT_CALL(*processedTrack, ProcessInput).Times(AtLeast(1)); 3322 EXPECT_CALL(checkpoint, Call(StrEq("before single iteration"))); 3323 EXPECT_CALL(*processedTrack, ProcessInput).Times(1); 3324 EXPECT_CALL(checkpoint, Call(StrEq("before empty iteration"))); 3325 EXPECT_CALL(checkpoint, Call(StrEq("after empty iteration"))); 3326 } 3327 DispatchFunction([&] { 3328 graph->AddTrack(processedTrack); 3329 processedTrack->AddAudioOutput(reinterpret_cast<void*>(1), nullptr); 3330 processedTrack->AddListener(fallbackListener); 3331 }); 3332 3333 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 3334 while (stream->State().isNothing()) { 3335 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3336 } 3337 EXPECT_EQ(*stream->State(), CUBEB_STATE_STARTED); 3338 // Wait for the AudioCallbackDriver to come into effect. 3339 while (fallbackListener->OnFallback()) { 3340 EXPECT_EQ(stream->ManualDataCallback(1), 3341 MockCubebStream::KeepProcessing::Yes); 3342 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3343 } 3344 // The graph is now run by ManualDataCallback(). 3345 GraphTime processedUpTo0 = processedTrack->GetEnd(); 3346 EXPECT_GT(processedUpTo0, 0u); 3347 checkpoint.Call("before single iteration"); 3348 // 128 matches the block size used by MediaTrackGraph and so is expected to 3349 // trigger another block of processing. 3350 EXPECT_EQ(stream->ManualDataCallback(128), 3351 MockCubebStream::KeepProcessing::Yes); 3352 GraphTime processedUpTo1 = processedTrack->GetEnd(); 3353 EXPECT_EQ(processedUpTo1, processedUpTo0 + 128); 3354 // Test that ProcessInput() is not called in a graph iteration with no 3355 // frames to be rendered. 3356 checkpoint.Call("before empty iteration"); 3357 EXPECT_EQ(stream->ManualDataCallback(0), 3358 MockCubebStream::KeepProcessing::Yes); 3359 checkpoint.Call("after empty iteration"); 3360 EXPECT_EQ(processedTrack->GetEnd(), processedUpTo1); 3361 3362 DispatchFunction([&] { processedTrack->Destroy(); }); 3363 ProcessEventQueue(); 3364 // Process the destroy message and drain the stream. 3365 auto destroyPromise = TakeN(cubeb->StreamDestroyEvent(), 1); 3366 while (stream->ManualDataCallback(0) == 3367 MockCubebStream::KeepProcessing::Yes) { 3368 } 3369 // Ensure the stream is no longer used by its MockCubeb before releasing our 3370 // reference, and before the next test might ForceSetCubebContext() to 3371 // destroy our cubeb. 3372 (void)WaitFor(destroyPromise).unwrap()[0]; 3373 } 3374 3375 TEST(TestAudioTrackGraph, DefaultOutputDeviceIDTracking) 3376 { 3377 #ifdef ANDROID 3378 GTEST_SKIP() << "On Android CubebDeviceEnumerator, not the cubeb backend, " 3379 "handles device enumeration, exposing only a single input " 3380 "and output device. Both with devid 0."; 3381 #else 3382 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); 3383 AddDevices(cubeb, 1, CUBEB_DEVICE_TYPE_INPUT); 3384 AddDevices(cubeb, 2, CUBEB_DEVICE_TYPE_OUTPUT); 3385 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); 3386 3387 MediaTrackGraphImpl* graph = MediaTrackGraphImpl::GetInstance( 3388 MediaTrackGraph::AUDIO_THREAD_DRIVER, /*Window ID*/ 1, 3389 CubebUtils::PreferredSampleRate(/* aShouldResistFingerprinting */ false), 3390 nullptr, GetMainThreadSerialEventTarget()); 3391 3392 // Mocks and expectations. 3393 RefPtr processedTrack = new MockProcessedMediaTrack(graph->GraphRate()); 3394 RefPtr<MockAudioDataListener> dataListener = 3395 MakeRefPtr<MockAudioDataListener>(); 3396 const Result<cubeb_input_processing_params, int> notSupportedResult( 3397 Err(CUBEB_ERROR_NOT_SUPPORTED)); 3398 3399 EXPECT_CALL(*processedTrack, ProcessInput).Times(AtLeast(1)); 3400 EXPECT_CALL(*dataListener, RequestedInputChannelCount) 3401 .WillRepeatedly(Return(2)); 3402 EXPECT_CALL(*dataListener, RequestedInputProcessingParams) 3403 .WillRepeatedly(Return(CUBEB_INPUT_PROCESSING_PARAM_NONE)); 3404 EXPECT_CALL(*dataListener, IsVoiceInput).WillRepeatedly(Return(true)); 3405 { 3406 InSequence s; 3407 EXPECT_CALL(*dataListener, NotifySetRequestedInputProcessingParamsResult( 3408 graph, 0, Eq(std::ref(notSupportedResult)))); 3409 EXPECT_CALL(*dataListener, Disconnect); 3410 } 3411 3412 // Add a track to maintain an output-only audio driver. 3413 RefPtr fallbackListener = new OnFallbackListener(processedTrack); 3414 DispatchFunction([&] { 3415 graph->AddTrack(processedTrack); 3416 processedTrack->AddAudioOutput(reinterpret_cast<void*>(1), nullptr); 3417 processedTrack->AddListener(fallbackListener); 3418 }); 3419 3420 RefPtr<SmartMockCubebStream> stream = WaitFor(cubeb->StreamInitEvent()); 3421 while (stream->State().isNothing()) { 3422 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3423 } 3424 EXPECT_EQ(*stream->State(), CUBEB_STATE_STARTED); 3425 // Wait for the AudioCallbackDriver to come into effect. 3426 while (fallbackListener->OnFallback()) { 3427 EXPECT_EQ(stream->ManualDataCallback(1), 3428 MockCubebStream::KeepProcessing::Yes); 3429 std::this_thread::sleep_for(std::chrono::milliseconds(1)); 3430 } 3431 3432 // The graph is now run by ManualDataCallback(). 3433 3434 EXPECT_EQ(graph->DefaultOutputDeviceID(), nullptr) 3435 << "Without an input track, the default output isn't tracked."; 3436 3437 // Add an input track. 3438 RefPtr<TestDeviceInputConsumerTrack> inputTrack; 3439 DispatchFunction([&] { 3440 processedTrack->RemoveListener(fallbackListener); 3441 inputTrack = TestDeviceInputConsumerTrack::Create(graph); 3442 inputTrack->ConnectDeviceInput(CubebUtils::AudioDeviceID(1), dataListener, 3443 PRINCIPAL_HANDLE_NONE); 3444 fallbackListener = new OnFallbackListener(inputTrack); 3445 inputTrack->AddListener(fallbackListener); 3446 }); 3447 auto initPromise = TakeN(cubeb->StreamInitEvent(), 1); 3448 ProcessEventQueue(); 3449 // Process the disconnect message, perform the switch, 3450 // and check that the second device is now used with the new graph driver. 3451 EXPECT_EQ(stream->ManualDataCallback(0), MockCubebStream::KeepProcessing::No); 3452 std::tie(stream) = WaitFor(initPromise).unwrap()[0]; 3453 EXPECT_TRUE(stream->mHasInput); 3454 EXPECT_TRUE(stream->mHasOutput); 3455 EXPECT_EQ(stream->InputChannels(), 2U); 3456 EXPECT_EQ(stream->GetInputDeviceID(), CubebUtils::AudioDeviceID(1)); 3457 EXPECT_EQ(stream->GetOutputDeviceID(), nullptr); 3458 stream->ManualDataCallback(0); 3459 3460 EXPECT_EQ(graph->DefaultOutputDeviceID(), CubebUtils::AudioDeviceID(2)) 3461 << "With an input track, we are tracking the default output device."; 3462 3463 cubeb->SetPreferredDevice(CubebUtils::AudioDeviceID(1), 3464 CUBEB_DEVICE_TYPE_OUTPUT); 3465 ProcessEventQueue(); 3466 EXPECT_EQ(graph->DefaultOutputDeviceID(), CubebUtils::AudioDeviceID(1)) 3467 << "Default output device tracking follows system changes."; 3468 3469 DispatchFunction([&] { 3470 inputTrack->RemoveListener(fallbackListener); 3471 inputTrack->Destroy(); 3472 processedTrack->Destroy(); 3473 }); 3474 ProcessEventQueue(); 3475 // Process the destroy message and drain the stream. 3476 auto destroyPromise = TakeN(cubeb->StreamDestroyEvent(), 1); 3477 while (stream->ManualDataCallback(0) == 3478 MockCubebStream::KeepProcessing::Yes) { 3479 } 3480 // Ensure the stream is no longer used by its MockCubeb before releasing our 3481 // reference, and before the next test might ForceSetCubebContext() to 3482 // destroy our cubeb. 3483 (void)WaitFor(destroyPromise).unwrap()[0]; 3484 #endif 3485 } 3486 3487 #undef Invoke 3488 #undef DispatchFunction 3489 #undef DispatchMethod