audio_state_unittest.cc (19299B)
1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "audio/audio_state.h" 12 13 #include <cmath> 14 #include <cstddef> 15 #include <cstdint> 16 #include <cstdlib> 17 #include <memory> 18 #include <numbers> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/functional/any_invocable.h" 23 #include "absl/strings/string_view.h" 24 #include "api/audio/audio_frame.h" 25 #include "api/audio/audio_frame_processor.h" 26 #include "api/audio/audio_mixer.h" 27 #include "api/location.h" 28 #include "api/make_ref_counted.h" 29 #include "api/ref_count.h" 30 #include "api/scoped_refptr.h" 31 #include "api/task_queue/task_queue_base.h" 32 #include "api/task_queue/task_queue_factory.h" 33 #include "api/task_queue/test/mock_task_queue_base.h" 34 #include "call/audio_state.h" 35 #include "call/test/mock_audio_receive_stream.h" 36 #include "call/test/mock_audio_send_stream.h" 37 #include "modules/async_audio_processing/async_audio_processing.h" 38 #include "modules/audio_device/include/mock_audio_device.h" 39 #include "modules/audio_mixer/audio_mixer_impl.h" 40 #include "modules/audio_processing/include/mock_audio_processing.h" 41 #include "rtc_base/task_queue_for_test.h" 42 #include "rtc_base/thread.h" 43 #include "test/gmock.h" 44 #include "test/gtest.h" 45 46 namespace webrtc { 47 namespace test { 48 namespace { 49 50 using ::testing::_; 51 using ::testing::InSequence; 52 using ::testing::Matcher; 53 using ::testing::NiceMock; 54 using ::testing::NotNull; 55 using ::testing::StrictMock; 56 using ::testing::Values; 57 58 constexpr int kSampleRate = 16000; 59 constexpr int kNumberOfChannels = 1; 60 61 struct FakeAsyncAudioProcessingHelper { 62 class FakeTaskQueue : public StrictMock<MockTaskQueueBase> { 63 public: 64 FakeTaskQueue() = default; 65 66 void Delete() override { delete this; } 67 void PostTaskImpl(absl::AnyInvocable<void() &&> task, 68 const PostTaskTraits& /*traits*/, 69 const Location& /*location*/) override { 70 std::move(task)(); 71 } 72 }; 73 74 class FakeTaskQueueFactory : public TaskQueueFactory { 75 public: 76 FakeTaskQueueFactory() = default; 77 ~FakeTaskQueueFactory() override = default; 78 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> CreateTaskQueue( 79 absl::string_view /* name */, 80 Priority /* priority */) const override { 81 return std::unique_ptr<TaskQueueBase, TaskQueueDeleter>( 82 new FakeTaskQueue()); 83 } 84 }; 85 86 class MockAudioFrameProcessor : public AudioFrameProcessor { 87 public: 88 ~MockAudioFrameProcessor() override = default; 89 90 MOCK_METHOD(void, ProcessCalled, ()); 91 MOCK_METHOD(void, SinkSet, ()); 92 MOCK_METHOD(void, SinkCleared, ()); 93 94 void Process(std::unique_ptr<AudioFrame> frame) override { 95 ProcessCalled(); 96 sink_callback_(std::move(frame)); 97 } 98 99 void SetSink(OnAudioFrameCallback sink_callback) override { 100 sink_callback_ = std::move(sink_callback); 101 if (sink_callback_ == nullptr) 102 SinkCleared(); 103 else 104 SinkSet(); 105 } 106 107 private: 108 OnAudioFrameCallback sink_callback_; 109 }; 110 111 NiceMock<MockAudioFrameProcessor> audio_frame_processor_; 112 FakeTaskQueueFactory task_queue_factory_; 113 114 scoped_refptr<AsyncAudioProcessing::Factory> CreateFactory() { 115 return make_ref_counted<AsyncAudioProcessing::Factory>( 116 audio_frame_processor_, task_queue_factory_); 117 } 118 }; 119 120 struct ConfigHelper { 121 struct Params { 122 bool use_null_audio_processing; 123 bool use_async_audio_processing; 124 }; 125 126 explicit ConfigHelper(const Params& params) 127 : audio_mixer(AudioMixerImpl::Create()) { 128 audio_state_config.audio_mixer = audio_mixer; 129 audio_state_config.audio_processing = 130 params.use_null_audio_processing 131 ? nullptr 132 : make_ref_counted<testing::NiceMock<MockAudioProcessing>>(); 133 audio_state_config.audio_device_module = 134 make_ref_counted<NiceMock<MockAudioDeviceModule>>(); 135 if (params.use_async_audio_processing) { 136 audio_state_config.async_audio_processing_factory = 137 async_audio_processing_helper_.CreateFactory(); 138 } 139 } 140 AudioState::Config& config() { return audio_state_config; } 141 scoped_refptr<AudioMixer> mixer() { return audio_mixer; } 142 NiceMock<FakeAsyncAudioProcessingHelper::MockAudioFrameProcessor>& 143 mock_audio_frame_processor() { 144 return async_audio_processing_helper_.audio_frame_processor_; 145 } 146 147 private: 148 AudioState::Config audio_state_config; 149 scoped_refptr<AudioMixer> audio_mixer; 150 FakeAsyncAudioProcessingHelper async_audio_processing_helper_; 151 }; 152 153 class FakeAudioSource : public AudioMixer::Source { 154 public: 155 int Ssrc() const override { return 0; } 156 157 int PreferredSampleRate() const override { return kSampleRate; } 158 159 MOCK_METHOD(AudioFrameInfo, 160 GetAudioFrameWithInfo, 161 (int sample_rate_hz, AudioFrame*), 162 (override)); 163 }; 164 165 std::vector<int16_t> Create10msTestData(int sample_rate_hz, 166 size_t num_channels) { 167 const int samples_per_channel = sample_rate_hz / 100; 168 std::vector<int16_t> audio_data(samples_per_channel * num_channels, 0); 169 // Fill the first channel with a 1kHz sine wave. 170 const float inc = (2 * std::numbers::pi_v<float> * 1000) / sample_rate_hz; 171 float w = 0.f; 172 for (int i = 0; i < samples_per_channel; ++i) { 173 audio_data[i * num_channels] = static_cast<int16_t>(32767.f * std::sin(w)); 174 w += inc; 175 } 176 return audio_data; 177 } 178 179 std::vector<uint32_t> ComputeChannelLevels(AudioFrame* audio_frame) { 180 const size_t num_channels = audio_frame->num_channels_; 181 const size_t samples_per_channel = audio_frame->samples_per_channel_; 182 std::vector<uint32_t> levels(num_channels, 0); 183 for (size_t i = 0; i < samples_per_channel; ++i) { 184 for (size_t j = 0; j < num_channels; ++j) { 185 levels[j] += std::abs(audio_frame->data()[i * num_channels + j]); 186 } 187 } 188 return levels; 189 } 190 } // namespace 191 192 class AudioStateTest : public ::testing::TestWithParam<ConfigHelper::Params> {}; 193 194 TEST_P(AudioStateTest, Create) { 195 ConfigHelper helper(GetParam()); 196 auto audio_state = AudioState::Create(helper.config()); 197 EXPECT_TRUE(audio_state.get()); 198 } 199 200 TEST_P(AudioStateTest, ConstructDestruct) { 201 ConfigHelper helper(GetParam()); 202 scoped_refptr<internal::AudioState> audio_state( 203 make_ref_counted<internal::AudioState>(helper.config())); 204 } 205 206 TEST_P(AudioStateTest, CreateUseDeleteOnDifferentThread) { 207 ConfigHelper helper(GetParam()); 208 scoped_refptr<AudioState> audio_state = AudioState::Create(helper.config()); 209 ASSERT_THAT(audio_state, NotNull()); 210 TaskQueueForTest queue; 211 queue.SendTask([&] { 212 // Attach to the current TQ. 213 audio_state->SetStereoChannelSwapping(true); 214 // Ensure the object gets deleted on the current thread. 215 EXPECT_EQ(audio_state.release()->Release(), 216 RefCountReleaseStatus::kDroppedLastRef); 217 }); 218 } 219 220 TEST_P(AudioStateTest, RecordedAudioArrivesAtSingleStream) { 221 ConfigHelper helper(GetParam()); 222 223 if (GetParam().use_async_audio_processing) { 224 EXPECT_CALL(helper.mock_audio_frame_processor(), SinkSet); 225 EXPECT_CALL(helper.mock_audio_frame_processor(), ProcessCalled); 226 EXPECT_CALL(helper.mock_audio_frame_processor(), SinkCleared); 227 } 228 229 scoped_refptr<internal::AudioState> audio_state( 230 make_ref_counted<internal::AudioState>(helper.config())); 231 232 MockAudioSendStream stream; 233 audio_state->AddSendingStream(&stream, 8000, 2); 234 235 EXPECT_CALL( 236 stream, 237 SendAudioDataForMock(::testing::AllOf( 238 ::testing::Field(&AudioFrame::sample_rate_hz_, ::testing::Eq(8000)), 239 ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(2u))))) 240 .WillOnce( 241 // Verify that channels are not swapped by default. 242 ::testing::Invoke([](AudioFrame* audio_frame) { 243 auto levels = ComputeChannelLevels(audio_frame); 244 EXPECT_LT(0u, levels[0]); 245 EXPECT_EQ(0u, levels[1]); 246 })); 247 MockAudioProcessing* ap = 248 GetParam().use_null_audio_processing 249 ? nullptr 250 : static_cast<MockAudioProcessing*>(audio_state->audio_processing()); 251 if (ap) { 252 EXPECT_CALL(*ap, set_stream_delay_ms(0)); 253 EXPECT_CALL(*ap, set_stream_key_pressed(false)); 254 EXPECT_CALL(*ap, ProcessStream(_, _, _, Matcher<int16_t*>(_))); 255 } 256 257 constexpr size_t kNumChannels = 2; 258 auto audio_data = Create10msTestData(kSampleRate, kNumChannels); 259 uint32_t new_mic_level = 667; 260 audio_state->audio_transport()->RecordedDataIsAvailable( 261 &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, 262 kSampleRate, 0, 0, 0, false, new_mic_level); 263 EXPECT_EQ(667u, new_mic_level); 264 265 audio_state->RemoveSendingStream(&stream); 266 } 267 268 TEST_P(AudioStateTest, RecordedAudioArrivesAtMultipleStreams) { 269 ConfigHelper helper(GetParam()); 270 271 if (GetParam().use_async_audio_processing) { 272 EXPECT_CALL(helper.mock_audio_frame_processor(), SinkSet); 273 EXPECT_CALL(helper.mock_audio_frame_processor(), ProcessCalled); 274 EXPECT_CALL(helper.mock_audio_frame_processor(), SinkCleared); 275 } 276 277 scoped_refptr<internal::AudioState> audio_state( 278 make_ref_counted<internal::AudioState>(helper.config())); 279 280 MockAudioSendStream stream_1; 281 MockAudioSendStream stream_2; 282 audio_state->AddSendingStream(&stream_1, 8001, 2); 283 audio_state->AddSendingStream(&stream_2, 32000, 1); 284 285 EXPECT_CALL( 286 stream_1, 287 SendAudioDataForMock(::testing::AllOf( 288 ::testing::Field(&AudioFrame::sample_rate_hz_, ::testing::Eq(16000)), 289 ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(1u))))) 290 .WillOnce( 291 // Verify that there is output signal. 292 ::testing::Invoke([](AudioFrame* audio_frame) { 293 auto levels = ComputeChannelLevels(audio_frame); 294 EXPECT_LT(0u, levels[0]); 295 })); 296 EXPECT_CALL( 297 stream_2, 298 SendAudioDataForMock(::testing::AllOf( 299 ::testing::Field(&AudioFrame::sample_rate_hz_, ::testing::Eq(16000)), 300 ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(1u))))) 301 .WillOnce( 302 // Verify that there is output signal. 303 ::testing::Invoke([](AudioFrame* audio_frame) { 304 auto levels = ComputeChannelLevels(audio_frame); 305 EXPECT_LT(0u, levels[0]); 306 })); 307 MockAudioProcessing* ap = 308 static_cast<MockAudioProcessing*>(audio_state->audio_processing()); 309 if (ap) { 310 EXPECT_CALL(*ap, set_stream_delay_ms(5)); 311 EXPECT_CALL(*ap, set_stream_key_pressed(true)); 312 EXPECT_CALL(*ap, ProcessStream(_, _, _, Matcher<int16_t*>(_))); 313 } 314 315 constexpr size_t kNumChannels = 1; 316 auto audio_data = Create10msTestData(kSampleRate, kNumChannels); 317 uint32_t new_mic_level = 667; 318 audio_state->audio_transport()->RecordedDataIsAvailable( 319 &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, 320 kSampleRate, 5, 0, 0, true, new_mic_level); 321 EXPECT_EQ(667u, new_mic_level); 322 323 audio_state->RemoveSendingStream(&stream_1); 324 audio_state->RemoveSendingStream(&stream_2); 325 } 326 327 TEST_P(AudioStateTest, EnableChannelSwap) { 328 constexpr size_t kNumChannels = 2; 329 330 ConfigHelper helper(GetParam()); 331 332 if (GetParam().use_async_audio_processing) { 333 EXPECT_CALL(helper.mock_audio_frame_processor(), SinkSet); 334 EXPECT_CALL(helper.mock_audio_frame_processor(), ProcessCalled); 335 EXPECT_CALL(helper.mock_audio_frame_processor(), SinkCleared); 336 } 337 338 scoped_refptr<internal::AudioState> audio_state( 339 make_ref_counted<internal::AudioState>(helper.config())); 340 341 audio_state->SetStereoChannelSwapping(true); 342 343 MockAudioSendStream stream; 344 audio_state->AddSendingStream(&stream, kSampleRate, kNumChannels); 345 346 EXPECT_CALL(stream, SendAudioDataForMock(_)) 347 .WillOnce( 348 // Verify that channels are swapped. 349 ::testing::Invoke([](AudioFrame* audio_frame) { 350 auto levels = ComputeChannelLevels(audio_frame); 351 EXPECT_EQ(0u, levels[0]); 352 EXPECT_LT(0u, levels[1]); 353 })); 354 355 auto audio_data = Create10msTestData(kSampleRate, kNumChannels); 356 uint32_t new_mic_level = 667; 357 audio_state->audio_transport()->RecordedDataIsAvailable( 358 &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, 359 kSampleRate, 0, 0, 0, false, new_mic_level); 360 EXPECT_EQ(667u, new_mic_level); 361 362 audio_state->RemoveSendingStream(&stream); 363 } 364 365 TEST_P(AudioStateTest, 366 QueryingTransportForAudioShouldResultInGetAudioCallOnMixerSource) { 367 ConfigHelper helper(GetParam()); 368 auto audio_state = AudioState::Create(helper.config()); 369 370 FakeAudioSource fake_source; 371 helper.mixer()->AddSource(&fake_source); 372 373 EXPECT_CALL(fake_source, GetAudioFrameWithInfo(_, _)) 374 .WillOnce( 375 ::testing::Invoke([](int sample_rate_hz, AudioFrame* audio_frame) { 376 audio_frame->sample_rate_hz_ = sample_rate_hz; 377 audio_frame->samples_per_channel_ = sample_rate_hz / 100; 378 audio_frame->num_channels_ = kNumberOfChannels; 379 return AudioMixer::Source::AudioFrameInfo::kNormal; 380 })); 381 382 int16_t audio_buffer[kSampleRate / 100 * kNumberOfChannels]; 383 size_t n_samples_out; 384 int64_t elapsed_time_ms; 385 int64_t ntp_time_ms; 386 audio_state->audio_transport()->NeedMorePlayData( 387 kSampleRate / 100, kNumberOfChannels * 2, kNumberOfChannels, kSampleRate, 388 audio_buffer, n_samples_out, &elapsed_time_ms, &ntp_time_ms); 389 } 390 391 TEST_P(AudioStateTest, StartRecordingDoesNothingWithoutStream) { 392 ConfigHelper helper(GetParam()); 393 scoped_refptr<internal::AudioState> audio_state( 394 make_ref_counted<internal::AudioState>(helper.config())); 395 396 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 397 helper.config().audio_device_module.get()); 398 399 EXPECT_CALL(*adm, InitRecording()).Times(0); 400 EXPECT_CALL(*adm, StartRecording()).Times(0); 401 EXPECT_CALL(*adm, StopRecording()).Times(1); 402 audio_state->SetRecording(false); 403 audio_state->SetRecording(true); 404 } 405 406 TEST_P(AudioStateTest, AddStreamDoesNothingIfRecordingDisabled) { 407 ConfigHelper helper(GetParam()); 408 scoped_refptr<internal::AudioState> audio_state( 409 make_ref_counted<internal::AudioState>(helper.config())); 410 411 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 412 helper.config().audio_device_module.get()); 413 414 EXPECT_CALL(*adm, StopRecording()).Times(2); 415 audio_state->SetRecording(false); 416 417 MockAudioSendStream stream; 418 EXPECT_CALL(*adm, StartRecording).Times(0); 419 audio_state->AddSendingStream(&stream, kSampleRate, kNumberOfChannels); 420 audio_state->RemoveSendingStream(&stream); 421 } 422 423 TEST_P(AudioStateTest, AlwaysCallInitRecordingBeforeStartRecording) { 424 ConfigHelper helper(GetParam()); 425 scoped_refptr<internal::AudioState> audio_state( 426 make_ref_counted<internal::AudioState>(helper.config())); 427 428 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 429 helper.config().audio_device_module.get()); 430 431 MockAudioSendStream stream; 432 { 433 InSequence s; 434 EXPECT_CALL(*adm, InitRecording()); 435 EXPECT_CALL(*adm, StartRecording()); 436 audio_state->AddSendingStream(&stream, kSampleRate, kNumberOfChannels); 437 } 438 439 EXPECT_CALL(*adm, StopRecording()); 440 audio_state->SetRecording(false); 441 442 { 443 InSequence s; 444 EXPECT_CALL(*adm, InitRecording()); 445 EXPECT_CALL(*adm, StartRecording()); 446 audio_state->SetRecording(true); 447 } 448 449 EXPECT_CALL(*adm, StopRecording()); 450 audio_state->RemoveSendingStream(&stream); 451 } 452 453 // The recording can also be initialized by WebRtcVoiceSendChannel 454 // options_.init_recording_on_send. Make sure StopRecording is still 455 // being called in this scenario. 456 TEST_P(AudioStateTest, CallStopRecordingIfRecordingIsInitialized) { 457 ConfigHelper helper(GetParam()); 458 scoped_refptr<internal::AudioState> audio_state( 459 make_ref_counted<internal::AudioState>(helper.config())); 460 461 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 462 helper.config().audio_device_module.get()); 463 464 audio_state->SetRecording(false); 465 466 EXPECT_CALL(*adm, StopRecording()); 467 audio_state->SetRecording(false); 468 } 469 470 TEST_P(AudioStateTest, StartPlayoutDoesNothingWithoutStream) { 471 ConfigHelper helper(GetParam()); 472 scoped_refptr<internal::AudioState> audio_state( 473 make_ref_counted<internal::AudioState>(helper.config())); 474 475 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 476 helper.config().audio_device_module.get()); 477 478 EXPECT_CALL(*adm, InitPlayout()).Times(0); 479 EXPECT_CALL(*adm, StartPlayout()).Times(0); 480 EXPECT_CALL(*adm, StopPlayout()).Times(1); 481 audio_state->SetPlayout(false); 482 483 audio_state->SetPlayout(true); 484 } 485 486 TEST_P(AudioStateTest, AlwaysCallInitPlayoutBeforeStartPlayout) { 487 ConfigHelper helper(GetParam()); 488 scoped_refptr<internal::AudioState> audio_state( 489 make_ref_counted<internal::AudioState>(helper.config())); 490 491 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 492 helper.config().audio_device_module.get()); 493 494 MockAudioReceiveStream stream; 495 { 496 InSequence s; 497 EXPECT_CALL(*adm, InitPlayout()); 498 EXPECT_CALL(*adm, StartPlayout()); 499 audio_state->AddReceivingStream(&stream); 500 } 501 502 // SetPlayout(false) starts the NullAudioPoller...which needs a thread. 503 ThreadManager::Instance()->WrapCurrentThread(); 504 505 EXPECT_CALL(*adm, StopPlayout()); 506 audio_state->SetPlayout(false); 507 508 { 509 InSequence s; 510 EXPECT_CALL(*adm, InitPlayout()); 511 EXPECT_CALL(*adm, StartPlayout()); 512 audio_state->SetPlayout(true); 513 } 514 515 // Playout without streams starts the NullAudioPoller... 516 // which needs a thread. 517 ThreadManager::Instance()->WrapCurrentThread(); 518 519 EXPECT_CALL(*adm, StopPlayout()); 520 audio_state->RemoveReceivingStream(&stream); 521 } 522 523 TEST_P(AudioStateTest, CallStopPlayoutIfPlayoutIsInitialized) { 524 ConfigHelper helper(GetParam()); 525 scoped_refptr<internal::AudioState> audio_state( 526 make_ref_counted<internal::AudioState>(helper.config())); 527 528 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 529 helper.config().audio_device_module.get()); 530 531 audio_state->SetPlayout(false); 532 533 EXPECT_CALL(*adm, StopPlayout()); 534 audio_state->SetPlayout(false); 535 } 536 537 TEST_P(AudioStateTest, AddStreamDoesNothingIfPlayoutDisabled) { 538 ConfigHelper helper(GetParam()); 539 scoped_refptr<internal::AudioState> audio_state( 540 make_ref_counted<internal::AudioState>(helper.config())); 541 542 auto* adm = reinterpret_cast<MockAudioDeviceModule*>( 543 helper.config().audio_device_module.get()); 544 545 EXPECT_CALL(*adm, StopPlayout()).Times(2); 546 audio_state->SetPlayout(false); 547 548 // AddReceivingStream with playout disabled start the NullAudioPoller... 549 // which needs a thread. 550 ThreadManager::Instance()->WrapCurrentThread(); 551 552 MockAudioReceiveStream stream; 553 audio_state->AddReceivingStream(&stream); 554 audio_state->RemoveReceivingStream(&stream); 555 } 556 557 INSTANTIATE_TEST_SUITE_P(AudioStateTest, 558 AudioStateTest, 559 Values(ConfigHelper::Params({false, false}), 560 ConfigHelper::Params({true, false}), 561 ConfigHelper::Params({false, true}), 562 ConfigHelper::Params({true, true}))); 563 564 } // namespace test 565 } // namespace webrtc