audio_processing_simulator.cc (20860B)
1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_processing/test/audio_processing_simulator.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <fstream> 16 #include <iostream> 17 #include <memory> 18 #include <string> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/base/nullability.h" 23 #include "absl/strings/string_view.h" 24 #include "api/audio/audio_processing.h" 25 #include "api/scoped_refptr.h" 26 #include "common_audio/channel_buffer.h" 27 #include "common_audio/include/audio_util.h" 28 #include "common_audio/wav_file.h" 29 #include "modules/audio_processing/aec_dump/aec_dump_factory.h" 30 #include "modules/audio_processing/logging/apm_data_dumper.h" 31 #include "modules/audio_processing/test/api_call_statistics.h" 32 #include "modules/audio_processing/test/fake_recording_device.h" 33 #include "modules/audio_processing/test/test_utils.h" 34 #include "rtc_base/checks.h" 35 #include "rtc_base/logging.h" 36 #include "rtc_base/strings/string_builder.h" 37 #include "rtc_base/time_utils.h" 38 39 namespace webrtc { 40 namespace test { 41 namespace { 42 43 std::string GetIndexedOutputWavFilename(absl::string_view wav_name, 44 int counter) { 45 StringBuilder ss; 46 ss << wav_name.substr(0, wav_name.size() - 4) << "_" << counter 47 << wav_name.substr(wav_name.size() - 4); 48 return ss.Release(); 49 } 50 51 void WriteEchoLikelihoodGraphFileHeader(std::ofstream* output_file) { 52 (*output_file) << "import numpy as np" << std::endl 53 << "import matplotlib.pyplot as plt" << std::endl 54 << "y = np.array(["; 55 } 56 57 void WriteEchoLikelihoodGraphFileFooter(std::ofstream* output_file) { 58 (*output_file) << "])" << std::endl 59 << "if __name__ == '__main__':" << std::endl 60 << " x = np.arange(len(y))*.01" << std::endl 61 << " plt.plot(x, y)" << std::endl 62 << " plt.ylabel('Echo likelihood')" << std::endl 63 << " plt.xlabel('Time (s)')" << std::endl 64 << " plt.show()" << std::endl; 65 } 66 67 // RAII class for execution time measurement. Updates the provided 68 // ApiCallStatistics based on the time between ScopedTimer creation and 69 // leaving the enclosing scope. 70 class ScopedTimer { 71 public: 72 ScopedTimer(ApiCallStatistics* api_call_statistics, 73 ApiCallStatistics::CallType call_type) 74 : start_time_(TimeNanos()), 75 call_type_(call_type), 76 api_call_statistics_(api_call_statistics) {} 77 78 ~ScopedTimer() { 79 api_call_statistics_->Add(TimeNanos() - start_time_, call_type_); 80 } 81 82 private: 83 const int64_t start_time_; 84 const ApiCallStatistics::CallType call_type_; 85 ApiCallStatistics* const api_call_statistics_; 86 }; 87 88 } // namespace 89 90 SimulationSettings::SimulationSettings() = default; 91 SimulationSettings::SimulationSettings(const SimulationSettings&) = default; 92 SimulationSettings::~SimulationSettings() = default; 93 94 AudioProcessingSimulator::AudioProcessingSimulator( 95 const SimulationSettings& settings, 96 absl_nonnull scoped_refptr<AudioProcessing> audio_processing) 97 : settings_(settings), 98 ap_(std::move(audio_processing)), 99 applied_input_volume_(settings.initial_mic_level), 100 fake_recording_device_( 101 settings.initial_mic_level, 102 settings_.simulate_mic_gain ? *settings.simulated_mic_kind : 0), 103 worker_queue_("file_writer_task_queue") { 104 RTC_CHECK(!settings_.dump_internal_data || WEBRTC_APM_DEBUG_DUMP == 1); 105 if (settings_.dump_start_frame || settings_.dump_end_frame) { 106 ApmDataDumper::SetActivated(!settings_.dump_start_frame); 107 } else { 108 ApmDataDumper::SetActivated(settings_.dump_internal_data); 109 } 110 111 if (settings_.dump_set_to_use) { 112 ApmDataDumper::SetDumpSetToUse(*settings_.dump_set_to_use); 113 } 114 115 if (settings_.dump_internal_data_output_dir.has_value()) { 116 ApmDataDumper::SetOutputDirectory( 117 settings_.dump_internal_data_output_dir.value()); 118 } 119 120 if (settings_.ed_graph_output_filename && 121 !settings_.ed_graph_output_filename->empty()) { 122 residual_echo_likelihood_graph_writer_.open( 123 *settings_.ed_graph_output_filename); 124 RTC_CHECK(residual_echo_likelihood_graph_writer_.is_open()); 125 WriteEchoLikelihoodGraphFileHeader(&residual_echo_likelihood_graph_writer_); 126 } 127 128 if (settings_.simulate_mic_gain) 129 RTC_LOG(LS_VERBOSE) << "Simulating analog mic gain"; 130 } 131 132 AudioProcessingSimulator::~AudioProcessingSimulator() { 133 if (residual_echo_likelihood_graph_writer_.is_open()) { 134 WriteEchoLikelihoodGraphFileFooter(&residual_echo_likelihood_graph_writer_); 135 residual_echo_likelihood_graph_writer_.close(); 136 } 137 } 138 139 void AudioProcessingSimulator::ProcessStream(bool fixed_interface) { 140 // Optionally simulate the input volume. 141 if (settings_.simulate_mic_gain) { 142 RTC_DCHECK(!settings_.use_analog_mic_gain_emulation); 143 // Set the input volume to simulate. 144 fake_recording_device_.SetMicLevel(applied_input_volume_); 145 146 if (settings_.aec_dump_input_filename && 147 aec_dump_applied_input_level_.has_value()) { 148 // For AEC dumps, use the applied input level, if recorded, to "virtually 149 // restore" the capture signal level before the input volume was applied. 150 fake_recording_device_.SetUndoMicLevel(*aec_dump_applied_input_level_); 151 } 152 153 // Apply the input volume. 154 if (fixed_interface) { 155 fake_recording_device_.SimulateAnalogGain(fwd_frame_.data); 156 } else { 157 fake_recording_device_.SimulateAnalogGain(in_buf_.get()); 158 } 159 } 160 161 // Let APM know which input volume was applied. 162 // Keep track of whether `set_stream_analog_level()` is called. 163 bool applied_input_volume_set = false; 164 if (settings_.simulate_mic_gain) { 165 // When the input volume is simulated, use the volume applied for 166 // simulation. 167 ap_->set_stream_analog_level(fake_recording_device_.MicLevel()); 168 applied_input_volume_set = true; 169 } else if (!settings_.use_analog_mic_gain_emulation) { 170 // Ignore the recommended input volume stored in `applied_input_volume_` and 171 // instead notify APM with the recorded input volume (if available). 172 if (settings_.aec_dump_input_filename && 173 aec_dump_applied_input_level_.has_value()) { 174 // The actually applied input volume is available in the AEC dump. 175 ap_->set_stream_analog_level(*aec_dump_applied_input_level_); 176 applied_input_volume_set = true; 177 } else if (!settings_.aec_dump_input_filename) { 178 // Wav files do not include any information about the actually applied 179 // input volume. Hence, use the recommended input volume stored in 180 // `applied_input_volume_`. 181 ap_->set_stream_analog_level(applied_input_volume_); 182 applied_input_volume_set = true; 183 } 184 } 185 186 // Post any scheduled runtime settings. 187 if (settings_.frame_for_sending_capture_output_used_false && 188 *settings_.frame_for_sending_capture_output_used_false == 189 static_cast<int>(num_process_stream_calls_)) { 190 ap_->PostRuntimeSetting( 191 AudioProcessing::RuntimeSetting::CreateCaptureOutputUsedSetting(false)); 192 } 193 if (settings_.frame_for_sending_capture_output_used_true && 194 *settings_.frame_for_sending_capture_output_used_true == 195 static_cast<int>(num_process_stream_calls_)) { 196 ap_->PostRuntimeSetting( 197 AudioProcessing::RuntimeSetting::CreateCaptureOutputUsedSetting(true)); 198 } 199 200 // Process the current audio frame. 201 if (fixed_interface) { 202 { 203 const auto st = ScopedTimer(&api_call_statistics_, 204 ApiCallStatistics::CallType::kCapture); 205 RTC_CHECK_EQ( 206 AudioProcessing::kNoError, 207 ap_->ProcessStream(fwd_frame_.data.data(), fwd_frame_.config, 208 fwd_frame_.config, fwd_frame_.data.data())); 209 } 210 fwd_frame_.CopyTo(out_buf_.get()); 211 } else { 212 const auto st = ScopedTimer(&api_call_statistics_, 213 ApiCallStatistics::CallType::kCapture); 214 RTC_CHECK_EQ(AudioProcessing::kNoError, 215 ap_->ProcessStream(in_buf_->channels(), in_config_, 216 out_config_, out_buf_->channels())); 217 } 218 219 // Retrieve the recommended input volume only if `set_stream_analog_level()` 220 // has been called to stick to the APM API contract. 221 if (applied_input_volume_set) { 222 applied_input_volume_ = ap_->recommended_stream_analog_level(); 223 } 224 225 if (buffer_memory_writer_) { 226 RTC_CHECK(!buffer_file_writer_); 227 buffer_memory_writer_->Write(*out_buf_); 228 } else if (buffer_file_writer_) { 229 RTC_CHECK(!buffer_memory_writer_); 230 buffer_file_writer_->Write(*out_buf_); 231 } 232 233 if (linear_aec_output_file_writer_) { 234 bool output_available = ap_->GetLinearAecOutput(linear_aec_output_buf_); 235 RTC_CHECK(output_available); 236 RTC_CHECK_GT(linear_aec_output_buf_.size(), 0); 237 RTC_CHECK_EQ(linear_aec_output_buf_[0].size(), 160); 238 for (size_t k = 0; k < linear_aec_output_buf_[0].size(); ++k) { 239 for (size_t ch = 0; ch < linear_aec_output_buf_.size(); ++ch) { 240 RTC_CHECK_EQ(linear_aec_output_buf_[ch].size(), 160); 241 float sample = FloatToFloatS16(linear_aec_output_buf_[ch][k]); 242 linear_aec_output_file_writer_->WriteSamples(&sample, 1); 243 } 244 } 245 } 246 247 if (residual_echo_likelihood_graph_writer_.is_open()) { 248 auto stats = ap_->GetStatistics(); 249 residual_echo_likelihood_graph_writer_ 250 << stats.residual_echo_likelihood.value_or(-1.f) << ", "; 251 } 252 253 ++num_process_stream_calls_; 254 } 255 256 void AudioProcessingSimulator::ProcessReverseStream(bool fixed_interface) { 257 if (fixed_interface) { 258 { 259 const auto st = ScopedTimer(&api_call_statistics_, 260 ApiCallStatistics::CallType::kRender); 261 RTC_CHECK_EQ( 262 AudioProcessing::kNoError, 263 ap_->ProcessReverseStream(rev_frame_.data.data(), rev_frame_.config, 264 rev_frame_.config, rev_frame_.data.data())); 265 } 266 rev_frame_.CopyTo(reverse_out_buf_.get()); 267 } else { 268 const auto st = ScopedTimer(&api_call_statistics_, 269 ApiCallStatistics::CallType::kRender); 270 RTC_CHECK_EQ(AudioProcessing::kNoError, 271 ap_->ProcessReverseStream( 272 reverse_in_buf_->channels(), reverse_in_config_, 273 reverse_out_config_, reverse_out_buf_->channels())); 274 } 275 276 if (reverse_buffer_file_writer_) { 277 reverse_buffer_file_writer_->Write(*reverse_out_buf_); 278 } 279 280 ++num_reverse_process_stream_calls_; 281 } 282 283 void AudioProcessingSimulator::SetupBuffersConfigsOutputs( 284 int input_sample_rate_hz, 285 int output_sample_rate_hz, 286 int reverse_input_sample_rate_hz, 287 int reverse_output_sample_rate_hz, 288 int input_num_channels, 289 int output_num_channels, 290 int reverse_input_num_channels, 291 int reverse_output_num_channels) { 292 in_config_ = StreamConfig(input_sample_rate_hz, input_num_channels); 293 in_buf_.reset(new ChannelBuffer<float>( 294 CheckedDivExact(input_sample_rate_hz, kChunksPerSecond), 295 input_num_channels)); 296 297 reverse_in_config_ = 298 StreamConfig(reverse_input_sample_rate_hz, reverse_input_num_channels); 299 reverse_in_buf_.reset(new ChannelBuffer<float>( 300 CheckedDivExact(reverse_input_sample_rate_hz, kChunksPerSecond), 301 reverse_input_num_channels)); 302 303 out_config_ = StreamConfig(output_sample_rate_hz, output_num_channels); 304 out_buf_.reset(new ChannelBuffer<float>( 305 CheckedDivExact(output_sample_rate_hz, kChunksPerSecond), 306 output_num_channels)); 307 308 reverse_out_config_ = 309 StreamConfig(reverse_output_sample_rate_hz, reverse_output_num_channels); 310 reverse_out_buf_.reset(new ChannelBuffer<float>( 311 CheckedDivExact(reverse_output_sample_rate_hz, kChunksPerSecond), 312 reverse_output_num_channels)); 313 314 fwd_frame_.SetFormat(input_sample_rate_hz, input_num_channels); 315 rev_frame_.SetFormat(reverse_input_sample_rate_hz, 316 reverse_input_num_channels); 317 318 if (settings_.use_verbose_logging) { 319 LogMessage::LogToDebug(LS_VERBOSE); 320 321 std::cout << "Sample rates:" << std::endl; 322 std::cout << " Forward input: " << input_sample_rate_hz << std::endl; 323 std::cout << " Forward output: " << output_sample_rate_hz << std::endl; 324 std::cout << " Reverse input: " << reverse_input_sample_rate_hz 325 << std::endl; 326 std::cout << " Reverse output: " << reverse_output_sample_rate_hz 327 << std::endl; 328 std::cout << "Number of channels: " << std::endl; 329 std::cout << " Forward input: " << input_num_channels << std::endl; 330 std::cout << " Forward output: " << output_num_channels << std::endl; 331 std::cout << " Reverse input: " << reverse_input_num_channels << std::endl; 332 std::cout << " Reverse output: " << reverse_output_num_channels 333 << std::endl; 334 } 335 336 SetupOutput(); 337 } 338 339 void AudioProcessingSimulator::SelectivelyToggleDataDumping( 340 int init_index, 341 int capture_frames_since_init) const { 342 if (!(settings_.dump_start_frame || settings_.dump_end_frame)) { 343 return; 344 } 345 346 if (settings_.init_to_process && *settings_.init_to_process != init_index) { 347 return; 348 } 349 350 if (settings_.dump_start_frame && 351 *settings_.dump_start_frame == capture_frames_since_init) { 352 ApmDataDumper::SetActivated(true); 353 } 354 355 if (settings_.dump_end_frame && 356 *settings_.dump_end_frame == capture_frames_since_init) { 357 ApmDataDumper::SetActivated(false); 358 } 359 } 360 361 void AudioProcessingSimulator::SetupOutput() { 362 if (settings_.output_filename) { 363 std::string filename; 364 if (settings_.store_intermediate_output) { 365 filename = GetIndexedOutputWavFilename(*settings_.output_filename, 366 output_reset_counter_); 367 } else { 368 filename = *settings_.output_filename; 369 } 370 371 std::unique_ptr<WavWriter> out_file( 372 new WavWriter(filename, out_config_.sample_rate_hz(), 373 static_cast<size_t>(out_config_.num_channels()), 374 settings_.wav_output_format)); 375 buffer_file_writer_.reset(new ChannelBufferWavWriter(std::move(out_file))); 376 } else if (settings_.aec_dump_input_string.has_value()) { 377 buffer_memory_writer_ = std::make_unique<ChannelBufferVectorWriter>( 378 settings_.processed_capture_samples); 379 } 380 381 if (settings_.linear_aec_output_filename) { 382 std::string filename; 383 if (settings_.store_intermediate_output) { 384 filename = GetIndexedOutputWavFilename( 385 *settings_.linear_aec_output_filename, output_reset_counter_); 386 } else { 387 filename = *settings_.linear_aec_output_filename; 388 } 389 390 linear_aec_output_file_writer_.reset( 391 new WavWriter(filename, 16000, out_config_.num_channels(), 392 settings_.wav_output_format)); 393 394 linear_aec_output_buf_.resize(out_config_.num_channels()); 395 } 396 397 if (settings_.reverse_output_filename) { 398 std::string filename; 399 if (settings_.store_intermediate_output) { 400 filename = GetIndexedOutputWavFilename(*settings_.reverse_output_filename, 401 output_reset_counter_); 402 } else { 403 filename = *settings_.reverse_output_filename; 404 } 405 406 std::unique_ptr<WavWriter> reverse_out_file( 407 new WavWriter(filename, reverse_out_config_.sample_rate_hz(), 408 static_cast<size_t>(reverse_out_config_.num_channels()), 409 settings_.wav_output_format)); 410 reverse_buffer_file_writer_.reset( 411 new ChannelBufferWavWriter(std::move(reverse_out_file))); 412 } 413 414 ++output_reset_counter_; 415 } 416 417 void AudioProcessingSimulator::DetachAecDump() { 418 if (settings_.aec_dump_output_filename) { 419 ap_->DetachAecDump(); 420 } 421 } 422 423 void AudioProcessingSimulator::ConfigureAudioProcessor() { 424 AudioProcessing::Config apm_config; 425 if (settings_.use_ts) { 426 apm_config.transient_suppression.enabled = *settings_.use_ts != 0; 427 } 428 if (settings_.multi_channel_render) { 429 apm_config.pipeline.multi_channel_render = *settings_.multi_channel_render; 430 } 431 432 if (settings_.multi_channel_capture) { 433 apm_config.pipeline.multi_channel_capture = 434 *settings_.multi_channel_capture; 435 } 436 437 if (settings_.use_agc2) { 438 apm_config.gain_controller2.enabled = *settings_.use_agc2; 439 if (settings_.agc2_fixed_gain_db) { 440 apm_config.gain_controller2.fixed_digital.gain_db = 441 *settings_.agc2_fixed_gain_db; 442 } 443 if (settings_.agc2_use_adaptive_gain) { 444 apm_config.gain_controller2.adaptive_digital.enabled = 445 *settings_.agc2_use_adaptive_gain; 446 } 447 if (settings_.agc2_use_input_volume_controller) { 448 apm_config.gain_controller2.input_volume_controller.enabled = 449 *settings_.agc2_use_input_volume_controller; 450 } 451 } 452 if (settings_.use_pre_amplifier) { 453 apm_config.pre_amplifier.enabled = *settings_.use_pre_amplifier; 454 if (settings_.pre_amplifier_gain_factor) { 455 apm_config.pre_amplifier.fixed_gain_factor = 456 *settings_.pre_amplifier_gain_factor; 457 } 458 } 459 460 if (settings_.use_analog_mic_gain_emulation) { 461 if (*settings_.use_analog_mic_gain_emulation) { 462 apm_config.capture_level_adjustment.enabled = true; 463 apm_config.capture_level_adjustment.analog_mic_gain_emulation.enabled = 464 true; 465 } else { 466 apm_config.capture_level_adjustment.analog_mic_gain_emulation.enabled = 467 false; 468 } 469 } 470 if (settings_.analog_mic_gain_emulation_initial_level) { 471 apm_config.capture_level_adjustment.analog_mic_gain_emulation 472 .initial_level = *settings_.analog_mic_gain_emulation_initial_level; 473 } 474 475 if (settings_.use_capture_level_adjustment) { 476 apm_config.capture_level_adjustment.enabled = 477 *settings_.use_capture_level_adjustment; 478 } 479 if (settings_.pre_gain_factor) { 480 apm_config.capture_level_adjustment.pre_gain_factor = 481 *settings_.pre_gain_factor; 482 } 483 if (settings_.post_gain_factor) { 484 apm_config.capture_level_adjustment.post_gain_factor = 485 *settings_.post_gain_factor; 486 } 487 488 const bool use_aec = settings_.use_aec && *settings_.use_aec; 489 const bool use_aecm = settings_.use_aecm && *settings_.use_aecm; 490 if (use_aec || use_aecm) { 491 apm_config.echo_canceller.enabled = true; 492 apm_config.echo_canceller.mobile_mode = use_aecm; 493 } 494 apm_config.echo_canceller.export_linear_aec_output = 495 !!settings_.linear_aec_output_filename; 496 497 if (settings_.use_hpf) { 498 apm_config.high_pass_filter.enabled = *settings_.use_hpf; 499 } 500 501 if (settings_.use_agc) { 502 apm_config.gain_controller1.enabled = *settings_.use_agc; 503 } 504 if (settings_.agc_mode) { 505 apm_config.gain_controller1.mode = 506 static_cast<AudioProcessing::Config::GainController1::Mode>( 507 *settings_.agc_mode); 508 } 509 if (settings_.use_agc_limiter) { 510 apm_config.gain_controller1.enable_limiter = *settings_.use_agc_limiter; 511 } 512 if (settings_.agc_target_level) { 513 apm_config.gain_controller1.target_level_dbfs = *settings_.agc_target_level; 514 } 515 if (settings_.agc_compression_gain) { 516 apm_config.gain_controller1.compression_gain_db = 517 *settings_.agc_compression_gain; 518 } 519 if (settings_.use_analog_agc) { 520 apm_config.gain_controller1.analog_gain_controller.enabled = 521 *settings_.use_analog_agc; 522 } 523 if (settings_.analog_agc_use_digital_adaptive_controller) { 524 apm_config.gain_controller1.analog_gain_controller.enable_digital_adaptive = 525 *settings_.analog_agc_use_digital_adaptive_controller; 526 } 527 528 if (settings_.maximum_internal_processing_rate) { 529 apm_config.pipeline.maximum_internal_processing_rate = 530 *settings_.maximum_internal_processing_rate; 531 } 532 533 if (settings_.use_ns) { 534 apm_config.noise_suppression.enabled = *settings_.use_ns; 535 } 536 if (settings_.ns_level) { 537 const int level = *settings_.ns_level; 538 RTC_CHECK_GE(level, 0); 539 RTC_CHECK_LE(level, 3); 540 apm_config.noise_suppression.level = 541 static_cast<AudioProcessing::Config::NoiseSuppression::Level>(level); 542 } 543 if (settings_.ns_analysis_on_linear_aec_output) { 544 apm_config.noise_suppression.analyze_linear_aec_output_when_available = 545 *settings_.ns_analysis_on_linear_aec_output; 546 } 547 548 ap_->ApplyConfig(apm_config); 549 550 if (settings_.use_ts) { 551 // Default to key pressed if activating the transient suppressor with 552 // continuous key events. 553 ap_->set_stream_key_pressed(*settings_.use_ts == 2); 554 } 555 556 if (settings_.aec_dump_output_filename) { 557 ap_->AttachAecDump(AecDumpFactory::Create( 558 *settings_.aec_dump_output_filename, -1, worker_queue_.Get())); 559 } 560 } 561 562 } // namespace test 563 } // namespace webrtc