block_processor.cc (11681B)
1 /* 2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #include "modules/audio_processing/aec3/block_processor.h" 11 12 #include <atomic> 13 #include <cstddef> 14 #include <memory> 15 #include <optional> 16 #include <utility> 17 18 #include "api/audio/echo_canceller3_config.h" 19 #include "api/audio/echo_control.h" 20 #include "api/environment/environment.h" 21 #include "modules/audio_processing/aec3/aec3_common.h" 22 #include "modules/audio_processing/aec3/block.h" 23 #include "modules/audio_processing/aec3/block_processor_metrics.h" 24 #include "modules/audio_processing/aec3/delay_estimate.h" 25 #include "modules/audio_processing/aec3/echo_path_variability.h" 26 #include "modules/audio_processing/aec3/echo_remover.h" 27 #include "modules/audio_processing/aec3/render_delay_buffer.h" 28 #include "modules/audio_processing/aec3/render_delay_controller.h" 29 #include "modules/audio_processing/logging/apm_data_dumper.h" 30 #include "rtc_base/checks.h" 31 #include "rtc_base/logging.h" 32 33 namespace webrtc { 34 namespace { 35 36 enum class BlockProcessorApiCall { kCapture, kRender }; 37 38 class BlockProcessorImpl final : public BlockProcessor { 39 public: 40 BlockProcessorImpl(const EchoCanceller3Config& config, 41 int sample_rate_hz, 42 size_t num_render_channels, 43 size_t num_capture_channels, 44 std::unique_ptr<RenderDelayBuffer> render_buffer, 45 std::unique_ptr<RenderDelayController> delay_controller, 46 std::unique_ptr<EchoRemover> echo_remover); 47 48 BlockProcessorImpl() = delete; 49 50 ~BlockProcessorImpl() override; 51 52 void ProcessCapture(bool echo_path_gain_change, 53 bool capture_signal_saturation, 54 Block* linear_output, 55 Block* capture_block) override; 56 57 void BufferRender(const Block& block) override; 58 59 void UpdateEchoLeakageStatus(bool leakage_detected) override; 60 61 void GetMetrics(EchoControl::Metrics* metrics) const override; 62 63 void SetAudioBufferDelay(int delay_ms) override; 64 void SetCaptureOutputUsage(bool capture_output_used) override; 65 66 private: 67 static std::atomic<int> instance_count_; 68 std::unique_ptr<ApmDataDumper> data_dumper_; 69 const EchoCanceller3Config config_; 70 bool capture_properly_started_ = false; 71 bool render_properly_started_ = false; 72 const size_t sample_rate_hz_; 73 std::unique_ptr<RenderDelayBuffer> render_buffer_; 74 std::unique_ptr<RenderDelayController> delay_controller_; 75 std::unique_ptr<EchoRemover> echo_remover_; 76 BlockProcessorMetrics metrics_; 77 RenderDelayBuffer::BufferingEvent render_event_; 78 size_t capture_call_counter_ = 0; 79 std::optional<DelayEstimate> estimated_delay_; 80 }; 81 82 std::atomic<int> BlockProcessorImpl::instance_count_(0); 83 84 BlockProcessorImpl::BlockProcessorImpl( 85 const EchoCanceller3Config& config, 86 int sample_rate_hz, 87 size_t /* num_render_channels */, 88 size_t /* num_capture_channels */, 89 std::unique_ptr<RenderDelayBuffer> render_buffer, 90 std::unique_ptr<RenderDelayController> delay_controller, 91 std::unique_ptr<EchoRemover> echo_remover) 92 : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), 93 config_(config), 94 sample_rate_hz_(sample_rate_hz), 95 render_buffer_(std::move(render_buffer)), 96 delay_controller_(std::move(delay_controller)), 97 echo_remover_(std::move(echo_remover)), 98 render_event_(RenderDelayBuffer::BufferingEvent::kNone) { 99 RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); 100 } 101 102 BlockProcessorImpl::~BlockProcessorImpl() = default; 103 104 void BlockProcessorImpl::ProcessCapture(bool echo_path_gain_change, 105 bool capture_signal_saturation, 106 Block* linear_output, 107 Block* capture_block) { 108 RTC_DCHECK(capture_block); 109 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->NumBands()); 110 111 capture_call_counter_++; 112 113 data_dumper_->DumpRaw("aec3_processblock_call_order", 114 static_cast<int>(BlockProcessorApiCall::kCapture)); 115 data_dumper_->DumpWav("aec3_processblock_capture_input", 116 capture_block->View(/*band=*/0, /*channel=*/0), 16000, 117 1); 118 119 if (render_properly_started_) { 120 if (!capture_properly_started_) { 121 capture_properly_started_ = true; 122 render_buffer_->Reset(); 123 if (delay_controller_) 124 delay_controller_->Reset(true); 125 } 126 } else { 127 // If no render data has yet arrived, do not process the capture signal. 128 render_buffer_->HandleSkippedCaptureProcessing(); 129 return; 130 } 131 132 EchoPathVariability echo_path_variability( 133 echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone, 134 false); 135 136 if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun && 137 render_properly_started_) { 138 echo_path_variability.delay_change = 139 EchoPathVariability::DelayAdjustment::kBufferFlush; 140 if (delay_controller_) 141 delay_controller_->Reset(true); 142 RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block " 143 << capture_call_counter_; 144 } 145 render_event_ = RenderDelayBuffer::BufferingEvent::kNone; 146 147 // Update the render buffers with any newly arrived render blocks and prepare 148 // the render buffers for reading the render data corresponding to the current 149 // capture block. 150 RenderDelayBuffer::BufferingEvent buffer_event = 151 render_buffer_->PrepareCaptureProcessing(); 152 // Reset the delay controller at render buffer underrun. 153 if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) { 154 if (delay_controller_) 155 delay_controller_->Reset(false); 156 } 157 158 data_dumper_->DumpWav("aec3_processblock_capture_input2", 159 capture_block->View(/*band=*/0, /*channel=*/0), 16000, 160 1); 161 162 bool has_delay_estimator = !config_.delay.use_external_delay_estimator; 163 if (has_delay_estimator) { 164 RTC_DCHECK(delay_controller_); 165 // Compute and apply the render delay required to achieve proper signal 166 // alignment. 167 estimated_delay_ = delay_controller_->GetDelay( 168 render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(), 169 *capture_block); 170 171 if (estimated_delay_) { 172 bool delay_change = 173 render_buffer_->AlignFromDelay(estimated_delay_->delay); 174 if (delay_change) { 175 LoggingSeverity log_level = 176 config_.delay.log_warning_on_delay_changes ? LS_WARNING : LS_INFO; 177 RTC_LOG_V(log_level) << "Delay changed to " << estimated_delay_->delay 178 << " at block " << capture_call_counter_; 179 echo_path_variability.delay_change = 180 EchoPathVariability::DelayAdjustment::kNewDetectedDelay; 181 } 182 } 183 184 echo_path_variability.clock_drift = delay_controller_->HasClockdrift(); 185 186 } else { 187 render_buffer_->AlignFromExternalDelay(); 188 } 189 190 // Remove the echo from the capture signal. 191 if (has_delay_estimator || render_buffer_->HasReceivedBufferDelay()) { 192 echo_remover_->ProcessCapture( 193 echo_path_variability, capture_signal_saturation, estimated_delay_, 194 render_buffer_->GetRenderBuffer(), linear_output, capture_block); 195 } 196 197 // Update the metrics. 198 metrics_.UpdateCapture(false); 199 } 200 201 void BlockProcessorImpl::BufferRender(const Block& block) { 202 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.NumBands()); 203 data_dumper_->DumpRaw("aec3_processblock_call_order", 204 static_cast<int>(BlockProcessorApiCall::kRender)); 205 data_dumper_->DumpWav("aec3_processblock_render_input", 206 block.View(/*band=*/0, /*channel=*/0), 16000, 1); 207 208 render_event_ = render_buffer_->Insert(block); 209 210 metrics_.UpdateRender(render_event_ != 211 RenderDelayBuffer::BufferingEvent::kNone); 212 213 render_properly_started_ = true; 214 if (delay_controller_) 215 delay_controller_->LogRenderCall(); 216 } 217 218 void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { 219 echo_remover_->UpdateEchoLeakageStatus(leakage_detected); 220 } 221 222 void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { 223 echo_remover_->GetMetrics(metrics); 224 constexpr int block_size_ms = 4; 225 std::optional<size_t> delay = render_buffer_->Delay(); 226 metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0; 227 } 228 229 void BlockProcessorImpl::SetAudioBufferDelay(int delay_ms) { 230 render_buffer_->SetAudioBufferDelay(delay_ms); 231 } 232 233 void BlockProcessorImpl::SetCaptureOutputUsage(bool capture_output_used) { 234 echo_remover_->SetCaptureOutputUsage(capture_output_used); 235 } 236 237 } // namespace 238 239 std::unique_ptr<BlockProcessor> BlockProcessor::Create( 240 const Environment& env, 241 const EchoCanceller3Config& config, 242 int sample_rate_hz, 243 size_t num_render_channels, 244 size_t num_capture_channels, 245 NeuralResidualEchoEstimator* neural_residual_echo_estimator) { 246 std::unique_ptr<RenderDelayBuffer> render_buffer( 247 RenderDelayBuffer::Create(config, sample_rate_hz, num_render_channels)); 248 std::unique_ptr<RenderDelayController> delay_controller; 249 if (!config.delay.use_external_delay_estimator) { 250 delay_controller.reset(RenderDelayController::Create(config, sample_rate_hz, 251 num_capture_channels)); 252 } 253 std::unique_ptr<EchoRemover> echo_remover = 254 EchoRemover::Create(env, config, sample_rate_hz, num_render_channels, 255 num_capture_channels, neural_residual_echo_estimator); 256 return Create(config, sample_rate_hz, num_render_channels, 257 num_capture_channels, std::move(render_buffer), 258 std::move(delay_controller), std::move(echo_remover)); 259 } 260 261 std::unique_ptr<BlockProcessor> BlockProcessor::Create( 262 const Environment& env, 263 const EchoCanceller3Config& config, 264 int sample_rate_hz, 265 size_t num_render_channels, 266 size_t num_capture_channels, 267 std::unique_ptr<RenderDelayBuffer> render_buffer, 268 NeuralResidualEchoEstimator* neural_residual_echo_estimator) { 269 std::unique_ptr<RenderDelayController> delay_controller; 270 if (!config.delay.use_external_delay_estimator) { 271 delay_controller.reset(RenderDelayController::Create(config, sample_rate_hz, 272 num_capture_channels)); 273 } 274 std::unique_ptr<EchoRemover> echo_remover = 275 EchoRemover::Create(env, config, sample_rate_hz, num_render_channels, 276 num_capture_channels, neural_residual_echo_estimator); 277 return Create(config, sample_rate_hz, num_render_channels, 278 num_capture_channels, std::move(render_buffer), 279 std::move(delay_controller), std::move(echo_remover)); 280 } 281 282 std::unique_ptr<BlockProcessor> BlockProcessor::Create( 283 const EchoCanceller3Config& config, 284 int sample_rate_hz, 285 size_t num_render_channels, 286 size_t num_capture_channels, 287 std::unique_ptr<RenderDelayBuffer> render_buffer, 288 std::unique_ptr<RenderDelayController> delay_controller, 289 std::unique_ptr<EchoRemover> echo_remover) { 290 return std::make_unique<BlockProcessorImpl>( 291 config, sample_rate_hz, num_render_channels, num_capture_channels, 292 std::move(render_buffer), std::move(delay_controller), 293 std::move(echo_remover)); 294 } 295 296 } // namespace webrtc