echo_canceller3.h (9359B)
1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 12 #define MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 13 14 #include <stddef.h> 15 16 #include <atomic> 17 #include <memory> 18 #include <optional> 19 #include <vector> 20 21 #include "api/array_view.h" 22 #include "api/audio/echo_canceller3_config.h" 23 #include "api/audio/echo_control.h" 24 #include "api/audio/neural_residual_echo_estimator.h" 25 #include "api/environment/environment.h" 26 #include "api/field_trials_view.h" 27 #include "modules/audio_processing/aec3/api_call_jitter_metrics.h" 28 #include "modules/audio_processing/aec3/block.h" 29 #include "modules/audio_processing/aec3/block_delay_buffer.h" 30 #include "modules/audio_processing/aec3/block_framer.h" 31 #include "modules/audio_processing/aec3/block_processor.h" 32 #include "modules/audio_processing/aec3/config_selector.h" 33 #include "modules/audio_processing/aec3/frame_blocker.h" 34 #include "modules/audio_processing/aec3/multi_channel_content_detector.h" 35 #include "modules/audio_processing/audio_buffer.h" 36 #include "modules/audio_processing/logging/apm_data_dumper.h" 37 #include "rtc_base/gtest_prod_util.h" 38 #include "rtc_base/race_checker.h" 39 #include "rtc_base/swap_queue.h" 40 #include "rtc_base/thread_annotations.h" 41 42 namespace webrtc { 43 44 // Method for adjusting config parameter dependencies. 45 // Only to be used externally to AEC3 for testing purposes. 46 // TODO(webrtc:5298): Move this to a separate file. 47 EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config, 48 const FieldTrialsView& field_trials); 49 50 // Functor for verifying the invariance of the frames being put into the render 51 // queue. 52 class Aec3RenderQueueItemVerifier { 53 public: 54 Aec3RenderQueueItemVerifier(size_t num_bands, 55 size_t num_channels, 56 size_t frame_length) 57 : num_bands_(num_bands), 58 num_channels_(num_channels), 59 frame_length_(frame_length) {} 60 61 bool operator()(const std::vector<std::vector<std::vector<float>>>& v) const { 62 if (v.size() != num_bands_) { 63 return false; 64 } 65 for (const auto& band : v) { 66 if (band.size() != num_channels_) { 67 return false; 68 } 69 for (const auto& channel : band) { 70 if (channel.size() != frame_length_) { 71 return false; 72 } 73 } 74 } 75 return true; 76 } 77 78 private: 79 const size_t num_bands_; 80 const size_t num_channels_; 81 const size_t frame_length_; 82 }; 83 84 // Main class for the echo canceller3. 85 // It does 4 things: 86 // -Receives 10 ms frames of band-split audio. 87 // -Provides the lower level echo canceller functionality with 88 // blocks of 64 samples of audio data. 89 // -Partially handles the jitter in the render and capture API 90 // call sequence. 91 // 92 // The class is supposed to be used in a non-concurrent manner apart from the 93 // AnalyzeRender call which can be called concurrently with the other methods. 94 class EchoCanceller3 : public EchoControl { 95 public: 96 EchoCanceller3(const Environment& env, 97 const EchoCanceller3Config& config, 98 const std::optional<EchoCanceller3Config>& multichannel_config, 99 NeuralResidualEchoEstimator* neural_residual_echo_estimator, 100 int sample_rate_hz, 101 size_t num_render_channels, 102 size_t num_capture_channels); 103 104 ~EchoCanceller3() override; 105 106 EchoCanceller3(const EchoCanceller3&) = delete; 107 EchoCanceller3& operator=(const EchoCanceller3&) = delete; 108 109 // Analyzes and stores an internal copy of the split-band domain render 110 // signal. 111 void AnalyzeRender(AudioBuffer* render) override { AnalyzeRender(*render); } 112 // Analyzes the full-band domain capture signal to detect signal saturation. 113 void AnalyzeCapture(AudioBuffer* capture) override { 114 AnalyzeCapture(*capture); 115 } 116 // Processes the split-band domain capture signal in order to remove any echo 117 // present in the signal. 118 void ProcessCapture(AudioBuffer* capture, bool level_change) override; 119 // As above, but also returns the linear filter output. 120 void ProcessCapture(AudioBuffer* capture, 121 AudioBuffer* linear_output, 122 bool level_change) override; 123 // Collect current metrics from the echo canceller. 124 Metrics GetMetrics() const override; 125 // Provides an optional external estimate of the audio buffer delay. 126 void SetAudioBufferDelay(int delay_ms) override; 127 128 // Specifies whether the capture output will be used. The purpose of this is 129 // to allow the echo controller to deactivate some of the processing when the 130 // resulting output is anyway not used, for instance when the endpoint is 131 // muted. 132 void SetCaptureOutputUsage(bool capture_output_used) override; 133 134 bool ActiveProcessing() const override; 135 136 // Signals whether an external detector has detected echo leakage from the 137 // echo canceller. 138 // Note that in the case echo leakage has been flagged, it should be unflagged 139 // once it is no longer occurring. 140 void UpdateEchoLeakageStatus(bool leakage_detected) { 141 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); 142 block_processor_->UpdateEchoLeakageStatus(leakage_detected); 143 } 144 145 private: 146 friend class EchoCanceller3Tester; 147 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, DetectionOfProperStereo); 148 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, 149 DetectionOfProperStereoUsingThreshold); 150 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, 151 DetectionOfProperStereoUsingHysteresis); 152 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, 153 StereoContentDetectionForMonoSignals); 154 155 class RenderWriter; 156 157 // (Re-)Initializes the selected subset of the EchoCanceller3 fields, at 158 // creation as well as during reconfiguration. 159 void Initialize(); 160 161 // Only for testing. Replaces the internal block processor. 162 void SetBlockProcessorForTesting( 163 std::unique_ptr<BlockProcessor> block_processor); 164 165 // Only for testing. Returns whether stereo processing is active. 166 bool StereoRenderProcessingActiveForTesting() const { 167 return multichannel_content_detector_.IsProperMultiChannelContentDetected(); 168 } 169 170 // Only for testing. 171 const EchoCanceller3Config& GetActiveConfigForTesting() const { 172 return config_selector_.active_config(); 173 } 174 175 // Empties the render SwapQueue. 176 void EmptyRenderQueue(); 177 178 // Analyzes and stores an internal copy of the split-band domain render 179 // signal. 180 void AnalyzeRender(const AudioBuffer& render); 181 // Analyzes the full-band domain capture signal to detect signal saturation. 182 void AnalyzeCapture(const AudioBuffer& capture); 183 184 const Environment env_; 185 RaceChecker capture_race_checker_; 186 RaceChecker render_race_checker_; 187 188 // State that is accessed by the AnalyzeRender call. 189 std::unique_ptr<RenderWriter> render_writer_ 190 RTC_GUARDED_BY(render_race_checker_); 191 192 // State that may be accessed by the capture thread. 193 static std::atomic<int> instance_count_; 194 std::unique_ptr<ApmDataDumper> data_dumper_; 195 const EchoCanceller3Config config_; 196 const int sample_rate_hz_; 197 const int num_bands_; 198 const size_t num_render_input_channels_; 199 size_t num_render_channels_to_aec_; 200 const size_t num_capture_channels_; 201 ConfigSelector config_selector_; 202 MultiChannelContentDetector multichannel_content_detector_; 203 NeuralResidualEchoEstimator* neural_residual_echo_estimator_; 204 std::unique_ptr<BlockFramer> linear_output_framer_ 205 RTC_GUARDED_BY(capture_race_checker_); 206 BlockFramer output_framer_ RTC_GUARDED_BY(capture_race_checker_); 207 FrameBlocker capture_blocker_ RTC_GUARDED_BY(capture_race_checker_); 208 std::unique_ptr<FrameBlocker> render_blocker_ 209 RTC_GUARDED_BY(capture_race_checker_); 210 SwapQueue<std::vector<std::vector<std::vector<float>>>, 211 Aec3RenderQueueItemVerifier> 212 render_transfer_queue_; 213 std::unique_ptr<BlockProcessor> block_processor_ 214 RTC_GUARDED_BY(capture_race_checker_); 215 std::vector<std::vector<std::vector<float>>> render_queue_output_frame_ 216 RTC_GUARDED_BY(capture_race_checker_); 217 bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) = 218 false; 219 Block render_block_ RTC_GUARDED_BY(capture_race_checker_); 220 std::unique_ptr<Block> linear_output_block_ 221 RTC_GUARDED_BY(capture_race_checker_); 222 Block capture_block_ RTC_GUARDED_BY(capture_race_checker_); 223 std::vector<std::vector<ArrayView<float>>> render_sub_frame_view_ 224 RTC_GUARDED_BY(capture_race_checker_); 225 std::vector<std::vector<ArrayView<float>>> linear_output_sub_frame_view_ 226 RTC_GUARDED_BY(capture_race_checker_); 227 std::vector<std::vector<ArrayView<float>>> capture_sub_frame_view_ 228 RTC_GUARDED_BY(capture_race_checker_); 229 std::unique_ptr<BlockDelayBuffer> block_delay_buffer_ 230 RTC_GUARDED_BY(capture_race_checker_); 231 ApiCallJitterMetrics api_call_metrics_ RTC_GUARDED_BY(capture_race_checker_); 232 }; 233 } // namespace webrtc 234 235 #endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_