neteq_impl.h (16849B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 12 #define MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 13 14 #include <cstddef> 15 #include <cstdint> 16 #include <map> 17 #include <memory> 18 #include <optional> 19 #include <vector> 20 21 #include "api/array_view.h" 22 #include "api/audio/audio_frame.h" 23 #include "api/audio_codecs/audio_decoder.h" 24 #include "api/audio_codecs/audio_decoder_factory.h" 25 #include "api/audio_codecs/audio_format.h" 26 #include "api/environment/environment.h" 27 #include "api/neteq/neteq.h" 28 #include "api/neteq/neteq_controller.h" 29 #include "api/neteq/neteq_controller_factory.h" 30 #include "api/neteq/tick_timer.h" 31 #include "api/rtp_headers.h" 32 #include "api/rtp_packet_info.h" 33 #include "api/scoped_refptr.h" 34 #include "api/units/timestamp.h" 35 #include "modules/audio_coding/neteq/audio_multi_vector.h" 36 #include "modules/audio_coding/neteq/packet.h" 37 #include "modules/audio_coding/neteq/packet_buffer.h" 38 #include "modules/audio_coding/neteq/random_vector.h" 39 #include "modules/audio_coding/neteq/statistics_calculator.h" 40 #include "rtc_base/buffer.h" 41 #include "rtc_base/synchronization/mutex.h" 42 #include "rtc_base/thread_annotations.h" 43 44 namespace webrtc { 45 46 // Forward declarations. 47 class Accelerate; 48 class BackgroundNoise; 49 class ComfortNoise; 50 class DecoderDatabase; 51 class DtmfBuffer; 52 class DtmfToneGenerator; 53 class Expand; 54 class Merge; 55 class NackTracker; 56 class Normal; 57 class RedPayloadSplitter; 58 class PreemptiveExpand; 59 class RandomVector; 60 class SyncBuffer; 61 class TimestampScaler; 62 struct AccelerateFactory; 63 struct DtmfEvent; 64 struct ExpandFactory; 65 struct PreemptiveExpandFactory; 66 67 class NetEqImpl : public NetEq { 68 public: 69 enum class OutputType { 70 kNormalSpeech, 71 kPLC, 72 kCNG, 73 kPLCCNG, 74 kVadPassive, 75 kCodecPLC 76 }; 77 78 enum Error : int { 79 kNoError = 0, 80 kOtherError, 81 kUnknownRtpPayloadType, 82 kDecoderNotFound, 83 kInvalidPointer, 84 kAccelerateError, 85 kPreemptiveExpandError, 86 kComfortNoiseErrorCode, 87 kDecoderErrorCode, 88 kOtherDecoderError, 89 kInvalidOperation, 90 kDtmfParsingError, 91 kDtmfInsertError, 92 kSampleUnderrun, 93 kDecodedTooMuch, 94 kRedundancySplitError, 95 kPacketBufferCorruption, 96 }; 97 98 struct Dependencies { 99 // The constructor populates the Dependencies struct with the default 100 // implementations of the objects. They can all be replaced by the user 101 // before sending the struct to the NetEqImpl constructor. However, there 102 // are dependencies between some of the classes inside the struct, so 103 // swapping out one may make it necessary to re-create another one. 104 Dependencies(const Environment& env, 105 const NetEq::Config& config, 106 scoped_refptr<AudioDecoderFactory> decoder_factory, 107 const NetEqControllerFactory& controller_factory); 108 ~Dependencies(); 109 110 const Environment env; 111 std::unique_ptr<TickTimer> tick_timer; 112 std::unique_ptr<StatisticsCalculator> stats; 113 std::unique_ptr<DecoderDatabase> decoder_database; 114 std::unique_ptr<DtmfBuffer> dtmf_buffer; 115 std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator; 116 std::unique_ptr<PacketBuffer> packet_buffer; 117 std::unique_ptr<NetEqController> neteq_controller; 118 std::unique_ptr<RedPayloadSplitter> red_payload_splitter; 119 std::unique_ptr<TimestampScaler> timestamp_scaler; 120 std::unique_ptr<AccelerateFactory> accelerate_factory; 121 std::unique_ptr<ExpandFactory> expand_factory; 122 std::unique_ptr<PreemptiveExpandFactory> preemptive_expand_factory; 123 }; 124 125 // Creates a new NetEqImpl object. 126 NetEqImpl(const NetEq::Config& config, 127 Dependencies&& deps, 128 bool create_components = true); 129 130 ~NetEqImpl() override; 131 132 NetEqImpl(const NetEqImpl&) = delete; 133 NetEqImpl& operator=(const NetEqImpl&) = delete; 134 135 int InsertPacket(const RTPHeader& rtp_header, 136 ArrayView<const uint8_t> payload) override { 137 return InsertPacket( 138 rtp_header, payload, 139 RtpPacketInfo(rtp_header, /*receive_time=*/Timestamp::MinusInfinity())); 140 } 141 142 // Inserts a new packet into NetEq. Returns 0 on success, -1 on failure. 143 int InsertPacket(const RTPHeader& rtp_header, 144 ArrayView<const uint8_t> payload, 145 const RtpPacketInfo& packet_info) override; 146 147 void InsertEmptyPacket(const RTPHeader& rtp_header) override; 148 149 int GetAudio( 150 AudioFrame* audio_frame, 151 bool* muted = nullptr, 152 int* current_sample_rate_hz = nullptr, 153 std::optional<Operation> action_override = std::nullopt) override; 154 155 void SetCodecs(const std::map<int, SdpAudioFormat>& codecs) override; 156 157 bool RegisterPayloadType(int rtp_payload_type, 158 const SdpAudioFormat& audio_format) override; 159 160 bool CreateDecoder(int rtp_payload_type) override; 161 162 // Removes `rtp_payload_type` from the codec database. Returns 0 on success, 163 // -1 on failure. 164 int RemovePayloadType(uint8_t rtp_payload_type) override; 165 166 void RemoveAllPayloadTypes() override; 167 168 bool SetMinimumDelay(int delay_ms) override; 169 170 bool SetMaximumDelay(int delay_ms) override; 171 172 bool SetBaseMinimumDelayMs(int delay_ms) override; 173 174 int GetBaseMinimumDelayMs() const override; 175 176 int TargetDelayMs() const override; 177 178 int FilteredCurrentDelayMs() const override; 179 180 // Writes the current network statistics to `stats`. The statistics are reset 181 // after the call. 182 int NetworkStatistics(NetEqNetworkStatistics* stats) override; 183 184 NetEqNetworkStatistics CurrentNetworkStatistics() const override; 185 186 NetEqLifetimeStatistics GetLifetimeStatistics() const override; 187 188 NetEqOperationsAndState GetOperationsAndState() const override; 189 190 std::optional<uint32_t> GetPlayoutTimestamp() const override; 191 192 int last_output_sample_rate_hz() const override; 193 194 std::optional<DecoderFormat> GetCurrentDecoderFormat() const override; 195 196 // Flushes both the packet buffer and the sync buffer. 197 void FlushBuffers() override; 198 199 void EnableNack(size_t max_nack_list_size) override; 200 201 void DisableNack() override; 202 203 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override; 204 205 int SyncBufferSizeMs() const override; 206 207 // This accessor method is only intended for testing purposes. 208 const SyncBuffer* sync_buffer_for_test() const; 209 Operation last_operation_for_test() const; 210 211 protected: 212 static const int kOutputSizeMs = 10; 213 static const size_t kMaxFrameSize = 5760; // 120 ms @ 48 kHz. 214 // TODO(hlundin): Provide a better value for kSyncBufferSize. 215 // Current value is kMaxFrameSize + 60 ms * 48 kHz, which is enough for 216 // calculating correlations of current frame against history. 217 static const size_t kSyncBufferSize = kMaxFrameSize + 60 * 48; 218 219 // Inserts a new packet into NetEq. This is used by the InsertPacket method 220 // above. Returns 0 on success, otherwise an error code. 221 // TODO(hlundin): Merge this with InsertPacket above? 222 Error InsertPacketInternal(const RTPHeader& rtp_header, 223 ArrayView<const uint8_t> payload, 224 const RtpPacketInfo& packet_info) 225 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 226 227 // Returns true if the payload type changed (this should be followed by 228 // resetting various state). Returns false if the current payload type is 229 // unknown or equal to `payload_type`. 230 bool MaybeChangePayloadType(uint8_t payload_type) 231 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 232 233 // Delivers 10 ms of audio data. The data is written to `audio_frame`. 234 // Returns 0 on success, otherwise an error code. 235 int GetAudioInternal(AudioFrame* audio_frame, 236 std::optional<Operation> action_override) 237 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 238 239 // Provides a decision to the GetAudioInternal method. The decision what to 240 // do is written to `operation`. Packets to decode are written to 241 // `packet_list`, and a DTMF event to play is written to `dtmf_event`. When 242 // DTMF should be played, `play_dtmf` is set to true by the method. 243 // Returns 0 on success, otherwise an error code. 244 int GetDecision(Operation* operation, 245 PacketList* packet_list, 246 DtmfEvent* dtmf_event, 247 bool* play_dtmf, 248 std::optional<Operation> action_override) 249 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 250 251 // Decodes the speech packets in `packet_list`, and writes the results to 252 // `decoded_buffer`, which is allocated to hold `decoded_buffer_length` 253 // elements. The length of the decoded data is written to `decoded_length`. 254 // The speech type -- speech or (codec-internal) comfort noise -- is written 255 // to `speech_type`. If `packet_list` contains any SID frames for RFC 3389 256 // comfort noise, those are not decoded. 257 int Decode(PacketList* packet_list, 258 Operation* operation, 259 int* decoded_length, 260 AudioDecoder::SpeechType* speech_type) 261 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 262 263 // Sub-method to Decode(). Performs codec internal CNG. 264 int DecodeCng(AudioDecoder* decoder, 265 int* decoded_length, 266 AudioDecoder::SpeechType* speech_type) 267 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 268 269 // Sub-method to Decode(). Performs the actual decoding. 270 int DecodeLoop(PacketList* packet_list, 271 const Operation& operation, 272 AudioDecoder* decoder, 273 int* decoded_length, 274 AudioDecoder::SpeechType* speech_type) 275 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 276 277 // Sub-method which calls the Normal class to perform the normal operation. 278 void DoNormal(const int16_t* decoded_buffer, 279 size_t decoded_length, 280 AudioDecoder::SpeechType speech_type, 281 bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 282 283 // Sub-method which calls the Merge class to perform the merge operation. 284 void DoMerge(int16_t* decoded_buffer, 285 size_t decoded_length, 286 AudioDecoder::SpeechType speech_type, 287 bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 288 289 bool DoCodecPlc() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 290 291 // Sub-method which calls the Expand class to perform the expand operation. 292 int DoExpand(bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 293 294 // Sub-method which calls the Accelerate class to perform the accelerate 295 // operation. 296 int DoAccelerate(int16_t* decoded_buffer, 297 size_t decoded_length, 298 AudioDecoder::SpeechType speech_type, 299 bool play_dtmf, 300 bool fast_accelerate) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 301 302 // Sub-method which calls the PreemptiveExpand class to perform the 303 // preemptive expand operation. 304 int DoPreemptiveExpand(int16_t* decoded_buffer, 305 size_t decoded_length, 306 AudioDecoder::SpeechType speech_type, 307 bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 308 309 // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort 310 // noise. `packet_list` can either contain one SID frame to update the 311 // noise parameters, or no payload at all, in which case the previously 312 // received parameters are used. 313 int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) 314 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 315 316 // Calls the audio decoder to generate codec-internal comfort noise when 317 // no packet was received. 318 void DoCodecInternalCng(const int16_t* decoded_buffer, size_t decoded_length) 319 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 320 321 // Calls the DtmfToneGenerator class to generate DTMF tones. 322 int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) 323 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 324 325 // Overdub DTMF on top of `output`. 326 int DtmfOverdub(const DtmfEvent& dtmf_event, 327 size_t num_channels, 328 int16_t* output) const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 329 330 // Extracts packets from `packet_buffer_` to produce at least 331 // `required_samples` samples. The packets are inserted into `packet_list`. 332 // Returns the number of samples that the packets in the list will produce, or 333 // -1 in case of an error. 334 int ExtractPackets(size_t required_samples, PacketList* packet_list) 335 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 336 337 // Resets various variables and objects to new values based on the sample rate 338 // `fs_hz` and `channels` number audio channels. 339 // If the sample rate, the number of channels or a combination thereof aren't 340 // supported, the function will fail on an RTC_CHECK. 341 void SetSampleRateAndChannels(int fs_hz, size_t channels) 342 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 343 344 // Returns the output type for the audio produced by the latest call to 345 // GetAudio(). 346 OutputType LastOutputType() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 347 348 // Updates Expand and Merge. 349 virtual void UpdatePlcComponents(int fs_hz, size_t channels) 350 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 351 352 NetEqNetworkStatistics CurrentNetworkStatisticsInternal() const 353 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 354 355 NetEqController::PacketArrivedInfo ToPacketArrivedInfo( 356 const Packet& packet) const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 357 358 const Environment env_; 359 360 mutable Mutex mutex_; 361 const std::unique_ptr<TickTimer> tick_timer_ RTC_GUARDED_BY(mutex_); 362 const std::unique_ptr<DecoderDatabase> decoder_database_ 363 RTC_GUARDED_BY(mutex_); 364 const std::unique_ptr<DtmfBuffer> dtmf_buffer_ RTC_GUARDED_BY(mutex_); 365 const std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator_ 366 RTC_GUARDED_BY(mutex_); 367 const std::unique_ptr<PacketBuffer> packet_buffer_ RTC_GUARDED_BY(mutex_); 368 const std::unique_ptr<RedPayloadSplitter> red_payload_splitter_ 369 RTC_GUARDED_BY(mutex_); 370 const std::unique_ptr<TimestampScaler> timestamp_scaler_ 371 RTC_GUARDED_BY(mutex_); 372 const std::unique_ptr<ExpandFactory> expand_factory_ RTC_GUARDED_BY(mutex_); 373 const std::unique_ptr<AccelerateFactory> accelerate_factory_ 374 RTC_GUARDED_BY(mutex_); 375 const std::unique_ptr<PreemptiveExpandFactory> preemptive_expand_factory_ 376 RTC_GUARDED_BY(mutex_); 377 const std::unique_ptr<StatisticsCalculator> stats_ RTC_GUARDED_BY(mutex_); 378 379 std::unique_ptr<BackgroundNoise> background_noise_ RTC_GUARDED_BY(mutex_); 380 std::unique_ptr<NetEqController> controller_ RTC_GUARDED_BY(mutex_); 381 std::unique_ptr<AudioMultiVector> algorithm_buffer_ RTC_GUARDED_BY(mutex_); 382 std::unique_ptr<SyncBuffer> sync_buffer_ RTC_GUARDED_BY(mutex_); 383 std::unique_ptr<Expand> expand_ RTC_GUARDED_BY(mutex_); 384 std::unique_ptr<Normal> normal_ RTC_GUARDED_BY(mutex_); 385 std::unique_ptr<Merge> merge_ RTC_GUARDED_BY(mutex_); 386 std::unique_ptr<Accelerate> accelerate_ RTC_GUARDED_BY(mutex_); 387 std::unique_ptr<PreemptiveExpand> preemptive_expand_ RTC_GUARDED_BY(mutex_); 388 RandomVector random_vector_ RTC_GUARDED_BY(mutex_); 389 std::unique_ptr<ComfortNoise> comfort_noise_ RTC_GUARDED_BY(mutex_); 390 int fs_hz_ RTC_GUARDED_BY(mutex_); 391 int fs_mult_ RTC_GUARDED_BY(mutex_); 392 int last_output_sample_rate_hz_ RTC_GUARDED_BY(mutex_); 393 size_t output_size_samples_ RTC_GUARDED_BY(mutex_); 394 size_t decoder_frame_length_ RTC_GUARDED_BY(mutex_); 395 Mode last_mode_ RTC_GUARDED_BY(mutex_); 396 Operation last_operation_ RTC_GUARDED_BY(mutex_); 397 std::optional<AudioDecoder::SpeechType> last_decoded_type_ 398 RTC_GUARDED_BY(mutex_); 399 size_t decoded_buffer_length_ RTC_GUARDED_BY(mutex_); 400 std::unique_ptr<int16_t[]> decoded_buffer_ RTC_GUARDED_BY(mutex_); 401 uint32_t playout_timestamp_ RTC_GUARDED_BY(mutex_); 402 bool new_codec_ RTC_GUARDED_BY(mutex_); 403 uint32_t timestamp_ RTC_GUARDED_BY(mutex_); 404 bool reset_decoder_ RTC_GUARDED_BY(mutex_); 405 std::optional<uint8_t> current_rtp_payload_type_ RTC_GUARDED_BY(mutex_); 406 std::optional<uint8_t> current_cng_rtp_payload_type_ RTC_GUARDED_BY(mutex_); 407 bool first_packet_ RTC_GUARDED_BY(mutex_); 408 bool enable_fast_accelerate_ RTC_GUARDED_BY(mutex_); 409 std::unique_ptr<NackTracker> nack_ RTC_GUARDED_BY(mutex_); 410 bool nack_enabled_ RTC_GUARDED_BY(mutex_); 411 const bool enable_muted_state_ RTC_GUARDED_BY(mutex_); 412 std::unique_ptr<TickTimer::Stopwatch> generated_noise_stopwatch_ 413 RTC_GUARDED_BY(mutex_); 414 std::vector<RtpPacketInfo> last_decoded_packet_infos_ RTC_GUARDED_BY(mutex_); 415 bool no_time_stretching_ RTC_GUARDED_BY(mutex_); // Only used for test. 416 BufferT<int16_t> concealment_audio_ RTC_GUARDED_BY(mutex_); 417 }; 418 419 } // namespace webrtc 420 #endif // MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_