channel_receive.cc (46914B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "audio/channel_receive.h" 12 13 #include <algorithm> 14 #include <cstddef> 15 #include <cstdint> 16 #include <map> 17 #include <memory> 18 #include <optional> 19 #include <string> 20 #include <utility> 21 #include <vector> 22 23 #include "api/array_view.h" 24 #include "api/audio/audio_device.h" 25 #include "api/audio/audio_mixer.h" 26 #include "api/audio_codecs/audio_codec_pair_id.h" 27 #include "api/audio_codecs/audio_decoder_factory.h" 28 #include "api/audio_codecs/audio_format.h" 29 #include "api/call/audio_sink.h" 30 #include "api/call/transport.h" 31 #include "api/crypto/crypto_options.h" 32 #include "api/crypto/frame_decryptor_interface.h" 33 #include "api/environment/environment.h" 34 #include "api/frame_transformer_interface.h" 35 #include "api/make_ref_counted.h" 36 #include "api/media_types.h" 37 #include "api/neteq/default_neteq_factory.h" 38 #include "api/neteq/neteq.h" 39 #include "api/neteq/neteq_factory.h" 40 #include "api/rtc_event_log/rtc_event_log.h" 41 #include "api/rtp_headers.h" 42 #include "api/rtp_packet_info.h" 43 #include "api/rtp_packet_infos.h" 44 #include "api/scoped_refptr.h" 45 #include "api/sequence_checker.h" 46 #include "api/task_queue/pending_task_safety_flag.h" 47 #include "api/task_queue/task_queue_base.h" 48 #include "api/transport/rtp/rtp_source.h" 49 #include "api/units/time_delta.h" 50 #include "api/units/timestamp.h" 51 #include "audio/audio_level.h" 52 #include "audio/channel_receive_frame_transformer_delegate.h" 53 #include "audio/utility/audio_frame_operations.h" 54 #include "call/syncable.h" 55 #include "logging/rtc_event_log/events/rtc_event_audio_playout.h" 56 #include "logging/rtc_event_log/events/rtc_event_neteq_set_minimum_delay.h" 57 #include "modules/audio_coding/acm2/acm_resampler.h" 58 #include "modules/audio_coding/acm2/call_statistics.h" 59 #include "modules/audio_coding/include/audio_coding_module_typedefs.h" 60 #include "modules/pacing/packet_router.h" 61 #include "modules/rtp_rtcp/include/receive_statistics.h" 62 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" 63 #include "modules/rtp_rtcp/include/rtcp_statistics.h" 64 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 65 #include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" 66 #include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" 67 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 68 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" 69 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" 70 #include "modules/rtp_rtcp/source/source_tracker.h" 71 #include "rtc_base/buffer.h" 72 #include "rtc_base/checks.h" 73 #include "rtc_base/logging.h" 74 #include "rtc_base/numerics/safe_conversions.h" 75 #include "rtc_base/numerics/sequence_number_unwrapper.h" 76 #include "rtc_base/race_checker.h" 77 #include "rtc_base/strings/string_builder.h" 78 #include "rtc_base/synchronization/mutex.h" 79 #include "rtc_base/system/no_unique_address.h" 80 #include "rtc_base/thread_annotations.h" 81 #include "rtc_base/trace_event.h" 82 #include "system_wrappers/include/metrics.h" 83 #include "system_wrappers/include/ntp_time.h" 84 85 namespace webrtc { 86 namespace voe { 87 88 namespace { 89 90 constexpr double kAudioSampleDurationSeconds = 0.01; 91 92 // Video Sync. 93 constexpr TimeDelta kVoiceEngineMinMinPlayoutDelay = TimeDelta::Zero(); 94 constexpr TimeDelta kVoiceEngineMaxMinPlayoutDelay = TimeDelta::Seconds(10); 95 96 std::unique_ptr<NetEq> CreateNetEq( 97 NetEqFactory* neteq_factory, 98 std::optional<AudioCodecPairId> codec_pair_id, 99 size_t jitter_buffer_max_packets, 100 bool jitter_buffer_fast_playout, 101 int jitter_buffer_min_delay_ms, 102 const Environment& env, 103 scoped_refptr<AudioDecoderFactory> decoder_factory) { 104 NetEq::Config config; 105 config.codec_pair_id = codec_pair_id; 106 config.max_packets_in_buffer = jitter_buffer_max_packets; 107 config.enable_fast_accelerate = jitter_buffer_fast_playout; 108 config.enable_muted_state = true; 109 config.min_delay_ms = jitter_buffer_min_delay_ms; 110 if (neteq_factory) { 111 return neteq_factory->Create(env, config, std::move(decoder_factory)); 112 } 113 return DefaultNetEqFactory().Create(env, config, std::move(decoder_factory)); 114 } 115 116 class ChannelReceive : public ChannelReceiveInterface, 117 public RtcpPacketTypeCounterObserver { 118 public: 119 // Used for receive streams. 120 ChannelReceive(const Environment& env, 121 NetEqFactory* neteq_factory, 122 AudioDeviceModule* audio_device_module, 123 Transport* rtcp_send_transport, 124 uint32_t local_ssrc, 125 uint32_t remote_ssrc, 126 size_t jitter_buffer_max_packets, 127 bool jitter_buffer_fast_playout, 128 int jitter_buffer_min_delay_ms, 129 bool enable_non_sender_rtt, 130 scoped_refptr<AudioDecoderFactory> decoder_factory, 131 std::optional<AudioCodecPairId> codec_pair_id, 132 scoped_refptr<FrameDecryptorInterface> frame_decryptor, 133 const CryptoOptions& crypto_options, 134 scoped_refptr<FrameTransformerInterface> frame_transformer, 135 RtcpEventObserver* rtcp_event_observer); 136 ~ChannelReceive() override; 137 138 void SetSink(AudioSinkInterface* sink) override; 139 140 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override; 141 142 // API methods 143 144 void StartPlayout() override; 145 void StopPlayout() override; 146 147 // Codecs 148 std::optional<std::pair<int, SdpAudioFormat>> GetReceiveCodec() 149 const override; 150 151 void ReceivedRTCPPacket(const uint8_t* data, size_t length) override; 152 153 // RtpPacketSinkInterface. 154 void OnRtpPacket(const RtpPacketReceived& packet) override; 155 156 // Muting, Volume and Level. 157 void SetChannelOutputVolumeScaling(float scaling) override; 158 int GetSpeechOutputLevelFullRange() const override; 159 // See description of "totalAudioEnergy" in the WebRTC stats spec: 160 // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy 161 double GetTotalOutputEnergy() const override; 162 double GetTotalOutputDuration() const override; 163 164 // Stats. 165 NetworkStatistics GetNetworkStatistics( 166 bool get_and_clear_legacy_stats) const override; 167 AudioDecodingCallStats GetDecodingCallStatistics() const override; 168 169 // Audio+Video Sync. 170 uint32_t GetDelayEstimate() const override; 171 bool SetMinimumPlayoutDelay(TimeDelta delay) override; 172 std::optional<Syncable::PlayoutInfo> GetPlayoutRtpTimestamp() const override; 173 void SetEstimatedPlayoutNtpTimestamp(NtpTime ntp_time, 174 Timestamp time) override; 175 std::optional<int64_t> GetCurrentEstimatedPlayoutNtpTimestampMs( 176 int64_t now_ms) const override; 177 178 // Audio quality. 179 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; 180 int GetBaseMinimumPlayoutDelayMs() const override; 181 182 // Produces the transport-related timestamps; current_delay_ms is left unset. 183 std::optional<Syncable::Info> GetSyncInfo() const override; 184 185 void RegisterReceiverCongestionControlObjects( 186 PacketRouter* packet_router) override; 187 void ResetReceiverCongestionControlObjects() override; 188 189 ChannelReceiveStatistics GetRTCPStatistics() const override; 190 void SetNACKStatus(bool enable, int max_packets) override; 191 void SetRtcpMode(RtcpMode mode) override; 192 void SetNonSenderRttMeasurement(bool enabled) override; 193 194 AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( 195 int sample_rate_hz, 196 AudioFrame* audio_frame) override; 197 198 int PreferredSampleRate() const override; 199 200 std::vector<RtpSource> GetSources() const override; 201 202 // Sets a frame transformer between the depacketizer and the decoder, to 203 // transform the received frames before decoding them. 204 void SetDepacketizerToDecoderFrameTransformer( 205 scoped_refptr<FrameTransformerInterface> frame_transformer) override; 206 207 void SetFrameDecryptor( 208 scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; 209 210 void OnLocalSsrcChange(uint32_t local_ssrc) override; 211 212 void RtcpPacketTypesCounterUpdated( 213 uint32_t ssrc, 214 const RtcpPacketTypeCounter& packet_counter) override; 215 216 private: 217 void ReceivePacket(const uint8_t* packet, 218 size_t packet_length, 219 const RTPHeader& header, 220 Timestamp receive_time) RTC_RUN_ON(worker_thread_checker_); 221 int ResendPackets(const uint16_t* sequence_numbers, int length); 222 void UpdatePlayoutTimestamp(bool rtcp, Timestamp now) 223 RTC_RUN_ON(worker_thread_checker_); 224 225 int GetRtpTimestampRateHz() const; 226 227 void OnReceivedPayloadData(ArrayView<const uint8_t> payload, 228 const RTPHeader& rtpHeader, 229 Timestamp receive_time) 230 RTC_RUN_ON(worker_thread_checker_); 231 232 void InitFrameTransformerDelegate( 233 scoped_refptr<FrameTransformerInterface> frame_transformer) 234 RTC_RUN_ON(worker_thread_checker_); 235 236 // Thread checkers document and lock usage of some methods to specific threads 237 // we know about. The goal is to eventually split up voe::ChannelReceive into 238 // parts with single-threaded semantics, and thereby reduce the need for 239 // locks. 240 RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_thread_checker_; 241 242 const Environment env_; 243 TaskQueueBase* const worker_thread_; 244 ScopedTaskSafety worker_safety_; 245 246 // Methods accessed from audio and video threads are checked for sequential- 247 // only access. We don't necessarily own and control these threads, so thread 248 // checkers cannot be used. E.g. Chromium may transfer "ownership" from one 249 // audio thread to another, but access is still sequential. 250 RaceChecker audio_thread_race_checker_; 251 Mutex callback_mutex_; 252 Mutex volume_settings_mutex_; 253 mutable Mutex call_stats_mutex_; 254 255 bool playing_ RTC_GUARDED_BY(worker_thread_checker_) = false; 256 257 // Indexed by payload type. 258 std::map<uint8_t, int> payload_type_frequencies_; 259 260 std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 261 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; 262 const uint32_t remote_ssrc_; 263 SourceTracker source_tracker_ RTC_GUARDED_BY(&worker_thread_checker_); 264 265 std::optional<uint32_t> last_received_rtp_timestamp_ 266 RTC_GUARDED_BY(&worker_thread_checker_); 267 std::optional<Timestamp> last_received_rtp_system_time_ 268 RTC_GUARDED_BY(&worker_thread_checker_); 269 270 const std::unique_ptr<NetEq> neteq_; // NetEq is thread-safe; no lock needed. 271 acm2::ResamplerHelper resampler_helper_ 272 RTC_GUARDED_BY(audio_thread_race_checker_); 273 acm2::CallStatistics call_stats_ RTC_GUARDED_BY(call_stats_mutex_); 274 AudioSinkInterface* audio_sink_ = nullptr; 275 AudioLevel _outputAudioLevel; 276 277 RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_); 278 279 // Timestamp of the audio pulled from NetEq. 280 std::optional<uint32_t> jitter_buffer_playout_timestamp_; 281 282 std::optional<Syncable::PlayoutInfo> playout_timestamp_ 283 RTC_GUARDED_BY(worker_thread_checker_); 284 uint32_t playout_delay_ms_ RTC_GUARDED_BY(worker_thread_checker_); 285 std::optional<NtpTime> playout_timestamp_ntp_ 286 RTC_GUARDED_BY(worker_thread_checker_); 287 std::optional<Timestamp> playout_timestamp_ntp_time_ 288 RTC_GUARDED_BY(worker_thread_checker_); 289 290 mutable Mutex ts_stats_lock_; 291 292 RtpTimestampUnwrapper rtp_ts_wraparound_handler_; 293 // The rtp timestamp of the first played out audio frame. 294 int64_t capture_start_rtp_time_stamp_; 295 // The capture ntp time (in local timebase) of the first played out audio 296 // frame. 297 int64_t capture_start_ntp_time_ms_ RTC_GUARDED_BY(ts_stats_lock_); 298 299 AudioDeviceModule* _audioDeviceModulePtr; 300 float _outputGain RTC_GUARDED_BY(volume_settings_mutex_); 301 302 PacketRouter* packet_router_ = nullptr; 303 304 SequenceChecker construction_thread_; 305 306 // E2EE Audio Frame Decryption 307 scoped_refptr<FrameDecryptorInterface> frame_decryptor_ 308 RTC_GUARDED_BY(worker_thread_checker_); 309 CryptoOptions crypto_options_; 310 311 AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ 312 RTC_GUARDED_BY(worker_thread_checker_); 313 314 CaptureClockOffsetUpdater capture_clock_offset_updater_ 315 RTC_GUARDED_BY(ts_stats_lock_); 316 317 scoped_refptr<ChannelReceiveFrameTransformerDelegate> 318 frame_transformer_delegate_; 319 320 // Counter that's used to control the frequency of reporting histograms 321 // from the `GetAudioFrameWithInfo` callback. 322 int audio_frame_interval_count_ RTC_GUARDED_BY(audio_thread_race_checker_) = 323 0; 324 // Controls how many callbacks we let pass by before reporting callback stats. 325 // A value of 100 means 100 callbacks, each one of which represents 10ms worth 326 // of data, so the stats reporting frequency will be 1Hz (modulo failures). 327 constexpr static int kHistogramReportingInterval = 100; 328 329 mutable Mutex rtcp_counter_mutex_; 330 RtcpPacketTypeCounter rtcp_packet_type_counter_ 331 RTC_GUARDED_BY(rtcp_counter_mutex_); 332 333 std::map<int, SdpAudioFormat> payload_type_map_; 334 }; 335 336 void ChannelReceive::OnReceivedPayloadData(ArrayView<const uint8_t> payload, 337 const RTPHeader& rtpHeader, 338 Timestamp receive_time) { 339 if (!playing_) { 340 // Avoid inserting into NetEQ when we are not playing. Count the 341 // packet as discarded. 342 343 // Tell source_tracker_ that the frame has been "delivered". Normally, this 344 // happens in AudioReceiveStreamInterface when audio frames are pulled out, 345 // but when playout is muted, nothing is pulling frames. The downside of 346 // this approach is that frames delivered this way won't be delayed for 347 // playout, and therefore will be unsynchronized with (a) audio delay when 348 // playing and (b) any audio/video synchronization. But the alternative is 349 // that muting playout also stops the SourceTracker from updating RtpSource 350 // information. 351 RtpPacketInfos::vector_type packet_vector = { 352 RtpPacketInfo(rtpHeader, receive_time)}; 353 source_tracker_.OnFrameDelivered(RtpPacketInfos(packet_vector), 354 env_.clock().CurrentTime()); 355 return; 356 } 357 358 // Push the incoming payload (parsed and ready for decoding) into NetEq. 359 if (payload.empty()) { 360 neteq_->InsertEmptyPacket(rtpHeader); 361 } else if (neteq_->InsertPacket(rtpHeader, payload, 362 RtpPacketInfo(rtpHeader, receive_time)) < 0) { 363 RTC_DLOG(LS_ERROR) << "ChannelReceive::OnReceivedPayloadData() unable to " 364 "insert packet into NetEq; PT = " 365 << static_cast<int>(rtpHeader.payloadType); 366 return; 367 } 368 369 TimeDelta round_trip_time = rtp_rtcp_->LastRtt().value_or(TimeDelta::Zero()); 370 371 std::vector<uint16_t> nack_list = neteq_->GetNackList(round_trip_time.ms()); 372 if (!nack_list.empty()) { 373 // Can't use nack_list.data() since it's not supported by all 374 // compilers. 375 ResendPackets(&(nack_list[0]), static_cast<int>(nack_list.size())); 376 } 377 } 378 379 void ChannelReceive::InitFrameTransformerDelegate( 380 scoped_refptr<FrameTransformerInterface> frame_transformer) { 381 RTC_DCHECK(frame_transformer); 382 RTC_DCHECK(!frame_transformer_delegate_); 383 RTC_DCHECK(worker_thread_->IsCurrent()); 384 385 // Pass a callback to ChannelReceive::OnReceivedPayloadData, to be called by 386 // the delegate to receive transformed audio. 387 ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback 388 receive_audio_callback = [this](ArrayView<const uint8_t> packet, 389 const RTPHeader& header, 390 Timestamp receive_time) { 391 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 392 OnReceivedPayloadData(packet, header, receive_time); 393 }; 394 frame_transformer_delegate_ = 395 make_ref_counted<ChannelReceiveFrameTransformerDelegate>( 396 std::move(receive_audio_callback), std::move(frame_transformer), 397 worker_thread_); 398 frame_transformer_delegate_->Init(); 399 } 400 401 AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( 402 int sample_rate_hz, 403 AudioFrame* audio_frame) { 404 TRACE_EVENT_BEGIN1("webrtc", "ChannelReceive::GetAudioFrameWithInfo", 405 "sample_rate_hz", sample_rate_hz); 406 RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_); 407 408 env_.event_log().Log(std::make_unique<RtcEventAudioPlayout>(remote_ssrc_)); 409 410 if ((neteq_->GetAudio(audio_frame) != NetEq::kOK) || 411 !resampler_helper_.MaybeResample(sample_rate_hz, audio_frame)) { 412 RTC_DLOG(LS_ERROR) 413 << "ChannelReceive::GetAudioFrame() PlayoutData10Ms() failed!"; 414 // In all likelihood, the audio in this frame is garbage. We return an 415 // error so that the audio mixer module doesn't add it to the mix. As 416 // a result, it won't be played out and the actions skipped here are 417 // irrelevant. 418 419 TRACE_EVENT_END1("webrtc", "ChannelReceive::GetAudioFrameWithInfo", "error", 420 1); 421 return AudioMixer::Source::AudioFrameInfo::kError; 422 } 423 424 { 425 MutexLock lock(&call_stats_mutex_); 426 call_stats_.DecodedByNetEq(audio_frame->speech_type_, audio_frame->muted()); 427 } 428 429 { 430 // Pass the audio buffers to an optional sink callback, before applying 431 // scaling/panning, as that applies to the mix operation. 432 // External recipients of the audio (e.g. via AudioTrack), will do their 433 // own mixing/dynamic processing. 434 MutexLock lock(&callback_mutex_); 435 if (audio_sink_) { 436 AudioSinkInterface::Data data( 437 audio_frame->data(), audio_frame->samples_per_channel_, 438 audio_frame->sample_rate_hz_, audio_frame->num_channels_, 439 audio_frame->timestamp_); 440 audio_sink_->OnData(data); 441 } 442 } 443 444 float output_gain = 1.0f; 445 { 446 MutexLock lock(&volume_settings_mutex_); 447 output_gain = _outputGain; 448 } 449 450 // Output volume scaling 451 if (output_gain < 0.99f || output_gain > 1.01f) { 452 // TODO(solenberg): Combine with mute state - this can cause clicks! 453 AudioFrameOperations::ScaleWithSat(output_gain, audio_frame); 454 } 455 456 // Measure audio level (0-9) 457 // TODO(henrik.lundin) Use the `muted` information here too. 458 // TODO(deadbeef): Use RmsLevel for `_outputAudioLevel` (see 459 // https://crbug.com/webrtc/7517). 460 _outputAudioLevel.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds); 461 462 if (capture_start_rtp_time_stamp_ < 0 && audio_frame->timestamp_ != 0) { 463 // The first frame with a valid rtp timestamp. 464 capture_start_rtp_time_stamp_ = audio_frame->timestamp_; 465 } 466 467 if (capture_start_rtp_time_stamp_ >= 0) { 468 // audio_frame.timestamp_ should be valid from now on. 469 // Compute elapsed time. 470 int64_t unwrap_timestamp = 471 rtp_ts_wraparound_handler_.Unwrap(audio_frame->timestamp_); 472 audio_frame->elapsed_time_ms_ = 473 (unwrap_timestamp - capture_start_rtp_time_stamp_) / 474 (GetRtpTimestampRateHz() / 1000); 475 476 { 477 MutexLock lock(&ts_stats_lock_); 478 // Compute ntp time. 479 audio_frame->ntp_time_ms_ = 480 ntp_estimator_.Estimate(audio_frame->timestamp_); 481 // `ntp_time_ms_` won't be valid until at least 2 RTCP SRs are received. 482 if (audio_frame->ntp_time_ms_ > 0) { 483 // Compute `capture_start_ntp_time_ms_` so that 484 // `capture_start_ntp_time_ms_` + `elapsed_time_ms_` == `ntp_time_ms_` 485 capture_start_ntp_time_ms_ = 486 audio_frame->ntp_time_ms_ - audio_frame->elapsed_time_ms_; 487 } 488 } 489 } 490 491 // Fill in local capture clock offset in `audio_frame->packet_infos_`. 492 RtpPacketInfos::vector_type packet_infos; 493 for (auto& packet_info : audio_frame->packet_infos_) { 494 RtpPacketInfo new_packet_info(packet_info); 495 if (packet_info.absolute_capture_time().has_value()) { 496 MutexLock lock(&ts_stats_lock_); 497 new_packet_info.set_local_capture_clock_offset( 498 CaptureClockOffsetUpdater::ConvertToTimeDelta( 499 capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset( 500 packet_info.absolute_capture_time() 501 ->estimated_capture_clock_offset))); 502 } 503 packet_infos.push_back(std::move(new_packet_info)); 504 } 505 audio_frame->packet_infos_ = RtpPacketInfos(std::move(packet_infos)); 506 if (!audio_frame->packet_infos_.empty()) { 507 RtpPacketInfos infos_copy = audio_frame->packet_infos_; 508 Timestamp delivery_time = env_.clock().CurrentTime(); 509 worker_thread_->PostTask( 510 SafeTask(worker_safety_.flag(), [this, infos_copy, delivery_time]() { 511 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 512 source_tracker_.OnFrameDelivered(infos_copy, delivery_time); 513 })); 514 } 515 516 ++audio_frame_interval_count_; 517 if (audio_frame_interval_count_ >= kHistogramReportingInterval) { 518 audio_frame_interval_count_ = 0; 519 worker_thread_->PostTask(SafeTask(worker_safety_.flag(), [this]() { 520 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 521 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.TargetJitterBufferDelayMs", 522 neteq_->TargetDelayMs()); 523 const int jitter_buffer_delay = neteq_->FilteredCurrentDelayMs(); 524 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDelayEstimateMs", 525 jitter_buffer_delay + playout_delay_ms_); 526 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverJitterBufferDelayMs", 527 jitter_buffer_delay); 528 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDeviceDelayMs", 529 playout_delay_ms_); 530 })); 531 } 532 533 TRACE_EVENT_END2("webrtc", "ChannelReceive::GetAudioFrameWithInfo", "gain", 534 output_gain, "muted", audio_frame->muted()); 535 return audio_frame->muted() ? AudioMixer::Source::AudioFrameInfo::kMuted 536 : AudioMixer::Source::AudioFrameInfo::kNormal; 537 } 538 539 int ChannelReceive::PreferredSampleRate() const { 540 RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_); 541 const std::optional<NetEq::DecoderFormat> decoder = 542 neteq_->GetCurrentDecoderFormat(); 543 const int last_packet_sample_rate_hz = decoder ? decoder->sample_rate_hz : 0; 544 // Return the bigger of playout and receive frequency in the ACM. 545 return std::max(last_packet_sample_rate_hz, 546 neteq_->last_output_sample_rate_hz()); 547 } 548 549 ChannelReceive::ChannelReceive( 550 const Environment& env, 551 NetEqFactory* neteq_factory, 552 AudioDeviceModule* audio_device_module, 553 Transport* rtcp_send_transport, 554 uint32_t local_ssrc, 555 uint32_t remote_ssrc, 556 size_t jitter_buffer_max_packets, 557 bool jitter_buffer_fast_playout, 558 int jitter_buffer_min_delay_ms, 559 bool enable_non_sender_rtt, 560 scoped_refptr<AudioDecoderFactory> decoder_factory, 561 std::optional<AudioCodecPairId> codec_pair_id, 562 scoped_refptr<FrameDecryptorInterface> frame_decryptor, 563 const CryptoOptions& crypto_options, 564 scoped_refptr<FrameTransformerInterface> frame_transformer, 565 RtcpEventObserver* rtcp_event_observer) 566 : env_(env), 567 worker_thread_(TaskQueueBase::Current()), 568 rtp_receive_statistics_(ReceiveStatistics::Create(&env_.clock())), 569 remote_ssrc_(remote_ssrc), 570 source_tracker_(&env_.clock()), 571 neteq_(CreateNetEq(neteq_factory, 572 codec_pair_id, 573 jitter_buffer_max_packets, 574 jitter_buffer_fast_playout, 575 jitter_buffer_min_delay_ms, 576 env_, 577 decoder_factory)), 578 _outputAudioLevel(), 579 ntp_estimator_(&env_.clock()), 580 playout_delay_ms_(0), 581 capture_start_rtp_time_stamp_(-1), 582 capture_start_ntp_time_ms_(-1), 583 _audioDeviceModulePtr(audio_device_module), 584 _outputGain(1.0f), 585 frame_decryptor_(frame_decryptor), 586 crypto_options_(crypto_options), 587 absolute_capture_time_interpolator_(&env_.clock()) { 588 RTC_DCHECK(audio_device_module); 589 590 rtp_receive_statistics_->EnableRetransmitDetection(remote_ssrc_, true); 591 RtpRtcpInterface::Configuration configuration; 592 configuration.audio = true; 593 configuration.receiver_only = true; 594 configuration.outgoing_transport = rtcp_send_transport; 595 configuration.receive_statistics = rtp_receive_statistics_.get(); 596 configuration.local_media_ssrc = local_ssrc; 597 configuration.rtcp_packet_type_counter_observer = this; 598 configuration.non_sender_rtt_measurement = enable_non_sender_rtt; 599 configuration.rtcp_event_observer = rtcp_event_observer; 600 601 if (frame_transformer) 602 InitFrameTransformerDelegate(std::move(frame_transformer)); 603 604 rtp_rtcp_ = std::make_unique<ModuleRtpRtcpImpl2>(env_, configuration); 605 rtp_rtcp_->SetRemoteSSRC(remote_ssrc_); 606 607 // Ensure that RTCP is enabled for the created channel. 608 rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound); 609 } 610 611 ChannelReceive::~ChannelReceive() { 612 RTC_DCHECK_RUN_ON(&construction_thread_); 613 614 // Resets the delegate's callback to ChannelReceive::OnReceivedPayloadData. 615 if (frame_transformer_delegate_) 616 frame_transformer_delegate_->Reset(); 617 618 StopPlayout(); 619 } 620 621 void ChannelReceive::SetSink(AudioSinkInterface* sink) { 622 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 623 MutexLock lock(&callback_mutex_); 624 audio_sink_ = sink; 625 } 626 627 void ChannelReceive::StartPlayout() { 628 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 629 playing_ = true; 630 } 631 632 void ChannelReceive::StopPlayout() { 633 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 634 playing_ = false; 635 _outputAudioLevel.ResetLevelFullRange(); 636 neteq_->FlushBuffers(); 637 } 638 639 std::optional<std::pair<int, SdpAudioFormat>> ChannelReceive::GetReceiveCodec() 640 const { 641 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 642 std::optional<NetEq::DecoderFormat> decoder = 643 neteq_->GetCurrentDecoderFormat(); 644 if (!decoder) { 645 return std::nullopt; 646 } 647 return std::make_pair(decoder->payload_type, decoder->sdp_format); 648 } 649 650 void ChannelReceive::SetReceiveCodecs( 651 const std::map<int, SdpAudioFormat>& codecs) { 652 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 653 for (const auto& kv : codecs) { 654 RTC_DCHECK_GE(kv.second.clockrate_hz, 1000); 655 payload_type_frequencies_[kv.first] = kv.second.clockrate_hz; 656 } 657 payload_type_map_ = codecs; 658 neteq_->SetCodecs(codecs); 659 } 660 661 void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) { 662 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 663 Timestamp now = env_.clock().CurrentTime(); 664 665 last_received_rtp_timestamp_ = packet.Timestamp(); 666 last_received_rtp_system_time_ = now; 667 668 // Store playout timestamp for the received RTP packet 669 UpdatePlayoutTimestamp(false, now); 670 671 const auto& it = payload_type_frequencies_.find(packet.PayloadType()); 672 if (it == payload_type_frequencies_.end()) 673 return; 674 // TODO(bugs.webrtc.org/7135): Set payload_type_frequency earlier, when packet 675 // is parsed. 676 RtpPacketReceived packet_copy(packet); 677 packet_copy.set_payload_type_frequency(it->second); 678 679 rtp_receive_statistics_->OnRtpPacket(packet_copy); 680 681 RTPHeader header; 682 packet_copy.GetHeader(&header); 683 684 // Interpolates absolute capture timestamp RTP header extension. 685 header.extension.absolute_capture_time = 686 absolute_capture_time_interpolator_.OnReceivePacket( 687 AbsoluteCaptureTimeInterpolator::GetSource(header.ssrc, 688 header.arrOfCSRCs), 689 header.timestamp, 690 saturated_cast<uint32_t>(packet_copy.payload_type_frequency()), 691 header.extension.absolute_capture_time); 692 693 ReceivePacket(packet_copy.data(), packet_copy.size(), header, 694 packet.arrival_time()); 695 } 696 697 void ChannelReceive::ReceivePacket(const uint8_t* packet, 698 size_t packet_length, 699 const RTPHeader& header, 700 Timestamp receive_time) { 701 const uint8_t* payload = packet + header.headerLength; 702 RTC_DCHECK_GE(packet_length, header.headerLength); 703 size_t payload_length = packet_length - header.headerLength; 704 705 size_t payload_data_length = payload_length - header.paddingLength; 706 707 // E2EE Custom Audio Frame Decryption (This is optional). 708 // Keep this buffer around for the lifetime of the OnReceivedPayloadData call. 709 Buffer decrypted_audio_payload; 710 if (frame_decryptor_ != nullptr) { 711 const size_t max_plaintext_size = frame_decryptor_->GetMaxPlaintextByteSize( 712 MediaType::AUDIO, payload_length); 713 decrypted_audio_payload.SetSize(max_plaintext_size); 714 715 const std::vector<uint32_t> csrcs(header.arrOfCSRCs, 716 header.arrOfCSRCs + header.numCSRCs); 717 const FrameDecryptorInterface::Result decrypt_result = 718 frame_decryptor_->Decrypt( 719 MediaType::AUDIO, csrcs, 720 /*additional_data=*/ 721 nullptr, ArrayView<const uint8_t>(payload, payload_data_length), 722 decrypted_audio_payload); 723 724 if (decrypt_result.IsOk()) { 725 decrypted_audio_payload.SetSize(decrypt_result.bytes_written); 726 } else { 727 // Interpret failures as a silent frame. 728 decrypted_audio_payload.SetSize(0); 729 } 730 731 payload = decrypted_audio_payload.data(); 732 payload_data_length = decrypted_audio_payload.size(); 733 } else if (crypto_options_.sframe.require_frame_encryption) { 734 RTC_DLOG(LS_ERROR) 735 << "FrameDecryptor required but not set, dropping packet"; 736 payload_data_length = 0; 737 } 738 739 ArrayView<const uint8_t> payload_data(payload, payload_data_length); 740 if (frame_transformer_delegate_) { 741 // Asynchronously transform the received payload. After the payload is 742 // transformed, the delegate will call OnReceivedPayloadData to handle it. 743 char buf[1024]; 744 SimpleStringBuilder mime_type(buf); 745 auto it = payload_type_map_.find(header.payloadType); 746 mime_type << MediaTypeToString(MediaType::AUDIO) << "/" 747 << (it != payload_type_map_.end() ? it->second.name 748 : "x-unknown"); 749 frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_, 750 mime_type.str(), receive_time); 751 } else { 752 OnReceivedPayloadData(payload_data, header, receive_time); 753 } 754 } 755 756 void ChannelReceive::ReceivedRTCPPacket(const uint8_t* data, size_t length) { 757 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 758 759 // Store playout timestamp for the received RTCP packet 760 UpdatePlayoutTimestamp(true, env_.clock().CurrentTime()); 761 762 // Deliver RTCP packet to RTP/RTCP module for parsing 763 rtp_rtcp_->IncomingRtcpPacket(MakeArrayView(data, length)); 764 765 std::optional<TimeDelta> rtt = rtp_rtcp_->LastRtt(); 766 if (!rtt.has_value()) { 767 // Waiting for valid RTT. 768 return; 769 } 770 771 std::optional<RtpRtcpInterface::SenderReportStats> last_sr = 772 rtp_rtcp_->GetSenderReportStats(); 773 if (!last_sr.has_value()) { 774 // Waiting for RTCP. 775 return; 776 } 777 778 { 779 MutexLock lock(&ts_stats_lock_); 780 ntp_estimator_.UpdateRtcpTimestamp(*rtt, last_sr->last_remote_ntp_timestamp, 781 last_sr->last_remote_rtp_timestamp); 782 std::optional<int64_t> remote_to_local_clock_offset = 783 ntp_estimator_.EstimateRemoteToLocalClockOffset(); 784 if (remote_to_local_clock_offset.has_value()) { 785 capture_clock_offset_updater_.SetRemoteToLocalClockOffset( 786 *remote_to_local_clock_offset); 787 } 788 } 789 } 790 791 int ChannelReceive::GetSpeechOutputLevelFullRange() const { 792 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 793 return _outputAudioLevel.LevelFullRange(); 794 } 795 796 double ChannelReceive::GetTotalOutputEnergy() const { 797 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 798 return _outputAudioLevel.TotalEnergy(); 799 } 800 801 double ChannelReceive::GetTotalOutputDuration() const { 802 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 803 return _outputAudioLevel.TotalDuration(); 804 } 805 806 void ChannelReceive::SetChannelOutputVolumeScaling(float scaling) { 807 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 808 MutexLock lock(&volume_settings_mutex_); 809 _outputGain = scaling; 810 } 811 812 void ChannelReceive::RegisterReceiverCongestionControlObjects( 813 PacketRouter* packet_router) { 814 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 815 RTC_DCHECK(packet_router); 816 RTC_DCHECK(!packet_router_); 817 constexpr bool remb_candidate = false; 818 packet_router->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); 819 packet_router_ = packet_router; 820 } 821 822 void ChannelReceive::ResetReceiverCongestionControlObjects() { 823 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 824 RTC_DCHECK(packet_router_); 825 packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); 826 packet_router_ = nullptr; 827 } 828 829 ChannelReceiveStatistics ChannelReceive::GetRTCPStatistics() const { 830 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 831 ChannelReceiveStatistics stats; 832 833 // The jitter statistics is updated for each received RTP packet and is based 834 // on received packets. 835 RtpReceiveStats rtp_stats; 836 StreamStatistician* statistician = 837 rtp_receive_statistics_->GetStatistician(remote_ssrc_); 838 if (statistician) { 839 rtp_stats = statistician->GetStats(); 840 } 841 842 stats.packets_lost = rtp_stats.packets_lost; 843 stats.jitter_ms = rtp_stats.interarrival_jitter.ms(); 844 845 // Data counters. 846 if (statistician) { 847 stats.payload_bytes_received = rtp_stats.packet_counter.payload_bytes; 848 stats.header_and_padding_bytes_received = 849 rtp_stats.packet_counter.header_bytes + 850 rtp_stats.packet_counter.padding_bytes; 851 stats.packets_received = rtp_stats.packet_counter.packets; 852 stats.packets_received_with_ect1 = 853 rtp_stats.packet_counter.packets_with_ect1; 854 stats.packets_received_with_ce = rtp_stats.packet_counter.packets_with_ce; 855 stats.last_packet_received = rtp_stats.last_packet_received; 856 } 857 858 { 859 MutexLock lock(&rtcp_counter_mutex_); 860 stats.nacks_sent = rtcp_packet_type_counter_.nack_packets; 861 } 862 863 // Timestamps. 864 { 865 MutexLock lock(&ts_stats_lock_); 866 stats.capture_start_ntp_time_ms = capture_start_ntp_time_ms_; 867 } 868 869 std::optional<RtpRtcpInterface::SenderReportStats> rtcp_sr_stats = 870 rtp_rtcp_->GetSenderReportStats(); 871 if (rtcp_sr_stats.has_value()) { 872 stats.last_sender_report_timestamp = rtcp_sr_stats->last_arrival_timestamp; 873 stats.last_sender_report_utc_timestamp = 874 Clock::NtpToUtc(rtcp_sr_stats->last_arrival_ntp_timestamp); 875 stats.last_sender_report_remote_utc_timestamp = 876 Clock::NtpToUtc(rtcp_sr_stats->last_remote_ntp_timestamp); 877 stats.sender_reports_packets_sent = rtcp_sr_stats->packets_sent; 878 stats.sender_reports_bytes_sent = rtcp_sr_stats->bytes_sent; 879 stats.sender_reports_reports_count = rtcp_sr_stats->reports_count; 880 } 881 882 std::optional<RtpRtcpInterface::NonSenderRttStats> non_sender_rtt_stats = 883 rtp_rtcp_->GetNonSenderRttStats(); 884 if (non_sender_rtt_stats.has_value()) { 885 stats.round_trip_time = non_sender_rtt_stats->round_trip_time; 886 stats.round_trip_time_measurements = 887 non_sender_rtt_stats->round_trip_time_measurements; 888 stats.total_round_trip_time = non_sender_rtt_stats->total_round_trip_time; 889 } 890 891 return stats; 892 } 893 894 void ChannelReceive::SetNACKStatus(bool enable, int max_packets) { 895 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 896 // None of these functions can fail. 897 if (enable) { 898 rtp_receive_statistics_->SetMaxReorderingThreshold(remote_ssrc_, 899 max_packets); 900 neteq_->EnableNack(max_packets); 901 } else { 902 rtp_receive_statistics_->SetMaxReorderingThreshold( 903 remote_ssrc_, kDefaultMaxReorderingThreshold); 904 neteq_->DisableNack(); 905 } 906 } 907 908 void ChannelReceive::SetRtcpMode(RtcpMode mode) { 909 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 910 rtp_rtcp_->SetRTCPStatus(mode); 911 } 912 913 void ChannelReceive::SetNonSenderRttMeasurement(bool enabled) { 914 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 915 rtp_rtcp_->SetNonSenderRttMeasurement(enabled); 916 } 917 918 // Called when we are missing one or more packets. 919 int ChannelReceive::ResendPackets(const uint16_t* sequence_numbers, 920 int length) { 921 return rtp_rtcp_->SendNACK(sequence_numbers, length); 922 } 923 924 void ChannelReceive::RtcpPacketTypesCounterUpdated( 925 uint32_t ssrc, 926 const RtcpPacketTypeCounter& packet_counter) { 927 if (ssrc != remote_ssrc_) { 928 return; 929 } 930 MutexLock lock(&rtcp_counter_mutex_); 931 rtcp_packet_type_counter_ = packet_counter; 932 } 933 934 void ChannelReceive::SetDepacketizerToDecoderFrameTransformer( 935 scoped_refptr<FrameTransformerInterface> frame_transformer) { 936 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 937 if (!frame_transformer) { 938 RTC_DCHECK_NOTREACHED() << "Not setting the transformer?"; 939 return; 940 } 941 if (frame_transformer_delegate_) { 942 // Depending on when the channel is created, the transformer might be set 943 // twice. Don't replace the delegate if it was already initialized. 944 // TODO(crbug.com/webrtc/15674): Prevent multiple calls during 945 // reconfiguration. 946 RTC_CHECK_EQ(frame_transformer_delegate_->FrameTransformer(), 947 frame_transformer); 948 return; 949 } 950 951 InitFrameTransformerDelegate(std::move(frame_transformer)); 952 } 953 954 void ChannelReceive::SetFrameDecryptor( 955 scoped_refptr<FrameDecryptorInterface> frame_decryptor) { 956 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 957 frame_decryptor_ = std::move(frame_decryptor); 958 } 959 960 void ChannelReceive::OnLocalSsrcChange(uint32_t local_ssrc) { 961 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 962 rtp_rtcp_->SetLocalSsrc(local_ssrc); 963 } 964 965 NetworkStatistics ChannelReceive::GetNetworkStatistics( 966 bool get_and_clear_legacy_stats) const { 967 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 968 NetworkStatistics acm_stat; 969 NetEqNetworkStatistics neteq_stat; 970 if (get_and_clear_legacy_stats) { 971 // NetEq function always returns zero, so we don't check the return value. 972 neteq_->NetworkStatistics(&neteq_stat); 973 974 acm_stat.currentExpandRate = neteq_stat.expand_rate; 975 acm_stat.currentSpeechExpandRate = neteq_stat.speech_expand_rate; 976 acm_stat.currentPreemptiveRate = neteq_stat.preemptive_rate; 977 acm_stat.currentAccelerateRate = neteq_stat.accelerate_rate; 978 acm_stat.currentSecondaryDecodedRate = neteq_stat.secondary_decoded_rate; 979 acm_stat.currentSecondaryDiscardedRate = 980 neteq_stat.secondary_discarded_rate; 981 acm_stat.meanWaitingTimeMs = neteq_stat.mean_waiting_time_ms; 982 acm_stat.maxWaitingTimeMs = neteq_stat.max_waiting_time_ms; 983 } else { 984 neteq_stat = neteq_->CurrentNetworkStatistics(); 985 acm_stat.currentExpandRate = 0; 986 acm_stat.currentSpeechExpandRate = 0; 987 acm_stat.currentPreemptiveRate = 0; 988 acm_stat.currentAccelerateRate = 0; 989 acm_stat.currentSecondaryDecodedRate = 0; 990 acm_stat.currentSecondaryDiscardedRate = 0; 991 acm_stat.meanWaitingTimeMs = -1; 992 acm_stat.maxWaitingTimeMs = 1; 993 } 994 acm_stat.currentBufferSize = neteq_stat.current_buffer_size_ms; 995 acm_stat.preferredBufferSize = neteq_stat.preferred_buffer_size_ms; 996 acm_stat.jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false; 997 998 NetEqLifetimeStatistics neteq_lifetime_stat = neteq_->GetLifetimeStatistics(); 999 acm_stat.totalSamplesReceived = neteq_lifetime_stat.total_samples_received; 1000 acm_stat.concealedSamples = neteq_lifetime_stat.concealed_samples; 1001 acm_stat.silentConcealedSamples = 1002 neteq_lifetime_stat.silent_concealed_samples; 1003 acm_stat.concealmentEvents = neteq_lifetime_stat.concealment_events; 1004 acm_stat.jitterBufferDelayMs = neteq_lifetime_stat.jitter_buffer_delay_ms; 1005 acm_stat.jitterBufferTargetDelayMs = 1006 neteq_lifetime_stat.jitter_buffer_target_delay_ms; 1007 acm_stat.jitterBufferMinimumDelayMs = 1008 neteq_lifetime_stat.jitter_buffer_minimum_delay_ms; 1009 acm_stat.jitterBufferEmittedCount = 1010 neteq_lifetime_stat.jitter_buffer_emitted_count; 1011 acm_stat.delayedPacketOutageSamples = 1012 neteq_lifetime_stat.delayed_packet_outage_samples; 1013 acm_stat.relativePacketArrivalDelayMs = 1014 neteq_lifetime_stat.relative_packet_arrival_delay_ms; 1015 acm_stat.interruptionCount = neteq_lifetime_stat.interruption_count; 1016 acm_stat.totalInterruptionDurationMs = 1017 neteq_lifetime_stat.total_interruption_duration_ms; 1018 acm_stat.insertedSamplesForDeceleration = 1019 neteq_lifetime_stat.inserted_samples_for_deceleration; 1020 acm_stat.removedSamplesForAcceleration = 1021 neteq_lifetime_stat.removed_samples_for_acceleration; 1022 acm_stat.fecPacketsReceived = neteq_lifetime_stat.fec_packets_received; 1023 acm_stat.fecPacketsDiscarded = neteq_lifetime_stat.fec_packets_discarded; 1024 acm_stat.totalProcessingDelayUs = 1025 neteq_lifetime_stat.total_processing_delay_us; 1026 acm_stat.packetsDiscarded = neteq_lifetime_stat.packets_discarded; 1027 1028 NetEqOperationsAndState neteq_operations_and_state = 1029 neteq_->GetOperationsAndState(); 1030 acm_stat.packetBufferFlushes = 1031 neteq_operations_and_state.packet_buffer_flushes; 1032 return acm_stat; 1033 } 1034 1035 AudioDecodingCallStats ChannelReceive::GetDecodingCallStatistics() const { 1036 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1037 MutexLock lock(&call_stats_mutex_); 1038 return call_stats_.GetDecodingStatistics(); 1039 } 1040 1041 uint32_t ChannelReceive::GetDelayEstimate() const { 1042 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1043 // Return the current jitter buffer delay + playout delay. 1044 return neteq_->FilteredCurrentDelayMs() + playout_delay_ms_; 1045 } 1046 1047 bool ChannelReceive::SetMinimumPlayoutDelay(TimeDelta delay) { 1048 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1049 // Limit to range accepted by both VoE and ACM, so we're at least getting as 1050 // close as possible, instead of failing. 1051 delay = std::clamp(delay, kVoiceEngineMinMinPlayoutDelay, 1052 kVoiceEngineMaxMinPlayoutDelay); 1053 if (!neteq_->SetMinimumDelay(delay.ms())) { 1054 RTC_DLOG(LS_ERROR) 1055 << "SetMinimumPlayoutDelay() failed to set min playout delay " << delay; 1056 return false; 1057 } 1058 return true; 1059 } 1060 1061 std::optional<Syncable::PlayoutInfo> ChannelReceive::GetPlayoutRtpTimestamp() 1062 const { 1063 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1064 return playout_timestamp_; 1065 } 1066 1067 void ChannelReceive::SetEstimatedPlayoutNtpTimestamp(NtpTime ntp_time, 1068 Timestamp time) { 1069 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1070 playout_timestamp_ntp_ = ntp_time; 1071 playout_timestamp_ntp_time_ = time; 1072 } 1073 1074 std::optional<int64_t> ChannelReceive::GetCurrentEstimatedPlayoutNtpTimestampMs( 1075 int64_t now_ms) const { 1076 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1077 if (!playout_timestamp_ntp_ || !playout_timestamp_ntp_time_) 1078 return std::nullopt; 1079 1080 int64_t elapsed_ms = now_ms - playout_timestamp_ntp_time_->ms(); 1081 return playout_timestamp_ntp_->ToMs() + elapsed_ms; 1082 } 1083 1084 bool ChannelReceive::SetBaseMinimumPlayoutDelayMs(int delay_ms) { 1085 env_.event_log().Log( 1086 std::make_unique<RtcEventNetEqSetMinimumDelay>(remote_ssrc_, delay_ms)); 1087 return neteq_->SetBaseMinimumDelayMs(delay_ms); 1088 } 1089 1090 int ChannelReceive::GetBaseMinimumPlayoutDelayMs() const { 1091 return neteq_->GetBaseMinimumDelayMs(); 1092 } 1093 1094 std::optional<Syncable::Info> ChannelReceive::GetSyncInfo() const { 1095 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1096 Syncable::Info info; 1097 std::optional<RtpRtcpInterface::SenderReportStats> last_sr = 1098 rtp_rtcp_->GetSenderReportStats(); 1099 if (!last_sr.has_value()) { 1100 return std::nullopt; 1101 } 1102 info.capture_time_ntp = last_sr->last_remote_ntp_timestamp; 1103 info.capture_time_rtp = last_sr->last_remote_rtp_timestamp; 1104 1105 if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_) { 1106 return std::nullopt; 1107 } 1108 info.latest_received_capture_rtp_timestamp = *last_received_rtp_timestamp_; 1109 info.latest_receive_time = *last_received_rtp_system_time_; 1110 1111 int jitter_buffer_delay = neteq_->FilteredCurrentDelayMs(); 1112 info.current_delay = 1113 TimeDelta::Millis(jitter_buffer_delay + playout_delay_ms_); 1114 1115 return info; 1116 } 1117 1118 void ChannelReceive::UpdatePlayoutTimestamp(bool rtcp, Timestamp now) { 1119 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1120 1121 jitter_buffer_playout_timestamp_ = neteq_->GetPlayoutTimestamp(); 1122 1123 if (!jitter_buffer_playout_timestamp_) { 1124 // This can happen if this channel has not received any RTP packets. In 1125 // this case, NetEq is not capable of computing a playout timestamp. 1126 return; 1127 } 1128 1129 uint16_t delay_ms = 0; 1130 if (_audioDeviceModulePtr->PlayoutDelay(&delay_ms) == -1) { 1131 RTC_DLOG(LS_WARNING) 1132 << "ChannelReceive::UpdatePlayoutTimestamp() failed to read" 1133 " playout delay from the ADM"; 1134 return; 1135 } 1136 1137 RTC_DCHECK(jitter_buffer_playout_timestamp_); 1138 uint32_t playout_timestamp = *jitter_buffer_playout_timestamp_; 1139 1140 // Remove the playout delay. 1141 playout_timestamp -= (delay_ms * (GetRtpTimestampRateHz() / 1000)); 1142 1143 if (!rtcp && (!playout_timestamp_.has_value() || 1144 playout_timestamp_->rtp_timestamp != playout_timestamp)) { 1145 playout_timestamp_ = {{.time = now, .rtp_timestamp = playout_timestamp}}; 1146 } 1147 playout_delay_ms_ = delay_ms; 1148 } 1149 1150 int ChannelReceive::GetRtpTimestampRateHz() const { 1151 const auto decoder_format = neteq_->GetCurrentDecoderFormat(); 1152 1153 // Default to the playout frequency if we've not gotten any packets yet. 1154 // TODO(ossu): Zero clock rate can only happen if we've added an external 1155 // decoder for a format we don't support internally. Remove once that way of 1156 // adding decoders is gone! 1157 // TODO(kwiberg): `decoder_format->sdp_format.clockrate_hz` is an RTP 1158 // clock rate as it should, but `neteq_->last_output_sample_rate_hz()` is a 1159 // codec sample rate, which is not always the same thing. 1160 return (decoder_format && decoder_format->sdp_format.clockrate_hz != 0) 1161 ? decoder_format->sdp_format.clockrate_hz 1162 : neteq_->last_output_sample_rate_hz(); 1163 } 1164 1165 std::vector<RtpSource> ChannelReceive::GetSources() const { 1166 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 1167 return source_tracker_.GetSources(); 1168 } 1169 1170 } // namespace 1171 1172 std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive( 1173 const Environment& env, 1174 NetEqFactory* neteq_factory, 1175 AudioDeviceModule* audio_device_module, 1176 Transport* rtcp_send_transport, 1177 uint32_t local_ssrc, 1178 uint32_t remote_ssrc, 1179 size_t jitter_buffer_max_packets, 1180 bool jitter_buffer_fast_playout, 1181 int jitter_buffer_min_delay_ms, 1182 bool enable_non_sender_rtt, 1183 scoped_refptr<AudioDecoderFactory> decoder_factory, 1184 std::optional<AudioCodecPairId> codec_pair_id, 1185 scoped_refptr<FrameDecryptorInterface> frame_decryptor, 1186 const CryptoOptions& crypto_options, 1187 scoped_refptr<FrameTransformerInterface> frame_transformer, 1188 RtcpEventObserver* rtcp_event_observer) { 1189 return std::make_unique<ChannelReceive>( 1190 env, neteq_factory, audio_device_module, rtcp_send_transport, local_ssrc, 1191 remote_ssrc, jitter_buffer_max_packets, jitter_buffer_fast_playout, 1192 jitter_buffer_min_delay_ms, enable_non_sender_rtt, decoder_factory, 1193 codec_pair_id, std::move(frame_decryptor), crypto_options, 1194 std::move(frame_transformer), rtcp_event_observer); 1195 } 1196 1197 } // namespace voe 1198 } // namespace webrtc