audio_receive_stream.cc (18208B)
1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "audio/audio_receive_stream.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <map> 16 #include <memory> 17 #include <optional> 18 #include <string> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/strings/string_view.h" 23 #include "api/array_view.h" 24 #include "api/audio/audio_frame.h" 25 #include "api/audio/audio_mixer.h" 26 #include "api/audio_codecs/audio_format.h" 27 #include "api/call/audio_sink.h" 28 #include "api/crypto/frame_decryptor_interface.h" 29 #include "api/environment/environment.h" 30 #include "api/frame_transformer_interface.h" 31 #include "api/neteq/neteq_factory.h" 32 #include "api/rtp_headers.h" 33 #include "api/scoped_refptr.h" 34 #include "api/sequence_checker.h" 35 #include "api/transport/rtp/rtp_source.h" 36 #include "api/units/time_delta.h" 37 #include "api/units/timestamp.h" 38 #include "audio/audio_send_stream.h" 39 #include "audio/audio_state.h" 40 #include "audio/channel_receive.h" 41 #include "audio/conversion.h" 42 #include "call/audio_state.h" 43 #include "call/rtp_config.h" 44 #include "call/rtp_stream_receiver_controller_interface.h" 45 #include "call/syncable.h" 46 #include "rtc_base/checks.h" 47 #include "rtc_base/logging.h" 48 #include "rtc_base/strings/string_builder.h" 49 #include "rtc_base/time_utils.h" 50 #include "system_wrappers/include/ntp_time.h" 51 52 namespace webrtc { 53 54 std::string AudioReceiveStreamInterface::Config::Rtp::ToString() const { 55 char ss_buf[1024]; 56 SimpleStringBuilder ss(ss_buf); 57 ss << "{remote_ssrc: " << remote_ssrc; 58 ss << ", local_ssrc: " << local_ssrc; 59 ss << ", nack: " << nack.ToString(); 60 ss << ", rtcp: " 61 << (rtcp_mode == RtcpMode::kCompound 62 ? "compound" 63 : (rtcp_mode == RtcpMode::kReducedSize ? "reducedSize" : "off")); 64 ss << ", rtcp_event_observer: " 65 << (rtcp_event_observer ? "(rtcp_event_observer)" : "nullptr"); 66 ss << '}'; 67 return ss.str(); 68 } 69 70 std::string AudioReceiveStreamInterface::Config::ToString() const { 71 char ss_buf[1024]; 72 SimpleStringBuilder ss(ss_buf); 73 ss << "{rtp: " << rtp.ToString(); 74 ss << ", rtcp_send_transport: " 75 << (rtcp_send_transport ? "(Transport)" : "null"); 76 if (!sync_group.empty()) { 77 ss << ", sync_group: " << sync_group; 78 } 79 ss << '}'; 80 return ss.str(); 81 } 82 83 namespace { 84 std::unique_ptr<voe::ChannelReceiveInterface> CreateChannelReceive( 85 const Environment& env, 86 AudioState* audio_state, 87 NetEqFactory* neteq_factory, 88 const AudioReceiveStreamInterface::Config& config) { 89 RTC_DCHECK(audio_state); 90 internal::AudioState* internal_audio_state = 91 static_cast<internal::AudioState*>(audio_state); 92 return voe::CreateChannelReceive( 93 env, neteq_factory, internal_audio_state->audio_device_module(), 94 config.rtcp_send_transport, config.rtp.local_ssrc, config.rtp.remote_ssrc, 95 config.jitter_buffer_max_packets, config.jitter_buffer_fast_accelerate, 96 config.jitter_buffer_min_delay_ms, config.enable_non_sender_rtt, 97 config.decoder_factory, config.codec_pair_id, 98 std::move(config.frame_decryptor), config.crypto_options, 99 std::move(config.frame_transformer), config.rtp.rtcp_event_observer); 100 } 101 } // namespace 102 103 AudioReceiveStreamImpl::AudioReceiveStreamImpl( 104 const Environment& env, 105 PacketRouter* packet_router, 106 NetEqFactory* neteq_factory, 107 const AudioReceiveStreamInterface::Config& config, 108 const scoped_refptr<AudioState>& audio_state) 109 : AudioReceiveStreamImpl( 110 env, 111 packet_router, 112 config, 113 audio_state, 114 CreateChannelReceive(env, audio_state.get(), neteq_factory, config)) { 115 } 116 117 AudioReceiveStreamImpl::AudioReceiveStreamImpl( 118 const Environment& env, 119 PacketRouter* packet_router, 120 const AudioReceiveStreamInterface::Config& config, 121 const scoped_refptr<AudioState>& audio_state, 122 std::unique_ptr<voe::ChannelReceiveInterface> channel_receive) 123 : env_(env), 124 config_(config), 125 audio_state_(audio_state), 126 channel_receive_(std::move(channel_receive)) { 127 RTC_LOG(LS_INFO) << "AudioReceiveStreamImpl: " << config.rtp.remote_ssrc; 128 RTC_DCHECK(config.decoder_factory); 129 RTC_DCHECK(config.rtcp_send_transport); 130 RTC_DCHECK(audio_state_); 131 RTC_DCHECK(channel_receive_); 132 133 RTC_DCHECK(packet_router); 134 // Configure bandwidth estimation. 135 channel_receive_->RegisterReceiverCongestionControlObjects(packet_router); 136 137 // Complete configuration. 138 // TODO(solenberg): Config NACK history window (which is a packet count), 139 // using the actual packet size for the configured codec. 140 channel_receive_->SetNACKStatus(config.rtp.nack.rtp_history_ms != 0, 141 config.rtp.nack.rtp_history_ms / 20); 142 channel_receive_->SetRtcpMode(config.rtp.rtcp_mode); 143 channel_receive_->SetReceiveCodecs(config.decoder_map); 144 // `frame_transformer` and `frame_decryptor` have been given to 145 // `channel_receive_` already. 146 } 147 148 AudioReceiveStreamImpl::~AudioReceiveStreamImpl() { 149 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 150 RTC_LOG(LS_INFO) << "~AudioReceiveStreamImpl: " << remote_ssrc(); 151 Stop(); 152 channel_receive_->ResetReceiverCongestionControlObjects(); 153 } 154 155 void AudioReceiveStreamImpl::RegisterWithTransport( 156 RtpStreamReceiverControllerInterface* receiver_controller) { 157 RTC_DCHECK_RUN_ON(&packet_sequence_checker_); 158 RTC_DCHECK(!rtp_stream_receiver_); 159 rtp_stream_receiver_ = receiver_controller->CreateReceiver( 160 remote_ssrc(), channel_receive_.get()); 161 } 162 163 void AudioReceiveStreamImpl::UnregisterFromTransport() { 164 RTC_DCHECK_RUN_ON(&packet_sequence_checker_); 165 rtp_stream_receiver_.reset(); 166 } 167 168 void AudioReceiveStreamImpl::ReconfigureForTesting( 169 const AudioReceiveStreamInterface::Config& config) { 170 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 171 172 // SSRC can't be changed mid-stream. 173 RTC_DCHECK_EQ(remote_ssrc(), config.rtp.remote_ssrc); 174 RTC_DCHECK_EQ(local_ssrc(), config.rtp.local_ssrc); 175 176 // Configuration parameters which cannot be changed. 177 RTC_DCHECK_EQ(config_.rtcp_send_transport, config.rtcp_send_transport); 178 // Decoder factory cannot be changed because it is configured at 179 // voe::Channel construction time. 180 RTC_DCHECK_EQ(config_.decoder_factory, config.decoder_factory); 181 182 // TODO(solenberg): Config NACK history window (which is a packet count), 183 // using the actual packet size for the configured codec. 184 RTC_DCHECK_EQ(config_.rtp.nack.rtp_history_ms, config.rtp.nack.rtp_history_ms) 185 << "Use SetUseTransportCcAndNackHistory"; 186 187 RTC_DCHECK(config_.decoder_map == config.decoder_map) << "Use SetDecoderMap"; 188 RTC_DCHECK_EQ(config_.frame_transformer, config.frame_transformer) 189 << "Use SetDepacketizerToDecoderFrameTransformer"; 190 191 config_ = config; 192 } 193 194 void AudioReceiveStreamImpl::Start() { 195 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 196 if (playing_) { 197 return; 198 } 199 RTC_LOG(LS_INFO) << "AudioReceiveStreamImpl::Start: " << remote_ssrc(); 200 channel_receive_->StartPlayout(); 201 playing_ = true; 202 audio_state()->AddReceivingStream(this); 203 } 204 205 void AudioReceiveStreamImpl::Stop() { 206 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 207 if (!playing_) { 208 return; 209 } 210 RTC_LOG(LS_INFO) << "AudioReceiveStreamImpl::Stop: " << remote_ssrc(); 211 channel_receive_->StopPlayout(); 212 playing_ = false; 213 audio_state()->RemoveReceivingStream(this); 214 } 215 216 bool AudioReceiveStreamImpl::IsRunning() const { 217 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 218 return playing_; 219 } 220 221 void AudioReceiveStreamImpl::SetDepacketizerToDecoderFrameTransformer( 222 scoped_refptr<FrameTransformerInterface> frame_transformer) { 223 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 224 channel_receive_->SetDepacketizerToDecoderFrameTransformer( 225 std::move(frame_transformer)); 226 } 227 228 void AudioReceiveStreamImpl::SetDecoderMap( 229 std::map<int, SdpAudioFormat> decoder_map) { 230 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 231 config_.decoder_map = std::move(decoder_map); 232 channel_receive_->SetReceiveCodecs(config_.decoder_map); 233 } 234 235 void AudioReceiveStreamImpl::SetNackHistory(int history_ms) { 236 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 237 RTC_DCHECK_GE(history_ms, 0); 238 239 if (config_.rtp.nack.rtp_history_ms == history_ms) 240 return; 241 242 config_.rtp.nack.rtp_history_ms = history_ms; 243 // TODO(solenberg): Config NACK history window (which is a packet count), 244 // using the actual packet size for the configured codec. 245 channel_receive_->SetNACKStatus(history_ms != 0, history_ms / 20); 246 } 247 248 void AudioReceiveStreamImpl::SetRtcpMode(RtcpMode mode) { 249 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 250 251 if (config_.rtp.rtcp_mode == mode) 252 return; 253 254 config_.rtp.rtcp_mode = mode; 255 channel_receive_->SetRtcpMode(mode); 256 } 257 258 void AudioReceiveStreamImpl::SetNonSenderRttMeasurement(bool enabled) { 259 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 260 config_.enable_non_sender_rtt = enabled; 261 channel_receive_->SetNonSenderRttMeasurement(enabled); 262 } 263 264 void AudioReceiveStreamImpl::SetFrameDecryptor( 265 scoped_refptr<FrameDecryptorInterface> frame_decryptor) { 266 // TODO(bugs.webrtc.org/11993): This is called via WebRtcAudioReceiveStream, 267 // expect to be called on the network thread. 268 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 269 channel_receive_->SetFrameDecryptor(std::move(frame_decryptor)); 270 } 271 272 AudioReceiveStreamInterface::Stats AudioReceiveStreamImpl::GetStats( 273 bool get_and_clear_legacy_stats) const { 274 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 275 AudioReceiveStreamInterface::Stats stats; 276 stats.remote_ssrc = remote_ssrc(); 277 278 auto receive_codec = channel_receive_->GetReceiveCodec(); 279 if (receive_codec) { 280 stats.codec_name = receive_codec->second.name; 281 stats.codec_payload_type = receive_codec->first; 282 } 283 284 ChannelReceiveStatistics channel_stats = 285 channel_receive_->GetRTCPStatistics(); 286 stats.payload_bytes_received = channel_stats.payload_bytes_received; 287 stats.header_and_padding_bytes_received = 288 channel_stats.header_and_padding_bytes_received; 289 stats.packets_received = channel_stats.packets_received; 290 stats.packets_received_with_ect1 = channel_stats.packets_received_with_ect1; 291 stats.packets_received_with_ce = channel_stats.packets_received_with_ce; 292 stats.packets_lost = channel_stats.packets_lost; 293 stats.jitter_ms = channel_stats.jitter_ms; 294 stats.nacks_sent = channel_stats.nacks_sent; 295 stats.capture_start_ntp_time_ms = channel_stats.capture_start_ntp_time_ms; 296 stats.last_packet_received = channel_stats.last_packet_received; 297 stats.last_sender_report_timestamp = 298 channel_stats.last_sender_report_timestamp; 299 stats.last_sender_report_utc_timestamp = 300 channel_stats.last_sender_report_utc_timestamp; 301 stats.last_sender_report_remote_utc_timestamp = 302 channel_stats.last_sender_report_remote_utc_timestamp; 303 stats.sender_reports_packets_sent = channel_stats.sender_reports_packets_sent; 304 stats.sender_reports_bytes_sent = channel_stats.sender_reports_bytes_sent; 305 stats.sender_reports_reports_count = 306 channel_stats.sender_reports_reports_count; 307 stats.round_trip_time = channel_stats.round_trip_time; 308 stats.round_trip_time_measurements = 309 channel_stats.round_trip_time_measurements; 310 stats.total_round_trip_time = channel_stats.total_round_trip_time; 311 312 stats.delay_estimate_ms = channel_receive_->GetDelayEstimate(); 313 stats.audio_level = channel_receive_->GetSpeechOutputLevelFullRange(); 314 stats.total_output_energy = channel_receive_->GetTotalOutputEnergy(); 315 stats.total_output_duration = channel_receive_->GetTotalOutputDuration(); 316 stats.estimated_playout_ntp_timestamp_ms = 317 channel_receive_->GetCurrentEstimatedPlayoutNtpTimestampMs( 318 env_.clock().TimeInMilliseconds()); 319 320 // Get jitter buffer and total delay (alg + jitter + playout) stats. 321 auto ns = channel_receive_->GetNetworkStatistics(get_and_clear_legacy_stats); 322 stats.packets_discarded = ns.packetsDiscarded; 323 stats.fec_packets_received = ns.fecPacketsReceived; 324 stats.fec_packets_discarded = ns.fecPacketsDiscarded; 325 stats.jitter_buffer_ms = ns.currentBufferSize; 326 stats.jitter_buffer_preferred_ms = ns.preferredBufferSize; 327 stats.total_samples_received = ns.totalSamplesReceived; 328 stats.concealed_samples = ns.concealedSamples; 329 stats.silent_concealed_samples = ns.silentConcealedSamples; 330 stats.concealment_events = ns.concealmentEvents; 331 stats.jitter_buffer_delay_seconds = 332 static_cast<double>(ns.jitterBufferDelayMs) / 333 static_cast<double>(kNumMillisecsPerSec); 334 stats.jitter_buffer_emitted_count = ns.jitterBufferEmittedCount; 335 stats.jitter_buffer_target_delay_seconds = 336 static_cast<double>(ns.jitterBufferTargetDelayMs) / 337 static_cast<double>(kNumMillisecsPerSec); 338 stats.jitter_buffer_minimum_delay_seconds = 339 static_cast<double>(ns.jitterBufferMinimumDelayMs) / 340 static_cast<double>(kNumMillisecsPerSec); 341 stats.inserted_samples_for_deceleration = ns.insertedSamplesForDeceleration; 342 stats.removed_samples_for_acceleration = ns.removedSamplesForAcceleration; 343 stats.total_processing_delay_seconds = 344 static_cast<double>(ns.totalProcessingDelayUs) / 345 static_cast<double>(kNumMicrosecsPerSec); 346 stats.expand_rate = Q14ToFloat(ns.currentExpandRate); 347 stats.speech_expand_rate = Q14ToFloat(ns.currentSpeechExpandRate); 348 stats.secondary_decoded_rate = Q14ToFloat(ns.currentSecondaryDecodedRate); 349 stats.secondary_discarded_rate = Q14ToFloat(ns.currentSecondaryDiscardedRate); 350 stats.accelerate_rate = Q14ToFloat(ns.currentAccelerateRate); 351 stats.preemptive_expand_rate = Q14ToFloat(ns.currentPreemptiveRate); 352 stats.jitter_buffer_flushes = ns.packetBufferFlushes; 353 stats.delayed_packet_outage_samples = ns.delayedPacketOutageSamples; 354 stats.relative_packet_arrival_delay_seconds = 355 static_cast<double>(ns.relativePacketArrivalDelayMs) / 356 static_cast<double>(kNumMillisecsPerSec); 357 stats.interruption_count = ns.interruptionCount; 358 stats.total_interruption_duration_ms = ns.totalInterruptionDurationMs; 359 360 auto ds = channel_receive_->GetDecodingCallStatistics(); 361 stats.decoding_calls_to_silence_generator = ds.calls_to_silence_generator; 362 stats.decoding_calls_to_neteq = ds.calls_to_neteq; 363 stats.decoding_normal = ds.decoded_normal; 364 stats.decoding_plc = ds.decoded_neteq_plc; 365 stats.decoding_codec_plc = ds.decoded_codec_plc; 366 stats.decoding_cng = ds.decoded_cng; 367 stats.decoding_plc_cng = ds.decoded_plc_cng; 368 stats.decoding_muted_output = ds.decoded_muted_output; 369 370 return stats; 371 } 372 373 void AudioReceiveStreamImpl::SetSink(AudioSinkInterface* sink) { 374 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 375 channel_receive_->SetSink(sink); 376 } 377 378 void AudioReceiveStreamImpl::SetGain(float gain) { 379 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 380 channel_receive_->SetChannelOutputVolumeScaling(gain); 381 } 382 383 bool AudioReceiveStreamImpl::SetBaseMinimumPlayoutDelayMs(int delay_ms) { 384 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 385 return channel_receive_->SetBaseMinimumPlayoutDelayMs(delay_ms); 386 } 387 388 int AudioReceiveStreamImpl::GetBaseMinimumPlayoutDelayMs() const { 389 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 390 return channel_receive_->GetBaseMinimumPlayoutDelayMs(); 391 } 392 393 std::vector<RtpSource> AudioReceiveStreamImpl::GetSources() const { 394 return channel_receive_->GetSources(); 395 } 396 397 AudioMixer::Source::AudioFrameInfo 398 AudioReceiveStreamImpl::GetAudioFrameWithInfo(int sample_rate_hz, 399 AudioFrame* audio_frame) { 400 return channel_receive_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame); 401 } 402 403 int AudioReceiveStreamImpl::Ssrc() const { 404 return remote_ssrc(); 405 } 406 407 int AudioReceiveStreamImpl::PreferredSampleRate() const { 408 return channel_receive_->PreferredSampleRate(); 409 } 410 411 uint32_t AudioReceiveStreamImpl::id() const { 412 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 413 return remote_ssrc(); 414 } 415 416 std::optional<Syncable::Info> AudioReceiveStreamImpl::GetInfo() const { 417 // TODO(bugs.webrtc.org/11993): This is called via RtpStreamsSynchronizer, 418 // expect to be called on the network thread. 419 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 420 return channel_receive_->GetSyncInfo(); 421 } 422 423 std::optional<Syncable::PlayoutInfo> 424 AudioReceiveStreamImpl::GetPlayoutRtpTimestamp() const { 425 // Called on video capture thread. 426 return channel_receive_->GetPlayoutRtpTimestamp(); 427 } 428 429 void AudioReceiveStreamImpl::SetEstimatedPlayoutNtpTimestamp(NtpTime ntp_time, 430 Timestamp time) { 431 // Called on video capture thread. 432 channel_receive_->SetEstimatedPlayoutNtpTimestamp(ntp_time, time); 433 } 434 435 bool AudioReceiveStreamImpl::SetMinimumPlayoutDelay(TimeDelta delay) { 436 // TODO(bugs.webrtc.org/11993): This is called via RtpStreamsSynchronizer, 437 // expect to be called on the network thread. 438 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 439 return channel_receive_->SetMinimumPlayoutDelay(delay); 440 } 441 442 void AudioReceiveStreamImpl::DeliverRtcp(ArrayView<const uint8_t> packet) { 443 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 444 channel_receive_->ReceivedRTCPPacket(packet.data(), packet.size()); 445 } 446 447 void AudioReceiveStreamImpl::SetSyncGroup(absl::string_view sync_group) { 448 RTC_DCHECK_RUN_ON(&packet_sequence_checker_); 449 config_.sync_group = std::string(sync_group); 450 } 451 452 void AudioReceiveStreamImpl::SetLocalSsrc(uint32_t local_ssrc) { 453 RTC_DCHECK_RUN_ON(&packet_sequence_checker_); 454 // TODO(tommi): Consider storing local_ssrc in one place. 455 config_.rtp.local_ssrc = local_ssrc; 456 channel_receive_->OnLocalSsrcChange(local_ssrc); 457 } 458 459 uint32_t AudioReceiveStreamImpl::local_ssrc() const { 460 RTC_DCHECK_RUN_ON(&packet_sequence_checker_); 461 return config_.rtp.local_ssrc; 462 } 463 464 const std::string& AudioReceiveStreamImpl::sync_group() const { 465 RTC_DCHECK_RUN_ON(&packet_sequence_checker_); 466 return config_.sync_group; 467 } 468 469 internal::AudioState* AudioReceiveStreamImpl::audio_state() const { 470 auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get()); 471 RTC_DCHECK(audio_state); 472 return audio_state; 473 } 474 } // namespace webrtc