audio_send_stream.cc (35445B)
1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "audio/audio_send_stream.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <memory> 16 #include <optional> 17 #include <string> 18 #include <utility> 19 #include <vector> 20 21 #include "absl/strings/str_cat.h" 22 #include "absl/strings/string_view.h" 23 #include "api/array_view.h" 24 #include "api/audio/audio_frame.h" 25 #include "api/audio/audio_processing.h" 26 #include "api/audio_codecs/audio_encoder.h" 27 #include "api/audio_codecs/audio_encoder_factory.h" 28 #include "api/audio_codecs/audio_format.h" 29 #include "api/call/bitrate_allocation.h" 30 #include "api/environment/environment.h" 31 #include "api/field_trials_view.h" 32 #include "api/function_view.h" 33 #include "api/rtc_error.h" 34 #include "api/rtc_event_log/rtc_event_log.h" 35 #include "api/rtp_parameters.h" 36 #include "api/rtp_sender_interface.h" 37 #include "api/scoped_refptr.h" 38 #include "api/sequence_checker.h" 39 #include "api/units/data_rate.h" 40 #include "api/units/data_size.h" 41 #include "api/units/time_delta.h" 42 #include "audio/audio_state.h" 43 #include "audio/channel_send.h" 44 #include "call/audio_state.h" 45 #include "call/bitrate_allocator.h" 46 #include "call/rtp_transport_controller_send_interface.h" 47 #include "common_audio/vad/include/vad.h" 48 #include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h" 49 #include "logging/rtc_event_log/rtc_stream_config.h" 50 #include "media/base/media_channel.h" 51 #include "media/base/media_constants.h" 52 #include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" 53 #include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" 54 #include "modules/rtp_rtcp/include/report_block_data.h" 55 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 56 #include "modules/rtp_rtcp/source/rtp_header_extensions.h" 57 #include "rtc_base/checks.h" 58 #include "rtc_base/experiments/struct_parameters_parser.h" 59 #include "rtc_base/logging.h" 60 #include "rtc_base/race_checker.h" 61 #include "rtc_base/synchronization/mutex.h" 62 #include "rtc_base/trace_event.h" 63 64 namespace webrtc { 65 namespace { 66 67 void UpdateEventLogStreamConfig(RtcEventLog& event_log, 68 const AudioSendStream::Config& config, 69 const AudioSendStream::Config* old_config) { 70 using SendCodecSpec = AudioSendStream::Config::SendCodecSpec; 71 // Only update if any of the things we log have changed. 72 auto payload_types_equal = [](const std::optional<SendCodecSpec>& a, 73 const std::optional<SendCodecSpec>& b) { 74 if (a.has_value() && b.has_value()) { 75 return a->format.name == b->format.name && 76 a->payload_type == b->payload_type; 77 } 78 return !a.has_value() && !b.has_value(); 79 }; 80 81 if (old_config && config.rtp.ssrc == old_config->rtp.ssrc && 82 config.rtp.extensions == old_config->rtp.extensions && 83 payload_types_equal(config.send_codec_spec, 84 old_config->send_codec_spec)) { 85 return; 86 } 87 88 auto rtclog_config = std::make_unique<rtclog::StreamConfig>(); 89 rtclog_config->local_ssrc = config.rtp.ssrc; 90 rtclog_config->rtp_extensions = config.rtp.extensions; 91 if (config.send_codec_spec) { 92 rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name, 93 config.send_codec_spec->payload_type, 0); 94 } 95 event_log.Log(std::make_unique<RtcEventAudioSendStreamConfig>( 96 std::move(rtclog_config))); 97 } 98 99 } // namespace 100 101 std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() { 102 return StructParametersParser::Create( // 103 "min", &min_bitrate, // 104 "max", &max_bitrate, // 105 "prio_rate", &priority_bitrate, // 106 "prio_rate_raw", &priority_bitrate_raw, // 107 "rate_prio", &bitrate_priority); 108 } 109 110 AudioAllocationConfig::AudioAllocationConfig( 111 const FieldTrialsView& field_trials) { 112 Parser()->Parse(field_trials.Lookup(kKey)); 113 if (priority_bitrate_raw && !priority_bitrate.IsZero()) { 114 RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually " 115 "exclusive but both were configured."; 116 } 117 } 118 119 namespace internal { 120 AudioSendStream::AudioSendStream( 121 const Environment& env, 122 const webrtc::AudioSendStream::Config& config, 123 const scoped_refptr<webrtc::AudioState>& audio_state, 124 RtpTransportControllerSendInterface* rtp_transport, 125 BitrateAllocatorInterface* bitrate_allocator, 126 RtcpRttStats* rtcp_rtt_stats, 127 const std::optional<RtpState>& suspended_rtp_state) 128 : AudioSendStream(env, 129 config, 130 audio_state, 131 rtp_transport, 132 bitrate_allocator, 133 suspended_rtp_state, 134 voe::CreateChannelSend(env, 135 config.send_transport, 136 rtcp_rtt_stats, 137 config.frame_encryptor.get(), 138 config.crypto_options, 139 config.rtp.extmap_allow_mixed, 140 config.rtcp_report_interval_ms, 141 config.rtp.ssrc, 142 config.frame_transformer, 143 rtp_transport)) {} 144 145 AudioSendStream::AudioSendStream( 146 const Environment& env, 147 const webrtc::AudioSendStream::Config& config, 148 const scoped_refptr<webrtc::AudioState>& audio_state, 149 RtpTransportControllerSendInterface* rtp_transport, 150 BitrateAllocatorInterface* bitrate_allocator, 151 const std::optional<RtpState>& suspended_rtp_state, 152 std::unique_ptr<voe::ChannelSendInterface> channel_send) 153 : env_(env), 154 allocate_audio_without_feedback_( 155 env_.field_trials().IsEnabled("WebRTC-Audio-ABWENoTWCC")), 156 enable_audio_alr_probing_( 157 !env_.field_trials().IsDisabled("WebRTC-Audio-AlrProbing")), 158 allocation_settings_(env_.field_trials()), 159 config_(Config(/*send_transport=*/nullptr)), 160 audio_state_(audio_state), 161 channel_send_(std::move(channel_send)), 162 use_legacy_overhead_calculation_( 163 env_.field_trials().IsEnabled("WebRTC-Audio-LegacyOverhead")), 164 enable_priority_bitrate_( 165 !env_.field_trials().IsDisabled("WebRTC-Audio-PriorityBitrate")), 166 bitrate_allocator_(bitrate_allocator), 167 rtp_transport_(rtp_transport), 168 rtp_rtcp_module_(channel_send_->GetRtpRtcp()), 169 suspended_rtp_state_(suspended_rtp_state) { 170 RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc; 171 RTC_DCHECK(audio_state_); 172 RTC_DCHECK(channel_send_); 173 RTC_DCHECK(bitrate_allocator_); 174 RTC_DCHECK(rtp_transport); 175 176 RTC_DCHECK(rtp_rtcp_module_); 177 178 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 179 ConfigureStream(config, true, nullptr); 180 } 181 182 AudioSendStream::~AudioSendStream() { 183 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 184 RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc; 185 RTC_DCHECK(!sending_); 186 channel_send_->ResetSenderCongestionControlObjects(); 187 } 188 189 const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const { 190 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 191 return config_; 192 } 193 194 void AudioSendStream::Reconfigure( 195 const webrtc::AudioSendStream::Config& new_config, 196 SetParametersCallback callback) { 197 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 198 ConfigureStream(new_config, false, std::move(callback)); 199 } 200 201 AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds( 202 const std::vector<RtpExtension>& extensions) { 203 ExtensionIds ids; 204 for (const auto& extension : extensions) { 205 if (extension.uri == RtpExtension::kAudioLevelUri) { 206 ids.audio_level = extension.id; 207 } else if (extension.uri == RtpExtension::kAbsSendTimeUri) { 208 ids.abs_send_time = extension.id; 209 } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) { 210 ids.transport_sequence_number = extension.id; 211 } else if (extension.uri == RtpExtension::kMidUri) { 212 ids.mid = extension.id; 213 } else if (extension.uri == RtpExtension::kRidUri) { 214 ids.rid = extension.id; 215 } else if (extension.uri == RtpExtension::kRepairedRidUri) { 216 ids.repaired_rid = extension.id; 217 } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) { 218 ids.abs_capture_time = extension.id; 219 } 220 } 221 return ids; 222 } 223 224 int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) { 225 return FindExtensionIds(config.rtp.extensions).transport_sequence_number; 226 } 227 228 void AudioSendStream::ConfigureStream( 229 const webrtc::AudioSendStream::Config& new_config, 230 bool first_time, 231 SetParametersCallback callback) { 232 RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: " 233 << new_config.ToString(); 234 UpdateEventLogStreamConfig(env_.event_log(), new_config, 235 first_time ? nullptr : &config_); 236 237 const auto& old_config = config_; 238 239 // Configuration parameters which cannot be changed. 240 RTC_DCHECK(first_time || 241 old_config.send_transport == new_config.send_transport); 242 RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc); 243 if (suspended_rtp_state_ && first_time) { 244 rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_); 245 } 246 if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) { 247 channel_send_->SetRTCP_CNAME(new_config.rtp.c_name); 248 } 249 250 // Enable the frame encryptor if a new frame encryptor has been provided. 251 if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) { 252 channel_send_->SetFrameEncryptor(new_config.frame_encryptor); 253 } 254 255 if (first_time || 256 new_config.frame_transformer != old_config.frame_transformer) { 257 channel_send_->SetEncoderToPacketizerFrameTransformer( 258 new_config.frame_transformer); 259 } 260 261 if (first_time || 262 new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) { 263 rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); 264 } 265 266 if (first_time || new_config.rtp.csrcs != old_config.rtp.csrcs) { 267 channel_send_->SetCsrcs(new_config.rtp.csrcs); 268 } 269 270 const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions); 271 const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions); 272 273 // Audio level indication 274 if (first_time || new_ids.audio_level != old_ids.audio_level) { 275 channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0, 276 new_ids.audio_level); 277 } 278 279 if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) { 280 absl::string_view uri = AbsoluteSendTime::Uri(); 281 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri); 282 if (new_ids.abs_send_time) { 283 rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, new_ids.abs_send_time); 284 } 285 } 286 287 bool transport_seq_num_id_changed = 288 new_ids.transport_sequence_number != old_ids.transport_sequence_number; 289 if (first_time || 290 (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) { 291 if (!first_time) { 292 channel_send_->ResetSenderCongestionControlObjects(); 293 } 294 295 if (!allocate_audio_without_feedback_ && 296 new_ids.transport_sequence_number != 0) { 297 rtp_rtcp_module_->RegisterRtpHeaderExtension( 298 TransportSequenceNumber::Uri(), new_ids.transport_sequence_number); 299 // Probing in application limited region is only used in combination with 300 // send side congestion control, wich depends on feedback packets which 301 // requires transport sequence numbers to be enabled. 302 // Optionally request ALR probing but do not override any existing 303 // request from other streams. 304 if (enable_audio_alr_probing_) { 305 rtp_transport_->EnablePeriodicAlrProbing(true); 306 } 307 } 308 channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_); 309 } 310 // MID RTP header extension. 311 if ((first_time || new_ids.mid != old_ids.mid || 312 new_config.rtp.mid != old_config.rtp.mid) && 313 new_ids.mid != 0 && !new_config.rtp.mid.empty()) { 314 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::Uri(), new_ids.mid); 315 rtp_rtcp_module_->SetMid(new_config.rtp.mid); 316 } 317 318 if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) { 319 absl::string_view uri = AbsoluteCaptureTimeExtension::Uri(); 320 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri); 321 if (new_ids.abs_capture_time) { 322 rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, 323 new_ids.abs_capture_time); 324 } 325 } 326 327 if (!ReconfigureSendCodec(new_config)) { 328 RTC_LOG(LS_ERROR) << "Failed to set up send codec state."; 329 330 webrtc::InvokeSetParametersCallback( 331 callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR, 332 "Failed to set up send codec state.")); 333 } 334 335 // Set currently known overhead (used in ANA, opus only). 336 UpdateOverheadPerPacket(); 337 338 channel_send_->CallEncoder([this](AudioEncoder* encoder) { 339 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 340 if (!encoder) { 341 return; 342 } 343 frame_length_range_ = encoder->GetFrameLengthRange(); 344 bitrate_range_ = encoder->GetBitrateRange(); 345 }); 346 347 if (sending_) { 348 ReconfigureBitrateObserver(new_config); 349 } 350 351 config_ = new_config; 352 353 webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK()); 354 } 355 356 void AudioSendStream::Start() { 357 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 358 if (sending_) { 359 return; 360 } 361 RTC_LOG(LS_INFO) << "AudioSendStream::Start: " << config_.rtp.ssrc; 362 if (!config_.has_dscp && config_.min_bitrate_bps != -1 && 363 config_.max_bitrate_bps != -1 && 364 (allocate_audio_without_feedback_ || 365 config_.include_in_congestion_control_allocation)) { 366 rtp_transport_->AccountForAudioPacketsInPacedSender(true); 367 rtp_transport_->IncludeOverheadInPacedSender(); 368 rtp_rtcp_module_->SetAsPartOfAllocation(true); 369 ConfigureBitrateObserver(); 370 } else { 371 rtp_rtcp_module_->SetAsPartOfAllocation(false); 372 } 373 channel_send_->StartSend(); 374 sending_ = true; 375 audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, 376 encoder_num_channels_); 377 } 378 379 void AudioSendStream::Stop() { 380 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 381 if (!sending_) { 382 return; 383 } 384 RTC_LOG(LS_INFO) << "AudioSendStream::Stop: " << config_.rtp.ssrc; 385 RemoveBitrateObserver(); 386 channel_send_->StopSend(); 387 sending_ = false; 388 audio_state()->RemoveSendingStream(this); 389 } 390 391 void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) { 392 RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); 393 RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0); 394 TRACE_EVENT0("webrtc", "AudioSendStream::SendAudioData"); 395 double duration = static_cast<double>(audio_frame->samples_per_channel_) / 396 audio_frame->sample_rate_hz_; 397 { 398 // Note: SendAudioData() passes the frame further down the pipeline and it 399 // may eventually get sent. But this method is invoked even if we are not 400 // connected, as long as we have an AudioSendStream (created as a result of 401 // an O/A exchange). This means that we are calculating audio levels whether 402 // or not we are sending samples. 403 // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats 404 // should move from send-streams to the local audio sources or tracks; a 405 // send-stream should not be required to read the microphone audio levels. 406 MutexLock lock(&audio_level_lock_); 407 audio_level_.ComputeLevel(*audio_frame, duration); 408 } 409 channel_send_->ProcessAndEncodeAudio(std::move(audio_frame)); 410 } 411 412 bool AudioSendStream::SendTelephoneEvent(int payload_type, 413 int payload_frequency, 414 int event, 415 int duration_ms) { 416 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 417 channel_send_->SetSendTelephoneEventPayloadType(payload_type, 418 payload_frequency); 419 return channel_send_->SendTelephoneEventOutband(event, duration_ms); 420 } 421 422 void AudioSendStream::SetMuted(bool muted) { 423 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 424 channel_send_->SetInputMute(muted); 425 } 426 427 webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { 428 return GetStats(true); 429 } 430 431 webrtc::AudioSendStream::Stats AudioSendStream::GetStats( 432 bool has_remote_tracks) const { 433 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 434 webrtc::AudioSendStream::Stats stats; 435 stats.local_ssrc = config_.rtp.ssrc; 436 stats.target_bitrate_bps = channel_send_->GetTargetBitrate(); 437 438 webrtc::ChannelSendStatistics channel_stats = 439 channel_send_->GetRTCPStatistics(); 440 stats.rtcp_packet_type_counts = channel_stats.rtcp_packet_type_counts; 441 stats.payload_bytes_sent = channel_stats.payload_bytes_sent; 442 stats.header_and_padding_bytes_sent = 443 channel_stats.header_and_padding_bytes_sent; 444 stats.retransmitted_bytes_sent = channel_stats.retransmitted_bytes_sent; 445 stats.packets_sent = channel_stats.packets_sent; 446 stats.packets_sent_with_ect1 = channel_stats.packets_sent_with_ect1; 447 stats.total_packet_send_delay = channel_stats.total_packet_send_delay; 448 stats.retransmitted_packets_sent = channel_stats.retransmitted_packets_sent; 449 // RTT isn't known until a RTCP report is received. Until then, VoiceEngine 450 // returns 0 to indicate an error value. 451 if (channel_stats.round_trip_time.ms() > 0) { 452 stats.rtt_ms = channel_stats.round_trip_time.ms(); 453 } 454 if (config_.send_codec_spec) { 455 const auto& spec = *config_.send_codec_spec; 456 stats.codec_name = spec.format.name; 457 stats.codec_payload_type = spec.payload_type; 458 459 // Get data from the last remote RTCP report. 460 for (const ReportBlockData& block : 461 channel_send_->GetRemoteRTCPReportBlocks()) { 462 // Lookup report for send ssrc only. 463 if (block.source_ssrc() == stats.local_ssrc) { 464 stats.packets_lost = block.cumulative_lost(); 465 stats.fraction_lost = block.fraction_lost(); 466 if (spec.format.clockrate_hz > 0) { 467 stats.jitter_ms = block.jitter(spec.format.clockrate_hz).ms(); 468 } 469 break; 470 } 471 } 472 } 473 474 { 475 MutexLock lock(&audio_level_lock_); 476 stats.audio_level = audio_level_.LevelFullRange(); 477 stats.total_input_energy = audio_level_.TotalEnergy(); 478 stats.total_input_duration = audio_level_.TotalDuration(); 479 } 480 481 stats.ana_statistics = channel_send_->GetANAStatistics(); 482 483 AudioProcessing* ap = audio_state_->audio_processing(); 484 if (ap) { 485 stats.apm_statistics = ap->GetStatistics(has_remote_tracks); 486 } 487 488 stats.report_block_datas = std::move(channel_stats.report_block_datas); 489 490 stats.nacks_received = channel_stats.nacks_received; 491 492 return stats; 493 } 494 495 void AudioSendStream::DeliverRtcp(ArrayView<const uint8_t> packet) { 496 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 497 channel_send_->ReceivedRTCPPacket(packet.data(), packet.size()); 498 // Poll if overhead has changed, which it can do if ack triggers us to stop 499 // sending mid/rid. 500 UpdateOverheadPerPacket(); 501 } 502 503 uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { 504 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 505 // Pick a target bitrate between the constraints. Overrules the allocator if 506 // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a 507 // higher than max to allow for e.g. extra FEC. 508 std::optional<TargetAudioBitrateConstraints> constraints = 509 GetMinMaxBitrateConstraints(); 510 if (constraints) { 511 update.target_bitrate.Clamp(constraints->min, constraints->max); 512 } 513 channel_send_->OnBitrateAllocation(update); 514 // The amount of audio protection is not exposed by the encoder, hence 515 // always returning 0. 516 return 0; 517 } 518 519 std::optional<DataRate> AudioSendStream::GetUsedRate() const { 520 return channel_send_->GetUsedRate(); 521 } 522 523 void AudioSendStream::SetTransportOverhead( 524 int transport_overhead_per_packet_bytes) { 525 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 526 transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes; 527 UpdateOverheadPerPacket(); 528 } 529 530 void AudioSendStream::UpdateOverheadPerPacket() { 531 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 532 size_t overhead_per_packet_bytes = 533 transport_overhead_per_packet_bytes_ + 534 rtp_rtcp_module_->ExpectedPerPacketOverhead(); 535 if (overhead_per_packet_ == overhead_per_packet_bytes) { 536 return; 537 } 538 overhead_per_packet_ = overhead_per_packet_bytes; 539 channel_send_->CallEncoder([&](AudioEncoder* encoder) { 540 encoder->OnReceivedOverhead(overhead_per_packet_bytes); 541 }); 542 if (registered_with_allocator_) { 543 ConfigureBitrateObserver(); 544 } 545 channel_send_->RegisterPacketOverhead(overhead_per_packet_bytes); 546 } 547 548 size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const { 549 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 550 return overhead_per_packet_; 551 } 552 553 RtpState AudioSendStream::GetRtpState() const { 554 return rtp_rtcp_module_->GetRtpState(); 555 } 556 557 const voe::ChannelSendInterface* AudioSendStream::GetChannel() const { 558 return channel_send_.get(); 559 } 560 561 internal::AudioState* AudioSendStream::audio_state() { 562 internal::AudioState* audio_state = 563 static_cast<internal::AudioState*>(audio_state_.get()); 564 RTC_DCHECK(audio_state); 565 return audio_state; 566 } 567 568 const internal::AudioState* AudioSendStream::audio_state() const { 569 internal::AudioState* audio_state = 570 static_cast<internal::AudioState*>(audio_state_.get()); 571 RTC_DCHECK(audio_state); 572 return audio_state; 573 } 574 575 void AudioSendStream::StoreEncoderProperties(int sample_rate_hz, 576 size_t num_channels) { 577 encoder_sample_rate_hz_ = sample_rate_hz; 578 encoder_num_channels_ = num_channels; 579 if (sending_) { 580 // Update AudioState's information about the stream. 581 audio_state()->AddSendingStream(this, sample_rate_hz, num_channels); 582 } 583 } 584 585 // Apply current codec settings to a single voe::Channel used for sending. 586 bool AudioSendStream::SetupSendCodec(const Config& new_config) { 587 RTC_DCHECK(new_config.send_codec_spec); 588 const auto& spec = *new_config.send_codec_spec; 589 590 RTC_DCHECK(new_config.encoder_factory); 591 std::unique_ptr<AudioEncoder> encoder = new_config.encoder_factory->Create( 592 env_, spec.format, 593 {.payload_type = spec.payload_type, 594 .codec_pair_id = new_config.codec_pair_id}); 595 596 if (!encoder) { 597 RTC_DLOG(LS_ERROR) << "Unable to create encoder for " 598 << absl::StrCat(spec.format); 599 return false; 600 } 601 602 // If a bitrate has been specified for the codec, use it over the 603 // codec's default. 604 if (spec.target_bitrate_bps) { 605 encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps); 606 } 607 608 // Enable ANA if configured (currently only used by Opus). 609 if (new_config.audio_network_adaptor_config) { 610 if (encoder->EnableAudioNetworkAdaptor( 611 *new_config.audio_network_adaptor_config)) { 612 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " 613 << new_config.rtp.ssrc; 614 } else { 615 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " 616 << new_config.rtp.ssrc; 617 } 618 } 619 620 // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled. 621 if (spec.cng_payload_type) { 622 AudioEncoderCngConfig cng_config; 623 cng_config.num_channels = encoder->NumChannels(); 624 cng_config.payload_type = *spec.cng_payload_type; 625 cng_config.speech_encoder = std::move(encoder); 626 cng_config.vad_mode = Vad::kVadNormal; 627 encoder = CreateComfortNoiseEncoder(std::move(cng_config)); 628 629 RegisterCngPayloadType(*spec.cng_payload_type, 630 new_config.send_codec_spec->format.clockrate_hz); 631 } 632 633 // Wrap the encoder in a RED encoder, if RED is enabled. 634 SdpAudioFormat format = spec.format; 635 if (spec.red_payload_type) { 636 AudioEncoderCopyRed::Config red_config; 637 red_config.payload_type = *spec.red_payload_type; 638 red_config.speech_encoder = std::move(encoder); 639 encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config), 640 env_.field_trials()); 641 format.name = kRedCodecName; 642 } 643 644 // Set currently known overhead (used in ANA, opus only). 645 // If overhead changes later, it will be updated in UpdateOverheadPerPacket. 646 if (overhead_per_packet_ > 0) { 647 encoder->OnReceivedOverhead(overhead_per_packet_); 648 } 649 650 StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels()); 651 channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, format, 652 std::move(encoder)); 653 654 return true; 655 } 656 657 bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) { 658 const auto& old_config = config_; 659 660 if (!new_config.send_codec_spec) { 661 // We cannot de-configure a send codec. So we will do nothing. 662 // By design, the send codec should have not been configured. 663 RTC_DCHECK(!old_config.send_codec_spec); 664 return true; 665 } 666 667 if (new_config.send_codec_spec == old_config.send_codec_spec && 668 new_config.audio_network_adaptor_config == 669 old_config.audio_network_adaptor_config) { 670 return true; 671 } 672 673 // If we have no encoder, or the format or payload type's changed, create a 674 // new encoder. 675 if (!old_config.send_codec_spec || 676 new_config.send_codec_spec->format != 677 old_config.send_codec_spec->format || 678 new_config.send_codec_spec->payload_type != 679 old_config.send_codec_spec->payload_type || 680 new_config.send_codec_spec->red_payload_type != 681 old_config.send_codec_spec->red_payload_type) { 682 return SetupSendCodec(new_config); 683 } 684 685 const std::optional<int>& new_target_bitrate_bps = 686 new_config.send_codec_spec->target_bitrate_bps; 687 // If a bitrate has been specified for the codec, use it over the 688 // codec's default. 689 if (new_target_bitrate_bps && 690 new_target_bitrate_bps != 691 old_config.send_codec_spec->target_bitrate_bps) { 692 channel_send_->CallEncoder([&](AudioEncoder* encoder) { 693 encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps); 694 }); 695 } 696 697 ReconfigureANA(new_config); 698 ReconfigureCNG(new_config); 699 700 return true; 701 } 702 703 void AudioSendStream::ReconfigureANA(const Config& new_config) { 704 if (new_config.audio_network_adaptor_config == 705 config_.audio_network_adaptor_config) { 706 return; 707 } 708 if (new_config.audio_network_adaptor_config) { 709 channel_send_->CallEncoder([&](AudioEncoder* encoder) { 710 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 711 if (encoder->EnableAudioNetworkAdaptor( 712 *new_config.audio_network_adaptor_config)) { 713 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " 714 << new_config.rtp.ssrc; 715 if (overhead_per_packet_ > 0) { 716 encoder->OnReceivedOverhead(overhead_per_packet_); 717 } 718 } else { 719 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " 720 << new_config.rtp.ssrc; 721 } 722 }); 723 } else { 724 channel_send_->CallEncoder( 725 [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); }); 726 RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC " 727 << new_config.rtp.ssrc; 728 } 729 } 730 731 void AudioSendStream::ReconfigureCNG(const Config& new_config) { 732 if (new_config.send_codec_spec->cng_payload_type == 733 config_.send_codec_spec->cng_payload_type) { 734 return; 735 } 736 737 // Register the CNG payload type if it's been added, don't do anything if CNG 738 // is removed. Payload types must not be redefined. 739 if (new_config.send_codec_spec->cng_payload_type) { 740 RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type, 741 new_config.send_codec_spec->format.clockrate_hz); 742 } 743 744 // Wrap or unwrap the encoder in an AudioEncoderCNG. 745 channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) { 746 std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr)); 747 auto sub_encoders = old_encoder->ReclaimContainedEncoders(); 748 if (!sub_encoders.empty()) { 749 // Replace enc with its sub encoder. We need to put the sub 750 // encoder in a temporary first, since otherwise the old value 751 // of enc would be destroyed before the new value got assigned, 752 // which would be bad since the new value is a part of the old 753 // value. 754 auto tmp = std::move(sub_encoders[0]); 755 old_encoder = std::move(tmp); 756 } 757 if (new_config.send_codec_spec->cng_payload_type) { 758 AudioEncoderCngConfig config; 759 config.speech_encoder = std::move(old_encoder); 760 config.num_channels = config.speech_encoder->NumChannels(); 761 config.payload_type = *new_config.send_codec_spec->cng_payload_type; 762 config.vad_mode = Vad::kVadNormal; 763 *encoder_ptr = CreateComfortNoiseEncoder(std::move(config)); 764 } else { 765 *encoder_ptr = std::move(old_encoder); 766 } 767 }); 768 } 769 770 void AudioSendStream::ReconfigureBitrateObserver( 771 const webrtc::AudioSendStream::Config& new_config) { 772 // Since the Config's default is for both of these to be -1, this test will 773 // allow us to configure the bitrate observer if the new config has bitrate 774 // limits set, but would only have us call RemoveBitrateObserver if we were 775 // previously configured with bitrate limits. 776 if (config_.min_bitrate_bps == new_config.min_bitrate_bps && 777 config_.max_bitrate_bps == new_config.max_bitrate_bps && 778 config_.bitrate_priority == new_config.bitrate_priority && 779 TransportSeqNumId(config_) == TransportSeqNumId(new_config) && 780 config_.audio_network_adaptor_config == 781 new_config.audio_network_adaptor_config) { 782 return; 783 } 784 785 if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 && 786 new_config.max_bitrate_bps != -1 && 787 new_config.include_in_congestion_control_allocation) { 788 rtp_transport_->AccountForAudioPacketsInPacedSender(true); 789 rtp_transport_->IncludeOverheadInPacedSender(); 790 // We may get a callback immediately as the observer is registered, so 791 // make sure the bitrate limits in config_ are up-to-date. 792 config_.min_bitrate_bps = new_config.min_bitrate_bps; 793 config_.max_bitrate_bps = new_config.max_bitrate_bps; 794 795 config_.bitrate_priority = new_config.bitrate_priority; 796 ConfigureBitrateObserver(); 797 rtp_rtcp_module_->SetAsPartOfAllocation(true); 798 } else { 799 rtp_transport_->AccountForAudioPacketsInPacedSender(false); 800 RemoveBitrateObserver(); 801 rtp_rtcp_module_->SetAsPartOfAllocation(false); 802 } 803 } 804 805 void AudioSendStream::ConfigureBitrateObserver() { 806 RTC_DCHECK_RUN_ON(&worker_thread_checker_); 807 // This either updates the current observer or adds a new observer. 808 // TODO(srte): Add overhead compensation here. 809 auto constraints = GetMinMaxBitrateConstraints(); 810 RTC_DCHECK(constraints.has_value()); 811 812 DataRate priority_bitrate = allocation_settings_.priority_bitrate; 813 if (use_legacy_overhead_calculation_) { 814 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) 815 constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12; 816 const TimeDelta kMinPacketDuration = TimeDelta::Millis(20); 817 DataRate max_overhead = 818 DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration; 819 priority_bitrate += max_overhead; 820 } else { 821 RTC_DCHECK(frame_length_range_); 822 const DataSize overhead_per_packet = DataSize::Bytes(overhead_per_packet_); 823 DataRate min_overhead = overhead_per_packet / frame_length_range_->second; 824 priority_bitrate += min_overhead; 825 } 826 827 if (allocation_settings_.priority_bitrate_raw) { 828 priority_bitrate = *allocation_settings_.priority_bitrate_raw; 829 } 830 831 if (!enable_priority_bitrate_) { 832 priority_bitrate = DataRate::BitsPerSec(0); 833 } 834 835 bitrate_allocator_->AddObserver( 836 this, 837 MediaStreamAllocationConfig{ 838 .min_bitrate_bps = constraints->min.bps<uint32_t>(), 839 .max_bitrate_bps = constraints->max.bps<uint32_t>(), 840 .pad_up_bitrate_bps = 0, 841 .priority_bitrate_bps = priority_bitrate.bps(), 842 .enforce_min_bitrate = true, 843 .bitrate_priority = allocation_settings_.bitrate_priority.value_or( 844 config_.bitrate_priority), 845 .rate_elasticity = TrackRateElasticity::kCanContributeUnusedRate}); 846 847 registered_with_allocator_ = true; 848 } 849 850 void AudioSendStream::RemoveBitrateObserver() { 851 registered_with_allocator_ = false; 852 bitrate_allocator_->RemoveObserver(this); 853 } 854 855 std::optional<AudioSendStream::TargetAudioBitrateConstraints> 856 AudioSendStream::GetMinMaxBitrateConstraints() const { 857 if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) { 858 RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps=" 859 << config_.min_bitrate_bps 860 << "; max_bitrate_bps=" << config_.max_bitrate_bps 861 << "; both expected greater or equal to 0"; 862 return std::nullopt; 863 } 864 TargetAudioBitrateConstraints constraints{ 865 .min = DataRate::BitsPerSec(config_.min_bitrate_bps), 866 .max = DataRate::BitsPerSec(config_.max_bitrate_bps)}; 867 868 // If bitrates were explicitly overriden via field trial, use those values. 869 if (allocation_settings_.min_bitrate) 870 constraints.min = *allocation_settings_.min_bitrate; 871 if (allocation_settings_.max_bitrate) 872 constraints.max = *allocation_settings_.max_bitrate; 873 874 // Use encoder defined bitrate range if available. 875 if (bitrate_range_) { 876 constraints.min = bitrate_range_->first; 877 constraints.max = bitrate_range_->second; 878 } 879 880 RTC_DCHECK_GE(constraints.min, DataRate::Zero()); 881 RTC_DCHECK_GE(constraints.max, DataRate::Zero()); 882 if (constraints.max < constraints.min) { 883 RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than " 884 << "TargetAudioBitrateConstraints::min"; 885 return std::nullopt; 886 } 887 if (use_legacy_overhead_calculation_) { 888 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) 889 const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12); 890 const TimeDelta kMaxFrameLength = 891 TimeDelta::Millis(60); // Based on Opus spec 892 const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength; 893 constraints.min += kMinOverhead; 894 constraints.max += kMinOverhead; 895 } else { 896 if (!frame_length_range_.has_value()) { 897 RTC_LOG(LS_WARNING) << "frame_length_range_ is not set"; 898 return std::nullopt; 899 } 900 const DataSize overhead_per_packet = DataSize::Bytes(overhead_per_packet_); 901 constraints.min += overhead_per_packet / frame_length_range_->second; 902 constraints.max += overhead_per_packet / frame_length_range_->first; 903 } 904 return constraints; 905 } 906 907 void AudioSendStream::RegisterCngPayloadType(int payload_type, 908 int clockrate_hz) { 909 channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz); 910 } 911 912 } // namespace internal 913 } // namespace webrtc