webrtc_video_engine.cc (155926B)
1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "media/engine/webrtc_video_engine.h" 12 13 #include <algorithm> 14 #include <cstdint> 15 #include <cstdio> 16 #include <functional> 17 #include <initializer_list> 18 #include <map> 19 #include <memory> 20 #include <optional> 21 #include <set> 22 #include <string> 23 #include <unordered_set> 24 #include <utility> 25 #include <vector> 26 27 #include "absl/algorithm/container.h" 28 #include "absl/functional/bind_front.h" 29 #include "absl/strings/match.h" 30 #include "absl/strings/string_view.h" 31 #include "api/array_view.h" 32 #include "api/crypto/crypto_options.h" 33 #include "api/crypto/frame_decryptor_interface.h" 34 #include "api/environment/environment.h" 35 #include "api/field_trials_view.h" 36 #include "api/frame_transformer_interface.h" 37 #include "api/make_ref_counted.h" 38 #include "api/media_stream_interface.h" 39 #include "api/media_types.h" 40 #include "api/priority.h" 41 #include "api/rtc_error.h" 42 #include "api/rtp_headers.h" 43 #include "api/rtp_parameters.h" 44 #include "api/rtp_sender_interface.h" 45 #include "api/rtp_transceiver_direction.h" 46 #include "api/scoped_refptr.h" 47 #include "api/sequence_checker.h" 48 #include "api/task_queue/pending_task_safety_flag.h" 49 #include "api/task_queue/task_queue_base.h" 50 #include "api/transport/rtp/rtp_source.h" 51 #include "api/units/time_delta.h" 52 #include "api/units/timestamp.h" 53 #include "api/video/recordable_encoded_frame.h" 54 #include "api/video/video_bitrate_allocator_factory.h" 55 #include "api/video/video_codec_type.h" 56 #include "api/video/video_sink_interface.h" 57 #include "api/video/video_source_interface.h" 58 #include "api/video_codecs/scalability_mode.h" 59 #include "api/video_codecs/sdp_video_format.h" 60 #include "api/video_codecs/video_codec.h" 61 #include "api/video_codecs/video_decoder_factory.h" 62 #include "api/video_codecs/video_encoder.h" 63 #include "api/video_codecs/video_encoder_factory.h" 64 #include "call/call.h" 65 #include "call/flexfec_receive_stream.h" 66 #include "call/packet_receiver.h" 67 #include "call/payload_type.h" 68 #include "call/payload_type_picker.h" 69 #include "call/receive_stream.h" 70 #include "call/rtp_config.h" 71 #include "call/rtp_transport_controller_send_interface.h" 72 #include "call/video_receive_stream.h" 73 #include "call/video_send_stream.h" 74 #include "common_video/frame_counts.h" 75 #include "media/base/codec.h" 76 #include "media/base/codec_comparators.h" 77 #include "media/base/media_channel.h" 78 #include "media/base/media_channel_impl.h" 79 #include "media/base/media_config.h" 80 #include "media/base/media_constants.h" 81 #include "media/base/media_engine.h" 82 #include "media/base/rid_description.h" 83 #include "media/base/rtp_utils.h" 84 #include "media/base/stream_params.h" 85 #include "media/engine/webrtc_media_engine.h" 86 #include "modules/rtp_rtcp/include/receive_statistics.h" 87 #include "modules/rtp_rtcp/include/rtcp_statistics.h" 88 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" 89 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 90 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 91 #include "modules/video_coding/svc/scalability_mode_util.h" 92 #include "rtc_base/checks.h" 93 #include "rtc_base/dscp.h" 94 #include "rtc_base/experiments/field_trial_parser.h" 95 #include "rtc_base/logging.h" 96 #include "rtc_base/network_route.h" 97 #include "rtc_base/socket.h" 98 #include "rtc_base/strings/string_builder.h" 99 #include "rtc_base/synchronization/mutex.h" 100 #include "rtc_base/time_utils.h" 101 #include "rtc_base/trace_event.h" 102 #include "video/config/video_encoder_config.h" 103 104 namespace webrtc { 105 106 namespace { 107 108 constexpr int64_t kUnsignaledSsrcCooldownMs = kNumMillisecsPerSec / 2; 109 110 // This constant is really an on/off, lower-level configurable NACK history 111 // duration hasn't been implemented. 112 const int kNackHistoryMs = 1000; 113 114 const int kDefaultRtcpReceiverReportSsrc = 1; 115 116 // Minimum time interval for logging stats. 117 const int64_t kStatsLogIntervalMs = 10000; 118 119 const char* StreamTypeToString(VideoSendStream::StreamStats::StreamType type) { 120 switch (type) { 121 case VideoSendStream::StreamStats::StreamType::kMedia: 122 return "kMedia"; 123 case VideoSendStream::StreamStats::StreamType::kRtx: 124 return "kRtx"; 125 case VideoSendStream::StreamStats::StreamType::kFlexfec: 126 return "kFlexfec"; 127 } 128 return nullptr; 129 } 130 131 void AddDefaultFeedbackParams(Codec* codec, const FieldTrialsView& trials) { 132 // Don't add any feedback params for RED and ULPFEC. 133 if (codec->name == kRedCodecName || codec->name == kUlpfecCodecName) 134 return; 135 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty)); 136 codec->AddFeedbackParam( 137 FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty)); 138 // Don't add any more feedback params for FLEXFEC. 139 if (codec->name == kFlexfecCodecName) 140 return; 141 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir)); 142 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty)); 143 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli)); 144 if (codec->name == kVp8CodecName && 145 trials.IsEnabled("WebRTC-RtcpLossNotification")) { 146 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamLntf, kParamValueEmpty)); 147 } 148 } 149 150 // Get the default set of supported codecs. 151 // is_decoder_factory is needed to keep track of the implict assumption that any 152 // H264 decoder also supports constrained base line profile. 153 // Also, is_decoder_factory is used to decide whether FlexFEC video format 154 // should be advertised as supported. 155 template <class T> 156 std::vector<SdpVideoFormat> GetDefaultSupportedFormats( 157 const T* factory, 158 bool is_decoder_factory, 159 const FieldTrialsView& trials) { 160 if (!factory) { 161 return {}; 162 } 163 164 std::vector<SdpVideoFormat> supported_formats = 165 factory->GetSupportedFormats(); 166 if (is_decoder_factory) { 167 AddH264ConstrainedBaselineProfileToSupportedFormats(&supported_formats); 168 } 169 170 if (supported_formats.empty()) 171 return supported_formats; 172 173 supported_formats.push_back(SdpVideoFormat(kRedCodecName)); 174 supported_formats.push_back(SdpVideoFormat(kUlpfecCodecName)); 175 // flexfec-03 is always supported as receive codec and as send codec 176 // only if WebRTC-FlexFEC-03-Advertised is enabled 177 if (is_decoder_factory || trials.IsEnabled("WebRTC-FlexFEC-03-Advertised")) { 178 SdpVideoFormat flexfec_format(kFlexfecCodecName); 179 // This value is currently arbitrarily set to 10 seconds. (The unit 180 // is microseconds.) This parameter MUST be present in the SDP, but 181 // we never use the actual value anywhere in our code however. 182 // TODO(brandtr): Consider honouring this value in the sender and receiver. 183 flexfec_format.parameters = {{kFlexfecFmtpRepairWindow, "10000000"}}; 184 supported_formats.push_back(flexfec_format); 185 } 186 return supported_formats; 187 } 188 189 // This function will assign dynamic payload types (in the range [96, 127] 190 // and then [35, 63]) to the input codecs, and also add ULPFEC, RED, FlexFEC, 191 // It will also add default feedback params to the codecs. 192 RTCErrorOr<Codec> AssignPayloadType(const SdpVideoFormat& format, 193 PayloadTypePicker& pt_mapper, 194 const FieldTrialsView& trials) { 195 Codec codec = CreateVideoCodec(format); 196 RTCErrorOr<PayloadType> result = 197 pt_mapper.SuggestMapping(codec, /* excluder= */ nullptr); 198 if (!result.ok()) { 199 return result.MoveError(); 200 } 201 codec.id = result.value(); 202 AddDefaultFeedbackParams(&codec, trials); 203 return codec; 204 } 205 206 // This function will add a associated RTX codec for a recognized primary codecs 207 // (VP8, VP9, AV1, H264, and RED). 208 RTCErrorOr<Codec> AddRtx(const Codec& primary_codec, 209 PayloadTypePicker& pt_mapper) { 210 Codec rtx_codec = CreateVideoRtxCodec(Codec::kIdNotSet, primary_codec.id); 211 RTCErrorOr<PayloadType> result = 212 pt_mapper.SuggestMapping(rtx_codec, /* excluder= */ nullptr); 213 if (!result.ok()) { 214 return result.MoveError(); 215 } 216 rtx_codec.id = result.value(); 217 return rtx_codec; 218 } 219 220 // TODO(kron): Perhaps it is better to move the implicit knowledge to the place 221 // where codecs are negotiated. 222 template <class T> 223 std::vector<Codec> GetPayloadTypesAndDefaultCodecs( 224 const T* factory, 225 bool is_decoder_factory, 226 bool include_rtx, 227 const FieldTrialsView& trials) { 228 auto supported_formats = 229 GetDefaultSupportedFormats(factory, is_decoder_factory, trials); 230 231 PayloadTypePicker pt_mapper; 232 std::unordered_set<int> used_payload_types; 233 std::vector<Codec> output_codecs; 234 for (const auto& supported_format : supported_formats) { 235 RTCErrorOr<Codec> result = 236 AssignPayloadType(supported_format, pt_mapper, trials); 237 if (!result.ok()) { 238 // TODO: https://issues.webrtc.org/360058654 - stop assigning PTs here. 239 // TODO: https://issues.webrtc.org/360058654 - Handle running out of IDs. 240 continue; 241 } 242 bool inserted = used_payload_types.insert(result.value().id).second; 243 if (!inserted) { 244 RTC_LOG(LS_WARNING) << "Factory produced duplicate codecs, ignoring " 245 << result.value() << " produced from " 246 << supported_format; 247 continue; 248 } 249 output_codecs.push_back(result.value()); 250 if (include_rtx) { 251 Codec::ResiliencyType resiliency_type = 252 result.value().GetResiliencyType(); 253 // FEC codecs do not use retransmission. 254 if (resiliency_type == Codec::ResiliencyType::kFlexfec || 255 resiliency_type == Codec::ResiliencyType::kUlpfec) { 256 continue; 257 } 258 259 RTCErrorOr<Codec> rtx_result = AddRtx(result.value(), pt_mapper); 260 if (!rtx_result.ok()) { 261 // TODO: https://issues.webrtc.org/360058654 - stop assigning PTs here. 262 // TODO: https://issues.webrtc.org/360058654 - Handle running out of 263 // IDs. 264 continue; 265 } 266 output_codecs.push_back(rtx_result.MoveValue()); 267 } 268 } 269 return output_codecs; 270 } 271 272 std::string CodecVectorToString(const std::vector<Codec>& codecs) { 273 StringBuilder out; 274 out << "{"; 275 for (size_t i = 0; i < codecs.size(); ++i) { 276 out << codecs[i].ToString(); 277 if (i != codecs.size() - 1) { 278 out << ", "; 279 } 280 } 281 out << "}"; 282 return out.Release(); 283 } 284 285 bool ValidateCodecFormats(const std::vector<Codec>& codecs) { 286 bool has_video = false; 287 for (size_t i = 0; i < codecs.size(); ++i) { 288 if (!codecs[i].ValidateCodecFormat()) { 289 return false; 290 } 291 if (codecs[i].IsMediaCodec()) { 292 has_video = true; 293 } 294 } 295 if (!has_video) { 296 RTC_LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: " 297 << CodecVectorToString(codecs); 298 return false; 299 } 300 return true; 301 } 302 303 bool ValidateStreamParams(const StreamParams& sp) { 304 if (sp.ssrcs.empty()) { 305 RTC_LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString(); 306 return false; 307 } 308 309 // Validate that a primary SSRC can only have one ssrc-group per semantics. 310 std::map<uint32_t, std::set<std::string>> primary_ssrc_to_semantics; 311 for (const auto& group : sp.ssrc_groups) { 312 auto result = primary_ssrc_to_semantics.try_emplace( 313 group.ssrcs[0], std::set<std::string>({group.semantics})); 314 if (!result.second) { 315 // A duplicate SSRC was found, check for duplicate semantics. 316 auto semantics_it = result.first->second.insert(group.semantics); 317 if (!semantics_it.second) { 318 RTC_LOG(LS_ERROR) << "Duplicate ssrc-group '" << group.semantics 319 << " for primary SSRC " << group.ssrcs[0] << " " 320 << sp.ToString(); 321 return false; 322 } 323 } 324 } 325 326 std::vector<uint32_t> primary_ssrcs; 327 sp.GetPrimarySsrcs(&primary_ssrcs); 328 for (const auto& semantic : 329 {kFidSsrcGroupSemantics, kFecFrSsrcGroupSemantics}) { 330 if (!sp.has_ssrc_group(semantic)) { 331 continue; 332 } 333 std::vector<uint32_t> secondary_ssrcs; 334 sp.GetSecondarySsrcs(semantic, primary_ssrcs, &secondary_ssrcs); 335 for (uint32_t secondary_ssrc : secondary_ssrcs) { 336 bool secondary_ssrc_present = false; 337 for (uint32_t sp_ssrc : sp.ssrcs) { 338 if (sp_ssrc == secondary_ssrc) { 339 secondary_ssrc_present = true; 340 break; 341 } 342 } 343 if (!secondary_ssrc_present) { 344 RTC_LOG(LS_ERROR) << "SSRC '" << secondary_ssrc 345 << "' missing from StreamParams ssrcs with semantics " 346 << semantic << ": " << sp.ToString(); 347 return false; 348 } 349 } 350 if (!secondary_ssrcs.empty() && 351 primary_ssrcs.size() != secondary_ssrcs.size()) { 352 RTC_LOG(LS_ERROR) 353 << semantic 354 << " secondary SSRCs exist, but don't cover all SSRCs (unsupported): " 355 << sp.ToString(); 356 return false; 357 } 358 } 359 for (const auto& group : sp.ssrc_groups) { 360 if (!(group.semantics == kFidSsrcGroupSemantics || 361 group.semantics == kSimSsrcGroupSemantics || 362 group.semantics == kFecFrSsrcGroupSemantics)) { 363 continue; 364 } 365 for (uint32_t group_ssrc : group.ssrcs) { 366 auto it = absl::c_find_if(sp.ssrcs, [&group_ssrc](uint32_t ssrc) { 367 return ssrc == group_ssrc; 368 }); 369 if (it == sp.ssrcs.end()) { 370 RTC_LOG(LS_ERROR) << "SSRC '" << group_ssrc 371 << "' missing from StreamParams ssrcs with semantics " 372 << group.semantics << ": " << sp.ToString(); 373 return false; 374 } 375 } 376 } 377 return true; 378 } 379 380 // Returns true if the given codec is disallowed from doing simulcast. 381 bool IsCodecDisabledForSimulcast(bool legacy_scalability_mode, 382 VideoCodecType codec_type) { 383 if (legacy_scalability_mode && 384 (codec_type == kVideoCodecVP9 || codec_type == kVideoCodecAV1)) { 385 return true; 386 } 387 388 return false; 389 } 390 391 bool IsLayerActive(const RtpEncodingParameters& layer) { 392 return layer.active && 393 (!layer.max_bitrate_bps || *layer.max_bitrate_bps > 0) && 394 (!layer.max_framerate || *layer.max_framerate > 0); 395 } 396 397 int NumActiveStreams(const RtpParameters& rtp_parameters) { 398 int res = 0; 399 for (size_t i = 0; i < rtp_parameters.encodings.size(); ++i) { 400 if (rtp_parameters.encodings[i].active) { 401 ++res; 402 } 403 } 404 return res; 405 } 406 407 std::optional<int> NumSpatialLayersFromEncoding( 408 const RtpParameters& rtp_parameters, 409 size_t idx) { 410 if (idx >= rtp_parameters.encodings.size()) 411 return std::nullopt; 412 413 std::optional<ScalabilityMode> scalability_mode = ScalabilityModeFromString( 414 rtp_parameters.encodings[idx].scalability_mode.value_or("")); 415 return scalability_mode 416 ? std::optional<int>( 417 ScalabilityModeToNumSpatialLayers(*scalability_mode)) 418 : std::nullopt; 419 } 420 421 std::map<uint32_t, VideoSendStream::StreamStats> 422 MergeInfoAboutOutboundRtpSubstreams( 423 const std::map<uint32_t, VideoSendStream::StreamStats>& substreams) { 424 std::map<uint32_t, VideoSendStream::StreamStats> rtp_substreams; 425 // Add substreams for all RTP media streams. 426 for (const auto& pair : substreams) { 427 uint32_t ssrc = pair.first; 428 const VideoSendStream::StreamStats& substream = pair.second; 429 switch (substream.type) { 430 case VideoSendStream::StreamStats::StreamType::kMedia: 431 break; 432 case VideoSendStream::StreamStats::StreamType::kRtx: 433 case VideoSendStream::StreamStats::StreamType::kFlexfec: 434 continue; 435 } 436 rtp_substreams.insert(std::make_pair(ssrc, substream)); 437 } 438 // Complement the kMedia substream stats with the associated kRtx and kFlexfec 439 // substream stats. 440 for (const auto& pair : substreams) { 441 switch (pair.second.type) { 442 case VideoSendStream::StreamStats::StreamType::kMedia: 443 continue; 444 case VideoSendStream::StreamStats::StreamType::kRtx: 445 case VideoSendStream::StreamStats::StreamType::kFlexfec: 446 break; 447 } 448 // The associated substream is an RTX or FlexFEC substream that is 449 // referencing an RTP media substream. 450 const VideoSendStream::StreamStats& associated_substream = pair.second; 451 RTC_DCHECK(associated_substream.referenced_media_ssrc.has_value()); 452 uint32_t media_ssrc = associated_substream.referenced_media_ssrc.value(); 453 if (substreams.find(media_ssrc) == substreams.end()) { 454 RTC_LOG(LS_WARNING) << "Substream [ssrc: " << pair.first << ", type: " 455 << StreamTypeToString(associated_substream.type) 456 << "] is associated with a media ssrc (" << media_ssrc 457 << ") that does not have StreamStats. Ignoring its " 458 << "RTP stats."; 459 continue; 460 } 461 VideoSendStream::StreamStats& rtp_substream = rtp_substreams[media_ssrc]; 462 463 // We only merge `rtp_stats`. All other metrics are not applicable for RTX 464 // and FlexFEC. 465 // TODO(hbos): kRtx and kFlexfec stats should use a separate struct to make 466 // it clear what is or is not applicable. 467 rtp_substream.rtp_stats.Add(associated_substream.rtp_stats); 468 } 469 return rtp_substreams; 470 } 471 472 bool IsActiveFromEncodings( 473 std::optional<uint32_t> ssrc, 474 const std::vector<RtpEncodingParameters>& encodings) { 475 if (ssrc.has_value()) { 476 // Report the `active` value of a specific ssrc, or false if an encoding 477 // with this ssrc does not exist. 478 auto encoding_it = std::find_if( 479 encodings.begin(), encodings.end(), 480 [ssrc = ssrc.value()](const RtpEncodingParameters& encoding) { 481 return encoding.ssrc.has_value() && encoding.ssrc.value() == ssrc; 482 }); 483 return encoding_it != encodings.end() ? encoding_it->active : false; 484 } 485 // If `ssrc` is not specified then any encoding being active counts as active. 486 for (const auto& encoding : encodings) { 487 if (encoding.active) { 488 return true; 489 } 490 } 491 return false; 492 } 493 494 bool IsScalabilityModeSupportedByCodec(const Codec& codec, 495 const std::string& scalability_mode, 496 const VideoSendStream::Config& config) { 497 return config.encoder_settings.encoder_factory 498 ->QueryCodecSupport(SdpVideoFormat(codec.name, codec.params), 499 scalability_mode) 500 .is_supported; 501 } 502 503 // Fallback to default value if the scalability mode is unset or unsupported by 504 // the codec. 505 void FallbackToDefaultScalabilityModeIfNotSupported( 506 const Codec& codec, 507 const VideoSendStream::Config& config, 508 std::vector<RtpEncodingParameters>& encodings) { 509 if (!absl::c_any_of(encodings, [](const RtpEncodingParameters& encoding) { 510 return encoding.scalability_mode && !encoding.scalability_mode->empty(); 511 })) { 512 // Fallback is only enabled if the scalability mode is configured for any of 513 // the encodings for now. 514 return; 515 } 516 if (config.encoder_settings.encoder_factory == nullptr) { 517 return; 518 } 519 for (auto& encoding : encodings) { 520 RTC_LOG(LS_INFO) << "Encoding scalability_mode: " 521 << encoding.scalability_mode.value_or("-"); 522 if (!encoding.active && !encoding.scalability_mode.has_value()) { 523 // Inactive encodings should not fallback since apps may only specify the 524 // scalability mode of the first encoding when the others are inactive. 525 continue; 526 } 527 528 if (!encoding.scalability_mode.has_value() || 529 !IsScalabilityModeSupportedByCodec(codec, *encoding.scalability_mode, 530 config)) { 531 encoding.scalability_mode = 532 (encoding.scalability_mode != 533 std::string(kDefaultScalabilityModeStr) && 534 IsScalabilityModeSupportedByCodec(codec, kDefaultScalabilityModeStr, 535 config)) 536 ? kDefaultScalabilityModeStr 537 : kNoLayeringScalabilityModeStr; 538 RTC_LOG(LS_INFO) << " -> " << *encoding.scalability_mode; 539 } 540 } 541 } 542 543 // Generate the list of codec parameters to pass down based on the negotiated 544 // "codecs". Note that VideoCodecSettings correspond to concrete codecs like 545 // VP8, VP9, H264 while VideoCodecs correspond also to "virtual" codecs like 546 // RTX, ULPFEC, FLEXFEC. 547 RTCErrorOr<std::vector<VideoCodecSettings>> MapCodecs( 548 const std::vector<Codec>& codecs) { 549 std::vector<VideoCodecSettings> video_codecs; 550 if (codecs.empty()) { 551 return video_codecs; 552 } 553 554 std::map<int, Codec::ResiliencyType> payload_codec_type; 555 // `rtx_mapping` maps video payload type to rtx payload type. 556 std::map<int, int> rtx_mapping; 557 std::map<int, int> rtx_time_mapping; 558 std::map<int, Codec> defined_codecs; 559 560 UlpfecConfig ulpfec_config; 561 std::optional<int> flexfec_payload_type; 562 563 for (const Codec& in_codec : codecs) { 564 const int payload_type = in_codec.id; 565 566 if (payload_codec_type.find(payload_type) != payload_codec_type.end()) { 567 if (MatchesWithCodecRules(defined_codecs.at(in_codec.id), in_codec)) { 568 // Ignore second occurence of the same codec. 569 // This can happen with multiple H.264 profiles. 570 continue; 571 } 572 RTC_LOG(LS_ERROR) << "Duplicate codec ID, rejecting " << in_codec 573 << " because " << defined_codecs.at(in_codec.id) 574 << " is earlier in the list."; 575 return RTCError(RTCErrorType::INVALID_PARAMETER, 576 "Duplicate codec ID with non-matching codecs"); 577 } 578 defined_codecs.insert({in_codec.id, in_codec}); 579 payload_codec_type[payload_type] = in_codec.GetResiliencyType(); 580 581 switch (in_codec.GetResiliencyType()) { 582 case Codec::ResiliencyType::kRed: { 583 if (ulpfec_config.red_payload_type != -1) { 584 RTC_LOG(LS_ERROR) 585 << "Duplicate RED codec: ignoring PT=" << payload_type 586 << " in favor of PT=" << ulpfec_config.red_payload_type 587 << " which was specified first."; 588 break; 589 } 590 ulpfec_config.red_payload_type = payload_type; 591 break; 592 } 593 594 case Codec::ResiliencyType::kUlpfec: { 595 if (ulpfec_config.ulpfec_payload_type != -1) { 596 RTC_LOG(LS_ERROR) 597 << "Duplicate ULPFEC codec: ignoring PT=" << payload_type 598 << " in favor of PT=" << ulpfec_config.ulpfec_payload_type 599 << " which was specified first."; 600 break; 601 } 602 ulpfec_config.ulpfec_payload_type = payload_type; 603 break; 604 } 605 606 case Codec::ResiliencyType::kFlexfec: { 607 if (flexfec_payload_type) { 608 RTC_LOG(LS_ERROR) 609 << "Duplicate FLEXFEC codec: ignoring PT=" << payload_type 610 << " in favor of PT=" << *flexfec_payload_type 611 << " which was specified first."; 612 break; 613 } 614 flexfec_payload_type = payload_type; 615 break; 616 } 617 618 case Codec::ResiliencyType::kRtx: { 619 int associated_payload_type; 620 if (!in_codec.GetParam(kCodecParamAssociatedPayloadType, 621 &associated_payload_type) || 622 !IsValidRtpPayloadType(associated_payload_type)) { 623 RTC_LOG(LS_ERROR) 624 << "RTX codec with invalid or no associated payload type: " 625 << in_codec.ToString(); 626 return RTCError(RTCErrorType::INVALID_PARAMETER, 627 "RTX codec with invalid APT"); 628 } 629 int rtx_time; 630 if (in_codec.GetParam(kCodecParamRtxTime, &rtx_time) && rtx_time > 0) { 631 rtx_time_mapping[associated_payload_type] = rtx_time; 632 } 633 rtx_mapping[associated_payload_type] = payload_type; 634 break; 635 } 636 637 case Codec::ResiliencyType::kNone: { 638 video_codecs.emplace_back(in_codec); 639 break; 640 } 641 } 642 } 643 644 // One of these codecs should have been a video codec. Only having FEC 645 // parameters into this code is a logic error. 646 RTC_DCHECK(!video_codecs.empty()); 647 648 for (const auto& entry : rtx_mapping) { 649 const int associated_payload_type = entry.first; 650 const int rtx_payload_type = entry.second; 651 auto it = payload_codec_type.find(associated_payload_type); 652 if (it == payload_codec_type.end()) { 653 RTC_LOG(LS_ERROR) << "RTX codec (PT=" << rtx_payload_type 654 << ") mapped to PT=" << associated_payload_type 655 << " which is not in the codec list."; 656 return RTCError(RTCErrorType::INVALID_PARAMETER, 657 "RTX codec with unlisted APT"); 658 } 659 const Codec::ResiliencyType associated_codec_type = it->second; 660 if (associated_codec_type != Codec::ResiliencyType::kNone && 661 associated_codec_type != Codec::ResiliencyType::kRed) { 662 RTC_LOG(LS_ERROR) 663 << "RTX PT=" << rtx_payload_type 664 << " not mapped to regular video codec or RED codec (PT=" 665 << associated_payload_type << ")."; 666 return RTCError(RTCErrorType::INVALID_PARAMETER, 667 "RTX codec with APT not video"); 668 } 669 670 if (associated_payload_type == ulpfec_config.red_payload_type) { 671 ulpfec_config.red_rtx_payload_type = rtx_payload_type; 672 } 673 } 674 675 for (VideoCodecSettings& codec_settings : video_codecs) { 676 const int payload_type = codec_settings.codec.id; 677 codec_settings.ulpfec = ulpfec_config; 678 codec_settings.flexfec_payload_type = flexfec_payload_type.value_or(-1); 679 auto it = rtx_mapping.find(payload_type); 680 if (it != rtx_mapping.end()) { 681 const int rtx_payload_type = it->second; 682 codec_settings.rtx_payload_type = rtx_payload_type; 683 684 auto rtx_time_it = rtx_time_mapping.find(payload_type); 685 if (rtx_time_it != rtx_time_mapping.end()) { 686 const int rtx_time = rtx_time_it->second; 687 if (rtx_time < kNackHistoryMs) { 688 codec_settings.rtx_time = rtx_time; 689 } else { 690 codec_settings.rtx_time = kNackHistoryMs; 691 } 692 } 693 } 694 } 695 696 return video_codecs; 697 } 698 699 bool NonFlexfecReceiveCodecsHaveChanged(std::vector<VideoCodecSettings> before, 700 std::vector<VideoCodecSettings> after) { 701 // The receive codec order doesn't matter, so we sort the codecs before 702 // comparing. This is necessary because currently the 703 // only way to change the send codec is to munge SDP, which causes 704 // the receive codec list to change order, which causes the streams 705 // to be recreates which causes a "blink" of black video. In order 706 // to support munging the SDP in this way without recreating receive 707 // streams, we ignore the order of the received codecs so that 708 // changing the order doesn't cause this "blink". 709 auto comparison = [](const VideoCodecSettings& codec1, 710 const VideoCodecSettings& codec2) { 711 return codec1.codec.id > codec2.codec.id; 712 }; 713 absl::c_sort(before, comparison); 714 absl::c_sort(after, comparison); 715 716 // Changes in FlexFEC payload type are handled separately in 717 // WebRtcVideoReceiveChannel::GetChangedReceiverParameters, so disregard 718 // FlexFEC in the comparison here. 719 return !absl::c_equal(before, after, 720 VideoCodecSettings::EqualsDisregardingFlexfec); 721 } 722 723 std::string CodecSettingsVectorToString( 724 const std::vector<VideoCodecSettings>& codecs) { 725 StringBuilder out; 726 out << "{"; 727 for (size_t i = 0; i < codecs.size(); ++i) { 728 out << codecs[i].codec.ToString(); 729 if (i != codecs.size() - 1) { 730 out << ", "; 731 } 732 } 733 out << "}"; 734 return out.Release(); 735 } 736 737 void ExtractCodecInformation( 738 ArrayView<const VideoCodecSettings> recv_codecs, 739 std::map<int, int>& rtx_associated_payload_types, 740 std::set<int>& raw_payload_types, 741 std::vector<VideoReceiveStreamInterface::Decoder>& decoders) { 742 RTC_DCHECK(!recv_codecs.empty()); 743 RTC_DCHECK(rtx_associated_payload_types.empty()); 744 RTC_DCHECK(raw_payload_types.empty()); 745 RTC_DCHECK(decoders.empty()); 746 747 for (const VideoCodecSettings& recv_codec : recv_codecs) { 748 decoders.emplace_back( 749 SdpVideoFormat(recv_codec.codec.name, recv_codec.codec.params), 750 recv_codec.codec.id); 751 rtx_associated_payload_types.emplace(recv_codec.rtx_payload_type, 752 recv_codec.codec.id); 753 if (recv_codec.codec.packetization == kPacketizationParamRaw) { 754 raw_payload_types.insert(recv_codec.codec.id); 755 } 756 } 757 } 758 759 int ParseReceiveBufferSize(const FieldTrialsView& trials) { 760 FieldTrialParameter<int> size_bytes("size_bytes", kVideoRtpRecvBufferSize); 761 ParseFieldTrial({&size_bytes}, trials.Lookup("WebRTC-ReceiveBufferSize")); 762 if (size_bytes.Get() < 10'000 || size_bytes.Get() > 10'000'000) { 763 RTC_LOG(LS_WARNING) << "WebRTC-ReceiveBufferSize out of bounds: " 764 << size_bytes.Get(); 765 return kVideoRtpRecvBufferSize; 766 } 767 return size_bytes.Get(); 768 } 769 770 RTCError ResolveSendCodecs( 771 const VideoCodecSettings& current_codec, 772 const std::vector<VideoCodecSettings>& current_codecs, 773 const std::vector<RtpEncodingParameters>& encodings, 774 const std::vector<VideoCodecSettings> negotiated_codecs, 775 std::vector<VideoCodecSettings>* resolved_codecs) { 776 RTC_DCHECK(resolved_codecs); 777 resolved_codecs->clear(); 778 for (size_t i = 0; i < encodings.size(); i++) { 779 const std::optional<RtpCodec>& requested_codec = encodings[i].codec; 780 std::optional<VideoCodecSettings> found_codec; 781 if (!requested_codec) { 782 found_codec = current_codec; 783 } else if (i < current_codecs.size()) { 784 const VideoCodecSettings& codec = current_codecs[i]; 785 if (IsSameRtpCodecIgnoringLevel(codec.codec, *requested_codec)) { 786 found_codec = codec; 787 } 788 } 789 if (!found_codec) { 790 RTC_DCHECK(requested_codec); 791 auto matched_codec = 792 absl::c_find_if(negotiated_codecs, [&](auto negotiated_codec) { 793 return IsSameRtpCodecIgnoringLevel(negotiated_codec.codec, 794 *requested_codec); 795 }); 796 if (matched_codec == negotiated_codecs.end()) { 797 return RTCError(RTCErrorType::INVALID_MODIFICATION, 798 "Attempted to use an unsupported codec for layer " + 799 std::to_string(i)); 800 } 801 found_codec = *matched_codec; 802 } 803 RTC_DCHECK(found_codec); 804 resolved_codecs->push_back(*found_codec); 805 } 806 return RTCError::OK(); 807 } 808 809 } // namespace 810 // --------------- WebRtcVideoEngine --------------------------- 811 812 WebRtcVideoEngine::WebRtcVideoEngine( 813 std::unique_ptr<VideoEncoderFactory> video_encoder_factory, 814 std::unique_ptr<VideoDecoderFactory> video_decoder_factory, 815 const FieldTrialsView& trials) 816 : decoder_factory_(std::move(video_decoder_factory)), 817 encoder_factory_(std::move(video_encoder_factory)), 818 trials_(trials) { 819 RTC_DLOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine()"; 820 } 821 822 WebRtcVideoEngine::~WebRtcVideoEngine() { 823 RTC_DLOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine"; 824 } 825 826 std::unique_ptr<VideoMediaSendChannelInterface> 827 WebRtcVideoEngine::CreateSendChannel( 828 const Environment& env, 829 Call* call, 830 const MediaConfig& config, 831 const VideoOptions& options, 832 const CryptoOptions& crypto_options, 833 VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) { 834 return std::make_unique<WebRtcVideoSendChannel>( 835 env, call, config, options, crypto_options, encoder_factory_.get(), 836 video_bitrate_allocator_factory); 837 } 838 std::unique_ptr<VideoMediaReceiveChannelInterface> 839 WebRtcVideoEngine::CreateReceiveChannel(const Environment& env, 840 Call* call, 841 const MediaConfig& config, 842 const VideoOptions& options, 843 const CryptoOptions& crypto_options) { 844 return std::make_unique<WebRtcVideoReceiveChannel>( 845 env, call, config, options, crypto_options, decoder_factory_.get()); 846 } 847 848 std::vector<Codec> WebRtcVideoEngine::LegacySendCodecs(bool include_rtx) const { 849 return GetPayloadTypesAndDefaultCodecs(encoder_factory_.get(), 850 /*is_decoder_factory=*/false, 851 include_rtx, trials_); 852 } 853 854 std::vector<Codec> WebRtcVideoEngine::LegacyRecvCodecs(bool include_rtx) const { 855 return GetPayloadTypesAndDefaultCodecs(decoder_factory_.get(), 856 /*is_decoder_factory=*/true, 857 include_rtx, trials_); 858 } 859 860 std::vector<RtpHeaderExtensionCapability> 861 WebRtcVideoEngine::GetRtpHeaderExtensions( 862 const webrtc::FieldTrialsView* field_trials) const { 863 // Use field trials from PeerConnection `field_trials` or from 864 // PeerConnectionFactory `trials_`. 865 const webrtc::FieldTrialsView& trials = 866 (field_trials != nullptr ? *field_trials : trials_); 867 868 std::vector<RtpHeaderExtensionCapability> result; 869 // id is *not* incremented for non-default extensions, UsedIds needs to 870 // resolve conflicts. 871 int id = 1; 872 for (const auto& uri : 873 {RtpExtension::kTimestampOffsetUri, RtpExtension::kAbsSendTimeUri, 874 RtpExtension::kVideoRotationUri, 875 RtpExtension::kTransportSequenceNumberUri, 876 RtpExtension::kPlayoutDelayUri, RtpExtension::kVideoContentTypeUri, 877 RtpExtension::kVideoTimingUri, RtpExtension::kColorSpaceUri, 878 RtpExtension::kMidUri, RtpExtension::kRidUri, 879 RtpExtension::kRepairedRidUri}) { 880 result.emplace_back(uri, id++, RtpTransceiverDirection::kSendRecv); 881 } 882 result.emplace_back(RtpExtension::kCorruptionDetectionUri, id++, 883 /*preferred_encrypt=*/true, 884 RtpTransceiverDirection::kStopped); 885 for (const auto& uri : {RtpExtension::kAbsoluteCaptureTimeUri}) { 886 result.emplace_back(uri, id, RtpTransceiverDirection::kStopped); 887 } 888 result.emplace_back(RtpExtension::kGenericFrameDescriptorUri00, id, 889 trials.IsEnabled("WebRTC-GenericDescriptorAdvertised") 890 ? RtpTransceiverDirection::kSendRecv 891 : RtpTransceiverDirection::kStopped); 892 result.emplace_back(RtpExtension::kDependencyDescriptorUri, id, 893 trials.IsEnabled("WebRTC-DependencyDescriptorAdvertised") 894 ? RtpTransceiverDirection::kSendRecv 895 : RtpTransceiverDirection::kStopped); 896 result.emplace_back(RtpExtension::kVideoLayersAllocationUri, id, 897 trials.IsEnabled("WebRTC-VideoLayersAllocationAdvertised") 898 ? RtpTransceiverDirection::kSendRecv 899 : RtpTransceiverDirection::kStopped); 900 901 // VideoFrameTrackingId is a test-only extension. 902 if (trials.IsEnabled("WebRTC-VideoFrameTrackingIdAdvertised")) { 903 result.emplace_back(RtpExtension::kVideoFrameTrackingIdUri, id, 904 RtpTransceiverDirection::kSendRecv); 905 } 906 return result; 907 } 908 909 // Free function, exported for testing 910 std::map<uint32_t, VideoSendStream::StreamStats> 911 MergeInfoAboutOutboundRtpSubstreamsForTesting( 912 const std::map<uint32_t, VideoSendStream::StreamStats>& substreams) { 913 return MergeInfoAboutOutboundRtpSubstreams(substreams); 914 } 915 916 // --------------- WebRtcVideoSendChannel ---------------------- 917 WebRtcVideoSendChannel::WebRtcVideoSendChannel( 918 const Environment& env, 919 Call* call, 920 const MediaConfig& config, 921 const VideoOptions& options, 922 const CryptoOptions& crypto_options, 923 VideoEncoderFactory* encoder_factory, 924 VideoBitrateAllocatorFactory* bitrate_allocator_factory) 925 : MediaChannelUtil(call->network_thread(), config.enable_dscp), 926 env_(env), 927 worker_thread_(call->worker_thread()), 928 sending_(false), 929 receiving_(false), 930 call_(call), 931 default_sink_(nullptr), 932 video_config_(config.video), 933 encoder_factory_(encoder_factory), 934 bitrate_allocator_factory_(bitrate_allocator_factory), 935 default_send_options_(options), 936 last_send_stats_log_ms_(-1), 937 last_receive_stats_log_ms_(-1), 938 discard_unknown_ssrc_packets_(env_.field_trials().IsEnabled( 939 "WebRTC-Video-DiscardPacketsWithUnknownSsrc")), 940 crypto_options_(crypto_options) { 941 RTC_DCHECK_RUN_ON(&thread_checker_); 942 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; 943 } 944 945 WebRtcVideoSendChannel::~WebRtcVideoSendChannel() { 946 for (auto& kv : send_streams_) 947 delete kv.second; 948 } 949 950 scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> 951 WebRtcVideoSendChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( 952 const Codec& codec) { 953 RTC_DCHECK_RUN_ON(&thread_checker_); 954 bool is_screencast = parameters_.options.is_screencast.value_or(false); 955 // No automatic resizing when using simulcast or screencast, or when 956 // disabled by field trial flag. 957 bool automatic_resize = !disable_automatic_resize_ && !is_screencast && 958 (parameters_.config.rtp.ssrcs.size() == 1 || 959 NumActiveStreams(rtp_parameters_) == 1); 960 961 bool denoising; 962 bool codec_default_denoising = false; 963 if (is_screencast) { 964 denoising = false; 965 } else { 966 // Use codec default if video_noise_reduction is unset. 967 codec_default_denoising = !parameters_.options.video_noise_reduction; 968 denoising = parameters_.options.video_noise_reduction.value_or(false); 969 } 970 971 if (absl::EqualsIgnoreCase(codec.name, kH264CodecName)) { 972 return nullptr; 973 } 974 if (absl::EqualsIgnoreCase(codec.name, kVp8CodecName)) { 975 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); 976 vp8_settings.automaticResizeOn = automatic_resize; 977 // VP8 denoising is enabled by default. 978 vp8_settings.denoisingOn = codec_default_denoising ? true : denoising; 979 return make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>( 980 vp8_settings); 981 } 982 if (absl::EqualsIgnoreCase(codec.name, kVp9CodecName)) { 983 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings(); 984 985 vp9_settings.numberOfSpatialLayers = std::min<unsigned char>( 986 parameters_.config.rtp.ssrcs.size(), kConferenceMaxNumSpatialLayers); 987 vp9_settings.numberOfTemporalLayers = 988 std::min<unsigned char>(parameters_.config.rtp.ssrcs.size() > 1 989 ? kConferenceDefaultNumTemporalLayers 990 : 1, 991 kConferenceMaxNumTemporalLayers); 992 993 // VP9 denoising is disabled by default. 994 vp9_settings.denoisingOn = codec_default_denoising ? true : denoising; 995 // Disable automatic resize if more than one spatial layer is requested. 996 bool vp9_automatic_resize = automatic_resize; 997 std::optional<int> num_spatial_layers = 998 NumSpatialLayersFromEncoding(rtp_parameters_, /*idx=*/0); 999 if (num_spatial_layers && *num_spatial_layers > 1) { 1000 vp9_automatic_resize = false; 1001 } 1002 vp9_settings.automaticResizeOn = vp9_automatic_resize; 1003 if (!is_screencast) { 1004 FieldTrialFlag interlayer_pred_experiment_enabled("Enabled"); 1005 FieldTrialEnum<InterLayerPredMode> inter_layer_pred_mode( 1006 "inter_layer_pred_mode", InterLayerPredMode::kOnKeyPic, 1007 {{"off", InterLayerPredMode::kOff}, 1008 {"on", InterLayerPredMode::kOn}, 1009 {"onkeypic", InterLayerPredMode::kOnKeyPic}}); 1010 ParseFieldTrial( 1011 {&interlayer_pred_experiment_enabled, &inter_layer_pred_mode}, 1012 env_.field_trials().Lookup("WebRTC-Vp9InterLayerPred")); 1013 if (interlayer_pred_experiment_enabled) { 1014 vp9_settings.interLayerPred = inter_layer_pred_mode; 1015 } else { 1016 // Limit inter-layer prediction to key pictures by default. 1017 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic; 1018 } 1019 1020 // TODO(webrtc:329396373): Remove after flexible mode is fully deployed. 1021 vp9_settings.flexibleMode = 1022 !env_.field_trials().IsDisabled("WebRTC-Video-Vp9FlexibleMode"); 1023 } else { 1024 // Multiple spatial layers vp9 screenshare needs flexible mode. 1025 vp9_settings.flexibleMode = vp9_settings.numberOfSpatialLayers > 1; 1026 vp9_settings.interLayerPred = InterLayerPredMode::kOn; 1027 } 1028 return make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>( 1029 vp9_settings); 1030 } 1031 if (absl::EqualsIgnoreCase(codec.name, kAv1CodecName)) { 1032 VideoCodecAV1 av1_settings = {.automatic_resize_on = automatic_resize}; 1033 if (NumSpatialLayersFromEncoding(rtp_parameters_, /*idx=*/0) > 1) { 1034 av1_settings.automatic_resize_on = false; 1035 } 1036 return make_ref_counted<VideoEncoderConfig::Av1EncoderSpecificSettings>( 1037 av1_settings); 1038 } 1039 return nullptr; 1040 } 1041 std::vector<VideoCodecSettings> WebRtcVideoSendChannel::SelectSendVideoCodecs( 1042 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const { 1043 std::vector<SdpVideoFormat> sdp_formats = 1044 encoder_factory_ ? encoder_factory_->GetImplementations() 1045 : std::vector<SdpVideoFormat>(); 1046 1047 // The returned vector holds the VideoCodecSettings in term of preference. 1048 // They are orderd by receive codec preference first and local implementation 1049 // preference second. 1050 std::vector<VideoCodecSettings> encoders; 1051 for (const VideoCodecSettings& remote_codec : remote_mapped_codecs) { 1052 for (auto format_it = sdp_formats.begin(); 1053 format_it != sdp_formats.end();) { 1054 // For H264, we will limit the encode level to the remote offered level 1055 // regardless if level asymmetry is allowed or not. This is strictly not 1056 // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2 1057 // since we should limit the encode level to the lower of local and remote 1058 // level when level asymmetry is not allowed. 1059 // For H.265, the level asymmetry is implicitly allowed. We need to make 1060 // sure the encode level is set to the remote offered level. 1061 if (format_it->IsSameCodec( 1062 {remote_codec.codec.name, remote_codec.codec.params})) { 1063 encoders.push_back(remote_codec); 1064 1065 // To allow the VideoEncoderFactory to keep information about which 1066 // implementation to instantitate when CreateEncoder is called the two 1067 // parmeter sets are merged. 1068 encoders.back().codec.params.insert(format_it->parameters.begin(), 1069 format_it->parameters.end()); 1070 1071 format_it = sdp_formats.erase(format_it); 1072 } else { 1073 ++format_it; 1074 } 1075 } 1076 } 1077 1078 return encoders; 1079 } 1080 1081 bool WebRtcVideoSendChannel::GetChangedSenderParameters( 1082 const VideoSenderParameters& params, 1083 ChangedSenderParameters* changed_params) const { 1084 if (!ValidateCodecFormats(params.codecs) || 1085 !ValidateRtpExtensions(params.extensions, send_rtp_extensions_)) { 1086 return false; 1087 } 1088 1089 auto result = MapCodecs(params.codecs); 1090 if (!result.ok()) { 1091 RTC_LOG(LS_ERROR) << "Failure in codec list, error = " << result.error(); 1092 return false; 1093 } 1094 std::vector<VideoCodecSettings> mapped_codecs = result.value(); 1095 1096 std::vector<VideoCodecSettings> negotiated_codecs = 1097 SelectSendVideoCodecs(mapped_codecs); 1098 1099 if (params.is_stream_active && negotiated_codecs.empty()) { 1100 // This is not a failure but will lead to the answer being rejected. 1101 RTC_LOG(LS_ERROR) << "No video codecs in common."; 1102 return true; 1103 } 1104 1105 // Never enable sending FlexFEC, unless we are in the experiment. 1106 if (!env_.field_trials().IsEnabled("WebRTC-FlexFEC-03")) { 1107 for (VideoCodecSettings& codec : negotiated_codecs) 1108 codec.flexfec_payload_type = -1; 1109 } 1110 1111 std::optional<VideoCodecSettings> force_codec; 1112 if (!send_streams_.empty()) { 1113 // Since we do not support mixed-codec simulcast yet, 1114 // all send streams must have the same codec value. 1115 auto rtp_parameters = send_streams_.begin()->second->GetRtpParameters(); 1116 if (rtp_parameters.encodings[0].codec) { 1117 auto matched_codec = 1118 absl::c_find_if(negotiated_codecs, [&](auto negotiated_codec) { 1119 return IsSameRtpCodec(negotiated_codec.codec, 1120 *rtp_parameters.encodings[0].codec); 1121 }); 1122 if (matched_codec != negotiated_codecs.end()) { 1123 force_codec = *matched_codec; 1124 } else { 1125 // The requested codec has been negotiated away, we clear it from the 1126 // parameters. 1127 for (auto& encoding : rtp_parameters.encodings) { 1128 encoding.codec.reset(); 1129 } 1130 send_streams_.begin()->second->SetRtpParameters(rtp_parameters, 1131 nullptr); 1132 } 1133 } 1134 } 1135 1136 if (negotiated_codecs_ != negotiated_codecs) { 1137 if (negotiated_codecs.empty()) { 1138 changed_params->send_codec = std::nullopt; 1139 } else if (force_codec) { 1140 changed_params->send_codec = force_codec; 1141 } else if (send_codec() != negotiated_codecs.front()) { 1142 changed_params->send_codec = negotiated_codecs.front(); 1143 } 1144 changed_params->negotiated_codecs = negotiated_codecs; 1145 } 1146 1147 // For mixed-codec simulcast 1148 std::vector<VideoCodecSettings> send_codecs; 1149 if (!send_streams_.empty() && !negotiated_codecs.empty()) { 1150 bool needs_update = false; 1151 auto rtp_parameters = send_streams_.begin()->second->GetRtpParameters(); 1152 for (auto& encoding : rtp_parameters.encodings) { 1153 if (encoding.codec) { 1154 auto matched_codec = 1155 absl::c_find_if(negotiated_codecs, [&](auto negotiated_codec) { 1156 return IsSameRtpCodec(negotiated_codec.codec, *encoding.codec); 1157 }); 1158 if (matched_codec != negotiated_codecs.end()) { 1159 send_codecs.push_back(*matched_codec); 1160 } else { 1161 // The requested codec has been negotiated away, we clear it from the 1162 // parameters. 1163 encoding.codec.reset(); 1164 needs_update = true; 1165 send_codecs.push_back(negotiated_codecs.front()); 1166 } 1167 } else { 1168 send_codecs.push_back(negotiated_codecs.front()); 1169 } 1170 } 1171 1172 if (needs_update) { 1173 send_streams_.begin()->second->SetRtpParameters(rtp_parameters, nullptr); 1174 } 1175 } 1176 1177 if (send_codecs_ != send_codecs) { 1178 changed_params->send_codecs = send_codecs; 1179 } 1180 1181 // Handle RTP header extensions. 1182 if (params.extmap_allow_mixed != ExtmapAllowMixed()) { 1183 changed_params->extmap_allow_mixed = params.extmap_allow_mixed; 1184 } 1185 std::vector<RtpExtension> filtered_extensions = 1186 FilterRtpExtensions(params.extensions, RtpExtension::IsSupportedForVideo, 1187 true, env_.field_trials()); 1188 if (send_rtp_extensions_ != filtered_extensions) { 1189 changed_params->rtp_header_extensions = 1190 std::optional<std::vector<RtpExtension>>(filtered_extensions); 1191 } 1192 1193 if (params.mid != send_params_.mid) { 1194 changed_params->mid = params.mid; 1195 } 1196 1197 // Handle max bitrate. 1198 if (params.max_bandwidth_bps != send_params_.max_bandwidth_bps && 1199 params.max_bandwidth_bps >= -1) { 1200 // 0 or -1 uncaps max bitrate. 1201 // TODO(pbos): Reconsider how 0 should be treated. It is not mentioned as a 1202 // special value and might very well be used for stopping sending. 1203 changed_params->max_bandwidth_bps = 1204 params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps; 1205 } 1206 1207 // Handle conference mode. 1208 if (params.conference_mode != send_params_.conference_mode) { 1209 changed_params->conference_mode = params.conference_mode; 1210 } 1211 1212 // Handle RTCP mode. 1213 if (params.rtcp.reduced_size != send_params_.rtcp.reduced_size) { 1214 changed_params->rtcp_mode = 1215 params.rtcp.reduced_size ? RtcpMode::kReducedSize : RtcpMode::kCompound; 1216 } 1217 1218 return true; 1219 } 1220 1221 bool WebRtcVideoSendChannel::SetSenderParameters( 1222 const VideoSenderParameters& params) { 1223 RTC_DCHECK_RUN_ON(&thread_checker_); 1224 TRACE_EVENT0("webrtc", "WebRtcVideoSendChannel::SetSenderParameters"); 1225 RTC_LOG(LS_INFO) << "SetSenderParameters: " << params.ToString(); 1226 ChangedSenderParameters changed_params; 1227 if (!GetChangedSenderParameters(params, &changed_params)) { 1228 return false; 1229 } 1230 1231 if (changed_params.negotiated_codecs) { 1232 for (const auto& send_codec : *changed_params.negotiated_codecs) 1233 RTC_LOG(LS_INFO) << "Negotiated codec: " << send_codec.codec.ToString(); 1234 } 1235 1236 send_params_ = params; 1237 return ApplyChangedParams(changed_params); 1238 } 1239 1240 void WebRtcVideoSendChannel::RequestEncoderFallback() { 1241 if (!worker_thread_->IsCurrent()) { 1242 worker_thread_->PostTask( 1243 SafeTask(task_safety_.flag(), [this] { RequestEncoderFallback(); })); 1244 return; 1245 } 1246 1247 RTC_DCHECK_RUN_ON(&thread_checker_); 1248 if (negotiated_codecs_.size() <= 1) { 1249 RTC_LOG(LS_WARNING) << "Encoder failed but no fallback codec is available"; 1250 return; 1251 } 1252 1253 ChangedSenderParameters params; 1254 params.negotiated_codecs = negotiated_codecs_; 1255 params.negotiated_codecs->erase(params.negotiated_codecs->begin()); 1256 params.send_codec = params.negotiated_codecs->front(); 1257 ApplyChangedParams(params); 1258 } 1259 1260 void WebRtcVideoSendChannel::RequestEncoderSwitch(const SdpVideoFormat& format, 1261 bool allow_default_fallback) { 1262 if (!worker_thread_->IsCurrent()) { 1263 worker_thread_->PostTask( 1264 SafeTask(task_safety_.flag(), [this, format, allow_default_fallback] { 1265 RequestEncoderSwitch(format, allow_default_fallback); 1266 })); 1267 return; 1268 } 1269 1270 RTC_DCHECK_RUN_ON(&thread_checker_); 1271 1272 for (const VideoCodecSettings& codec_setting : negotiated_codecs_) { 1273 if (format.IsSameCodec( 1274 {codec_setting.codec.name, codec_setting.codec.params})) { 1275 VideoCodecSettings new_codec_setting = codec_setting; 1276 for (const auto& kv : format.parameters) { 1277 new_codec_setting.codec.params[kv.first] = kv.second; 1278 } 1279 1280 if (send_codec() == new_codec_setting) { 1281 // Already using this codec, no switch required. 1282 return; 1283 } 1284 1285 ChangedSenderParameters params; 1286 params.send_codec = new_codec_setting; 1287 ApplyChangedParams(params); 1288 return; 1289 } 1290 } 1291 1292 RTC_LOG(LS_WARNING) << "Failed to switch encoder to: " << format.ToString() 1293 << ". Is default fallback allowed: " 1294 << allow_default_fallback; 1295 1296 if (allow_default_fallback) { 1297 RequestEncoderFallback(); 1298 } 1299 } 1300 1301 bool WebRtcVideoSendChannel::ApplyChangedParams( 1302 const ChangedSenderParameters& changed_params) { 1303 RTC_DCHECK_RUN_ON(&thread_checker_); 1304 if (changed_params.negotiated_codecs) 1305 negotiated_codecs_ = *changed_params.negotiated_codecs; 1306 1307 if (changed_params.send_codec) 1308 send_codec() = changed_params.send_codec; 1309 1310 if (changed_params.send_codecs) { 1311 send_codecs_ = *changed_params.send_codecs; 1312 } else { 1313 send_codecs_.clear(); 1314 } 1315 1316 if (changed_params.extmap_allow_mixed) { 1317 SetExtmapAllowMixed(*changed_params.extmap_allow_mixed); 1318 } 1319 if (changed_params.rtp_header_extensions) { 1320 send_rtp_extensions_ = *changed_params.rtp_header_extensions; 1321 } 1322 1323 if (changed_params.send_codec || changed_params.max_bandwidth_bps) { 1324 if (send_params_.max_bandwidth_bps == -1) { 1325 // Unset the global max bitrate (max_bitrate_bps) if max_bandwidth_bps is 1326 // -1, which corresponds to no "b=AS" attribute in SDP. Note that the 1327 // global max bitrate may be set below in GetBitrateConfigForCodec, from 1328 // the codec max bitrate. 1329 // TODO(pbos): This should be reconsidered (codec max bitrate should 1330 // probably not affect global call max bitrate). 1331 bitrate_config_.max_bitrate_bps = -1; 1332 } 1333 1334 if (send_codec()) { 1335 // TODO(holmer): Changing the codec parameters shouldn't necessarily mean 1336 // that we change the min/max of bandwidth estimation. Reevaluate this. 1337 bitrate_config_ = GetBitrateConfigForCodec(send_codec()->codec); 1338 if (!changed_params.send_codec) { 1339 // If the codec isn't changing, set the start bitrate to -1 which means 1340 // "unchanged" so that BWE isn't affected. 1341 bitrate_config_.start_bitrate_bps = -1; 1342 } 1343 } 1344 1345 if (send_params_.max_bandwidth_bps >= 0) { 1346 // Note that max_bandwidth_bps intentionally takes priority over the 1347 // bitrate config for the codec. This allows FEC to be applied above the 1348 // codec target bitrate. 1349 // TODO(pbos): Figure out whether b=AS means max bitrate for this 1350 // WebRtcVideoSendChannel (in which case we're good), or per sender 1351 // (SSRC), in which case this should not set a BitrateConstraints but 1352 // rather reconfigure all senders. 1353 bitrate_config_.max_bitrate_bps = send_params_.max_bandwidth_bps == 0 1354 ? -1 1355 : send_params_.max_bandwidth_bps; 1356 } 1357 1358 call_->GetTransportControllerSend()->SetSdpBitrateParameters( 1359 bitrate_config_); 1360 } 1361 1362 for (auto& kv : send_streams_) { 1363 kv.second->SetSenderParameters(changed_params); 1364 } 1365 if (changed_params.send_codec || changed_params.rtcp_mode) { 1366 if (send_codec_changed_callback_) { 1367 send_codec_changed_callback_(); 1368 } 1369 } 1370 return true; 1371 } 1372 1373 RtpParameters WebRtcVideoSendChannel::GetRtpSendParameters( 1374 uint32_t ssrc) const { 1375 RTC_DCHECK_RUN_ON(&thread_checker_); 1376 auto it = send_streams_.find(ssrc); 1377 if (it == send_streams_.end()) { 1378 RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream " 1379 "with ssrc " 1380 << ssrc << " which doesn't exist."; 1381 return RtpParameters(); 1382 } 1383 1384 RtpParameters rtp_params = it->second->GetRtpParameters(); 1385 // Need to add the common list of codecs to the send stream-specific 1386 // RTP parameters. 1387 for (const Codec& codec : send_params_.codecs) { 1388 if (send_codec() && send_codec()->codec.id == codec.id) { 1389 // Put the current send codec to the front of the codecs list. 1390 RTC_DCHECK_EQ(codec.name, send_codec()->codec.name); 1391 rtp_params.codecs.insert(rtp_params.codecs.begin(), 1392 codec.ToCodecParameters()); 1393 } else { 1394 rtp_params.codecs.push_back(codec.ToCodecParameters()); 1395 } 1396 } 1397 1398 return rtp_params; 1399 } 1400 1401 RTCError WebRtcVideoSendChannel::SetRtpSendParameters( 1402 uint32_t ssrc, 1403 const RtpParameters& parameters, 1404 SetParametersCallback callback) { 1405 RTC_DCHECK_RUN_ON(&thread_checker_); 1406 TRACE_EVENT0("webrtc", "WebRtcVideoSendChannel::SetRtpSendParameters"); 1407 auto it = send_streams_.find(ssrc); 1408 if (it == send_streams_.end()) { 1409 RTC_LOG(LS_ERROR) << "Attempting to set RTP send parameters for stream " 1410 "with ssrc " 1411 << ssrc << " which doesn't exist."; 1412 return InvokeSetParametersCallback(callback, 1413 RTCError(RTCErrorType::INTERNAL_ERROR)); 1414 } 1415 1416 // TODO(deadbeef): Handle setting parameters with a list of codecs in a 1417 // different order (which should change the send codec). 1418 RtpParameters current_parameters = GetRtpSendParameters(ssrc); 1419 if (current_parameters.codecs != parameters.codecs) { 1420 RTC_DLOG(LS_ERROR) << "Using SetParameters to change the set of codecs " 1421 "is not currently supported."; 1422 return InvokeSetParametersCallback(callback, 1423 RTCError(RTCErrorType::INTERNAL_ERROR)); 1424 } 1425 1426 if (!parameters.encodings.empty()) { 1427 // Note that these values come from: 1428 // https://tools.ietf.org/html/draft-ietf-tsvwg-rtcweb-qos-16#section-5 1429 // TODO(deadbeef): Change values depending on whether we are sending a 1430 // keyframe or non-keyframe. 1431 DiffServCodePoint new_dscp = DSCP_DEFAULT; 1432 switch (parameters.encodings[0].network_priority) { 1433 case Priority::kVeryLow: 1434 new_dscp = DSCP_CS1; 1435 break; 1436 case Priority::kLow: 1437 new_dscp = DSCP_DEFAULT; 1438 break; 1439 case Priority::kMedium: 1440 new_dscp = DSCP_AF42; 1441 break; 1442 case Priority::kHigh: 1443 new_dscp = DSCP_AF41; 1444 break; 1445 } 1446 1447 if (send_codec_ && 1448 std::any_of(parameters.encodings.begin(), parameters.encodings.end(), 1449 [](const auto& e) { return e.codec; })) { 1450 std::vector<VideoCodecSettings> send_codecs; 1451 auto error = 1452 ResolveSendCodecs(*send_codec_, send_codecs_, parameters.encodings, 1453 negotiated_codecs_, &send_codecs); 1454 if (!error.ok()) { 1455 return InvokeSetParametersCallback(callback, error); 1456 } 1457 1458 if (send_codecs_ != send_codecs) { 1459 ChangedSenderParameters params; 1460 if (!send_codecs.empty()) { 1461 params.send_codec = send_codecs[0]; 1462 } 1463 params.send_codecs = send_codecs; 1464 ApplyChangedParams(params); 1465 } 1466 } 1467 1468 SetPreferredDscp(new_dscp); 1469 } 1470 1471 return it->second->SetRtpParameters(parameters, std::move(callback)); 1472 } 1473 std::optional<Codec> WebRtcVideoSendChannel::GetSendCodec() const { 1474 RTC_DCHECK_RUN_ON(&thread_checker_); 1475 if (!send_codec()) { 1476 RTC_LOG(LS_VERBOSE) << "GetSendCodec: No send codec set."; 1477 return std::nullopt; 1478 } 1479 return send_codec()->codec; 1480 } 1481 1482 bool WebRtcVideoSendChannel::SetSend(bool send) { 1483 RTC_DCHECK_RUN_ON(&thread_checker_); 1484 TRACE_EVENT0("webrtc", "WebRtcVideoSendChannel::SetSend"); 1485 RTC_LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false"); 1486 if (send && !send_codec()) { 1487 RTC_DLOG(LS_ERROR) << "SetSend(true) called before setting codec."; 1488 return false; 1489 } 1490 for (const auto& kv : send_streams_) { 1491 kv.second->SetSend(send); 1492 } 1493 sending_ = send; 1494 return true; 1495 } 1496 1497 bool WebRtcVideoSendChannel::SetVideoSend( 1498 uint32_t ssrc, 1499 const VideoOptions* options, 1500 VideoSourceInterface<VideoFrame>* source) { 1501 RTC_DCHECK_RUN_ON(&thread_checker_); 1502 TRACE_EVENT0("webrtc", "SetVideoSend"); 1503 RTC_DCHECK(ssrc != 0); 1504 RTC_LOG(LS_INFO) << "SetVideoSend (ssrc= " << ssrc << ", options: " 1505 << (options ? options->ToString() : "nullptr") 1506 << ", source = " << (source ? "(source)" : "nullptr") << ")"; 1507 1508 const auto& kv = send_streams_.find(ssrc); 1509 if (kv == send_streams_.end()) { 1510 // Allow unknown ssrc only if source is null. 1511 RTC_CHECK(source == nullptr); 1512 RTC_LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1513 return false; 1514 } 1515 1516 return kv->second->SetVideoSend(options, source); 1517 } 1518 1519 bool WebRtcVideoSendChannel::ValidateSendSsrcAvailability( 1520 const StreamParams& sp) const { 1521 for (uint32_t ssrc : sp.ssrcs) { 1522 if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) { 1523 RTC_LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc 1524 << "' already exists."; 1525 return false; 1526 } 1527 } 1528 return true; 1529 } 1530 bool WebRtcVideoSendChannel::AddSendStream(const StreamParams& sp) { 1531 RTC_DCHECK_RUN_ON(&thread_checker_); 1532 1533 RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); 1534 if (!ValidateStreamParams(sp)) 1535 return false; 1536 1537 if (!ValidateSendSsrcAvailability(sp)) 1538 return false; 1539 1540 for (uint32_t used_ssrc : sp.ssrcs) 1541 send_ssrcs_.insert(used_ssrc); 1542 1543 VideoSendStream::Config config(transport()); 1544 1545 for (const RidDescription& rid : sp.rids()) { 1546 config.rtp.rids.push_back(rid.rid); 1547 } 1548 1549 config.suspend_below_min_bitrate = video_config_.suspend_below_min_bitrate; 1550 config.periodic_alr_bandwidth_probing = 1551 video_config_.periodic_alr_bandwidth_probing; 1552 config.encoder_settings.experiment_cpu_load_estimator = 1553 video_config_.experiment_cpu_load_estimator; 1554 config.encoder_settings.encoder_factory = encoder_factory_; 1555 config.encoder_settings.bitrate_allocator_factory = 1556 bitrate_allocator_factory_; 1557 config.encoder_settings.encoder_switch_request_callback = this; 1558 1559 config.crypto_options = crypto_options_; 1560 config.rtp.extmap_allow_mixed = ExtmapAllowMixed(); 1561 config.rtcp_report_interval_ms = video_config_.rtcp_report_interval_ms; 1562 config.rtp.enable_send_packet_batching = 1563 video_config_.enable_send_packet_batching; 1564 1565 WebRtcVideoSendStream* stream = new WebRtcVideoSendStream( 1566 env_, call_, sp, std::move(config), default_send_options_, 1567 video_config_.enable_cpu_adaptation, bitrate_config_.max_bitrate_bps, 1568 send_codec(), send_codecs_, send_rtp_extensions_, send_params_); 1569 1570 uint32_t ssrc = sp.first_ssrc(); 1571 RTC_DCHECK(ssrc != 0); 1572 send_streams_[ssrc] = stream; 1573 1574 if (ssrc_list_changed_callback_) { 1575 ssrc_list_changed_callback_(send_ssrcs_); 1576 } 1577 1578 if (sending_) { 1579 stream->SetSend(true); 1580 } 1581 1582 return true; 1583 } 1584 1585 bool WebRtcVideoSendChannel::RemoveSendStream(uint32_t ssrc) { 1586 RTC_DCHECK_RUN_ON(&thread_checker_); 1587 RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc; 1588 1589 WebRtcVideoSendStream* removed_stream; 1590 auto it = send_streams_.find(ssrc); 1591 if (it == send_streams_.end()) { 1592 return false; 1593 } 1594 1595 for (uint32_t old_ssrc : it->second->GetSsrcs()) 1596 send_ssrcs_.erase(old_ssrc); 1597 1598 removed_stream = it->second; 1599 send_streams_.erase(it); 1600 1601 // Switch receiver report SSRCs, in case the one in use is no longer valid. 1602 if (ssrc_list_changed_callback_) { 1603 ssrc_list_changed_callback_(send_ssrcs_); 1604 } 1605 1606 delete removed_stream; 1607 1608 return true; 1609 } 1610 1611 bool WebRtcVideoSendChannel::GetStats(VideoMediaSendInfo* info) { 1612 RTC_DCHECK_RUN_ON(&thread_checker_); 1613 TRACE_EVENT0("webrtc", "WebRtcVideoSendChannel::GetSendStats"); 1614 1615 info->Clear(); 1616 if (send_streams_.empty()) { 1617 return true; 1618 } 1619 1620 // Log stats periodically. 1621 bool log_stats = false; 1622 int64_t now_ms = env_.clock().TimeInMilliseconds(); 1623 if (last_send_stats_log_ms_ == -1 || 1624 now_ms - last_send_stats_log_ms_ > kStatsLogIntervalMs) { 1625 last_send_stats_log_ms_ = now_ms; 1626 log_stats = true; 1627 } 1628 1629 info->Clear(); 1630 FillSenderStats(info, log_stats); 1631 FillSendCodecStats(info); 1632 // TODO(holmer): We should either have rtt available as a metric on 1633 // VideoSend/ReceiveStreams, or we should remove rtt from VideoSenderInfo. 1634 Call::Stats stats = call_->GetStats(); 1635 if (stats.rtt_ms != -1) { 1636 for (size_t i = 0; i < info->senders.size(); ++i) { 1637 info->senders[i].rtt_ms = stats.rtt_ms; 1638 } 1639 for (size_t i = 0; i < info->aggregated_senders.size(); ++i) { 1640 info->aggregated_senders[i].rtt_ms = stats.rtt_ms; 1641 } 1642 } 1643 1644 if (log_stats) 1645 RTC_LOG(LS_INFO) << stats.ToString(now_ms); 1646 1647 return true; 1648 } 1649 void WebRtcVideoSendChannel::FillSenderStats( 1650 VideoMediaSendInfo* video_media_info, 1651 bool log_stats) { 1652 for (const auto& it : send_streams_) { 1653 auto infos = it.second->GetPerLayerVideoSenderInfos(log_stats); 1654 if (infos.empty()) 1655 continue; 1656 video_media_info->aggregated_senders.push_back( 1657 it.second->GetAggregatedVideoSenderInfo(infos)); 1658 for (auto&& info : infos) { 1659 video_media_info->senders.push_back(info); 1660 } 1661 } 1662 } 1663 1664 void WebRtcVideoSendChannel::FillBitrateInfo( 1665 BandwidthEstimationInfo* bwe_info) { 1666 RTC_DCHECK_RUN_ON(&thread_checker_); 1667 for (const auto& it : send_streams_) { 1668 it.second->FillBitrateInfo(bwe_info); 1669 } 1670 } 1671 1672 void WebRtcVideoSendChannel::FillSendCodecStats( 1673 VideoMediaSendInfo* video_media_info) { 1674 RTC_DCHECK_RUN_ON(&thread_checker_); 1675 if (!send_codec()) { 1676 return; 1677 } 1678 // Note: since RTP stats don't account for RTX and FEC separately (see 1679 // https://w3c.github.io/webrtc-stats/#dom-rtcstatstype-outbound-rtp) 1680 // we can omit the codec information for those here and only insert the 1681 // primary codec that is being used to send here. 1682 video_media_info->send_codecs.insert(std::make_pair( 1683 send_codec()->codec.id, send_codec()->codec.ToCodecParameters())); 1684 } 1685 1686 void WebRtcVideoSendChannel::OnPacketSent(const SentPacketInfo& sent_packet) { 1687 RTC_DCHECK_RUN_ON(&network_thread_checker_); 1688 // TODO(tommi): We shouldn't need to go through call_ to deliver this 1689 // notification. We should already have direct access to 1690 // video_send_delay_stats_ and transport_send_ptr_ via `stream_`. 1691 // So we should be able to remove OnSentPacket from Call and handle this per 1692 // channel instead. At the moment Call::OnSentPacket calls OnSentPacket for 1693 // the video stats, for all sent packets, including audio, which causes 1694 // unnecessary lookups. 1695 call_->OnSentPacket(sent_packet); 1696 } 1697 1698 void WebRtcVideoSendChannel::OnReadyToSend(bool ready) { 1699 RTC_DCHECK_RUN_ON(&network_thread_checker_); 1700 RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); 1701 call_->SignalChannelNetworkState(MediaType::VIDEO, 1702 ready ? kNetworkUp : kNetworkDown); 1703 } 1704 1705 void WebRtcVideoSendChannel::OnNetworkRouteChanged( 1706 absl::string_view transport_name, 1707 const NetworkRoute& network_route) { 1708 RTC_DCHECK_RUN_ON(&network_thread_checker_); 1709 worker_thread_->PostTask(SafeTask( 1710 task_safety_.flag(), 1711 [this, name = std::string(transport_name), route = network_route] { 1712 RTC_DCHECK_RUN_ON(&thread_checker_); 1713 RtpTransportControllerSendInterface* transport = 1714 call_->GetTransportControllerSend(); 1715 transport->OnNetworkRouteChanged(name, route); 1716 transport->OnTransportOverheadChanged(route.packet_overhead); 1717 })); 1718 } 1719 1720 void WebRtcVideoSendChannel::SetInterface(MediaChannelNetworkInterface* iface) { 1721 RTC_DCHECK_RUN_ON(&network_thread_checker_); 1722 MediaChannelUtil::SetInterface(iface); 1723 1724 // Speculative change to increase the outbound socket buffer size. 1725 // In b/15152257, we are seeing a significant number of packets discarded 1726 // due to lack of socket buffer space, although it's not yet clear what the 1727 // ideal value should be. 1728 const std::string group_name_send_buf_size = 1729 env_.field_trials().Lookup("WebRTC-SendBufferSizeBytes"); 1730 int send_buffer_size = kVideoRtpSendBufferSize; 1731 if (!group_name_send_buf_size.empty() && 1732 (sscanf(group_name_send_buf_size.c_str(), "%d", &send_buffer_size) != 1 || 1733 send_buffer_size <= 0)) { 1734 RTC_LOG(LS_WARNING) << "Invalid send buffer size: " 1735 << group_name_send_buf_size; 1736 send_buffer_size = kVideoRtpSendBufferSize; 1737 } 1738 1739 MediaChannelUtil::SetOption(MediaChannelNetworkInterface::ST_RTP, 1740 Socket::OPT_SNDBUF, send_buffer_size); 1741 } 1742 1743 void WebRtcVideoSendChannel::SetFrameEncryptor( 1744 uint32_t ssrc, 1745 scoped_refptr<FrameEncryptorInterface> frame_encryptor) { 1746 RTC_DCHECK_RUN_ON(&thread_checker_); 1747 auto matching_stream = send_streams_.find(ssrc); 1748 if (matching_stream != send_streams_.end()) { 1749 matching_stream->second->SetFrameEncryptor(frame_encryptor); 1750 } else { 1751 RTC_LOG(LS_ERROR) << "No stream found to attach frame encryptor"; 1752 } 1753 } 1754 1755 void WebRtcVideoSendChannel::SetEncoderSelector( 1756 uint32_t ssrc, 1757 VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) { 1758 RTC_DCHECK_RUN_ON(&thread_checker_); 1759 auto matching_stream = send_streams_.find(ssrc); 1760 if (matching_stream != send_streams_.end()) { 1761 matching_stream->second->SetEncoderSelector(encoder_selector); 1762 } else { 1763 RTC_LOG(LS_ERROR) << "No stream found to attach encoder selector"; 1764 } 1765 } 1766 1767 WebRtcVideoSendChannel::WebRtcVideoSendStream::VideoSendStreamParameters:: 1768 VideoSendStreamParameters( 1769 VideoSendStream::Config config, 1770 const VideoOptions& options, 1771 int max_bitrate_bps, 1772 const std::optional<VideoCodecSettings>& codec_settings, 1773 const std::vector<VideoCodecSettings>& codec_settings_list) 1774 : config(std::move(config)), 1775 options(options), 1776 max_bitrate_bps(max_bitrate_bps), 1777 conference_mode(false), 1778 codec_settings(codec_settings), 1779 codec_settings_list(codec_settings_list) {} 1780 1781 WebRtcVideoSendChannel::WebRtcVideoSendStream::WebRtcVideoSendStream( 1782 const Environment& env, 1783 Call* call, 1784 const StreamParams& sp, 1785 VideoSendStream::Config config, 1786 const VideoOptions& options, 1787 bool enable_cpu_overuse_detection, 1788 int max_bitrate_bps, 1789 const std::optional<VideoCodecSettings>& codec_settings, 1790 const std::vector<VideoCodecSettings>& codec_settings_list, 1791 const std::optional<std::vector<RtpExtension>>& rtp_extensions, 1792 // TODO(deadbeef): Don't duplicate information between send_params, 1793 // rtp_extensions, options, etc. 1794 const VideoSenderParameters& send_params) 1795 : env_(env), 1796 worker_thread_(call->worker_thread()), 1797 ssrcs_(sp.ssrcs), 1798 ssrc_groups_(sp.ssrc_groups), 1799 call_(call), 1800 enable_cpu_overuse_detection_(enable_cpu_overuse_detection), 1801 source_(nullptr), 1802 stream_(nullptr), 1803 parameters_(std::move(config), 1804 options, 1805 max_bitrate_bps, 1806 codec_settings, 1807 codec_settings_list), 1808 rtp_parameters_(CreateRtpParametersWithEncodings(sp)), 1809 sending_(false), 1810 disable_automatic_resize_(env_.field_trials().IsEnabled( 1811 "WebRTC-Video-DisableAutomaticResize")) { 1812 // Maximum packet size may come in RtpConfig from external transport, for 1813 // example from QuicTransportInterface implementation, so do not exceed 1814 // given max_packet_size. 1815 parameters_.config.rtp.max_packet_size = 1816 std::min<size_t>(parameters_.config.rtp.max_packet_size, kVideoMtu); 1817 parameters_.conference_mode = send_params.conference_mode; 1818 1819 sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); 1820 1821 // ValidateStreamParams should prevent this from happening. 1822 RTC_CHECK(!parameters_.config.rtp.ssrcs.empty()); 1823 rtp_parameters_.encodings[0].ssrc = parameters_.config.rtp.ssrcs[0]; 1824 1825 // RTX. 1826 sp.GetFidSsrcs(parameters_.config.rtp.ssrcs, 1827 ¶meters_.config.rtp.rtx.ssrcs); 1828 1829 // FlexFEC SSRCs. 1830 // TODO(brandtr): This code needs to be generalized when we add support for 1831 // multistream protection. 1832 if (env_.field_trials().IsEnabled("WebRTC-FlexFEC-03")) { 1833 uint32_t flexfec_ssrc; 1834 bool flexfec_enabled = false; 1835 for (uint32_t primary_ssrc : parameters_.config.rtp.ssrcs) { 1836 if (sp.GetFecFrSsrc(primary_ssrc, &flexfec_ssrc)) { 1837 if (flexfec_enabled) { 1838 RTC_LOG(LS_INFO) 1839 << "Multiple FlexFEC streams in local SDP, but " 1840 "our implementation only supports a single FlexFEC " 1841 "stream. Will not enable FlexFEC for proposed " 1842 "stream with SSRC: " 1843 << flexfec_ssrc << "."; 1844 continue; 1845 } 1846 1847 flexfec_enabled = true; 1848 parameters_.config.rtp.flexfec.ssrc = flexfec_ssrc; 1849 parameters_.config.rtp.flexfec.protected_media_ssrcs = {primary_ssrc}; 1850 } 1851 } 1852 } 1853 1854 parameters_.config.rtp.c_name = sp.cname; 1855 if (rtp_extensions) { 1856 parameters_.config.rtp.extensions = *rtp_extensions; 1857 rtp_parameters_.header_extensions = *rtp_extensions; 1858 } 1859 parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size 1860 ? RtcpMode::kReducedSize 1861 : RtcpMode::kCompound; 1862 parameters_.config.rtp.mid = send_params.mid; 1863 rtp_parameters_.rtcp.reduced_size = send_params.rtcp.reduced_size; 1864 1865 if (codec_settings) { 1866 SetCodec(*codec_settings, codec_settings_list); 1867 } 1868 } 1869 1870 WebRtcVideoSendChannel::WebRtcVideoSendStream::~WebRtcVideoSendStream() { 1871 if (stream_ != nullptr) { 1872 call_->DestroyVideoSendStream(stream_); 1873 } 1874 } 1875 1876 bool WebRtcVideoSendChannel::WebRtcVideoSendStream::SetVideoSend( 1877 const VideoOptions* options, 1878 VideoSourceInterface<VideoFrame>* source) { 1879 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetVideoSend"); 1880 RTC_DCHECK_RUN_ON(&thread_checker_); 1881 1882 bool reconfiguration_needed = false; 1883 if (options) { 1884 VideoOptions old_options = parameters_.options; 1885 parameters_.options.SetAll(*options); 1886 if (parameters_.options.is_screencast.value_or(false) != 1887 old_options.is_screencast.value_or(false) && 1888 parameters_.codec_settings) { 1889 // If screen content settings change, we may need to recreate the codec 1890 // instance so that the correct type is used. 1891 1892 SetCodec(*parameters_.codec_settings, parameters_.codec_settings_list); 1893 // Mark screenshare parameter as being updated, then test for any other 1894 // changes that may require codec reconfiguration. 1895 old_options.is_screencast = options->is_screencast; 1896 } 1897 if (parameters_.options != old_options) { 1898 reconfiguration_needed = true; 1899 } 1900 } 1901 1902 if (source_ && stream_) { 1903 stream_->SetSource(nullptr, DegradationPreference::DISABLED); 1904 if (source && source != source_) { 1905 reconfiguration_needed = true; 1906 } 1907 } 1908 1909 if (reconfiguration_needed) { 1910 ReconfigureEncoder(nullptr); 1911 } 1912 1913 // Switch to the new source. 1914 source_ = source; 1915 if (source && stream_) { 1916 stream_->SetSource(source_, GetDegradationPreference()); 1917 } 1918 return true; 1919 } 1920 1921 DegradationPreference 1922 WebRtcVideoSendChannel::WebRtcVideoSendStream::GetDegradationPreference() 1923 const { 1924 // Do not adapt resolution for screen content as this will likely 1925 // result in blurry and unreadable text. 1926 // `this` acts like a VideoSource to make sure SinkWants are handled on the 1927 // correct thread. 1928 if (!enable_cpu_overuse_detection_) { 1929 return DegradationPreference::DISABLED; 1930 } 1931 1932 DegradationPreference degradation_preference; 1933 if (rtp_parameters_.degradation_preference.has_value()) { 1934 degradation_preference = *rtp_parameters_.degradation_preference; 1935 } else { 1936 if (parameters_.options.content_hint == 1937 VideoTrackInterface::ContentHint::kFluid) { 1938 degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE; 1939 } else if (parameters_.options.is_screencast.value_or(false) || 1940 parameters_.options.content_hint == 1941 VideoTrackInterface::ContentHint::kDetailed || 1942 parameters_.options.content_hint == 1943 VideoTrackInterface::ContentHint::kText) { 1944 degradation_preference = DegradationPreference::MAINTAIN_RESOLUTION; 1945 } else if (env_.field_trials().IsEnabled( 1946 "WebRTC-Video-BalancedDegradation")) { 1947 // Standard wants balanced by default, but it needs to be tuned first. 1948 degradation_preference = DegradationPreference::BALANCED; 1949 } else { 1950 // Keep MAINTAIN_FRAMERATE by default until BALANCED has been tuned for 1951 // all codecs and launched. 1952 degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE; 1953 } 1954 } 1955 1956 return degradation_preference; 1957 } 1958 1959 const std::vector<uint32_t>& 1960 WebRtcVideoSendChannel::WebRtcVideoSendStream::GetSsrcs() const { 1961 return ssrcs_; 1962 } 1963 1964 void WebRtcVideoSendChannel::WebRtcVideoSendStream::SetCodec( 1965 const VideoCodecSettings& codec_settings, 1966 const std::vector<VideoCodecSettings>& codec_settings_list) { 1967 RTC_DCHECK_RUN_ON(&thread_checker_); 1968 FallbackToDefaultScalabilityModeIfNotSupported( 1969 codec_settings.codec, parameters_.config, rtp_parameters_.encodings); 1970 1971 parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec); 1972 RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0); 1973 1974 parameters_.config.rtp.payload_name = codec_settings.codec.name; 1975 parameters_.config.rtp.payload_type = codec_settings.codec.id; 1976 parameters_.config.rtp.raw_payload = 1977 codec_settings.codec.packetization == kPacketizationParamRaw; 1978 parameters_.config.rtp.ulpfec = codec_settings.ulpfec; 1979 parameters_.config.rtp.flexfec.payload_type = 1980 codec_settings.flexfec_payload_type; 1981 1982 // Set RTX payload type if RTX is enabled. 1983 if (!parameters_.config.rtp.rtx.ssrcs.empty()) { 1984 if (codec_settings.rtx_payload_type == -1) { 1985 RTC_LOG(LS_WARNING) 1986 << "RTX SSRCs configured but there's no configured RTX " 1987 "payload type. Ignoring."; 1988 parameters_.config.rtp.rtx.ssrcs.clear(); 1989 } else { 1990 parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type; 1991 } 1992 } 1993 1994 const bool has_lntf = HasLntf(codec_settings.codec); 1995 parameters_.config.rtp.lntf.enabled = has_lntf; 1996 parameters_.config.encoder_settings.capabilities.loss_notification = has_lntf; 1997 1998 parameters_.config.rtp.nack.rtp_history_ms = 1999 HasNack(codec_settings.codec) ? kNackHistoryMs : 0; 2000 2001 parameters_.codec_settings = codec_settings; 2002 2003 // Settings for mixed-codec simulcast. 2004 if (codec_settings_list.empty()) { 2005 parameters_.config.rtp.stream_configs.clear(); 2006 } else { 2007 if (parameters_.config.rtp.ssrcs.size() == codec_settings_list.size()) { 2008 parameters_.config.rtp.stream_configs.resize( 2009 parameters_.config.rtp.ssrcs.size()); 2010 for (size_t i = 0; i < codec_settings_list.size(); i++) { 2011 auto& stream_config = parameters_.config.rtp.stream_configs[i]; 2012 const auto& cs = codec_settings_list[i]; 2013 stream_config.ssrc = parameters_.config.rtp.ssrcs[i]; 2014 if (i < parameters_.config.rtp.rids.size()) { 2015 stream_config.rid = parameters_.config.rtp.rids[i]; 2016 } 2017 stream_config.payload_name = cs.codec.name; 2018 stream_config.payload_type = cs.codec.id; 2019 stream_config.raw_payload = 2020 cs.codec.packetization == kPacketizationParamRaw; 2021 if (i < parameters_.config.rtp.rtx.ssrcs.size()) { 2022 auto& rtx = stream_config.rtx.emplace( 2023 decltype(stream_config.rtx)::value_type()); 2024 rtx.ssrc = parameters_.config.rtp.rtx.ssrcs[i]; 2025 rtx.payload_type = cs.rtx_payload_type; 2026 } 2027 } 2028 } else { 2029 // TODO(crbug.com/378724147): We need to investigate when it 2030 // has mismatched sizes. 2031 RTC_DCHECK_EQ(parameters_.config.rtp.ssrcs.size(), 2032 codec_settings_list.size()); 2033 2034 RTC_LOG(LS_ERROR) << "Mismatched sizes between codec_settings_list:" 2035 << codec_settings_list.size() 2036 << ", parameters_.config.rtp.ssrcs:" 2037 << parameters_.config.rtp.ssrcs.size(); 2038 } 2039 } 2040 2041 parameters_.codec_settings_list = codec_settings_list; 2042 2043 // TODO(bugs.webrtc.org/8830): Avoid recreation, it should be enough to call 2044 // ReconfigureEncoder. 2045 RTC_LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetCodec."; 2046 RecreateWebRtcStream(); 2047 } 2048 2049 void WebRtcVideoSendChannel::WebRtcVideoSendStream::SetSenderParameters( 2050 const ChangedSenderParameters& params) { 2051 RTC_DCHECK_RUN_ON(&thread_checker_); 2052 // `recreate_stream` means construction-time parameters have changed and the 2053 // sending stream needs to be reset with the new config. 2054 bool recreate_stream = false; 2055 if (params.rtcp_mode) { 2056 parameters_.config.rtp.rtcp_mode = *params.rtcp_mode; 2057 rtp_parameters_.rtcp.reduced_size = 2058 parameters_.config.rtp.rtcp_mode == RtcpMode::kReducedSize; 2059 recreate_stream = true; 2060 } 2061 if (params.extmap_allow_mixed) { 2062 parameters_.config.rtp.extmap_allow_mixed = *params.extmap_allow_mixed; 2063 recreate_stream = true; 2064 } 2065 if (params.rtp_header_extensions) { 2066 parameters_.config.rtp.extensions = *params.rtp_header_extensions; 2067 rtp_parameters_.header_extensions = *params.rtp_header_extensions; 2068 recreate_stream = true; 2069 } 2070 if (params.mid) { 2071 parameters_.config.rtp.mid = *params.mid; 2072 recreate_stream = true; 2073 } 2074 if (params.max_bandwidth_bps) { 2075 parameters_.max_bitrate_bps = *params.max_bandwidth_bps; 2076 ReconfigureEncoder(nullptr); 2077 } 2078 if (params.conference_mode) { 2079 parameters_.conference_mode = *params.conference_mode; 2080 } 2081 2082 // Set codecs and options. 2083 if (params.send_codec) { 2084 SetCodec(*params.send_codec, 2085 params.send_codecs.value_or(std::vector<VideoCodecSettings>())); 2086 recreate_stream = false; // SetCodec has already recreated the stream. 2087 } else if (params.conference_mode && parameters_.codec_settings) { 2088 SetCodec(*parameters_.codec_settings, parameters_.codec_settings_list); 2089 recreate_stream = false; // SetCodec has already recreated the stream. 2090 } 2091 if (recreate_stream) { 2092 RTC_LOG(LS_INFO) 2093 << "RecreateWebRtcStream (send) because of SetSenderParameters"; 2094 RecreateWebRtcStream(); 2095 } 2096 } 2097 2098 RTCError WebRtcVideoSendChannel::WebRtcVideoSendStream::SetRtpParameters( 2099 const RtpParameters& new_parameters, 2100 SetParametersCallback callback) { 2101 RTC_DCHECK_RUN_ON(&thread_checker_); 2102 // This is checked higher in the stack (RtpSender), so this is only checking 2103 // for users accessing the private APIs or tests, not specification 2104 // conformance. 2105 // TODO(orphis): Migrate tests to later make this a DCHECK only 2106 RTCError error = CheckRtpParametersInvalidModificationAndValues( 2107 rtp_parameters_, new_parameters, env_.field_trials()); 2108 if (!error.ok()) { 2109 return InvokeSetParametersCallback(callback, error); 2110 } 2111 2112 bool new_param = false; 2113 for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) { 2114 if ((new_parameters.encodings[i].min_bitrate_bps != 2115 rtp_parameters_.encodings[i].min_bitrate_bps) || 2116 (new_parameters.encodings[i].max_bitrate_bps != 2117 rtp_parameters_.encodings[i].max_bitrate_bps) || 2118 (new_parameters.encodings[i].max_framerate != 2119 rtp_parameters_.encodings[i].max_framerate) || 2120 (new_parameters.encodings[i].scale_resolution_down_by != 2121 rtp_parameters_.encodings[i].scale_resolution_down_by) || 2122 (new_parameters.encodings[i].num_temporal_layers != 2123 rtp_parameters_.encodings[i].num_temporal_layers) || 2124 (new_parameters.encodings[i].scale_resolution_down_to != 2125 rtp_parameters_.encodings[i].scale_resolution_down_to) || 2126 (new_parameters.encodings[i].scalability_mode != 2127 rtp_parameters_.encodings[i].scalability_mode) || 2128 (new_parameters.encodings[i].codec != 2129 rtp_parameters_.encodings[i].codec)) { 2130 new_param = true; 2131 break; 2132 } 2133 } 2134 2135 bool new_degradation_preference = false; 2136 if (new_parameters.degradation_preference != 2137 rtp_parameters_.degradation_preference) { 2138 new_degradation_preference = true; 2139 } 2140 2141 // Some fields (e.g. bitrate priority) only need to update the bitrate 2142 // allocator which is updated via ReconfigureEncoder (however, note that the 2143 // actual encoder should only be reconfigured if needed). 2144 bool reconfigure_encoder = 2145 new_param || (new_parameters.encodings[0].bitrate_priority != 2146 rtp_parameters_.encodings[0].bitrate_priority); 2147 2148 // Note that the simulcast encoder adapter relies on the fact that layers 2149 // de/activation triggers encoder reinitialization. 2150 bool new_send_state = false; 2151 for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) { 2152 bool new_active = IsLayerActive(new_parameters.encodings[i]); 2153 bool old_active = IsLayerActive(rtp_parameters_.encodings[i]); 2154 if (new_active != old_active) { 2155 new_send_state = true; 2156 } 2157 } 2158 bool new_csrcs = 2159 (new_parameters.encodings[0].csrcs.has_value() && 2160 new_parameters.encodings[0].csrcs != rtp_parameters_.encodings[0].csrcs); 2161 2162 rtp_parameters_ = new_parameters; 2163 // Codecs are currently handled at the WebRtcVideoSendChannel level. 2164 rtp_parameters_.codecs.clear(); 2165 if (reconfigure_encoder || new_send_state) { 2166 // Callback responsibility is delegated to ReconfigureEncoder() 2167 ReconfigureEncoder(std::move(callback)); 2168 callback = nullptr; 2169 } 2170 if (new_degradation_preference) { 2171 if (source_ && stream_) { 2172 stream_->SetSource(source_, GetDegradationPreference()); 2173 } 2174 } 2175 if (stream_ && new_csrcs) { 2176 stream_->SetCsrcs(rtp_parameters_.encodings[0].csrcs.value()); 2177 } 2178 // Check if a key frame was requested via setParameters. 2179 std::vector<std::string> key_frames_requested_by_rid; 2180 for (const auto& encoding : rtp_parameters_.encodings) { 2181 if (encoding.request_key_frame) { 2182 key_frames_requested_by_rid.push_back(encoding.rid); 2183 } 2184 } 2185 if (!key_frames_requested_by_rid.empty()) { 2186 if (key_frames_requested_by_rid.size() == 1 && 2187 key_frames_requested_by_rid[0].empty()) { 2188 // For non-simulcast cases there is no rid, 2189 // request a keyframe on all layers. 2190 key_frames_requested_by_rid.clear(); 2191 } 2192 GenerateKeyFrame(key_frames_requested_by_rid); 2193 } 2194 return InvokeSetParametersCallback(callback, RTCError::OK()); 2195 } 2196 2197 RtpParameters WebRtcVideoSendChannel::WebRtcVideoSendStream::GetRtpParameters() 2198 const { 2199 RTC_DCHECK_RUN_ON(&thread_checker_); 2200 return rtp_parameters_; 2201 } 2202 2203 void WebRtcVideoSendChannel::WebRtcVideoSendStream::SetFrameEncryptor( 2204 scoped_refptr<FrameEncryptorInterface> frame_encryptor) { 2205 RTC_DCHECK_RUN_ON(&thread_checker_); 2206 parameters_.config.frame_encryptor = frame_encryptor; 2207 if (stream_) { 2208 RTC_LOG(LS_INFO) 2209 << "RecreateWebRtcStream (send) because of SetFrameEncryptor, ssrc=" 2210 << parameters_.config.rtp.ssrcs[0]; 2211 RecreateWebRtcStream(); 2212 } 2213 } 2214 2215 void WebRtcVideoSendChannel::WebRtcVideoSendStream::SetEncoderSelector( 2216 VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) { 2217 RTC_DCHECK_RUN_ON(&thread_checker_); 2218 parameters_.config.encoder_selector = encoder_selector; 2219 if (stream_) { 2220 RTC_LOG(LS_INFO) 2221 << "RecreateWebRtcStream (send) because of SetEncoderSelector, ssrc=" 2222 << parameters_.config.rtp.ssrcs[0]; 2223 RecreateWebRtcStream(); 2224 } 2225 } 2226 2227 void WebRtcVideoSendChannel::WebRtcVideoSendStream::UpdateSendState() { 2228 RTC_DCHECK_RUN_ON(&thread_checker_); 2229 if (sending_) { 2230 RTC_DCHECK(stream_ != nullptr); 2231 // This allows the the Stream to be used. Ie, DTLS is connected and the 2232 // RtpTransceiver direction allows sending. 2233 stream_->Start(); 2234 } else { 2235 if (stream_ != nullptr) { 2236 stream_->Stop(); 2237 } 2238 } 2239 } 2240 2241 VideoEncoderConfig 2242 WebRtcVideoSendChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig( 2243 const Codec& codec) const { 2244 RTC_DCHECK_RUN_ON(&thread_checker_); 2245 VideoEncoderConfig encoder_config; 2246 encoder_config.codec_type = PayloadStringToCodecType(codec.name); 2247 encoder_config.video_format = SdpVideoFormat(codec.name, codec.params); 2248 2249 bool is_screencast = parameters_.options.is_screencast.value_or(false); 2250 if (is_screencast) { 2251 encoder_config.min_transmit_bitrate_bps = 2252 1000 * parameters_.options.screencast_min_bitrate_kbps.value_or(0); 2253 encoder_config.content_type = VideoEncoderConfig::ContentType::kScreen; 2254 } else { 2255 encoder_config.min_transmit_bitrate_bps = 0; 2256 encoder_config.content_type = 2257 VideoEncoderConfig::ContentType::kRealtimeVideo; 2258 } 2259 2260 // By default, the stream count for the codec configuration should match the 2261 // number of negotiated ssrcs but this may be capped below depending on the 2262 // `legacy_scalability_mode` and codec used. 2263 encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size(); 2264 bool legacy_scalability_mode = true; 2265 for (const RtpEncodingParameters& encoding : rtp_parameters_.encodings) { 2266 if (encoding.scalability_mode.has_value() && 2267 (encoding.scale_resolution_down_by.has_value() || 2268 encoding.scale_resolution_down_to.has_value())) { 2269 legacy_scalability_mode = false; 2270 break; 2271 } 2272 } 2273 // Maybe limit the number of simulcast layers depending on 2274 // `legacy_scalability_mode`, codec types (VP9/AV1). This path only exists 2275 // for backwards compatibility and will one day be deleted. If you want SVC, 2276 // please specify with the `scalability_mode` API instead amd disabling all 2277 // but one encoding. 2278 if (IsCodecDisabledForSimulcast(legacy_scalability_mode, 2279 encoder_config.codec_type)) { 2280 encoder_config.number_of_streams = 1; 2281 } 2282 2283 // parameters_.max_bitrate comes from the max bitrate set at the SDP 2284 // (m-section) level with the attribute "b=AS." Note that stream max bitrate 2285 // is the RtpSender's max bitrate, but each individual encoding may also have 2286 // its own max bitrate specified by SetParameters. 2287 int stream_max_bitrate = parameters_.max_bitrate_bps; 2288 // The codec max bitrate comes from the "x-google-max-bitrate" parameter 2289 // attribute set in the SDP for a specific codec. It only has an effect if 2290 // max bitrate is not specified through other means. 2291 bool encodings_has_max_bitrate = false; 2292 for (const auto& encoding : rtp_parameters_.encodings) { 2293 if (encoding.active && encoding.max_bitrate_bps.value_or(0) > 0) { 2294 encodings_has_max_bitrate = true; 2295 break; 2296 } 2297 } 2298 int codec_max_bitrate_kbps; 2299 if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps) && 2300 stream_max_bitrate == -1 && !encodings_has_max_bitrate) { 2301 stream_max_bitrate = codec_max_bitrate_kbps * 1000; 2302 } 2303 encoder_config.max_bitrate_bps = stream_max_bitrate; 2304 2305 // The encoder config's default bitrate priority is set to 1.0, 2306 // unless it is set through the sender's encoding parameters. 2307 // The bitrate priority, which is used in the bitrate allocation, is done 2308 // on a per sender basis, so we use the first encoding's value. 2309 encoder_config.bitrate_priority = 2310 rtp_parameters_.encodings[0].bitrate_priority; 2311 2312 // Application-controlled state is held in the encoder_config's 2313 // simulcast_layers. Currently this is used to control which simulcast layers 2314 // are active and for configuring the min/max bitrate and max framerate. 2315 // The encoder_config's simulcast_layers is also used for non-simulcast (when 2316 // there is a single layer). 2317 RTC_DCHECK_GE(rtp_parameters_.encodings.size(), 2318 encoder_config.number_of_streams); 2319 RTC_DCHECK_GT(encoder_config.number_of_streams, 0); 2320 2321 // Copy all provided constraints. 2322 encoder_config.simulcast_layers.resize(rtp_parameters_.encodings.size()); 2323 for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) { 2324 encoder_config.simulcast_layers[i].active = 2325 rtp_parameters_.encodings[i].active; 2326 encoder_config.simulcast_layers[i].scalability_mode = 2327 ScalabilityModeFromString( 2328 rtp_parameters_.encodings[i].scalability_mode.value_or("")); 2329 if (rtp_parameters_.encodings[i].min_bitrate_bps) { 2330 encoder_config.simulcast_layers[i].min_bitrate_bps = 2331 *rtp_parameters_.encodings[i].min_bitrate_bps; 2332 } 2333 if (rtp_parameters_.encodings[i].max_bitrate_bps) { 2334 encoder_config.simulcast_layers[i].max_bitrate_bps = 2335 *rtp_parameters_.encodings[i].max_bitrate_bps; 2336 } 2337 if (rtp_parameters_.encodings[i].max_framerate) { 2338 encoder_config.simulcast_layers[i].max_framerate = 2339 *rtp_parameters_.encodings[i].max_framerate; 2340 } 2341 if (rtp_parameters_.encodings[i].scale_resolution_down_by) { 2342 encoder_config.simulcast_layers[i].scale_resolution_down_by = 2343 *rtp_parameters_.encodings[i].scale_resolution_down_by; 2344 } 2345 if (rtp_parameters_.encodings[i].num_temporal_layers) { 2346 encoder_config.simulcast_layers[i].num_temporal_layers = 2347 *rtp_parameters_.encodings[i].num_temporal_layers; 2348 } 2349 encoder_config.simulcast_layers[i].scale_resolution_down_to = 2350 rtp_parameters_.encodings[i].scale_resolution_down_to; 2351 } 2352 2353 encoder_config.legacy_conference_mode = parameters_.conference_mode; 2354 2355 encoder_config.is_quality_scaling_allowed = 2356 !disable_automatic_resize_ && !is_screencast && 2357 (parameters_.config.rtp.ssrcs.size() == 1 || 2358 NumActiveStreams(rtp_parameters_) == 1); 2359 2360 // Ensure frame dropping is always enabled. 2361 encoder_config.frame_drop_enabled = true; 2362 2363 int max_qp = -1; 2364 if (codec.GetParam(kCodecParamMaxQuantization, &max_qp) && max_qp > 0) { 2365 encoder_config.max_qp = max_qp; 2366 } 2367 2368 return encoder_config; 2369 } 2370 2371 void WebRtcVideoSendChannel::WebRtcVideoSendStream::ReconfigureEncoder( 2372 SetParametersCallback callback) { 2373 RTC_DCHECK_RUN_ON(&thread_checker_); 2374 if (!stream_) { 2375 // The VideoSendStream `stream_` has not yet been created but other 2376 // parameters has changed. 2377 InvokeSetParametersCallback(callback, RTCError::OK()); 2378 return; 2379 } 2380 2381 RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0); 2382 2383 RTC_CHECK(parameters_.codec_settings); 2384 VideoCodecSettings codec_settings = *parameters_.codec_settings; 2385 2386 FallbackToDefaultScalabilityModeIfNotSupported( 2387 codec_settings.codec, parameters_.config, rtp_parameters_.encodings); 2388 2389 // Latest config, with and without encoder specfic settings. 2390 VideoEncoderConfig encoder_config = 2391 CreateVideoEncoderConfig(codec_settings.codec); 2392 encoder_config.encoder_specific_settings = 2393 ConfigureVideoEncoderSettings(codec_settings.codec); 2394 VideoEncoderConfig encoder_config_with_specifics = encoder_config.Copy(); 2395 encoder_config.encoder_specific_settings = nullptr; 2396 2397 // When switching between legacy SVC (3 encodings interpreted as 1 stream with 2398 // 3 spatial layers) and the standard API (3 encodings = 3 streams and spatial 2399 // layers specified by `scalability_mode`), the number of streams can change. 2400 bool num_streams_changed = parameters_.encoder_config.number_of_streams != 2401 encoder_config.number_of_streams; 2402 parameters_.encoder_config = std::move(encoder_config); 2403 2404 if (num_streams_changed) { 2405 // The app is switching between legacy and standard modes, recreate instead 2406 // of reconfiguring to avoid number of streams not matching in lower layers. 2407 RecreateWebRtcStream(); 2408 InvokeSetParametersCallback(callback, RTCError::OK()); 2409 return; 2410 } 2411 2412 stream_->ReconfigureVideoEncoder(std::move(encoder_config_with_specifics), 2413 std::move(callback)); 2414 } 2415 2416 void WebRtcVideoSendChannel::WebRtcVideoSendStream::SetSend(bool send) { 2417 RTC_DCHECK_RUN_ON(&thread_checker_); 2418 sending_ = send; 2419 UpdateSendState(); 2420 } 2421 2422 std::vector<VideoSenderInfo> 2423 WebRtcVideoSendChannel::WebRtcVideoSendStream::GetPerLayerVideoSenderInfos( 2424 bool log_stats) { 2425 RTC_DCHECK_RUN_ON(&thread_checker_); 2426 VideoSenderInfo common_info; 2427 if (parameters_.codec_settings) { 2428 common_info.codec_name = parameters_.codec_settings->codec.name; 2429 common_info.codec_payload_type = parameters_.codec_settings->codec.id; 2430 } 2431 // If SVC is used, one stream is configured but multiple encodings exist. This 2432 // is not spec-compliant, but it is how we've implemented SVC so this affects 2433 // how the RTP stream's "active" value is determined. 2434 bool is_svc = (parameters_.encoder_config.number_of_streams == 1 && 2435 rtp_parameters_.encodings.size() > 1); 2436 std::vector<VideoSenderInfo> infos; 2437 VideoSendStream::Stats stats; 2438 if (stream_ == nullptr) { 2439 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) { 2440 common_info.add_ssrc(ssrc); 2441 } 2442 infos.push_back(common_info); 2443 return infos; 2444 } else { 2445 stats = stream_->GetStats(); 2446 if (log_stats) 2447 RTC_LOG(LS_INFO) << stats.ToString(env_.clock().TimeInMilliseconds()); 2448 2449 // Metrics that are in common for all substreams. 2450 common_info.adapt_changes = stats.number_of_cpu_adapt_changes; 2451 common_info.adapt_reason = 2452 stats.cpu_limited_resolution ? ADAPTREASON_CPU : ADAPTREASON_NONE; 2453 common_info.has_entered_low_resolution = stats.has_entered_low_resolution; 2454 2455 // Get bandwidth limitation info from stream_->GetStats(). 2456 // Input resolution (output from video_adapter) can be further scaled down 2457 // or higher video layer(s) can be dropped due to bitrate constraints. 2458 // Note, adapt_changes only include changes from the video_adapter. 2459 if (stats.bw_limited_resolution) 2460 common_info.adapt_reason |= ADAPTREASON_BANDWIDTH; 2461 2462 common_info.quality_limitation_reason = stats.quality_limitation_reason; 2463 common_info.quality_limitation_durations_ms = 2464 stats.quality_limitation_durations_ms; 2465 common_info.quality_limitation_resolution_changes = 2466 stats.quality_limitation_resolution_changes; 2467 common_info.encoder_implementation_name = stats.encoder_implementation_name; 2468 common_info.ssrc_groups = ssrc_groups_; 2469 common_info.frames = stats.frames; 2470 common_info.framerate_input = stats.input_frame_rate; 2471 common_info.avg_encode_ms = stats.avg_encode_time_ms; 2472 common_info.encode_usage_percent = stats.encode_usage_percent; 2473 common_info.nominal_bitrate = stats.media_bitrate_bps; 2474 common_info.content_type = stats.content_type; 2475 common_info.aggregated_framerate_sent = stats.encode_frame_rate; 2476 common_info.aggregated_huge_frames_sent = stats.huge_frames_sent; 2477 common_info.power_efficient_encoder = stats.power_efficient_encoder; 2478 2479 // The "typical case" where `substreams` exist because we have negotiated 2480 // and connected is handled below, but prior to that `substreams` is empty. 2481 // In this case we still need to return one "info" per SSRC and set a few 2482 // stats that should never be missing. 2483 if (stats.substreams.empty()) { 2484 size_t encoding_index = 0; 2485 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) { 2486 auto info = common_info; 2487 info.add_ssrc(ssrc); 2488 info.rid = parameters_.config.rtp.GetRidForSsrc(ssrc); 2489 info.encoding_index = encoding_index; 2490 info.active = IsActiveFromEncodings( 2491 !is_svc ? std::optional<uint32_t>(ssrc) : std::nullopt, 2492 rtp_parameters_.encodings); 2493 ++encoding_index; 2494 infos.push_back(info); 2495 } 2496 return infos; 2497 } 2498 } 2499 // Merge `stats.substreams`, which may contain additional SSRCs for RTX or 2500 // Flexfec, with media SSRCs. This results in a set of substreams that match 2501 // with the outbound-rtp stats objects. 2502 auto outbound_rtp_substreams = 2503 MergeInfoAboutOutboundRtpSubstreams(stats.substreams); 2504 // The streams are ordered by SSRC, but the SSRCs are randomly assigned so we 2505 // need map for index lookup by SSRC. 2506 std::map<uint32_t, size_t> encoding_index_by_ssrc; 2507 for (size_t i = 0; i < parameters_.config.rtp.ssrcs.size(); ++i) { 2508 encoding_index_by_ssrc[parameters_.config.rtp.ssrcs[i]] = i; 2509 } 2510 for (const auto& pair : outbound_rtp_substreams) { 2511 auto info = common_info; 2512 uint32_t ssrc = pair.first; 2513 info.add_ssrc(ssrc); 2514 info.rid = parameters_.config.rtp.GetRidForSsrc(ssrc); 2515 if (encoding_index_by_ssrc.find(ssrc) != encoding_index_by_ssrc.end()) { 2516 info.encoding_index = encoding_index_by_ssrc[ssrc]; 2517 } 2518 info.active = IsActiveFromEncodings( 2519 !is_svc ? std::optional<uint32_t>(ssrc) : std::nullopt, 2520 rtp_parameters_.encodings); 2521 auto stream_stats = pair.second; 2522 RTC_DCHECK_EQ(stream_stats.type, 2523 VideoSendStream::StreamStats::StreamType::kMedia); 2524 info.payload_bytes_sent = stream_stats.rtp_stats.transmitted.payload_bytes; 2525 info.header_and_padding_bytes_sent = 2526 stream_stats.rtp_stats.transmitted.header_bytes + 2527 stream_stats.rtp_stats.transmitted.padding_bytes; 2528 info.packets_sent = stream_stats.rtp_stats.transmitted.packets; 2529 info.packets_sent_with_ect1 = 2530 stream_stats.rtp_stats.transmitted.packets_with_ect1; 2531 info.total_packet_send_delay += 2532 stream_stats.rtp_stats.transmitted.total_packet_delay; 2533 info.send_frame_width = stream_stats.width; 2534 info.send_frame_height = stream_stats.height; 2535 info.key_frames_encoded = stream_stats.frame_counts.key_frames; 2536 info.framerate_sent = stream_stats.encode_frame_rate; 2537 info.frames_encoded = stream_stats.frames_encoded; 2538 info.frames_sent = stream_stats.frames_encoded; 2539 info.retransmitted_bytes_sent = 2540 stream_stats.rtp_stats.retransmitted.payload_bytes; 2541 info.retransmitted_packets_sent = 2542 stream_stats.rtp_stats.retransmitted.packets; 2543 info.firs_received = stream_stats.rtcp_packet_type_counts.fir_packets; 2544 info.nacks_received = stream_stats.rtcp_packet_type_counts.nack_packets; 2545 info.plis_received = stream_stats.rtcp_packet_type_counts.pli_packets; 2546 if (stream_stats.report_block_data.has_value()) { 2547 info.packets_lost = stream_stats.report_block_data->cumulative_lost(); 2548 info.fraction_lost = stream_stats.report_block_data->fraction_lost(); 2549 info.report_block_datas.push_back(*stream_stats.report_block_data); 2550 } 2551 info.qp_sum = stream_stats.qp_sum; 2552 info.psnr_sum = stream_stats.psnr_sum; 2553 info.psnr_measurements = stream_stats.psnr_measurements; 2554 info.total_encode_time_ms = stream_stats.total_encode_time_ms; 2555 info.total_encoded_bytes_target = stream_stats.total_encoded_bytes_target; 2556 info.huge_frames_sent = stream_stats.huge_frames_sent; 2557 info.scalability_mode = stream_stats.scalability_mode; 2558 info.target_bitrate = stream_stats.target_bitrate; 2559 infos.push_back(info); 2560 } 2561 return infos; 2562 } 2563 2564 VideoSenderInfo 2565 WebRtcVideoSendChannel::WebRtcVideoSendStream::GetAggregatedVideoSenderInfo( 2566 const std::vector<VideoSenderInfo>& infos) const { 2567 RTC_DCHECK_RUN_ON(&thread_checker_); 2568 RTC_CHECK(!infos.empty()); 2569 if (infos.size() == 1) { 2570 return infos[0]; 2571 } 2572 VideoSenderInfo info = infos[0]; 2573 info.encoding_index = std::nullopt; // An aggregated info has no index. 2574 info.local_stats.clear(); 2575 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) { 2576 info.add_ssrc(ssrc); 2577 } 2578 info.framerate_sent = info.aggregated_framerate_sent; 2579 info.huge_frames_sent = info.aggregated_huge_frames_sent; 2580 2581 for (size_t i = 1; i < infos.size(); i++) { 2582 info.key_frames_encoded += infos[i].key_frames_encoded; 2583 info.payload_bytes_sent += infos[i].payload_bytes_sent; 2584 info.header_and_padding_bytes_sent += 2585 infos[i].header_and_padding_bytes_sent; 2586 info.packets_sent += infos[i].packets_sent; 2587 info.total_packet_send_delay += infos[i].total_packet_send_delay; 2588 info.retransmitted_bytes_sent += infos[i].retransmitted_bytes_sent; 2589 info.retransmitted_packets_sent += infos[i].retransmitted_packets_sent; 2590 info.packets_lost += infos[i].packets_lost; 2591 if (infos[i].send_frame_width > info.send_frame_width) 2592 info.send_frame_width = infos[i].send_frame_width; 2593 if (infos[i].send_frame_height > info.send_frame_height) 2594 info.send_frame_height = infos[i].send_frame_height; 2595 info.firs_received += infos[i].firs_received; 2596 info.nacks_received += infos[i].nacks_received; 2597 info.plis_received += infos[i].plis_received; 2598 if (!infos[i].report_block_datas.empty()) 2599 info.report_block_datas.push_back(infos[i].report_block_datas[0]); 2600 if (infos[i].qp_sum) { 2601 if (!info.qp_sum) { 2602 info.qp_sum = 0; 2603 } 2604 info.qp_sum = *info.qp_sum + *infos[i].qp_sum; 2605 } 2606 if (infos[i].psnr_measurements > 0) { 2607 info.psnr_measurements += infos[i].psnr_measurements; 2608 info.psnr_sum.y += infos[i].psnr_sum.y; 2609 info.psnr_sum.u += infos[i].psnr_sum.u; 2610 info.psnr_sum.v += infos[i].psnr_sum.v; 2611 } 2612 info.frames_encoded += infos[i].frames_encoded; 2613 info.frames_sent += infos[i].frames_sent; 2614 info.total_encode_time_ms += infos[i].total_encode_time_ms; 2615 info.total_encoded_bytes_target += infos[i].total_encoded_bytes_target; 2616 } 2617 return info; 2618 } 2619 2620 void WebRtcVideoSendChannel::WebRtcVideoSendStream::FillBitrateInfo( 2621 BandwidthEstimationInfo* bwe_info) { 2622 RTC_DCHECK_RUN_ON(&thread_checker_); 2623 if (stream_ == nullptr) { 2624 return; 2625 } 2626 VideoSendStream::Stats stats = stream_->GetStats(); 2627 for (const auto& it : stats.substreams) { 2628 bwe_info->transmit_bitrate += it.second.total_bitrate_bps; 2629 bwe_info->retransmit_bitrate += it.second.retransmit_bitrate_bps; 2630 } 2631 bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps; 2632 bwe_info->actual_enc_bitrate += stats.media_bitrate_bps; 2633 } 2634 2635 void WebRtcVideoSendChannel::WebRtcVideoSendStream:: 2636 SetEncoderToPacketizerFrameTransformer( 2637 scoped_refptr<FrameTransformerInterface> frame_transformer) { 2638 RTC_DCHECK_RUN_ON(&thread_checker_); 2639 parameters_.config.frame_transformer = std::move(frame_transformer); 2640 if (stream_) 2641 RecreateWebRtcStream(); 2642 } 2643 2644 void WebRtcVideoSendChannel::WebRtcVideoSendStream::RecreateWebRtcStream() { 2645 RTC_DCHECK_RUN_ON(&thread_checker_); 2646 2647 RTC_CHECK(parameters_.codec_settings); 2648 RTC_DCHECK_EQ((parameters_.encoder_config.content_type == 2649 VideoEncoderConfig::ContentType::kScreen), 2650 parameters_.options.is_screencast.value_or(false)) 2651 << "encoder content type inconsistent with screencast option"; 2652 parameters_.encoder_config.encoder_specific_settings = 2653 ConfigureVideoEncoderSettings(parameters_.codec_settings->codec); 2654 2655 VideoSendStream::Config config = parameters_.config.Copy(); 2656 if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) { 2657 RTC_LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " 2658 "payload type the set codec. Ignoring RTX."; 2659 config.rtp.rtx.ssrcs.clear(); 2660 } 2661 if (parameters_.encoder_config.number_of_streams == 1) { 2662 // SVC is used instead of simulcast. Remove unnecessary SSRCs. 2663 if (config.rtp.ssrcs.size() > 1) { 2664 config.rtp.ssrcs.resize(1); 2665 if (config.rtp.rtx.ssrcs.size() > 1) { 2666 config.rtp.rtx.ssrcs.resize(1); 2667 } 2668 } 2669 } 2670 2671 if (RtpExtension::FindHeaderExtensionByUri( 2672 config.rtp.extensions, RtpExtension::kCorruptionDetectionUri, 2673 RtpExtension::kRequireEncryptedExtension)) { 2674 config.encoder_settings.enable_frame_instrumentation_generator = true; 2675 } 2676 2677 if (stream_ != nullptr) { 2678 // TODO: webrtc:40644448 - Make sure the stats are not updated between 2679 // GetStats and DestroyVideoSendStream. 2680 VideoSendStream::Stats stats = stream_->GetStats(); 2681 call_->DestroyVideoSendStream(stream_); 2682 stream_ = call_->CreateVideoSendStream(std::move(config), 2683 parameters_.encoder_config.Copy()); 2684 stream_->SetStats(stats); 2685 } else { 2686 stream_ = call_->CreateVideoSendStream(std::move(config), 2687 parameters_.encoder_config.Copy()); 2688 } 2689 if (!rtp_parameters_.encodings.empty() && 2690 rtp_parameters_.encodings[0].csrcs.has_value()) { 2691 stream_->SetCsrcs(rtp_parameters_.encodings[0].csrcs.value()); 2692 } 2693 2694 parameters_.encoder_config.encoder_specific_settings = nullptr; 2695 2696 // Calls stream_->StartPerRtpStream() to start the VideoSendStream 2697 // if necessary conditions are met. 2698 UpdateSendState(); 2699 2700 // Attach the source after starting the send stream to prevent frames from 2701 // being injected into a not-yet initializated video stream encoder. 2702 if (source_) { 2703 stream_->SetSource(source_, GetDegradationPreference()); 2704 } 2705 } 2706 2707 void WebRtcVideoSendChannel::WebRtcVideoSendStream::GenerateKeyFrame( 2708 const std::vector<std::string>& rids) { 2709 RTC_DCHECK_RUN_ON(&thread_checker_); 2710 if (stream_ != nullptr) { 2711 stream_->GenerateKeyFrame(rids); 2712 } else { 2713 RTC_LOG(LS_WARNING) 2714 << "Absent send stream; ignoring request to generate keyframe."; 2715 } 2716 } 2717 2718 void WebRtcVideoSendChannel::GenerateSendKeyFrame( 2719 uint32_t ssrc, 2720 const std::vector<std::string>& rids) { 2721 RTC_DCHECK_RUN_ON(&thread_checker_); 2722 auto it = send_streams_.find(ssrc); 2723 if (it != send_streams_.end()) { 2724 it->second->GenerateKeyFrame(rids); 2725 } else { 2726 RTC_LOG(LS_ERROR) 2727 << "Absent send stream; ignoring key frame generation for ssrc " 2728 << ssrc; 2729 } 2730 } 2731 2732 void WebRtcVideoSendChannel::SetEncoderToPacketizerFrameTransformer( 2733 uint32_t ssrc, 2734 scoped_refptr<FrameTransformerInterface> frame_transformer) { 2735 RTC_DCHECK_RUN_ON(&thread_checker_); 2736 auto matching_stream = send_streams_.find(ssrc); 2737 if (matching_stream != send_streams_.end()) { 2738 matching_stream->second->SetEncoderToPacketizerFrameTransformer( 2739 std::move(frame_transformer)); 2740 } 2741 } 2742 2743 // ------------------------ WebRtcVideoReceiveChannel --------------------- 2744 WebRtcVideoReceiveChannel::WebRtcVideoReceiveChannel( 2745 const Environment& env, 2746 Call* call, 2747 const MediaConfig& config, 2748 const VideoOptions& options, 2749 const CryptoOptions& crypto_options, 2750 VideoDecoderFactory* decoder_factory) 2751 : MediaChannelUtil(call->network_thread(), config.enable_dscp), 2752 env_(env), 2753 worker_thread_(call->worker_thread()), 2754 receiving_(false), 2755 call_(call), 2756 default_sink_(nullptr), 2757 video_config_(config.video), 2758 decoder_factory_(decoder_factory), 2759 default_send_options_(options), 2760 last_receive_stats_log_ms_(-1), 2761 discard_unknown_ssrc_packets_(env_.field_trials().IsEnabled( 2762 "WebRTC-Video-DiscardPacketsWithUnknownSsrc")), 2763 crypto_options_(crypto_options), 2764 receive_buffer_size_(ParseReceiveBufferSize(env_.field_trials())) { 2765 RTC_DCHECK_RUN_ON(&thread_checker_); 2766 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; 2767 // Crash if MapCodecs fails. 2768 recv_codecs_ = MapCodecs(GetPayloadTypesAndDefaultCodecs( 2769 decoder_factory_, /*is_decoder_factory=*/true, 2770 /*include_rtx=*/true, env_.field_trials())) 2771 .value(); 2772 recv_flexfec_payload_type_ = 2773 recv_codecs_.empty() ? 0 : recv_codecs_.front().flexfec_payload_type; 2774 } 2775 2776 WebRtcVideoReceiveChannel::~WebRtcVideoReceiveChannel() { 2777 for (auto& kv : receive_streams_) 2778 delete kv.second; 2779 } 2780 2781 void WebRtcVideoReceiveChannel::SetReceiverFeedbackParameters( 2782 bool lntf_enabled, 2783 bool nack_enabled, 2784 RtcpMode rtcp_mode, 2785 std::optional<int> rtx_time) { 2786 RTC_DCHECK_RUN_ON(&thread_checker_); 2787 2788 // Update receive feedback parameters from new codec or RTCP mode. 2789 for (auto& kv : receive_streams_) { 2790 RTC_DCHECK(kv.second != nullptr); 2791 kv.second->SetFeedbackParameters(lntf_enabled, nack_enabled, rtcp_mode, 2792 rtx_time); 2793 } 2794 // Store for future creation of receive streams 2795 rtp_config_.lntf.enabled = lntf_enabled; 2796 if (nack_enabled) { 2797 rtp_config_.nack.rtp_history_ms = kNackHistoryMs; 2798 } else { 2799 rtp_config_.nack.rtp_history_ms = 0; 2800 } 2801 rtp_config_.rtcp_mode = rtcp_mode; 2802 // Note: There is no place in config to store rtx_time. 2803 } 2804 2805 RtpParameters WebRtcVideoReceiveChannel::GetRtpReceiverParameters( 2806 uint32_t ssrc) const { 2807 RTC_DCHECK_RUN_ON(&thread_checker_); 2808 RtpParameters rtp_params; 2809 auto it = receive_streams_.find(ssrc); 2810 if (it == receive_streams_.end()) { 2811 RTC_LOG(LS_WARNING) 2812 << "Attempting to get RTP receive parameters for stream " 2813 "with SSRC " 2814 << ssrc << " which doesn't exist."; 2815 return RtpParameters(); 2816 } 2817 rtp_params = it->second->GetRtpParameters(); 2818 rtp_params.header_extensions = recv_rtp_extensions_; 2819 2820 // Add codecs, which any stream is prepared to receive. 2821 for (const Codec& codec : recv_params_.codecs) { 2822 rtp_params.codecs.push_back(codec.ToCodecParameters()); 2823 } 2824 2825 return rtp_params; 2826 } 2827 2828 RtpParameters WebRtcVideoReceiveChannel::GetDefaultRtpReceiveParameters() 2829 const { 2830 RTC_DCHECK_RUN_ON(&thread_checker_); 2831 RtpParameters rtp_params; 2832 if (!default_sink_) { 2833 // Getting parameters on a default, unsignaled video receive stream but 2834 // because we've not configured to receive such a stream, `encodings` is 2835 // empty. 2836 return rtp_params; 2837 } 2838 rtp_params.encodings.emplace_back(); 2839 2840 // Add codecs, which any stream is prepared to receive. 2841 for (const Codec& codec : recv_params_.codecs) { 2842 rtp_params.codecs.push_back(codec.ToCodecParameters()); 2843 } 2844 2845 return rtp_params; 2846 } 2847 2848 bool WebRtcVideoReceiveChannel::GetChangedReceiverParameters( 2849 const VideoReceiverParameters& params, 2850 ChangedReceiverParameters* changed_params) const { 2851 if (!ValidateCodecFormats(params.codecs) || 2852 !ValidateRtpExtensions(params.extensions, recv_rtp_extensions_)) { 2853 return false; 2854 } 2855 2856 // Handle receive codecs. 2857 auto result = MapCodecs(params.codecs); 2858 if (!result.ok()) { 2859 RTC_LOG(LS_ERROR) << "GetChangedReceiverParameters called without valid " 2860 "video codecs, error =" 2861 << result.error(); 2862 return false; 2863 } 2864 const std::vector<VideoCodecSettings> mapped_codecs = result.value(); 2865 2866 // Verify that every mapped codec is supported locally. 2867 if (params.is_stream_active) { 2868 const std::vector<Codec> local_supported_codecs = 2869 GetPayloadTypesAndDefaultCodecs(decoder_factory_, 2870 /*is_decoder_factory=*/true, 2871 /*include_rtx=*/true, 2872 env_.field_trials()); 2873 for (const VideoCodecSettings& mapped_codec : mapped_codecs) { 2874 if (!FindMatchingVideoCodec(local_supported_codecs, mapped_codec.codec)) { 2875 RTC_LOG(LS_ERROR) << "GetChangedReceiverParameters called with " 2876 "unsupported video codec: " 2877 << mapped_codec.codec.ToString(); 2878 return false; 2879 } 2880 } 2881 } 2882 2883 if (NonFlexfecReceiveCodecsHaveChanged(recv_codecs_, mapped_codecs)) { 2884 changed_params->codec_settings = 2885 std::optional<std::vector<VideoCodecSettings>>(mapped_codecs); 2886 } 2887 2888 // Handle RTP header extensions. 2889 std::vector<RtpExtension> filtered_extensions = 2890 FilterRtpExtensions(params.extensions, RtpExtension::IsSupportedForVideo, 2891 false, env_.field_trials()); 2892 if (filtered_extensions != recv_rtp_extensions_) { 2893 changed_params->rtp_header_extensions = 2894 std::optional<std::vector<RtpExtension>>(filtered_extensions); 2895 } 2896 2897 int flexfec_payload_type = mapped_codecs.front().flexfec_payload_type; 2898 if (flexfec_payload_type != recv_flexfec_payload_type_) { 2899 changed_params->flexfec_payload_type = flexfec_payload_type; 2900 } 2901 2902 return true; 2903 } 2904 2905 bool WebRtcVideoReceiveChannel::SetReceiverParameters( 2906 const VideoReceiverParameters& params) { 2907 RTC_DCHECK_RUN_ON(&thread_checker_); 2908 TRACE_EVENT0("webrtc", "WebRtcVideoReceiveChannel::SetReceiverParameters"); 2909 RTC_LOG(LS_INFO) << "SetReceiverParameters: " << params.ToString(); 2910 ChangedReceiverParameters changed_params; 2911 if (!GetChangedReceiverParameters(params, &changed_params)) { 2912 return false; 2913 } 2914 if (changed_params.flexfec_payload_type) { 2915 RTC_DLOG(LS_INFO) << "Changing FlexFEC payload type (recv) from " 2916 << recv_flexfec_payload_type_ << " to " 2917 << *changed_params.flexfec_payload_type; 2918 recv_flexfec_payload_type_ = *changed_params.flexfec_payload_type; 2919 } 2920 if (changed_params.rtp_header_extensions) { 2921 recv_rtp_extensions_ = *changed_params.rtp_header_extensions; 2922 recv_rtp_extension_map_ = RtpHeaderExtensionMap(recv_rtp_extensions_); 2923 } 2924 if (changed_params.codec_settings) { 2925 RTC_DLOG(LS_INFO) << "Changing recv codecs from " 2926 << CodecSettingsVectorToString(recv_codecs_) << " to " 2927 << CodecSettingsVectorToString( 2928 *changed_params.codec_settings); 2929 recv_codecs_ = *changed_params.codec_settings; 2930 } 2931 2932 for (auto& kv : receive_streams_) { 2933 kv.second->SetReceiverParameters(changed_params); 2934 } 2935 recv_params_ = params; 2936 return true; 2937 } 2938 2939 void WebRtcVideoReceiveChannel::SetReceiverReportSsrc(uint32_t ssrc) { 2940 RTC_DCHECK_RUN_ON(&thread_checker_); 2941 if (ssrc == rtcp_receiver_report_ssrc_) 2942 return; 2943 2944 rtcp_receiver_report_ssrc_ = ssrc; 2945 for (auto& [unused, receive_stream] : receive_streams_) 2946 receive_stream->SetLocalSsrc(ssrc); 2947 } 2948 2949 void WebRtcVideoReceiveChannel::ChooseReceiverReportSsrc( 2950 const std::set<uint32_t>& choices) { 2951 RTC_DCHECK_RUN_ON(&thread_checker_); 2952 // If we can continue using the current receiver report, do so. 2953 if (choices.find(rtcp_receiver_report_ssrc_) != choices.end()) { 2954 return; 2955 } 2956 // Go back to the default if list has been emptied. 2957 if (choices.empty()) { 2958 SetReceiverReportSsrc(kDefaultRtcpReceiverReportSsrc); 2959 return; 2960 } 2961 // Any number is as good as any other. 2962 SetReceiverReportSsrc(*choices.begin()); 2963 } 2964 2965 void WebRtcVideoReceiveChannel::SetReceive(bool receive) { 2966 RTC_DCHECK_RUN_ON(&thread_checker_); 2967 TRACE_EVENT0("webrtc", "WebRtcVideoReceiveChannel::SetReceive"); 2968 RTC_LOG(LS_VERBOSE) << "SetReceive: " << (receive ? "true" : "false"); 2969 for (const auto& kv : receive_streams_) { 2970 if (receive) { 2971 kv.second->StartReceiveStream(); 2972 } else { 2973 kv.second->StopReceiveStream(); 2974 } 2975 } 2976 receiving_ = receive; 2977 } 2978 2979 bool WebRtcVideoReceiveChannel::ValidateReceiveSsrcAvailability( 2980 const StreamParams& sp) const { 2981 for (uint32_t ssrc : sp.ssrcs) { 2982 if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) { 2983 RTC_LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc 2984 << "' already exists."; 2985 return false; 2986 } 2987 } 2988 return true; 2989 } 2990 2991 void WebRtcVideoReceiveChannel::DeleteReceiveStream( 2992 WebRtcVideoReceiveStream* stream) { 2993 for (uint32_t old_ssrc : stream->GetSsrcs()) 2994 receive_ssrcs_.erase(old_ssrc); 2995 delete stream; 2996 } 2997 2998 bool WebRtcVideoReceiveChannel::AddRecvStream(const StreamParams& sp) { 2999 return AddRecvStream(sp, false); 3000 } 3001 3002 bool WebRtcVideoReceiveChannel::AddRecvStream(const StreamParams& sp, 3003 bool default_stream) { 3004 RTC_DCHECK_RUN_ON(&thread_checker_); 3005 3006 RTC_LOG(LS_INFO) << "AddRecvStream" 3007 << (default_stream ? " (default stream)" : "") << ": " 3008 << sp.ToString(); 3009 if (!sp.has_ssrcs()) { 3010 // This is a StreamParam with unsignaled SSRCs. Store it, so it can be used 3011 // later when we know the SSRC on the first packet arrival. 3012 unsignaled_stream_params_ = sp; 3013 return true; 3014 } 3015 3016 if (!ValidateStreamParams(sp)) 3017 return false; 3018 3019 for (uint32_t ssrc : sp.ssrcs) { 3020 // Remove running stream if this was a default stream. 3021 const auto& prev_stream = receive_streams_.find(ssrc); 3022 if (prev_stream != receive_streams_.end()) { 3023 if (default_stream || !prev_stream->second->IsDefaultStream()) { 3024 RTC_LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc 3025 << "' already exists."; 3026 return false; 3027 } 3028 DeleteReceiveStream(prev_stream->second); 3029 receive_streams_.erase(prev_stream); 3030 } 3031 } 3032 3033 if (!ValidateReceiveSsrcAvailability(sp)) 3034 return false; 3035 3036 for (uint32_t used_ssrc : sp.ssrcs) 3037 receive_ssrcs_.insert(used_ssrc); 3038 3039 VideoReceiveStreamInterface::Config config(transport(), decoder_factory_); 3040 FlexfecReceiveStream::Config flexfec_config(transport()); 3041 ConfigureReceiverRtp(&config, &flexfec_config, sp); 3042 3043 config.crypto_options = crypto_options_; 3044 config.enable_prerenderer_smoothing = 3045 video_config_.enable_prerenderer_smoothing; 3046 if (!sp.stream_ids().empty()) { 3047 config.sync_group = sp.stream_ids()[0]; 3048 } 3049 3050 if (unsignaled_frame_transformer_ && !config.frame_transformer) 3051 config.frame_transformer = unsignaled_frame_transformer_; 3052 3053 auto receive_stream = new WebRtcVideoReceiveStream( 3054 env_, call_, sp, std::move(config), default_stream, recv_codecs_, 3055 flexfec_config); 3056 if (receiving_) { 3057 receive_stream->StartReceiveStream(); 3058 } 3059 receive_streams_[sp.first_ssrc()] = receive_stream; 3060 return true; 3061 } 3062 3063 void WebRtcVideoReceiveChannel::ConfigureReceiverRtp( 3064 VideoReceiveStreamInterface::Config* config, 3065 FlexfecReceiveStream::Config* flexfec_config, 3066 const StreamParams& sp) const { 3067 uint32_t ssrc = sp.first_ssrc(); 3068 3069 config->rtp.remote_ssrc = ssrc; 3070 config->rtp.local_ssrc = rtcp_receiver_report_ssrc_; 3071 3072 // TODO(pbos): This protection is against setting the same local ssrc as 3073 // remote which is not permitted by the lower-level API. RTCP requires a 3074 // corresponding sender SSRC. Figure out what to do when we don't have 3075 // (receive-only) or know a good local SSRC. 3076 if (config->rtp.remote_ssrc == config->rtp.local_ssrc) { 3077 if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) { 3078 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc; 3079 } else { 3080 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1; 3081 } 3082 } 3083 3084 // The mode and rtx time is determined by a call to the configuration 3085 // function. 3086 config->rtp.rtcp_mode = rtp_config_.rtcp_mode; 3087 3088 sp.GetFidSsrc(ssrc, &config->rtp.rtx_ssrc); 3089 3090 // TODO(brandtr): Generalize when we add support for multistream protection. 3091 flexfec_config->payload_type = recv_flexfec_payload_type_; 3092 if (!env_.field_trials().IsDisabled("WebRTC-FlexFEC-03-Advertised") && 3093 sp.GetFecFrSsrc(ssrc, &flexfec_config->rtp.remote_ssrc)) { 3094 flexfec_config->protected_media_ssrcs = {ssrc}; 3095 flexfec_config->rtp.local_ssrc = config->rtp.local_ssrc; 3096 flexfec_config->rtcp_mode = config->rtp.rtcp_mode; 3097 } 3098 } 3099 3100 bool WebRtcVideoReceiveChannel::RemoveRecvStream(uint32_t ssrc) { 3101 RTC_DCHECK_RUN_ON(&thread_checker_); 3102 RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; 3103 3104 auto stream = receive_streams_.find(ssrc); 3105 if (stream == receive_streams_.end()) { 3106 RTC_LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc; 3107 return false; 3108 } 3109 DeleteReceiveStream(stream->second); 3110 receive_streams_.erase(stream); 3111 3112 return true; 3113 } 3114 3115 void WebRtcVideoReceiveChannel::ResetUnsignaledRecvStream() { 3116 RTC_DCHECK_RUN_ON(&thread_checker_); 3117 RTC_LOG(LS_INFO) << "ResetUnsignaledRecvStream."; 3118 unsignaled_stream_params_ = StreamParams(); 3119 last_unsignalled_ssrc_creation_time_ms_ = std::nullopt; 3120 3121 // Delete any created default streams. This is needed to avoid SSRC collisions 3122 // in Call's RtpDemuxer, in the case that `this` has created a default video 3123 // receiver, and then some other WebRtcVideoReceiveChannel gets the SSRC 3124 // signaled in the corresponding Unified Plan "m=" section. 3125 auto it = receive_streams_.begin(); 3126 while (it != receive_streams_.end()) { 3127 if (it->second->IsDefaultStream()) { 3128 DeleteReceiveStream(it->second); 3129 receive_streams_.erase(it++); 3130 } else { 3131 ++it; 3132 } 3133 } 3134 } 3135 3136 std::optional<uint32_t> WebRtcVideoReceiveChannel::GetUnsignaledSsrc() const { 3137 RTC_DCHECK_RUN_ON(&thread_checker_); 3138 std::optional<uint32_t> ssrc; 3139 for (auto it = receive_streams_.begin(); it != receive_streams_.end(); ++it) { 3140 if (it->second->IsDefaultStream()) { 3141 ssrc.emplace(it->first); 3142 break; 3143 } 3144 } 3145 return ssrc; 3146 } 3147 3148 void WebRtcVideoReceiveChannel::OnDemuxerCriteriaUpdatePending() { 3149 RTC_DCHECK_RUN_ON(&thread_checker_); 3150 ++demuxer_criteria_id_; 3151 } 3152 3153 void WebRtcVideoReceiveChannel::OnDemuxerCriteriaUpdateComplete() { 3154 RTC_DCHECK_RUN_ON(&thread_checker_); 3155 ++demuxer_criteria_completed_id_; 3156 } 3157 3158 bool WebRtcVideoReceiveChannel::SetSink(uint32_t ssrc, 3159 VideoSinkInterface<VideoFrame>* sink) { 3160 RTC_DCHECK_RUN_ON(&thread_checker_); 3161 RTC_LOG(LS_INFO) << "SetSink: ssrc:" << ssrc << " " 3162 << (sink ? "(ptr)" : "nullptr"); 3163 3164 auto it = receive_streams_.find(ssrc); 3165 if (it == receive_streams_.end()) { 3166 return false; 3167 } 3168 3169 it->second->SetSink(sink); 3170 return true; 3171 } 3172 3173 void WebRtcVideoReceiveChannel::SetDefaultSink( 3174 VideoSinkInterface<VideoFrame>* sink) { 3175 RTC_DCHECK_RUN_ON(&thread_checker_); 3176 RTC_LOG(LS_INFO) << "SetDefaultSink: " << (sink ? "(ptr)" : "nullptr"); 3177 default_sink_ = sink; 3178 } 3179 3180 bool WebRtcVideoReceiveChannel::GetStats(VideoMediaReceiveInfo* info) { 3181 RTC_DCHECK_RUN_ON(&thread_checker_); 3182 TRACE_EVENT0("webrtc", "WebRtcVideoReceiveChannel::GetStats"); 3183 3184 info->Clear(); 3185 if (receive_streams_.empty()) { 3186 return true; 3187 } 3188 3189 // Log stats periodically. 3190 bool log_stats = false; 3191 int64_t now_ms = env_.clock().TimeInMilliseconds(); 3192 if (last_receive_stats_log_ms_ == -1 || 3193 now_ms - last_receive_stats_log_ms_ > kStatsLogIntervalMs) { 3194 last_receive_stats_log_ms_ = now_ms; 3195 log_stats = true; 3196 } 3197 3198 FillReceiverStats(info, log_stats); 3199 FillReceiveCodecStats(info); 3200 3201 return true; 3202 } 3203 3204 void WebRtcVideoReceiveChannel::FillReceiverStats( 3205 VideoMediaReceiveInfo* video_media_info, 3206 bool log_stats) { 3207 for (const auto& it : receive_streams_) { 3208 video_media_info->receivers.push_back( 3209 it.second->GetVideoReceiverInfo(log_stats)); 3210 } 3211 } 3212 3213 void WebRtcVideoReceiveChannel::FillReceiveCodecStats( 3214 VideoMediaReceiveInfo* video_media_info) { 3215 for (const auto& receiver : video_media_info->receivers) { 3216 auto codec = 3217 absl::c_find_if(recv_params_.codecs, [&receiver](const Codec& c) { 3218 return receiver.codec_payload_type && 3219 *receiver.codec_payload_type == c.id; 3220 }); 3221 if (codec != recv_params_.codecs.end()) { 3222 video_media_info->receive_codecs.insert( 3223 std::make_pair(codec->id, codec->ToCodecParameters())); 3224 } 3225 } 3226 } 3227 3228 void WebRtcVideoReceiveChannel::OnPacketReceived( 3229 const RtpPacketReceived& packet) { 3230 // Note: the network_thread_checker may refer to the worker thread if the two 3231 // threads are combined, but this is either always true or always false 3232 // depending on configuration set at object initialization. 3233 RTC_DCHECK_RUN_ON(&network_thread_checker_); 3234 3235 // TODO(crbug.com/1373439): Stop posting to the worker thread when the 3236 // combined network/worker project launches. 3237 if (TaskQueueBase::Current() != worker_thread_) { 3238 worker_thread_->PostTask( 3239 SafeTask(task_safety_.flag(), [this, packet = packet]() mutable { 3240 RTC_DCHECK_RUN_ON(&thread_checker_); 3241 ProcessReceivedPacket(std::move(packet)); 3242 })); 3243 } else { 3244 RTC_DCHECK_RUN_ON(&thread_checker_); 3245 ProcessReceivedPacket(packet); 3246 } 3247 } 3248 3249 bool WebRtcVideoReceiveChannel::MaybeCreateDefaultReceiveStream( 3250 const RtpPacketReceived& packet) { 3251 if (discard_unknown_ssrc_packets_) { 3252 return false; 3253 } 3254 3255 if (packet.PayloadType() == recv_flexfec_payload_type_) { 3256 return false; 3257 } 3258 3259 // Ignore unknown ssrcs if there is a demuxer criteria update pending. 3260 // During a demuxer update we may receive ssrcs that were recently 3261 // removed or we may receve ssrcs that were recently configured for a 3262 // different video channel. 3263 if (demuxer_criteria_id_ != demuxer_criteria_completed_id_) { 3264 return false; 3265 } 3266 3267 // See if this payload_type is registered as one that usually gets its 3268 // own SSRC (RTX) or at least is safe to drop either way (FEC). If it 3269 // is, and it wasn't handled above by DeliverPacket, that means we don't 3270 // know what stream it associates with, and we shouldn't ever create an 3271 // implicit channel for these. 3272 bool is_rtx_payload = false; 3273 for (auto& codec : recv_codecs_) { 3274 if (packet.PayloadType() == codec.ulpfec.red_rtx_payload_type || 3275 packet.PayloadType() == codec.ulpfec.ulpfec_payload_type) { 3276 return false; 3277 } 3278 3279 if (packet.PayloadType() == codec.rtx_payload_type) { 3280 is_rtx_payload = true; 3281 break; 3282 } 3283 } 3284 3285 if (is_rtx_payload) { 3286 // As we don't support receiving simulcast there can only be one RTX 3287 // stream, which will be associated with unsignaled media stream. 3288 std::optional<uint32_t> current_default_ssrc = GetUnsignaledSsrc(); 3289 if (current_default_ssrc) { 3290 FindReceiveStream(*current_default_ssrc)->UpdateRtxSsrc(packet.Ssrc()); 3291 return true; 3292 } 3293 // Default media SSRC not known yet. Drop the packet. 3294 // BWE has already been notified of this received packet. 3295 return false; 3296 } 3297 // Ignore unknown ssrcs if we recently created an unsignalled receive 3298 // stream since this shouldn't happen frequently. Getting into a state 3299 // of creating decoders on every packet eats up processing time (e.g. 3300 // https://crbug.com/1069603) and this cooldown prevents that. 3301 if (last_unsignalled_ssrc_creation_time_ms_.has_value()) { 3302 int64_t now_ms = env_.clock().TimeInMilliseconds(); 3303 if (now_ms - last_unsignalled_ssrc_creation_time_ms_.value() < 3304 kUnsignaledSsrcCooldownMs) { 3305 // We've already created an unsignalled ssrc stream within the last 3306 // 0.5 s, ignore with a warning. 3307 RTC_LOG(LS_WARNING) 3308 << "Another unsignalled ssrc packet arrived shortly after the " 3309 << "creation of an unsignalled ssrc stream. Dropping packet."; 3310 return false; 3311 } 3312 } 3313 3314 // RTX SSRC not yet known. 3315 ReCreateDefaultReceiveStream(packet.Ssrc(), std::nullopt); 3316 last_unsignalled_ssrc_creation_time_ms_ = env_.clock().TimeInMilliseconds(); 3317 return true; 3318 } 3319 3320 void WebRtcVideoReceiveChannel::ReCreateDefaultReceiveStream( 3321 uint32_t ssrc, 3322 std::optional<uint32_t> rtx_ssrc) { 3323 RTC_DCHECK_RUN_ON(&thread_checker_); 3324 3325 std::optional<uint32_t> default_recv_ssrc = GetUnsignaledSsrc(); 3326 if (default_recv_ssrc) { 3327 RTC_LOG(LS_INFO) << "Destroying old default receive stream for SSRC=" 3328 << ssrc << "."; 3329 RemoveRecvStream(*default_recv_ssrc); 3330 } 3331 3332 StreamParams sp = unsignaled_stream_params(); 3333 sp.ssrcs.push_back(ssrc); 3334 if (rtx_ssrc) { 3335 sp.AddFidSsrc(ssrc, *rtx_ssrc); 3336 } 3337 RTC_LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc 3338 << "."; 3339 if (!AddRecvStream(sp, /*default_stream=*/true)) { 3340 RTC_LOG(LS_WARNING) << "Could not create default receive stream."; 3341 } 3342 3343 // SSRC 0 returns default_recv_base_minimum_delay_ms. 3344 const int unsignaled_ssrc = 0; 3345 int default_recv_base_minimum_delay_ms = 3346 GetBaseMinimumPlayoutDelayMs(unsignaled_ssrc).value_or(0); 3347 // Set base minimum delay if it was set before for the default receive 3348 // stream. 3349 SetBaseMinimumPlayoutDelayMs(ssrc, default_recv_base_minimum_delay_ms); 3350 SetSink(ssrc, default_sink_); 3351 } 3352 3353 void WebRtcVideoReceiveChannel::SetInterface( 3354 MediaChannelNetworkInterface* iface) { 3355 RTC_DCHECK_RUN_ON(&network_thread_checker_); 3356 MediaChannelUtil::SetInterface(iface); 3357 // Set the RTP recv/send buffer to a bigger size. 3358 MediaChannelUtil::SetOption(MediaChannelNetworkInterface::ST_RTP, 3359 Socket::OPT_RCVBUF, receive_buffer_size_); 3360 } 3361 3362 void WebRtcVideoReceiveChannel::SetFrameDecryptor( 3363 uint32_t ssrc, 3364 scoped_refptr<FrameDecryptorInterface> frame_decryptor) { 3365 RTC_DCHECK_RUN_ON(&thread_checker_); 3366 auto matching_stream = receive_streams_.find(ssrc); 3367 if (matching_stream != receive_streams_.end()) { 3368 matching_stream->second->SetFrameDecryptor(frame_decryptor); 3369 } 3370 } 3371 3372 bool WebRtcVideoReceiveChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, 3373 int delay_ms) { 3374 RTC_DCHECK_RUN_ON(&thread_checker_); 3375 std::optional<uint32_t> default_ssrc = GetUnsignaledSsrc(); 3376 3377 // SSRC of 0 represents the default receive stream. 3378 if (ssrc == 0) { 3379 default_recv_base_minimum_delay_ms_ = delay_ms; 3380 } 3381 3382 if (ssrc == 0 && !default_ssrc) { 3383 return true; 3384 } 3385 3386 if (ssrc == 0 && default_ssrc) { 3387 ssrc = default_ssrc.value(); 3388 } 3389 3390 auto stream = receive_streams_.find(ssrc); 3391 if (stream != receive_streams_.end()) { 3392 stream->second->SetBaseMinimumPlayoutDelayMs(delay_ms); 3393 return true; 3394 } else { 3395 RTC_LOG(LS_ERROR) << "No stream found to set base minimum playout delay"; 3396 return false; 3397 } 3398 } 3399 3400 std::optional<int> WebRtcVideoReceiveChannel::GetBaseMinimumPlayoutDelayMs( 3401 uint32_t ssrc) const { 3402 RTC_DCHECK_RUN_ON(&thread_checker_); 3403 // SSRC of 0 represents the default receive stream. 3404 if (ssrc == 0) { 3405 return default_recv_base_minimum_delay_ms_; 3406 } 3407 3408 auto stream = receive_streams_.find(ssrc); 3409 if (stream != receive_streams_.end()) { 3410 return stream->second->GetBaseMinimumPlayoutDelayMs(); 3411 } else { 3412 RTC_LOG(LS_ERROR) << "No stream found to get base minimum playout delay"; 3413 return std::nullopt; 3414 } 3415 } 3416 3417 std::vector<RtpSource> WebRtcVideoReceiveChannel::GetSources( 3418 uint32_t ssrc) const { 3419 RTC_DCHECK_RUN_ON(&thread_checker_); 3420 auto it = receive_streams_.find(ssrc); 3421 if (it == receive_streams_.end()) { 3422 // TODO(bugs.webrtc.org/9781): Investigate standard compliance 3423 // with sources for streams that has been removed. 3424 RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:" 3425 << ssrc << " which doesn't exist."; 3426 return {}; 3427 } 3428 return it->second->GetSources(); 3429 } 3430 3431 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( 3432 const Environment& env, 3433 Call* call, 3434 const StreamParams& sp, 3435 VideoReceiveStreamInterface::Config config, 3436 bool default_stream, 3437 const std::vector<VideoCodecSettings>& recv_codecs, 3438 const FlexfecReceiveStream::Config& flexfec_config) 3439 : env_(env), 3440 call_(call), 3441 stream_params_(sp), 3442 stream_(nullptr), 3443 default_stream_(default_stream), 3444 config_(std::move(config)), 3445 flexfec_config_(flexfec_config), 3446 flexfec_stream_(nullptr), 3447 sink_(nullptr), 3448 first_frame_timestamp_(-1), 3449 estimated_remote_start_ntp_time_ms_(0), 3450 receiving_(false) { 3451 RTC_DCHECK(config_.decoder_factory); 3452 RTC_DCHECK(config_.decoders.empty()) 3453 << "Decoder info is supplied via `recv_codecs`"; 3454 3455 ExtractCodecInformation(recv_codecs, config_.rtp.rtx_associated_payload_types, 3456 config_.rtp.raw_payload_types, config_.decoders); 3457 const VideoCodecSettings& codec = recv_codecs.front(); 3458 config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type; 3459 config_.rtp.red_payload_type = codec.ulpfec.red_payload_type; 3460 config_.rtp.lntf.enabled = HasLntf(codec.codec); 3461 config_.rtp.nack.rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0; 3462 if (codec.rtx_time && config_.rtp.nack.rtp_history_ms != 0) { 3463 config_.rtp.nack.rtp_history_ms = *codec.rtx_time; 3464 } 3465 3466 config_.rtp.rtcp_xr.receiver_reference_time_report = HasRrtr(codec.codec); 3467 3468 if (codec.ulpfec.red_rtx_payload_type != -1) { 3469 config_.rtp 3470 .rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] = 3471 codec.ulpfec.red_payload_type; 3472 } 3473 3474 config_.renderer = this; 3475 flexfec_config_.payload_type = flexfec_config.payload_type; 3476 3477 CreateReceiveStream(); 3478 } 3479 3480 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3481 ~WebRtcVideoReceiveStream() { 3482 call_->DestroyVideoReceiveStream(stream_); 3483 if (flexfec_stream_) 3484 call_->DestroyFlexfecReceiveStream(flexfec_stream_); 3485 } 3486 3487 VideoReceiveStreamInterface& 3488 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::stream() { 3489 RTC_DCHECK(stream_); 3490 return *stream_; 3491 } 3492 3493 FlexfecReceiveStream* 3494 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::flexfec_stream() { 3495 return flexfec_stream_; 3496 } 3497 3498 const std::vector<uint32_t>& 3499 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::GetSsrcs() const { 3500 return stream_params_.ssrcs; 3501 } 3502 3503 std::vector<RtpSource> 3504 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::GetSources() { 3505 RTC_DCHECK(stream_); 3506 return stream_->GetSources(); 3507 } 3508 3509 RtpParameters 3510 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::GetRtpParameters() const { 3511 RtpParameters rtp_parameters; 3512 3513 std::vector<uint32_t> primary_ssrcs; 3514 stream_params_.GetPrimarySsrcs(&primary_ssrcs); 3515 for (uint32_t ssrc : primary_ssrcs) { 3516 rtp_parameters.encodings.emplace_back(); 3517 rtp_parameters.encodings.back().ssrc = ssrc; 3518 } 3519 3520 rtp_parameters.rtcp.reduced_size = 3521 config_.rtp.rtcp_mode == RtcpMode::kReducedSize; 3522 3523 return rtp_parameters; 3524 } 3525 3526 bool WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::ReconfigureCodecs( 3527 const std::vector<VideoCodecSettings>& recv_codecs) { 3528 RTC_DCHECK(stream_); 3529 RTC_DCHECK(!recv_codecs.empty()); 3530 3531 std::map<int, int> rtx_associated_payload_types; 3532 std::set<int> raw_payload_types; 3533 std::vector<VideoReceiveStreamInterface::Decoder> decoders; 3534 ExtractCodecInformation(recv_codecs, rtx_associated_payload_types, 3535 raw_payload_types, decoders); 3536 3537 const auto& codec = recv_codecs.front(); 3538 3539 if (config_.rtp.red_payload_type != codec.ulpfec.red_payload_type || 3540 config_.rtp.ulpfec_payload_type != codec.ulpfec.ulpfec_payload_type) { 3541 config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type; 3542 config_.rtp.red_payload_type = codec.ulpfec.red_payload_type; 3543 stream_->SetProtectionPayloadTypes(config_.rtp.red_payload_type, 3544 config_.rtp.ulpfec_payload_type); 3545 } 3546 3547 const bool has_lntf = HasLntf(codec.codec); 3548 if (config_.rtp.lntf.enabled != has_lntf) { 3549 config_.rtp.lntf.enabled = has_lntf; 3550 stream_->SetLossNotificationEnabled(has_lntf); 3551 } 3552 3553 int new_history_ms = config_.rtp.nack.rtp_history_ms; 3554 const int rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0; 3555 if (rtp_history_ms != config_.rtp.nack.rtp_history_ms) { 3556 new_history_ms = rtp_history_ms; 3557 } 3558 3559 // The rtx-time parameter can be used to override the hardcoded default for 3560 // the NACK buffer length. 3561 if (codec.rtx_time && new_history_ms != 0) { 3562 new_history_ms = *codec.rtx_time; 3563 } 3564 3565 if (config_.rtp.nack.rtp_history_ms != new_history_ms) { 3566 config_.rtp.nack.rtp_history_ms = new_history_ms; 3567 stream_->SetNackHistory(TimeDelta::Millis(new_history_ms)); 3568 } 3569 3570 const bool has_rtr = HasRrtr(codec.codec); 3571 if (has_rtr != config_.rtp.rtcp_xr.receiver_reference_time_report) { 3572 config_.rtp.rtcp_xr.receiver_reference_time_report = has_rtr; 3573 stream_->SetRtcpXr(config_.rtp.rtcp_xr); 3574 } 3575 3576 if (codec.ulpfec.red_rtx_payload_type != -1) { 3577 rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] = 3578 codec.ulpfec.red_payload_type; 3579 } 3580 3581 if (config_.rtp.rtx_associated_payload_types != 3582 rtx_associated_payload_types) { 3583 stream_->SetAssociatedPayloadTypes(rtx_associated_payload_types); 3584 rtx_associated_payload_types.swap(config_.rtp.rtx_associated_payload_types); 3585 } 3586 3587 bool recreate_needed = false; 3588 3589 if (raw_payload_types != config_.rtp.raw_payload_types) { 3590 raw_payload_types.swap(config_.rtp.raw_payload_types); 3591 recreate_needed = true; 3592 } 3593 3594 if (decoders != config_.decoders) { 3595 decoders.swap(config_.decoders); 3596 recreate_needed = true; 3597 } 3598 3599 return recreate_needed; 3600 } 3601 3602 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetFeedbackParameters( 3603 bool lntf_enabled, 3604 bool nack_enabled, 3605 RtcpMode rtcp_mode, 3606 std::optional<int> rtx_time) { 3607 RTC_DCHECK(stream_); 3608 3609 if (config_.rtp.rtcp_mode != rtcp_mode) { 3610 config_.rtp.rtcp_mode = rtcp_mode; 3611 stream_->SetRtcpMode(rtcp_mode); 3612 3613 flexfec_config_.rtcp_mode = rtcp_mode; 3614 if (flexfec_stream_) { 3615 flexfec_stream_->SetRtcpMode(rtcp_mode); 3616 } 3617 } 3618 3619 config_.rtp.lntf.enabled = lntf_enabled; 3620 stream_->SetLossNotificationEnabled(lntf_enabled); 3621 3622 int nack_history_ms = nack_enabled ? rtx_time.value_or(kNackHistoryMs) : 0; 3623 config_.rtp.nack.rtp_history_ms = nack_history_ms; 3624 stream_->SetNackHistory(TimeDelta::Millis(nack_history_ms)); 3625 } 3626 3627 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetFlexFecPayload( 3628 int payload_type) { 3629 // TODO(bugs.webrtc.org/11993, tommi): See if it is better to always have a 3630 // flexfec stream object around and instead of recreating the video stream, 3631 // reconfigure the flexfec object from within the rtp callback (soon to be on 3632 // the network thread). 3633 if (flexfec_stream_) { 3634 if (flexfec_stream_->payload_type() == payload_type) { 3635 RTC_DCHECK_EQ(flexfec_config_.payload_type, payload_type); 3636 return; 3637 } 3638 3639 flexfec_config_.payload_type = payload_type; 3640 flexfec_stream_->SetPayloadType(payload_type); 3641 3642 if (payload_type == -1) { 3643 stream_->SetFlexFecProtection(nullptr); 3644 call_->DestroyFlexfecReceiveStream(flexfec_stream_); 3645 flexfec_stream_ = nullptr; 3646 } 3647 } else if (payload_type != -1) { 3648 flexfec_config_.payload_type = payload_type; 3649 if (flexfec_config_.IsCompleteAndEnabled()) { 3650 flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_); 3651 stream_->SetFlexFecProtection(flexfec_stream_); 3652 } 3653 } else { 3654 // Noop. No flexfec stream exists and "new" payload_type == -1. 3655 RTC_DCHECK(!flexfec_config_.IsCompleteAndEnabled()); 3656 flexfec_config_.payload_type = payload_type; 3657 } 3658 } 3659 3660 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetReceiverParameters( 3661 const ChangedReceiverParameters& params) { 3662 RTC_DCHECK(stream_); 3663 bool video_needs_recreation = false; 3664 if (params.codec_settings) { 3665 video_needs_recreation = ReconfigureCodecs(*params.codec_settings); 3666 } 3667 3668 if (params.flexfec_payload_type) 3669 SetFlexFecPayload(*params.flexfec_payload_type); 3670 3671 if (video_needs_recreation) { 3672 RecreateReceiveStream(); 3673 } else { 3674 RTC_DLOG_F(LS_INFO) << "No receive stream recreate needed."; 3675 } 3676 } 3677 3678 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3679 RecreateReceiveStream() { 3680 RTC_DCHECK_RUN_ON(&thread_checker_); 3681 RTC_DCHECK(stream_); 3682 std::optional<int> base_minimum_playout_delay_ms; 3683 std::optional<VideoReceiveStreamInterface::RecordingState> recording_state; 3684 if (stream_) { 3685 base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs(); 3686 recording_state = stream_->SetAndGetRecordingState( 3687 VideoReceiveStreamInterface::RecordingState(), 3688 /*generate_key_frame=*/false); 3689 call_->DestroyVideoReceiveStream(stream_); 3690 stream_ = nullptr; 3691 } 3692 3693 if (flexfec_stream_) { 3694 call_->DestroyFlexfecReceiveStream(flexfec_stream_); 3695 flexfec_stream_ = nullptr; 3696 } 3697 3698 CreateReceiveStream(); 3699 3700 if (base_minimum_playout_delay_ms) { 3701 stream_->SetBaseMinimumPlayoutDelayMs( 3702 base_minimum_playout_delay_ms.value()); 3703 } 3704 if (recording_state) { 3705 stream_->SetAndGetRecordingState(std::move(*recording_state), 3706 /*generate_key_frame=*/false); 3707 } 3708 if (receiving_) { 3709 StartReceiveStream(); 3710 } 3711 } 3712 3713 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3714 CreateReceiveStream() { 3715 RTC_DCHECK(!stream_); 3716 RTC_DCHECK(!flexfec_stream_); 3717 if (flexfec_config_.IsCompleteAndEnabled()) { 3718 flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_); 3719 } 3720 3721 VideoReceiveStreamInterface::Config config = config_.Copy(); 3722 config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr); 3723 config.rtp.packet_sink_ = flexfec_stream_; 3724 stream_ = call_->CreateVideoReceiveStream(std::move(config)); 3725 } 3726 3727 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StartReceiveStream() { 3728 RTC_DCHECK_RUN_ON(&thread_checker_); 3729 receiving_ = true; 3730 stream_->Start(); 3731 } 3732 3733 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StopReceiveStream() { 3734 RTC_DCHECK_RUN_ON(&thread_checker_); 3735 receiving_ = false; 3736 stream_->Stop(); 3737 RecreateReceiveStream(); 3738 } 3739 3740 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::OnFrame( 3741 const VideoFrame& frame) { 3742 MutexLock lock(&sink_lock_); 3743 3744 int64_t time_now_ms = env_.clock().TimeInMilliseconds(); 3745 if (first_frame_timestamp_ < 0) 3746 first_frame_timestamp_ = time_now_ms; 3747 int64_t elapsed_time_ms = time_now_ms - first_frame_timestamp_; 3748 if (frame.ntp_time_ms() > 0) 3749 estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms; 3750 3751 if (sink_ == nullptr) { 3752 RTC_LOG(LS_WARNING) 3753 << "VideoReceiveStreamInterface not connected to a VideoSink."; 3754 return; 3755 } 3756 3757 sink_->OnFrame(frame); 3758 } 3759 3760 bool WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::IsDefaultStream() 3761 const { 3762 return default_stream_; 3763 } 3764 3765 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetFrameDecryptor( 3766 scoped_refptr<FrameDecryptorInterface> frame_decryptor) { 3767 config_.frame_decryptor = frame_decryptor; 3768 if (stream_) { 3769 RTC_LOG(LS_INFO) 3770 << "Setting FrameDecryptor (recv) because of SetFrameDecryptor, " 3771 "remote_ssrc=" 3772 << config_.rtp.remote_ssrc; 3773 stream_->SetFrameDecryptor(frame_decryptor); 3774 } 3775 } 3776 3777 bool WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3778 SetBaseMinimumPlayoutDelayMs(int delay_ms) { 3779 return stream_ ? stream_->SetBaseMinimumPlayoutDelayMs(delay_ms) : false; 3780 } 3781 3782 int WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3783 GetBaseMinimumPlayoutDelayMs() const { 3784 return stream_ ? stream_->GetBaseMinimumPlayoutDelayMs() : 0; 3785 } 3786 3787 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetSink( 3788 VideoSinkInterface<VideoFrame>* sink) { 3789 MutexLock lock(&sink_lock_); 3790 sink_ = sink; 3791 } 3792 3793 VideoReceiverInfo 3794 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo( 3795 bool log_stats) { 3796 VideoReceiverInfo info; 3797 info.ssrc_groups = stream_params_.ssrc_groups; 3798 info.add_ssrc(config_.rtp.remote_ssrc); 3799 VideoReceiveStreamInterface::Stats stats = stream_->GetStats(); 3800 info.decoder_implementation_name = stats.decoder_implementation_name; 3801 info.power_efficient_decoder = stats.power_efficient_decoder; 3802 if (stats.current_payload_type != -1) { 3803 info.codec_payload_type = stats.current_payload_type; 3804 auto decoder_it = absl::c_find_if(config_.decoders, [&](const auto& d) { 3805 return d.payload_type == stats.current_payload_type; 3806 }); 3807 if (decoder_it != config_.decoders.end()) 3808 info.codec_name = decoder_it->video_format.name; 3809 } 3810 info.payload_bytes_received = stats.rtp_stats.packet_counter.payload_bytes; 3811 info.header_and_padding_bytes_received = 3812 stats.rtp_stats.packet_counter.header_bytes + 3813 stats.rtp_stats.packet_counter.padding_bytes; 3814 info.packets_received = stats.rtp_stats.packet_counter.packets; 3815 info.packets_received_with_ect1 = 3816 stats.rtp_stats.packet_counter.packets_with_ect1; 3817 info.packets_received_with_ce = 3818 stats.rtp_stats.packet_counter.packets_with_ce; 3819 info.packets_lost = stats.rtp_stats.packets_lost; 3820 info.jitter_ms = stats.rtp_stats.jitter / (kVideoCodecClockrate / 1000); 3821 3822 info.framerate_received = stats.network_frame_rate; 3823 info.framerate_decoded = stats.decode_frame_rate; 3824 info.framerate_output = stats.render_frame_rate; 3825 info.frame_width = stats.width; 3826 info.frame_height = stats.height; 3827 3828 { 3829 MutexLock frame_cs(&sink_lock_); 3830 info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_; 3831 } 3832 3833 info.decode_ms = stats.decode_ms; 3834 info.max_decode_ms = stats.max_decode_ms; 3835 info.current_delay_ms = stats.current_delay_ms; 3836 info.target_delay_ms = stats.target_delay_ms; 3837 info.jitter_buffer_ms = stats.jitter_buffer_ms; 3838 info.jitter_buffer_delay_seconds = 3839 stats.jitter_buffer_delay.seconds<double>(); 3840 info.jitter_buffer_target_delay_seconds = 3841 stats.jitter_buffer_target_delay.seconds<double>(); 3842 info.jitter_buffer_emitted_count = stats.jitter_buffer_emitted_count; 3843 info.jitter_buffer_minimum_delay_seconds = 3844 stats.jitter_buffer_minimum_delay.seconds<double>(); 3845 info.min_playout_delay_ms = stats.min_playout_delay_ms; 3846 info.render_delay_ms = stats.render_delay_ms; 3847 info.frames_received = 3848 stats.frame_counts.key_frames + stats.frame_counts.delta_frames; 3849 info.frames_dropped = stats.frames_dropped; 3850 info.frames_decoded = stats.frames_decoded; 3851 info.key_frames_decoded = stats.frame_counts.key_frames; 3852 info.frames_rendered = stats.frames_rendered; 3853 info.qp_sum = stats.qp_sum; 3854 info.corruption_score_sum = stats.corruption_score_sum; 3855 info.corruption_score_squared_sum = stats.corruption_score_squared_sum; 3856 info.corruption_score_count = stats.corruption_score_count; 3857 info.total_decode_time = stats.total_decode_time; 3858 info.total_processing_delay = stats.total_processing_delay; 3859 info.total_assembly_time = stats.total_assembly_time; 3860 info.frames_assembled_from_multiple_packets = 3861 stats.frames_assembled_from_multiple_packets; 3862 info.last_packet_received = stats.rtp_stats.last_packet_received; 3863 info.estimated_playout_ntp_timestamp_ms = 3864 stats.estimated_playout_ntp_timestamp_ms; 3865 info.first_frame_received_to_decoded_ms = 3866 stats.first_frame_received_to_decoded_ms; 3867 info.total_inter_frame_delay = stats.total_inter_frame_delay; 3868 info.total_squared_inter_frame_delay = stats.total_squared_inter_frame_delay; 3869 info.interframe_delay_max_ms = stats.interframe_delay_max_ms; 3870 info.freeze_count = stats.freeze_count; 3871 info.pause_count = stats.pause_count; 3872 info.total_freezes_duration_ms = stats.total_freezes_duration_ms; 3873 info.total_pauses_duration_ms = stats.total_pauses_duration_ms; 3874 3875 info.content_type = stats.content_type; 3876 3877 info.firs_sent = stats.rtcp_packet_type_counts.fir_packets; 3878 info.plis_sent = stats.rtcp_packet_type_counts.pli_packets; 3879 info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets; 3880 // TODO(bugs.webrtc.org/10662): Add stats for LNTF. 3881 3882 info.timing_frame_info = stats.timing_frame_info; 3883 3884 if (stats.rtx_rtp_stats.has_value()) { 3885 info.retransmitted_packets_received = 3886 stats.rtx_rtp_stats->packet_counter.packets; 3887 info.retransmitted_bytes_received = 3888 stats.rtx_rtp_stats->packet_counter.payload_bytes; 3889 // RTX information gets added to primary counters. 3890 info.payload_bytes_received += 3891 stats.rtx_rtp_stats->packet_counter.payload_bytes; 3892 info.header_and_padding_bytes_received += 3893 stats.rtx_rtp_stats->packet_counter.header_bytes + 3894 stats.rtx_rtp_stats->packet_counter.padding_bytes; 3895 info.packets_received += stats.rtx_rtp_stats->packet_counter.packets; 3896 } 3897 3898 if (flexfec_stream_) { 3899 const ReceiveStatistics* fec_stats = flexfec_stream_->GetStats(); 3900 if (fec_stats) { 3901 const StreamStatistician* statistican = 3902 fec_stats->GetStatistician(flexfec_config_.rtp.remote_ssrc); 3903 if (statistican) { 3904 const RtpReceiveStats fec_rtp_stats = statistican->GetStats(); 3905 info.fec_packets_received = fec_rtp_stats.packet_counter.packets; 3906 // TODO(bugs.webrtc.org/15250): implement fecPacketsDiscarded. 3907 info.fec_bytes_received = fec_rtp_stats.packet_counter.payload_bytes; 3908 // FEC information gets added to primary counters. 3909 info.payload_bytes_received += 3910 fec_rtp_stats.packet_counter.payload_bytes; 3911 info.header_and_padding_bytes_received += 3912 fec_rtp_stats.packet_counter.header_bytes + 3913 fec_rtp_stats.packet_counter.padding_bytes; 3914 info.packets_received += fec_rtp_stats.packet_counter.packets; 3915 } else { 3916 info.fec_packets_received = 0; 3917 } 3918 } 3919 } 3920 3921 // remote-outbound-rtp stats. 3922 info.last_sender_report_timestamp = stats.last_sender_report_timestamp; 3923 info.last_sender_report_utc_timestamp = 3924 stats.last_sender_report_utc_timestamp; 3925 info.last_sender_report_remote_utc_timestamp = 3926 stats.last_sender_report_remote_utc_timestamp; 3927 info.sender_reports_packets_sent = stats.sender_reports_packets_sent; 3928 info.sender_reports_bytes_sent = stats.sender_reports_bytes_sent; 3929 info.sender_reports_reports_count = stats.sender_reports_reports_count; 3930 // TODO(bugs.webrtc.org/12529): RTT-related fields are missing and can only be 3931 // present if DLRR is enabled. 3932 3933 if (log_stats) 3934 RTC_LOG(LS_INFO) << stats.ToString(env_.clock().TimeInMilliseconds()); 3935 3936 return info; 3937 } 3938 3939 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3940 SetRecordableEncodedFrameCallback( 3941 std::function<void(const RecordableEncodedFrame&)> callback) { 3942 if (stream_) { 3943 stream_->SetAndGetRecordingState( 3944 VideoReceiveStreamInterface::RecordingState(std::move(callback)), 3945 /*generate_key_frame=*/true); 3946 } else { 3947 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded " 3948 "frame sink"; 3949 } 3950 } 3951 3952 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3953 ClearRecordableEncodedFrameCallback() { 3954 if (stream_) { 3955 stream_->SetAndGetRecordingState( 3956 VideoReceiveStreamInterface::RecordingState(), 3957 /*generate_key_frame=*/false); 3958 } else { 3959 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded " 3960 "frame sink"; 3961 } 3962 } 3963 3964 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() { 3965 if (stream_) { 3966 stream_->GenerateKeyFrame(); 3967 } else { 3968 RTC_LOG(LS_ERROR) 3969 << "Absent receive stream; ignoring key frame generation request."; 3970 } 3971 } 3972 3973 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: 3974 SetDepacketizerToDecoderFrameTransformer( 3975 scoped_refptr<FrameTransformerInterface> frame_transformer) { 3976 config_.frame_transformer = frame_transformer; 3977 if (stream_) 3978 stream_->SetDepacketizerToDecoderFrameTransformer(frame_transformer); 3979 } 3980 3981 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetLocalSsrc( 3982 uint32_t ssrc) { 3983 config_.rtp.local_ssrc = ssrc; 3984 call_->OnLocalSsrcUpdated(stream(), ssrc); 3985 if (flexfec_stream_) 3986 call_->OnLocalSsrcUpdated(*flexfec_stream_, ssrc); 3987 } 3988 3989 void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::UpdateRtxSsrc( 3990 uint32_t ssrc) { 3991 stream_->UpdateRtxSsrc(ssrc); 3992 } 3993 WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream* 3994 WebRtcVideoReceiveChannel::FindReceiveStream(uint32_t ssrc) { 3995 if (ssrc == 0) { 3996 std::optional<uint32_t> default_ssrc = GetUnsignaledSsrc(); 3997 if (!default_ssrc) { 3998 return nullptr; 3999 } 4000 ssrc = *default_ssrc; 4001 } 4002 auto it = receive_streams_.find(ssrc); 4003 if (it != receive_streams_.end()) { 4004 return it->second; 4005 } 4006 return nullptr; 4007 } 4008 4009 // RTC_RUN_ON(worker_thread_) 4010 void WebRtcVideoReceiveChannel::ProcessReceivedPacket( 4011 RtpPacketReceived packet) { 4012 // TODO(bugs.webrtc.org/11993): This code is very similar to what 4013 // WebRtcVoiceMediaChannel::OnPacketReceived does. For maintainability and 4014 // consistency it would be good to move the interaction with call_->Receiver() 4015 // to a common implementation and provide a callback on the worker thread 4016 // for the exception case (DELIVERY_UNKNOWN_SSRC) and how retry is attempted. 4017 // TODO(bugs.webrtc.org/7135): extensions in `packet` is currently set 4018 // in RtpTransport and does not neccessarily include extensions specific 4019 // to this channel/MID. Also see comment in 4020 // BaseChannel::MaybeUpdateDemuxerAndRtpExtensions_w. 4021 // It would likely be good if extensions where merged per BUNDLE and 4022 // applied directly in RtpTransport::DemuxPacket; 4023 packet.IdentifyExtensions(recv_rtp_extension_map_); 4024 packet.set_payload_type_frequency(kVideoPayloadTypeFrequency); 4025 if (!packet.arrival_time().IsFinite()) { 4026 packet.set_arrival_time(env_.clock().CurrentTime()); 4027 } 4028 4029 call_->Receiver()->DeliverRtpPacket( 4030 MediaType::VIDEO, std::move(packet), 4031 absl::bind_front( 4032 &WebRtcVideoReceiveChannel::MaybeCreateDefaultReceiveStream, this)); 4033 } 4034 4035 void WebRtcVideoReceiveChannel::SetRecordableEncodedFrameCallback( 4036 uint32_t ssrc, 4037 std::function<void(const RecordableEncodedFrame&)> callback) { 4038 RTC_DCHECK_RUN_ON(&thread_checker_); 4039 WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); 4040 if (stream) { 4041 stream->SetRecordableEncodedFrameCallback(std::move(callback)); 4042 } else { 4043 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded " 4044 "frame sink for ssrc " 4045 << ssrc; 4046 } 4047 } 4048 4049 void WebRtcVideoReceiveChannel::ClearRecordableEncodedFrameCallback( 4050 uint32_t ssrc) { 4051 RTC_DCHECK_RUN_ON(&thread_checker_); 4052 WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); 4053 if (stream) { 4054 stream->ClearRecordableEncodedFrameCallback(); 4055 } else { 4056 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded " 4057 "frame sink for ssrc " 4058 << ssrc; 4059 } 4060 } 4061 4062 void WebRtcVideoReceiveChannel::RequestRecvKeyFrame(uint32_t ssrc) { 4063 RTC_DCHECK_RUN_ON(&thread_checker_); 4064 WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); 4065 if (stream) { 4066 return stream->GenerateKeyFrame(); 4067 } else { 4068 RTC_LOG(LS_ERROR) 4069 << "Absent receive stream; ignoring key frame generation for ssrc " 4070 << ssrc; 4071 } 4072 } 4073 4074 void WebRtcVideoReceiveChannel::SetDepacketizerToDecoderFrameTransformer( 4075 uint32_t ssrc, 4076 scoped_refptr<FrameTransformerInterface> frame_transformer) { 4077 RTC_DCHECK(frame_transformer); 4078 RTC_DCHECK_RUN_ON(&thread_checker_); 4079 if (ssrc == 0) { 4080 // If the receiver is unsignaled, save the frame transformer and set it 4081 // when the stream is associated with an ssrc. 4082 unsignaled_frame_transformer_ = std::move(frame_transformer); 4083 return; 4084 } 4085 4086 auto matching_stream = receive_streams_.find(ssrc); 4087 if (matching_stream != receive_streams_.end()) { 4088 matching_stream->second->SetDepacketizerToDecoderFrameTransformer( 4089 std::move(frame_transformer)); 4090 } 4091 } 4092 4093 // ------------------------- VideoCodecSettings -------------------- 4094 4095 VideoCodecSettings::VideoCodecSettings(const Codec& codec) 4096 : codec(codec), flexfec_payload_type(-1), rtx_payload_type(-1) {} 4097 4098 bool VideoCodecSettings::operator==(const VideoCodecSettings& other) const { 4099 return codec == other.codec && ulpfec == other.ulpfec && 4100 flexfec_payload_type == other.flexfec_payload_type && 4101 rtx_payload_type == other.rtx_payload_type && 4102 rtx_time == other.rtx_time; 4103 } 4104 4105 bool VideoCodecSettings::EqualsDisregardingFlexfec( 4106 const VideoCodecSettings& a, 4107 const VideoCodecSettings& b) { 4108 return a.codec == b.codec && a.ulpfec == b.ulpfec && 4109 a.rtx_payload_type == b.rtx_payload_type && a.rtx_time == b.rtx_time; 4110 } 4111 4112 bool VideoCodecSettings::operator!=(const VideoCodecSettings& other) const { 4113 return !(*this == other); 4114 } 4115 4116 } // namespace webrtc