rtp_sender_video.cc (37366B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/rtp_rtcp/source/rtp_sender_video.h" 12 13 #include <cstdint> 14 #include <cstdlib> 15 #include <cstring> 16 #include <memory> 17 #include <optional> 18 #include <utility> 19 #include <variant> 20 #include <vector> 21 22 #include "absl/algorithm/container.h" 23 #include "absl/memory/memory.h" 24 #include "api/array_view.h" 25 #include "api/crypto/frame_encryptor_interface.h" 26 #include "api/field_trials_view.h" 27 #include "api/make_ref_counted.h" 28 #include "api/media_types.h" 29 #include "api/transport/rtp/corruption_detection_message.h" 30 #include "api/transport/rtp/dependency_descriptor.h" 31 #include "api/units/data_rate.h" 32 #include "api/units/frequency.h" 33 #include "api/units/time_delta.h" 34 #include "api/units/timestamp.h" 35 #include "api/video/encoded_image.h" 36 #include "api/video/video_codec_type.h" 37 #include "api/video/video_content_type.h" 38 #include "api/video/video_frame_type.h" 39 #include "api/video/video_layers_allocation.h" 40 #include "api/video/video_rotation.h" 41 #include "api/video/video_timing.h" 42 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 43 #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" 44 #include "modules/rtp_rtcp/source/corruption_detection_extension.h" 45 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" 46 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" 47 #include "modules/rtp_rtcp/source/rtp_format.h" 48 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" 49 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" 50 #include "modules/rtp_rtcp/source/rtp_header_extensions.h" 51 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" 52 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" 53 #include "modules/rtp_rtcp/source/rtp_video_header.h" 54 #include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h" 55 #include "modules/rtp_rtcp/source/video_fec_generator.h" 56 #include "modules/video_coding/codecs/h264/include/h264_globals.h" 57 #include "modules/video_coding/codecs/interface/common_constants.h" 58 #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" 59 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" 60 #include "rtc_base/buffer.h" 61 #include "rtc_base/checks.h" 62 #include "rtc_base/experiments/field_trial_parser.h" 63 #include "rtc_base/logging.h" 64 #include "rtc_base/race_checker.h" 65 #include "rtc_base/synchronization/mutex.h" 66 #include "system_wrappers/include/ntp_time.h" 67 68 namespace webrtc { 69 70 namespace { 71 constexpr size_t kRedForFecHeaderLength = 1; 72 constexpr TimeDelta kMaxUnretransmittableFrameInterval = 73 TimeDelta::Millis(33 * 4); 74 75 void BuildRedPayload(const RtpPacketToSend& media_packet, 76 RtpPacketToSend* red_packet) { 77 uint8_t* red_payload = red_packet->AllocatePayload( 78 kRedForFecHeaderLength + media_packet.payload_size()); 79 RTC_DCHECK(red_payload); 80 red_payload[0] = media_packet.PayloadType(); 81 82 auto media_payload = media_packet.payload(); 83 memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(), 84 media_payload.size()); 85 } 86 87 bool MinimizeDescriptor(RTPVideoHeader* video_header) { 88 if (auto* vp8 = 89 std::get_if<RTPVideoHeaderVP8>(&video_header->video_type_header)) { 90 // Set minimum fields the RtpPacketizer is using to create vp8 packets. 91 // nonReference is the only field that doesn't require extra space. 92 bool non_reference = vp8->nonReference; 93 vp8->InitRTPVideoHeaderVP8(); 94 vp8->nonReference = non_reference; 95 return true; 96 } 97 return false; 98 } 99 100 bool IsBaseLayer(const RTPVideoHeader& video_header) { 101 // For AV1 & H.265 we fetch temporal index from the generic descriptor. 102 if (video_header.generic) { 103 const auto& generic = video_header.generic.value(); 104 return (generic.temporal_index == 0 || 105 generic.temporal_index == kNoTemporalIdx); 106 } 107 switch (video_header.codec) { 108 case kVideoCodecVP8: { 109 const auto& vp8 = 110 std::get<RTPVideoHeaderVP8>(video_header.video_type_header); 111 return (vp8.temporalIdx == 0 || vp8.temporalIdx == kNoTemporalIdx); 112 } 113 case kVideoCodecVP9: { 114 const auto& vp9 = 115 std::get<RTPVideoHeaderVP9>(video_header.video_type_header); 116 return (vp9.temporal_idx == 0 || vp9.temporal_idx == kNoTemporalIdx); 117 } 118 case kVideoCodecH264: 119 // TODO(kron): Implement logic for H264 once WebRTC supports temporal 120 // layers for H264. 121 break; 122 // These codecs do not have codec-specifics, from which we can fetch 123 // temporal index. 124 case kVideoCodecH265: 125 case kVideoCodecAV1: 126 case kVideoCodecGeneric: 127 break; 128 } 129 return true; 130 } 131 132 std::optional<VideoPlayoutDelay> LoadVideoPlayoutDelayOverride( 133 const FieldTrialsView* key_value_config) { 134 RTC_DCHECK(key_value_config); 135 FieldTrialOptional<int> playout_delay_min_ms("min_ms", std::nullopt); 136 FieldTrialOptional<int> playout_delay_max_ms("max_ms", std::nullopt); 137 ParseFieldTrial({&playout_delay_max_ms, &playout_delay_min_ms}, 138 key_value_config->Lookup("WebRTC-ForceSendPlayoutDelay")); 139 return playout_delay_max_ms && playout_delay_min_ms 140 ? std::make_optional<VideoPlayoutDelay>( 141 TimeDelta::Millis(*playout_delay_min_ms), 142 TimeDelta::Millis(*playout_delay_max_ms)) 143 : std::nullopt; 144 } 145 146 // Some packets can be skipped and the stream can still be decoded. Those 147 // packets are less likely to be retransmitted if they are lost. 148 bool PacketWillLikelyBeRequestedForRestransmissionIfLost( 149 const RTPVideoHeader& video_header) { 150 return IsBaseLayer(video_header) && 151 !(video_header.generic.has_value() 152 ? absl::c_linear_search( 153 video_header.generic->decode_target_indications, 154 DecodeTargetIndication::kDiscardable) 155 : false); 156 } 157 158 } // namespace 159 160 RTPSenderVideo::RTPSenderVideo(const Config& config) 161 : rtp_sender_(config.rtp_sender), 162 clock_(config.clock), 163 retransmission_settings_( 164 config.enable_retransmit_all_layers 165 ? kRetransmitAllLayers 166 : (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)), 167 last_rotation_(kVideoRotation_0), 168 transmit_color_space_next_frame_(false), 169 send_allocation_(SendVideoLayersAllocation::kDontSend), 170 playout_delay_pending_(false), 171 forced_playout_delay_(LoadVideoPlayoutDelayOverride(config.field_trials)), 172 red_payload_type_(config.red_payload_type), 173 fec_type_(config.fec_type), 174 fec_overhead_bytes_(config.fec_overhead_bytes), 175 post_encode_overhead_bitrate_(/*max_window_size=*/TimeDelta::Seconds(1)), 176 frame_encryptor_(config.frame_encryptor), 177 require_frame_encryption_(config.require_frame_encryption), 178 generic_descriptor_auth_experiment_( 179 !config.field_trials->IsDisabled("WebRTC-GenericDescriptorAuth")), 180 raw_packetization_(config.raw_packetization), 181 absolute_capture_time_sender_(config.clock), 182 frame_transformer_delegate_( 183 config.frame_transformer 184 ? make_ref_counted<RTPSenderVideoFrameTransformerDelegate>( 185 this, 186 config.frame_transformer, 187 rtp_sender_->SSRC(), 188 rtp_sender_->Rid(), 189 config.task_queue_factory) 190 : nullptr) { 191 if (frame_transformer_delegate_) 192 frame_transformer_delegate_->Init(); 193 } 194 195 RTPSenderVideo::~RTPSenderVideo() { 196 if (frame_transformer_delegate_) 197 frame_transformer_delegate_->Reset(); 198 } 199 200 void RTPSenderVideo::LogAndSendToNetwork( 201 std::vector<std::unique_ptr<RtpPacketToSend>> packets, 202 size_t encoder_output_size) { 203 { 204 MutexLock lock(&stats_mutex_); 205 size_t packetized_payload_size = 0; 206 for (const auto& packet : packets) { 207 if (*packet->packet_type() == RtpPacketMediaType::kVideo) { 208 packetized_payload_size += packet->payload_size(); 209 } 210 } 211 // AV1 and H264 packetizers may produce less packetized bytes than 212 // unpacketized. 213 if (packetized_payload_size >= encoder_output_size) { 214 post_encode_overhead_bitrate_.Update( 215 packetized_payload_size - encoder_output_size, clock_->CurrentTime()); 216 } 217 } 218 219 rtp_sender_->EnqueuePackets(std::move(packets)); 220 } 221 222 size_t RTPSenderVideo::FecPacketOverhead() const { 223 size_t overhead = fec_overhead_bytes_; 224 if (red_enabled()) { 225 // The RED overhead is due to a small header. 226 overhead += kRedForFecHeaderLength; 227 228 if (fec_type_ == VideoFecGenerator::FecType::kUlpFec) { 229 // For ULPFEC, the overhead is the FEC headers plus RED for FEC header 230 // (see above) plus anything in RTP header beyond the 12 bytes base header 231 // (CSRC list, extensions...) 232 // This reason for the header extensions to be included here is that 233 // from an FEC viewpoint, they are part of the payload to be protected. 234 // (The base RTP header is already protected by the FEC header.) 235 overhead += 236 rtp_sender_->FecOrPaddingPacketMaxRtpHeaderLength() - kRtpHeaderSize; 237 } 238 } 239 return overhead; 240 } 241 242 void RTPSenderVideo::SetRetransmissionSetting(int32_t retransmission_settings) { 243 RTC_DCHECK_RUNS_SERIALIZED(&send_checker_); 244 retransmission_settings_ = retransmission_settings; 245 } 246 247 void RTPSenderVideo::SetVideoStructure( 248 const FrameDependencyStructure* video_structure) { 249 if (frame_transformer_delegate_) { 250 frame_transformer_delegate_->SetVideoStructureUnderLock(video_structure); 251 return; 252 } 253 SetVideoStructureInternal(video_structure); 254 } 255 256 void RTPSenderVideo::SetVideoStructureAfterTransformation( 257 const FrameDependencyStructure* video_structure) { 258 SetVideoStructureInternal(video_structure); 259 } 260 261 void RTPSenderVideo::SetVideoStructureInternal( 262 const FrameDependencyStructure* video_structure) { 263 RTC_DCHECK_RUNS_SERIALIZED(&send_checker_); 264 if (video_structure == nullptr) { 265 video_structure_ = nullptr; 266 return; 267 } 268 // Simple sanity checks video structure is set up. 269 RTC_DCHECK_GT(video_structure->num_decode_targets, 0); 270 RTC_DCHECK_GT(video_structure->templates.size(), 0); 271 272 int structure_id = 0; 273 if (video_structure_) { 274 if (*video_structure_ == *video_structure) { 275 // Same structure (just a new key frame), no update required. 276 return; 277 } 278 // When setting different video structure make sure structure_id is updated 279 // so that templates from different structures do not collide. 280 static constexpr int kMaxTemplates = 64; 281 structure_id = 282 (video_structure_->structure_id + video_structure_->templates.size()) % 283 kMaxTemplates; 284 } 285 286 video_structure_ = 287 std::make_unique<FrameDependencyStructure>(*video_structure); 288 video_structure_->structure_id = structure_id; 289 } 290 291 void RTPSenderVideo::SetVideoLayersAllocation( 292 VideoLayersAllocation allocation) { 293 if (frame_transformer_delegate_) { 294 frame_transformer_delegate_->SetVideoLayersAllocationUnderLock( 295 std::move(allocation)); 296 return; 297 } 298 SetVideoLayersAllocationInternal(std::move(allocation)); 299 } 300 301 void RTPSenderVideo::SetVideoLayersAllocationAfterTransformation( 302 VideoLayersAllocation allocation) { 303 SetVideoLayersAllocationInternal(std::move(allocation)); 304 } 305 306 void RTPSenderVideo::SetVideoLayersAllocationInternal( 307 VideoLayersAllocation allocation) { 308 RTC_DCHECK_RUNS_SERIALIZED(&send_checker_); 309 if (!allocation_ || allocation.active_spatial_layers.size() != 310 allocation_->active_spatial_layers.size()) { 311 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution; 312 } else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) { 313 send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution; 314 } 315 if (send_allocation_ == SendVideoLayersAllocation::kSendWithoutResolution) { 316 // Check if frame rate changed more than 5fps since the last time the 317 // extension was sent with frame rate and resolution. 318 for (size_t i = 0; i < allocation.active_spatial_layers.size(); ++i) { 319 if (abs(static_cast<int>( 320 allocation.active_spatial_layers[i].frame_rate_fps) - 321 static_cast<int>( 322 last_full_sent_allocation_->active_spatial_layers[i] 323 .frame_rate_fps)) > 5) { 324 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution; 325 break; 326 } 327 } 328 } 329 allocation_ = std::move(allocation); 330 } 331 332 void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header, 333 bool first_packet, 334 bool last_packet, 335 RtpPacketToSend* packet) const { 336 // Send color space when changed or if the frame is a key frame. Keep 337 // sending color space information until the first base layer frame to 338 // guarantee that the information is retrieved by the receiver. 339 bool set_color_space = 340 video_header.color_space != last_color_space_ || 341 video_header.frame_type == VideoFrameType::kVideoFrameKey || 342 transmit_color_space_next_frame_; 343 // Color space requires two-byte header extensions if HDR metadata is 344 // included. Therefore, it's best to add this extension first so that the 345 // other extensions in the same packet are written as two-byte headers at 346 // once. 347 if (last_packet && set_color_space && video_header.color_space) 348 packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value()); 349 350 // According to 351 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ 352 // ts_126114v120700p.pdf Section 7.4.5: 353 // The MTSI client shall add the payload bytes as defined in this clause 354 // onto the last RTP packet in each group of packets which make up a key 355 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265 356 // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP 357 // packet in each group of packets which make up another type of frame 358 // (e.g. a P-Frame) only if the current value is different from the previous 359 // value sent. 360 // Set rotation when key frame or when changed (to follow standard). 361 // Or when different from 0 (to follow current receiver implementation). 362 bool set_video_rotation = 363 video_header.frame_type == VideoFrameType::kVideoFrameKey || 364 video_header.rotation != last_rotation_ || 365 video_header.rotation != kVideoRotation_0; 366 if (last_packet && set_video_rotation) 367 packet->SetExtension<VideoOrientation>(video_header.rotation); 368 369 // Report content type only for key frames. 370 if (last_packet && 371 video_header.frame_type == VideoFrameType::kVideoFrameKey && 372 video_header.content_type != VideoContentType::UNSPECIFIED) 373 packet->SetExtension<VideoContentTypeExtension>(video_header.content_type); 374 375 if (last_packet && 376 video_header.video_timing.flags != VideoSendTiming::kInvalid) 377 packet->SetExtension<VideoTimingExtension>(video_header.video_timing); 378 379 // If transmitted, add to all packets; ack logic depends on this. 380 if (playout_delay_pending_ && current_playout_delay_.has_value()) { 381 packet->SetExtension<PlayoutDelayLimits>(*current_playout_delay_); 382 } 383 384 if (first_packet && video_header.absolute_capture_time.has_value()) { 385 packet->SetExtension<AbsoluteCaptureTimeExtension>( 386 *video_header.absolute_capture_time); 387 } 388 389 if (video_header.generic) { 390 bool extension_is_set = false; 391 if (packet->IsRegistered<RtpDependencyDescriptorExtension>() && 392 video_structure_ != nullptr) { 393 DependencyDescriptor descriptor; 394 descriptor.first_packet_in_frame = first_packet; 395 descriptor.last_packet_in_frame = last_packet; 396 descriptor.frame_number = video_header.generic->frame_id & 0xFFFF; 397 descriptor.frame_dependencies.spatial_id = 398 video_header.generic->spatial_index; 399 descriptor.frame_dependencies.temporal_id = 400 video_header.generic->temporal_index; 401 for (int64_t dep : video_header.generic->dependencies) { 402 descriptor.frame_dependencies.frame_diffs.push_back( 403 video_header.generic->frame_id - dep); 404 } 405 descriptor.frame_dependencies.chain_diffs = 406 video_header.generic->chain_diffs; 407 descriptor.frame_dependencies.decode_target_indications = 408 video_header.generic->decode_target_indications; 409 RTC_DCHECK_EQ( 410 descriptor.frame_dependencies.decode_target_indications.size(), 411 video_structure_->num_decode_targets); 412 413 if (first_packet) { 414 descriptor.active_decode_targets_bitmask = 415 active_decode_targets_tracker_.ActiveDecodeTargetsBitmask(); 416 } 417 // VP9 mark all layer frames of the first picture as kVideoFrameKey, 418 // Structure should be attached to the descriptor to lowest spatial layer 419 // when inter layer dependency is used, i.e. L structures; or to all 420 // layers when inter layer dependency is not used, i.e. S structures. 421 // Distinguish these two cases by checking if there are any dependencies. 422 if (video_header.frame_type == VideoFrameType::kVideoFrameKey && 423 video_header.generic->dependencies.empty() && first_packet) { 424 // To avoid extra structure copy, temporary share ownership of the 425 // video_structure with the dependency descriptor. 426 descriptor.attached_structure = 427 absl::WrapUnique(video_structure_.get()); 428 } 429 extension_is_set = packet->SetExtension<RtpDependencyDescriptorExtension>( 430 *video_structure_, 431 active_decode_targets_tracker_.ActiveChainsBitmask(), descriptor); 432 433 // Remove the temporary shared ownership. 434 descriptor.attached_structure.release(); 435 } 436 437 // Do not use generic frame descriptor when dependency descriptor is stored. 438 if (packet->IsRegistered<RtpGenericFrameDescriptorExtension00>() && 439 !extension_is_set) { 440 RtpGenericFrameDescriptor generic_descriptor; 441 generic_descriptor.SetFirstPacketInSubFrame(first_packet); 442 generic_descriptor.SetLastPacketInSubFrame(last_packet); 443 444 if (first_packet) { 445 generic_descriptor.SetFrameId( 446 static_cast<uint16_t>(video_header.generic->frame_id)); 447 for (int64_t dep : video_header.generic->dependencies) { 448 generic_descriptor.AddFrameDependencyDiff( 449 video_header.generic->frame_id - dep); 450 } 451 452 uint8_t spatial_bitmask = 1 << video_header.generic->spatial_index; 453 generic_descriptor.SetSpatialLayersBitmask(spatial_bitmask); 454 455 generic_descriptor.SetTemporalLayer( 456 video_header.generic->temporal_index); 457 458 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) { 459 generic_descriptor.SetResolution(video_header.width, 460 video_header.height); 461 } 462 } 463 464 packet->SetExtension<RtpGenericFrameDescriptorExtension00>( 465 generic_descriptor); 466 } 467 } 468 469 if (packet->IsRegistered<RtpVideoLayersAllocationExtension>() && 470 first_packet && 471 send_allocation_ != SendVideoLayersAllocation::kDontSend && 472 (video_header.frame_type == VideoFrameType::kVideoFrameKey || 473 PacketWillLikelyBeRequestedForRestransmissionIfLost(video_header))) { 474 VideoLayersAllocation allocation = allocation_.value(); 475 allocation.resolution_and_frame_rate_is_valid = 476 send_allocation_ == SendVideoLayersAllocation::kSendWithResolution; 477 packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation); 478 } 479 480 if (first_packet && video_header.video_frame_tracking_id) { 481 packet->SetExtension<VideoFrameTrackingIdExtension>( 482 *video_header.video_frame_tracking_id); 483 } 484 485 if (last_packet && video_header.frame_instrumentation_data) { 486 packet->SetExtension<CorruptionDetectionExtension>( 487 CorruptionDetectionMessage::FromFrameInstrumentationData( 488 *video_header.frame_instrumentation_data)); 489 } 490 } 491 492 bool RTPSenderVideo::SendVideo(int payload_type, 493 std::optional<VideoCodecType> codec_type, 494 uint32_t rtp_timestamp, 495 Timestamp capture_time, 496 ArrayView<const uint8_t> payload, 497 size_t encoder_output_size, 498 RTPVideoHeader video_header, 499 TimeDelta expected_retransmission_time, 500 std::vector<uint32_t> csrcs) { 501 RTC_CHECK_RUNS_SERIALIZED(&send_checker_); 502 503 // TODO(b/446768451): Add a check that Codec type can only be absent when 504 // using raw packetization once downstream projects have been updated. 505 506 if (video_header.frame_type == VideoFrameType::kEmptyFrame) 507 return true; 508 509 if (payload.empty()) 510 return false; 511 512 if (!rtp_sender_->SendingMedia()) { 513 return false; 514 } 515 516 int32_t retransmission_settings = retransmission_settings_; 517 if (codec_type == VideoCodecType::kVideoCodecH264) { 518 // Backward compatibility for older receivers without temporal layer logic. 519 retransmission_settings = kRetransmitBaseLayer | kRetransmitHigherLayers; 520 } 521 const uint8_t temporal_id = GetTemporalId(video_header); 522 // TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be 523 // replaced by expected_retransmission_time.IsFinite(). 524 const bool allow_retransmission = 525 expected_retransmission_time.IsFinite() && 526 AllowRetransmission(temporal_id, retransmission_settings, 527 expected_retransmission_time); 528 529 MaybeUpdateCurrentPlayoutDelay(video_header); 530 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) { 531 if (current_playout_delay_.has_value()) { 532 // Force playout delay on key-frames, if set. 533 playout_delay_pending_ = true; 534 } 535 if (allocation_) { 536 // Send the bitrate allocation on every key frame. 537 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution; 538 } 539 } 540 541 if (video_structure_ != nullptr && video_header.generic) { 542 active_decode_targets_tracker_.OnFrame( 543 video_structure_->decode_target_protected_by_chain, 544 video_header.generic->active_decode_targets, 545 video_header.frame_type == VideoFrameType::kVideoFrameKey, 546 video_header.generic->frame_id, video_header.generic->chain_diffs); 547 } 548 549 // No FEC protection for upper temporal layers, if used. 550 const bool use_fec = fec_type_.has_value() && 551 (temporal_id == 0 || temporal_id == kNoTemporalIdx); 552 553 // Maximum size of packet including rtp headers. 554 // Extra space left in case packet will be resent using fec or rtx. 555 int packet_capacity = rtp_sender_->MaxRtpPacketSize(); 556 if (use_fec) { 557 packet_capacity -= FecPacketOverhead(); 558 } 559 if (allow_retransmission) { 560 packet_capacity -= rtp_sender_->RtxPacketOverhead(); 561 } 562 563 std::unique_ptr<RtpPacketToSend> single_packet = 564 rtp_sender_->AllocatePacket(csrcs); 565 RTC_DCHECK_LE(packet_capacity, single_packet->capacity()); 566 single_packet->SetPayloadType(payload_type); 567 single_packet->SetTimestamp(rtp_timestamp); 568 if (capture_time.IsFinite()) 569 single_packet->set_capture_time(capture_time); 570 571 // Construct the absolute capture time extension if not provided. 572 if (!video_header.absolute_capture_time.has_value() && 573 capture_time.IsFinite()) { 574 video_header.absolute_capture_time.emplace(); 575 video_header.absolute_capture_time->absolute_capture_timestamp = 576 Int64MsToUQ32x32( 577 clock_->ConvertTimestampToNtpTime(capture_time).ToMs()); 578 video_header.absolute_capture_time->estimated_capture_clock_offset = 0; 579 } 580 581 // Let `absolute_capture_time_sender_` decide if the extension should be sent. 582 if (video_header.absolute_capture_time.has_value()) { 583 video_header.absolute_capture_time = 584 absolute_capture_time_sender_.OnSendPacket( 585 AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(), csrcs), 586 single_packet->Timestamp(), kVideoPayloadTypeFrequency, 587 NtpTime( 588 video_header.absolute_capture_time->absolute_capture_timestamp), 589 video_header.absolute_capture_time->estimated_capture_clock_offset); 590 } 591 592 auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet); 593 auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet); 594 auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet); 595 // Simplest way to estimate how much extensions would occupy is to set them. 596 AddRtpHeaderExtensions(video_header, 597 /*first_packet=*/true, /*last_packet=*/true, 598 single_packet.get()); 599 if (video_structure_ != nullptr && 600 single_packet->IsRegistered<RtpDependencyDescriptorExtension>() && 601 !single_packet->HasExtension<RtpDependencyDescriptorExtension>()) { 602 RTC_DCHECK_EQ(video_header.frame_type, VideoFrameType::kVideoFrameKey); 603 // Disable attaching dependency descriptor to delta packets (including 604 // non-first packet of a key frame) when it wasn't attached to a key frame, 605 // as dependency descriptor can't be usable in such case. 606 // This can also happen when the descriptor is larger than 15 bytes and 607 // two-byte header extensions are not negotiated using extmap-allow-mixed. 608 RTC_LOG(LS_WARNING) << "Disable dependency descriptor because failed to " 609 "attach it to a key frame."; 610 video_structure_ = nullptr; 611 } 612 613 AddRtpHeaderExtensions(video_header, 614 /*first_packet=*/true, /*last_packet=*/false, 615 first_packet.get()); 616 AddRtpHeaderExtensions(video_header, 617 /*first_packet=*/false, /*last_packet=*/false, 618 middle_packet.get()); 619 AddRtpHeaderExtensions(video_header, 620 /*first_packet=*/false, /*last_packet=*/true, 621 last_packet.get()); 622 623 RTC_DCHECK_GT(packet_capacity, single_packet->headers_size()); 624 RTC_DCHECK_GT(packet_capacity, first_packet->headers_size()); 625 RTC_DCHECK_GT(packet_capacity, middle_packet->headers_size()); 626 RTC_DCHECK_GT(packet_capacity, last_packet->headers_size()); 627 RtpPacketizer::PayloadSizeLimits limits; 628 limits.max_payload_len = packet_capacity - middle_packet->headers_size(); 629 630 RTC_DCHECK_GE(single_packet->headers_size(), middle_packet->headers_size()); 631 limits.single_packet_reduction_len = 632 single_packet->headers_size() - middle_packet->headers_size(); 633 634 RTC_DCHECK_GE(first_packet->headers_size(), middle_packet->headers_size()); 635 limits.first_packet_reduction_len = 636 first_packet->headers_size() - middle_packet->headers_size(); 637 638 RTC_DCHECK_GE(last_packet->headers_size(), middle_packet->headers_size()); 639 limits.last_packet_reduction_len = 640 last_packet->headers_size() - middle_packet->headers_size(); 641 642 bool has_generic_descriptor = 643 first_packet->HasExtension<RtpGenericFrameDescriptorExtension00>() || 644 first_packet->HasExtension<RtpDependencyDescriptorExtension>(); 645 646 // Minimization of the vp8 descriptor may erase temporal_id, so use 647 // `temporal_id` rather than reference `video_header` beyond this point. 648 if (has_generic_descriptor) { 649 MinimizeDescriptor(&video_header); 650 } 651 652 Buffer encrypted_video_payload; 653 if (frame_encryptor_ != nullptr) { 654 const size_t max_ciphertext_size = 655 frame_encryptor_->GetMaxCiphertextByteSize(MediaType::VIDEO, 656 payload.size()); 657 encrypted_video_payload.SetSize(max_ciphertext_size); 658 659 size_t bytes_written = 0; 660 661 // Enable header authentication if the field trial isn't disabled. 662 std::vector<uint8_t> additional_data; 663 if (generic_descriptor_auth_experiment_) { 664 additional_data = RtpDescriptorAuthentication(video_header); 665 } 666 667 if (frame_encryptor_->Encrypt( 668 MediaType::VIDEO, first_packet->Ssrc(), additional_data, payload, 669 encrypted_video_payload, &bytes_written) != 0) { 670 return false; 671 } 672 673 encrypted_video_payload.SetSize(bytes_written); 674 payload = encrypted_video_payload; 675 } else if (require_frame_encryption_) { 676 RTC_LOG(LS_WARNING) 677 << "No FrameEncryptor is attached to this video sending stream but " 678 "one is required since require_frame_encryptor is set"; 679 } 680 681 std::unique_ptr<RtpPacketizer> packetizer = 682 RtpPacketizer::Create(raw_packetization_ ? std::nullopt : codec_type, 683 payload, limits, video_header); 684 685 const size_t num_packets = packetizer->NumPackets(); 686 687 if (num_packets == 0) 688 return false; 689 690 bool first_frame = first_frame_sent_(); 691 std::vector<std::unique_ptr<RtpPacketToSend>> rtp_packets; 692 for (size_t i = 0; i < num_packets; ++i) { 693 std::unique_ptr<RtpPacketToSend> packet; 694 int expected_payload_capacity; 695 // Choose right packet template: 696 if (num_packets == 1) { 697 packet = std::move(single_packet); 698 expected_payload_capacity = 699 limits.max_payload_len - limits.single_packet_reduction_len; 700 } else if (i == 0) { 701 packet = std::move(first_packet); 702 expected_payload_capacity = 703 limits.max_payload_len - limits.first_packet_reduction_len; 704 } else if (i == num_packets - 1) { 705 packet = std::move(last_packet); 706 expected_payload_capacity = 707 limits.max_payload_len - limits.last_packet_reduction_len; 708 } else { 709 packet = std::make_unique<RtpPacketToSend>(*middle_packet); 710 expected_payload_capacity = limits.max_payload_len; 711 } 712 713 packet->set_first_packet_of_frame(i == 0); 714 715 if (!packetizer->NextPacket(packet.get())) 716 return false; 717 RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity); 718 719 packet->set_allow_retransmission(allow_retransmission); 720 packet->set_is_key_frame(video_header.frame_type == 721 VideoFrameType::kVideoFrameKey); 722 723 // Put packetization finish timestamp into extension. 724 if (packet->HasExtension<VideoTimingExtension>()) { 725 packet->set_packetization_finish_time(clock_->CurrentTime()); 726 } 727 728 packet->set_fec_protect_packet(use_fec); 729 730 if (red_enabled()) { 731 // TODO(sprang): Consider packetizing directly into packets with the RED 732 // header already in place, to avoid this copy. 733 std::unique_ptr<RtpPacketToSend> red_packet(new RtpPacketToSend(*packet)); 734 BuildRedPayload(*packet, red_packet.get()); 735 red_packet->SetPayloadType(*red_payload_type_); 736 red_packet->set_is_red(true); 737 738 // Append `red_packet` instead of `packet` to output. 739 red_packet->set_packet_type(RtpPacketMediaType::kVideo); 740 red_packet->set_allow_retransmission(packet->allow_retransmission()); 741 rtp_packets.emplace_back(std::move(red_packet)); 742 } else { 743 packet->set_packet_type(RtpPacketMediaType::kVideo); 744 rtp_packets.emplace_back(std::move(packet)); 745 } 746 747 if (first_frame) { 748 if (i == 0) { 749 RTC_LOG(LS_INFO) 750 << "Sent first RTP packet of the first video frame (pre-pacer)"; 751 } 752 if (i == num_packets - 1) { 753 RTC_LOG(LS_INFO) 754 << "Sent last RTP packet of the first video frame (pre-pacer)"; 755 } 756 } 757 } 758 759 LogAndSendToNetwork(std::move(rtp_packets), encoder_output_size); 760 761 // Update details about the last sent frame. 762 last_rotation_ = video_header.rotation; 763 764 if (video_header.color_space != last_color_space_) { 765 last_color_space_ = video_header.color_space; 766 transmit_color_space_next_frame_ = !IsBaseLayer(video_header); 767 } else { 768 transmit_color_space_next_frame_ = 769 transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false; 770 } 771 772 if (video_header.frame_type == VideoFrameType::kVideoFrameKey || 773 PacketWillLikelyBeRequestedForRestransmissionIfLost(video_header)) { 774 // This frame will likely be delivered, no need to populate playout 775 // delay extensions until it changes again. 776 playout_delay_pending_ = false; 777 if (send_allocation_ == SendVideoLayersAllocation::kSendWithResolution) { 778 last_full_sent_allocation_ = allocation_; 779 } 780 send_allocation_ = SendVideoLayersAllocation::kDontSend; 781 } 782 783 return true; 784 } 785 786 bool RTPSenderVideo::SendEncodedImage(int payload_type, 787 std::optional<VideoCodecType> codec_type, 788 uint32_t rtp_timestamp, 789 const EncodedImage& encoded_image, 790 RTPVideoHeader video_header, 791 TimeDelta expected_retransmission_time, 792 const std::vector<uint32_t>& csrcs) { 793 if (frame_transformer_delegate_) { 794 // The frame will be sent async once transformed. 795 return frame_transformer_delegate_->TransformFrame( 796 payload_type, codec_type, rtp_timestamp, encoded_image, video_header, 797 expected_retransmission_time, csrcs); 798 } 799 return SendVideo(payload_type, codec_type, rtp_timestamp, 800 encoded_image.CaptureTime(), encoded_image, 801 encoded_image.size(), video_header, 802 expected_retransmission_time, csrcs); 803 } 804 805 DataRate RTPSenderVideo::PostEncodeOverhead() const { 806 MutexLock lock(&stats_mutex_); 807 return post_encode_overhead_bitrate_.Rate(clock_->CurrentTime()) 808 .value_or(DataRate::Zero()); 809 } 810 811 bool RTPSenderVideo::AllowRetransmission( 812 uint8_t temporal_id, 813 int32_t retransmission_settings, 814 TimeDelta expected_retransmission_time) { 815 if (retransmission_settings == kRetransmitOff) 816 return false; 817 818 MutexLock lock(&stats_mutex_); 819 // Media packet storage. 820 if ((retransmission_settings & kConditionallyRetransmitHigherLayers) && 821 UpdateConditionalRetransmit(temporal_id, expected_retransmission_time)) { 822 retransmission_settings |= kRetransmitHigherLayers; 823 } 824 825 if (temporal_id == kNoTemporalIdx) 826 return true; 827 828 if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0) 829 return true; 830 831 if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0) 832 return true; 833 834 return false; 835 } 836 837 uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) { 838 struct TemporalIdGetter { 839 uint8_t operator()(const RTPVideoHeaderVP8& vp8) { return vp8.temporalIdx; } 840 uint8_t operator()(const RTPVideoHeaderVP9& vp9) { 841 return vp9.temporal_idx; 842 } 843 uint8_t operator()(const RTPVideoHeaderH264&) { return kNoTemporalIdx; } 844 uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) { 845 return kNoTemporalIdx; 846 } 847 uint8_t operator()(const std::monostate&) { return kNoTemporalIdx; } 848 }; 849 return std::visit(TemporalIdGetter(), header.video_type_header); 850 } 851 852 bool RTPSenderVideo::UpdateConditionalRetransmit( 853 uint8_t temporal_id, 854 TimeDelta expected_retransmission_time) { 855 Timestamp now = clock_->CurrentTime(); 856 // Update stats for any temporal layer. 857 TemporalLayerStats* current_layer_stats = 858 &frame_stats_by_temporal_layer_[temporal_id]; 859 current_layer_stats->frame_rate.Update(now); 860 TimeDelta tl_frame_interval = now - current_layer_stats->last_frame_time; 861 current_layer_stats->last_frame_time = now; 862 863 // Conditional retransmit only applies to upper layers. 864 if (temporal_id != kNoTemporalIdx && temporal_id > 0) { 865 if (tl_frame_interval >= kMaxUnretransmittableFrameInterval) { 866 // Too long since a retransmittable frame in this layer, enable NACK 867 // protection. 868 return true; 869 } else { 870 // Estimate when the next frame of any lower layer will be sent. 871 Timestamp expected_next_frame_time = Timestamp::PlusInfinity(); 872 for (int i = temporal_id - 1; i >= 0; --i) { 873 TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i]; 874 std::optional<Frequency> rate = stats->frame_rate.Rate(now); 875 if (rate > Frequency::Zero()) { 876 Timestamp tl_next = stats->last_frame_time + 1 / *rate; 877 if (tl_next - now > -expected_retransmission_time && 878 tl_next < expected_next_frame_time) { 879 expected_next_frame_time = tl_next; 880 } 881 } 882 } 883 884 if (expected_next_frame_time - now > expected_retransmission_time) { 885 // The next frame in a lower layer is expected at a later time (or 886 // unable to tell due to lack of data) than a retransmission is 887 // estimated to be able to arrive, so allow this packet to be nacked. 888 return true; 889 } 890 } 891 } 892 893 return false; 894 } 895 896 void RTPSenderVideo::MaybeUpdateCurrentPlayoutDelay( 897 const RTPVideoHeader& header) { 898 std::optional<VideoPlayoutDelay> requested_delay = 899 forced_playout_delay_.has_value() ? forced_playout_delay_ 900 : header.playout_delay; 901 902 if (!requested_delay.has_value()) { 903 return; 904 } 905 906 current_playout_delay_ = requested_delay; 907 playout_delay_pending_ = true; 908 } 909 910 } // namespace webrtc