rtp_sender_video.h (11088B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 12 #define MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 13 14 #include <cstddef> 15 #include <cstdint> 16 #include <map> 17 #include <memory> 18 #include <optional> 19 #include <vector> 20 21 #include "api/array_view.h" 22 #include "api/field_trials_view.h" 23 #include "api/frame_transformer_interface.h" 24 #include "api/scoped_refptr.h" 25 #include "api/task_queue/task_queue_factory.h" 26 #include "api/transport/rtp/dependency_descriptor.h" 27 #include "api/units/data_rate.h" 28 #include "api/units/time_delta.h" 29 #include "api/units/timestamp.h" 30 #include "api/video/color_space.h" 31 #include "api/video/encoded_image.h" 32 #include "api/video/video_codec_type.h" 33 #include "api/video/video_layers_allocation.h" 34 #include "api/video/video_rotation.h" 35 #include "api/video/video_timing.h" 36 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 37 #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" 38 #include "modules/rtp_rtcp/source/active_decode_targets_helper.h" 39 #include "modules/rtp_rtcp/source/rtp_sender.h" 40 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" 41 #include "modules/rtp_rtcp/source/rtp_video_header.h" 42 #include "modules/rtp_rtcp/source/video_fec_generator.h" 43 #include "rtc_base/bitrate_tracker.h" 44 #include "rtc_base/frequency_tracker.h" 45 #include "rtc_base/one_time_event.h" 46 #include "rtc_base/race_checker.h" 47 #include "rtc_base/synchronization/mutex.h" 48 #include "rtc_base/thread_annotations.h" 49 #include "system_wrappers/include/clock.h" 50 51 namespace webrtc { 52 53 class FrameEncryptorInterface; 54 class RtpPacketizer; 55 class RtpPacketToSend; 56 57 // kConditionallyRetransmitHigherLayers allows retransmission of video frames 58 // in higher layers if either the last frame in that layer was too far back in 59 // time, or if we estimate that a new frame will be available in a lower layer 60 // in a shorter time than it would take to request and receive a retransmission. 61 enum RetransmissionMode : uint8_t { 62 kRetransmitOff = 0x0, 63 kRetransmitBaseLayer = 0x2, 64 kRetransmitHigherLayers = 0x4, 65 kRetransmitAllLayers = 0x6, 66 kConditionallyRetransmitHigherLayers = 0x8 67 }; 68 69 class RTPSenderVideo : public RTPVideoFrameSenderInterface { 70 public: 71 static constexpr TimeDelta kTLRateWindowSize = TimeDelta::Millis(2'500); 72 73 struct Config { 74 Config() = default; 75 Config(const Config&) = delete; 76 Config(Config&&) = default; 77 78 // All members of this struct, with the exception of `field_trials`, are 79 // expected to outlive the RTPSenderVideo object they are passed to. 80 Clock* clock = nullptr; 81 RTPSender* rtp_sender = nullptr; 82 // Some FEC data is duplicated here in preparation of moving FEC to 83 // the egress stage. 84 std::optional<VideoFecGenerator::FecType> fec_type; 85 size_t fec_overhead_bytes = 0; // Per packet max FEC overhead. 86 FrameEncryptorInterface* frame_encryptor = nullptr; 87 bool require_frame_encryption = false; 88 bool enable_retransmit_all_layers = false; 89 std::optional<int> red_payload_type; 90 const FieldTrialsView* field_trials = nullptr; 91 scoped_refptr<FrameTransformerInterface> frame_transformer; 92 TaskQueueFactory* task_queue_factory = nullptr; 93 bool raw_packetization = false; 94 }; 95 96 explicit RTPSenderVideo(const Config& config); 97 98 virtual ~RTPSenderVideo(); 99 100 // `capture_time` and `clock::CurrentTime` should be using the same epoch. 101 // `expected_retransmission_time.IsFinite()` -> retransmission allowed. 102 // `encoder_output_size` is the size of the video frame as it came out of the 103 // video encoder, excluding any additional overhead. 104 // Calls to this method are assumed to be externally serialized. 105 bool SendVideo(int payload_type, 106 std::optional<VideoCodecType> codec_type, 107 uint32_t rtp_timestamp, 108 Timestamp capture_time, 109 ArrayView<const uint8_t> payload, 110 size_t encoder_output_size, 111 RTPVideoHeader video_header, 112 TimeDelta expected_retransmission_time, 113 std::vector<uint32_t> csrcs) override; 114 115 bool SendEncodedImage(int payload_type, 116 std::optional<VideoCodecType> codec_type, 117 uint32_t rtp_timestamp, 118 const EncodedImage& encoded_image, 119 RTPVideoHeader video_header, 120 TimeDelta expected_retransmission_time, 121 const std::vector<uint32_t>& csrcs = {}); 122 123 // Configures video structures produced by encoder to send using the 124 // dependency descriptor rtp header extension. Next call to SendVideo should 125 // have video_header.frame_type == kVideoFrameKey. 126 // All calls to SendVideo after this call must use video_header compatible 127 // with the video_structure. 128 void SetVideoStructure(const FrameDependencyStructure* video_structure); 129 // Should only be used by a RTPSenderVideoFrameTransformerDelegate and exists 130 // to ensure correct syncronization. 131 void SetVideoStructureAfterTransformation( 132 const FrameDependencyStructure* video_structure) override; 133 134 // Sets current active VideoLayersAllocation. The allocation will be sent 135 // using the rtp video layers allocation extension. The allocation will be 136 // sent in full on every key frame. The allocation will be sent once on a 137 // none discardable delta frame per call to this method and will not contain 138 // resolution and frame rate. 139 void SetVideoLayersAllocation(VideoLayersAllocation allocation); 140 // Should only be used by a RTPSenderVideoFrameTransformerDelegate and exists 141 // to ensure correct syncronization. 142 void SetVideoLayersAllocationAfterTransformation( 143 VideoLayersAllocation allocation) override; 144 145 // Returns the current post encode overhead rate, in bps. Note that this is 146 // the payload overhead, eg the VP8 payload headers and any other added 147 // metadata added by transforms. It does not include the RTP headers or 148 // extensions. 149 // TODO(sprang): Consider moving this to RtpSenderEgress so it's in the same 150 // place as the other rate stats. 151 DataRate PostEncodeOverhead() const; 152 153 // 'retransmission_mode' is either a value of enum RetransmissionMode, or 154 // computed with bitwise operators on values of enum RetransmissionMode. 155 void SetRetransmissionSetting(int32_t retransmission_settings); 156 157 protected: 158 static uint8_t GetTemporalId(const RTPVideoHeader& header); 159 bool AllowRetransmission(uint8_t temporal_id, 160 int32_t retransmission_settings, 161 TimeDelta expected_retransmission_time); 162 163 private: 164 struct TemporalLayerStats { 165 FrequencyTracker frame_rate{kTLRateWindowSize}; 166 Timestamp last_frame_time = Timestamp::Zero(); 167 }; 168 169 enum class SendVideoLayersAllocation { 170 kSendWithResolution, 171 kSendWithoutResolution, 172 kDontSend 173 }; 174 175 void SetVideoStructureInternal( 176 const FrameDependencyStructure* video_structure); 177 void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation); 178 179 void AddRtpHeaderExtensions(const RTPVideoHeader& video_header, 180 bool first_packet, 181 bool last_packet, 182 RtpPacketToSend* packet) const 183 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 184 185 size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 186 187 void LogAndSendToNetwork( 188 std::vector<std::unique_ptr<RtpPacketToSend>> packets, 189 size_t encoder_output_size); 190 191 bool red_enabled() const { return red_payload_type_.has_value(); } 192 193 bool UpdateConditionalRetransmit(uint8_t temporal_id, 194 TimeDelta expected_retransmission_time) 195 RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_mutex_); 196 197 void MaybeUpdateCurrentPlayoutDelay(const RTPVideoHeader& header) 198 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 199 200 RTPSender* const rtp_sender_; 201 Clock* const clock_; 202 203 // These members should only be accessed from within SendVideo() to avoid 204 // potential race conditions. 205 RaceChecker send_checker_; 206 int32_t retransmission_settings_ RTC_GUARDED_BY(send_checker_); 207 VideoRotation last_rotation_ RTC_GUARDED_BY(send_checker_); 208 std::optional<ColorSpace> last_color_space_ RTC_GUARDED_BY(send_checker_); 209 bool transmit_color_space_next_frame_ RTC_GUARDED_BY(send_checker_); 210 std::unique_ptr<FrameDependencyStructure> video_structure_ 211 RTC_GUARDED_BY(send_checker_); 212 std::optional<VideoLayersAllocation> allocation_ 213 RTC_GUARDED_BY(send_checker_); 214 // Flag indicating if we should send `allocation_`. 215 SendVideoLayersAllocation send_allocation_ RTC_GUARDED_BY(send_checker_); 216 std::optional<VideoLayersAllocation> last_full_sent_allocation_ 217 RTC_GUARDED_BY(send_checker_); 218 219 // Current target playout delay. 220 std::optional<VideoPlayoutDelay> current_playout_delay_ 221 RTC_GUARDED_BY(send_checker_); 222 // Flag indicating if we need to send `current_playout_delay_` in order 223 // to guarantee it gets delivered. 224 bool playout_delay_pending_; 225 // Set by the field trial WebRTC-ForceSendPlayoutDelay to override the playout 226 // delay of outgoing video frames. 227 const std::optional<VideoPlayoutDelay> forced_playout_delay_; 228 229 // Should never be held when calling out of this class. 230 Mutex mutex_; 231 232 const std::optional<int> red_payload_type_; 233 std::optional<VideoFecGenerator::FecType> fec_type_; 234 const size_t fec_overhead_bytes_; // Per packet max FEC overhead. 235 236 mutable Mutex stats_mutex_; 237 BitrateTracker post_encode_overhead_bitrate_ RTC_GUARDED_BY(stats_mutex_); 238 239 std::map<int, TemporalLayerStats> frame_stats_by_temporal_layer_ 240 RTC_GUARDED_BY(stats_mutex_); 241 242 OneTimeEvent first_frame_sent_; 243 244 // E2EE Custom Video Frame Encryptor (optional) 245 FrameEncryptorInterface* const frame_encryptor_ = nullptr; 246 // If set to true will require all outgoing frames to pass through an 247 // initialized frame_encryptor_ before being sent out of the network. 248 // Otherwise these payloads will be dropped. 249 const bool require_frame_encryption_; 250 // Set to true if the generic descriptor should be authenticated. 251 const bool generic_descriptor_auth_experiment_; 252 253 const bool raw_packetization_; 254 255 AbsoluteCaptureTimeSender absolute_capture_time_sender_ 256 RTC_GUARDED_BY(send_checker_); 257 // Tracks updates to the active decode targets and decides when active decode 258 // targets bitmask should be attached to the dependency descriptor. 259 ActiveDecodeTargetsHelper active_decode_targets_tracker_; 260 261 const scoped_refptr<RTPSenderVideoFrameTransformerDelegate> 262 frame_transformer_delegate_; 263 }; 264 265 } // namespace webrtc 266 267 #endif // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_