rtp_video_sender.h (9732B)
1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_RTP_VIDEO_SENDER_H_ 12 #define CALL_RTP_VIDEO_SENDER_H_ 13 14 #include <cstddef> 15 #include <cstdint> 16 #include <map> 17 #include <memory> 18 #include <vector> 19 20 #include "absl/base/nullability.h" 21 #include "api/array_view.h" 22 #include "api/call/bitrate_allocation.h" 23 #include "api/call/transport.h" 24 #include "api/crypto/crypto_options.h" 25 #include "api/environment/environment.h" 26 #include "api/fec_controller.h" 27 #include "api/frame_transformer_interface.h" 28 #include "api/scoped_refptr.h" 29 #include "api/sequence_checker.h" 30 #include "api/task_queue/pending_task_safety_flag.h" 31 #include "api/task_queue/task_queue_base.h" 32 #include "api/units/data_rate.h" 33 #include "api/units/data_size.h" 34 #include "api/units/frequency.h" 35 #include "api/video/encoded_image.h" 36 #include "api/video/video_layers_allocation.h" 37 #include "api/video_codecs/video_encoder.h" 38 #include "call/rtp_config.h" 39 #include "call/rtp_payload_params.h" 40 #include "call/rtp_transport_controller_send_interface.h" 41 #include "call/rtp_video_sender_interface.h" 42 #include "common_video/frame_counts.h" 43 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 44 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" 45 #include "modules/rtp_rtcp/source/rtp_sender.h" 46 #include "modules/rtp_rtcp/source/rtp_sender_video.h" 47 #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" 48 #include "modules/rtp_rtcp/source/video_fec_generator.h" 49 #include "rtc_base/rate_limiter.h" 50 #include "rtc_base/synchronization/mutex.h" 51 #include "rtc_base/system/no_unique_address.h" 52 #include "rtc_base/thread_annotations.h" 53 54 namespace webrtc { 55 56 class FrameEncryptorInterface; 57 class RtpTransportControllerSendInterface; 58 59 namespace webrtc_internal_rtp_video_sender { 60 // RTP state for a single simulcast stream. Internal to the implementation of 61 // RtpVideoSender. 62 struct RtpStreamSender { 63 RtpStreamSender(std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp, 64 std::unique_ptr<RTPSenderVideo> sender_video, 65 std::unique_ptr<VideoFecGenerator> fec_generator); 66 ~RtpStreamSender(); 67 68 RtpStreamSender(RtpStreamSender&&) = default; 69 RtpStreamSender& operator=(RtpStreamSender&&) = default; 70 71 // Note: Needs pointer stability. 72 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp; 73 std::unique_ptr<RTPSenderVideo> sender_video; 74 std::unique_ptr<VideoFecGenerator> fec_generator; 75 }; 76 77 } // namespace webrtc_internal_rtp_video_sender 78 79 // RtpVideoSender routes outgoing data to the correct sending RTP module, based 80 // on the simulcast layer in RTPVideoHeader. 81 class RtpVideoSender : public RtpVideoSenderInterface, 82 public VCMProtectionCallback, 83 public StreamFeedbackObserver { 84 public: 85 // Rtp modules are assumed to be sorted in simulcast index order. 86 RtpVideoSender( 87 const Environment& env, 88 TaskQueueBase* absl_nonnull transport_queue, 89 const std::map<uint32_t, RtpState>& suspended_ssrcs, 90 const std::map<uint32_t, RtpPayloadState>& states, 91 const RtpConfig& rtp_config, 92 int rtcp_report_interval_ms, 93 Transport* send_transport, 94 const RtpSenderObservers& observers, 95 RtpTransportControllerSendInterface* transport, 96 RateLimiter* retransmission_limiter, // move inside RtpTransport 97 std::unique_ptr<FecController> fec_controller, 98 FrameEncryptorInterface* frame_encryptor, 99 const CryptoOptions& crypto_options, // move inside RtpTransport 100 scoped_refptr<FrameTransformerInterface> frame_transformer); 101 ~RtpVideoSender() override; 102 103 RtpVideoSender(const RtpVideoSender&) = delete; 104 RtpVideoSender& operator=(const RtpVideoSender&) = delete; 105 106 void SetSending(bool enabled) RTC_LOCKS_EXCLUDED(mutex_) override; 107 bool IsActive() RTC_LOCKS_EXCLUDED(mutex_) override; 108 109 void OnNetworkAvailability(bool network_available) 110 RTC_LOCKS_EXCLUDED(mutex_) override; 111 std::map<uint32_t, RtpState> GetRtpStates() const 112 RTC_LOCKS_EXCLUDED(mutex_) override; 113 std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const 114 RTC_LOCKS_EXCLUDED(mutex_) override; 115 116 void DeliverRtcp(ArrayView<const uint8_t> packet) 117 RTC_LOCKS_EXCLUDED(mutex_) override; 118 // Implements webrtc::VCMProtectionCallback. 119 int ProtectionRequest(const FecProtectionParams* delta_params, 120 const FecProtectionParams* key_params, 121 uint32_t* sent_video_rate_bps, 122 uint32_t* sent_nack_rate_bps, 123 uint32_t* sent_fec_rate_bps) 124 RTC_LOCKS_EXCLUDED(mutex_) override; 125 126 // 'retransmission_mode' is either a value of enum RetransmissionMode, or 127 // computed with bitwise operators on values of enum RetransmissionMode. 128 void SetRetransmissionMode(int retransmission_mode) 129 RTC_LOCKS_EXCLUDED(mutex_) override; 130 131 // Implements FecControllerOverride. 132 void SetFecAllowed(bool fec_allowed) RTC_LOCKS_EXCLUDED(mutex_) override; 133 134 // Implements EncodedImageCallback. 135 // Returns 0 if the packet was routed / sent, -1 otherwise. 136 EncodedImageCallback::Result OnEncodedImage( 137 const EncodedImage& encoded_image, 138 const CodecSpecificInfo* codec_specific_info) 139 RTC_LOCKS_EXCLUDED(mutex_) override; 140 141 void OnBitrateAllocationUpdated(const VideoBitrateAllocation& bitrate) 142 RTC_LOCKS_EXCLUDED(mutex_) override; 143 void OnVideoLayersAllocationUpdated( 144 const VideoLayersAllocation& layers) override; 145 void OnTransportOverheadChanged(size_t transport_overhead_bytes_per_packet) 146 RTC_LOCKS_EXCLUDED(mutex_) override; 147 void OnBitrateUpdated(BitrateAllocationUpdate update, int framerate) 148 RTC_LOCKS_EXCLUDED(mutex_) override; 149 uint32_t GetPayloadBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 150 uint32_t GetProtectionBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 151 void SetEncodingData(size_t width, size_t height, size_t num_temporal_layers) 152 RTC_LOCKS_EXCLUDED(mutex_) override; 153 154 // Sets the list of CSRCs to be included in every packet. If more than 155 // kRtpCsrcSize CSRCs are provided, only the first kRtpCsrcSize elements are 156 // kept. 157 void SetCsrcs(ArrayView<const uint32_t> csrcs) 158 RTC_LOCKS_EXCLUDED(mutex_) override; 159 160 std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( 161 uint32_t ssrc, 162 ArrayView<const uint16_t> sequence_numbers) const 163 RTC_LOCKS_EXCLUDED(mutex_) override; 164 165 // From StreamFeedbackObserver. 166 void OnPacketFeedbackVector( 167 std::vector<StreamPacketInfo> packet_feedback_vector) 168 RTC_LOCKS_EXCLUDED(mutex_) override; 169 170 private: 171 bool IsActiveLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 172 void SetActiveModulesLocked(bool sending) 173 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 174 void UpdateModuleSendingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 175 void ConfigureProtection(); 176 void ConfigureSsrcs(const std::map<uint32_t, RtpState>& suspended_ssrcs); 177 bool NackEnabled() const; 178 DataRate GetPostEncodeOverhead() const; 179 DataRate CalculateOverheadRate(DataRate data_rate, 180 DataSize packet_size, 181 DataSize overhead_per_packet, 182 Frequency framerate) const; 183 void SetModuleIsActive(bool sending, RtpRtcpInterface& rtp_module) 184 RTC_RUN_ON(transport_checker_); 185 186 const Environment env_; 187 const bool use_frame_rate_for_overhead_; 188 const bool has_packet_feedback_; 189 190 // Semantically equivalent to checking for `transport_->GetWorkerQueue()` 191 // but some tests need to be updated to call from the correct context. 192 RTC_NO_UNIQUE_ADDRESS SequenceChecker transport_checker_; 193 TaskQueueBase& transport_queue_; 194 195 // TODO(bugs.webrtc.org/13517): Remove mutex_ once RtpVideoSender runs on the 196 // transport task queue. 197 mutable Mutex mutex_; 198 bool active_ RTC_GUARDED_BY(mutex_); 199 200 const std::unique_ptr<FecController> fec_controller_; 201 bool fec_allowed_ RTC_GUARDED_BY(mutex_); 202 203 // Rtp modules are assumed to be sorted in simulcast index order. 204 const std::vector<webrtc_internal_rtp_video_sender::RtpStreamSender> 205 rtp_streams_; 206 const RtpConfig rtp_config_; 207 RtpTransportControllerSendInterface* const transport_; 208 209 // The list of CSRCs to be included when sending an encoded image. 210 std::vector<uint32_t> csrcs_ RTC_GUARDED_BY(mutex_); 211 212 // When using the generic descriptor we want all simulcast streams to share 213 // one frame id space (so that the SFU can switch stream without having to 214 // rewrite the frame id), therefore `shared_frame_id` has to live in a place 215 // where we are aware of all the different streams. 216 int64_t shared_frame_id_ = 0; 217 const bool independent_frame_ids_; 218 std::vector<RtpPayloadParams> params_ RTC_GUARDED_BY(mutex_); 219 220 size_t transport_overhead_bytes_per_packet_ RTC_GUARDED_BY(mutex_); 221 uint32_t protection_bitrate_bps_; 222 uint32_t encoder_target_rate_bps_; 223 224 std::vector<bool> loss_mask_vector_ RTC_GUARDED_BY(mutex_); 225 226 std::vector<FrameCounts> frame_counts_ RTC_GUARDED_BY(mutex_); 227 FrameCountObserver* const frame_count_observer_; 228 229 // Effectively const map from SSRC to RtpRtcp, for all media SSRCs. 230 // This map is set at construction time and never changed, but it's 231 // non-trivial to make it properly const. 232 std::map<uint32_t, RtpRtcpInterface*> ssrc_to_rtp_module_; 233 234 ScopedTaskSafety safety_; 235 }; 236 237 } // namespace webrtc 238 239 #endif // CALL_RTP_VIDEO_SENDER_H_