video_send_stream.h (10974B)
1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_VIDEO_SEND_STREAM_H_ 12 #define CALL_VIDEO_SEND_STREAM_H_ 13 14 #include <stdint.h> 15 16 #include <map> 17 #include <optional> 18 #include <string> 19 #include <vector> 20 21 #include "api/adaptation/resource.h" 22 #include "api/array_view.h" 23 #include "api/call/transport.h" 24 #include "api/crypto/crypto_options.h" 25 #include "api/frame_transformer_interface.h" 26 #include "api/rtp_parameters.h" 27 #include "api/rtp_sender_setparameters_callback.h" 28 #include "api/scoped_refptr.h" 29 #include "api/units/data_rate.h" 30 #include "api/video/encoded_image.h" 31 #include "api/video/video_content_type.h" 32 #include "api/video/video_frame.h" 33 #include "api/video/video_source_interface.h" 34 #include "api/video/video_stream_encoder_settings.h" 35 #include "api/video_codecs/scalability_mode.h" 36 #include "api/video_codecs/video_encoder_factory.h" 37 #include "call/rtp_config.h" 38 #include "common_video/frame_counts.h" 39 #include "common_video/include/quality_limitation_reason.h" 40 #include "modules/rtp_rtcp/include/report_block_data.h" 41 #include "modules/rtp_rtcp/include/rtcp_statistics.h" 42 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 43 #include "rtc_base/checks.h" 44 #include "video/config/video_encoder_config.h" 45 46 namespace webrtc { 47 48 class FrameEncryptorInterface; 49 50 class VideoSendStream { 51 public: 52 // Multiple StreamStats objects are present if simulcast is used (multiple 53 // kMedia streams) or if RTX or FlexFEC is negotiated. Multiple SVC layers, on 54 // the other hand, does not cause additional StreamStats. 55 struct StreamStats { 56 enum class StreamType { 57 // A media stream is an RTP stream for audio or video. Retransmissions and 58 // FEC is either sent over the same SSRC or negotiated to be sent over 59 // separate SSRCs, in which case separate StreamStats objects exist with 60 // references to this media stream's SSRC. 61 kMedia, 62 // RTX streams are streams dedicated to retransmissions. They have a 63 // dependency on a single kMedia stream: `referenced_media_ssrc`. 64 kRtx, 65 // FlexFEC streams are streams dedicated to FlexFEC. They have a 66 // dependency on a single kMedia stream: `referenced_media_ssrc`. 67 kFlexfec, 68 }; 69 70 StreamStats(); 71 ~StreamStats(); 72 73 std::string ToString() const; 74 75 StreamType type = StreamType::kMedia; 76 // If `type` is kRtx or kFlexfec this value is present. The referenced SSRC 77 // is the kMedia stream that this stream is performing retransmissions or 78 // FEC for. If `type` is kMedia, this value is null. 79 std::optional<uint32_t> referenced_media_ssrc; 80 FrameCounts frame_counts; 81 int width = 0; 82 int height = 0; 83 // TODO(holmer): Move bitrate_bps out to the webrtc::Call layer. 84 int total_bitrate_bps = 0; 85 int retransmit_bitrate_bps = 0; 86 // `avg_delay_ms` and `max_delay_ms` are only used in tests. Consider 87 // deleting. 88 int avg_delay_ms = 0; 89 int max_delay_ms = 0; 90 StreamDataCounters rtp_stats; 91 RtcpPacketTypeCounter rtcp_packet_type_counts; 92 // A snapshot of the most recent Report Block with additional data of 93 // interest to statistics. Used to implement RTCRemoteInboundRtpStreamStats. 94 std::optional<ReportBlockData> report_block_data; 95 double encode_frame_rate = 0.0; 96 int frames_encoded = 0; 97 std::optional<uint64_t> qp_sum; 98 EncodedImage::Psnr psnr_sum; 99 uint64_t psnr_measurements = 0; 100 uint64_t total_encode_time_ms = 0; 101 uint64_t total_encoded_bytes_target = 0; 102 uint32_t huge_frames_sent = 0; 103 std::optional<ScalabilityMode> scalability_mode; 104 // The target bitrate is what we tell the encoder to produce. What the 105 // encoder actually produces is the sum of encoded bytes. 106 std::optional<DataRate> target_bitrate; 107 }; 108 109 struct Stats { 110 Stats(); 111 ~Stats(); 112 std::string ToString(int64_t time_ms) const; 113 std::optional<std::string> encoder_implementation_name; 114 double input_frame_rate = 0; 115 int encode_frame_rate = 0; 116 int avg_encode_time_ms = 0; 117 int encode_usage_percent = 0; 118 uint32_t frames_encoded = 0; 119 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime 120 uint64_t total_encode_time_ms = 0; 121 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget 122 uint64_t total_encoded_bytes_target = 0; 123 uint32_t frames = 0; 124 uint32_t frames_dropped_by_capturer = 0; 125 uint32_t frames_dropped_by_bad_timestamp = 0; 126 uint32_t frames_dropped_by_encoder_queue = 0; 127 uint32_t frames_dropped_by_rate_limiter = 0; 128 uint32_t frames_dropped_by_congestion_window = 0; 129 uint32_t frames_dropped_by_encoder = 0; 130 // Metric only used by legacy getStats()'s BWE. 131 // - Similar to `StreamStats::target_bitrate` except this is for the whole 132 // stream as opposed to being per substream (per SSRC). 133 // - Unlike what you would expect, it is not equal to the sum of all 134 // substream targets and may sometimes over-report e.g. webrtc:392424845. 135 int target_media_bitrate_bps = 0; 136 // Bitrate the encoder is actually producing. 137 int media_bitrate_bps = 0; 138 bool suspended = false; 139 bool bw_limited_resolution = false; 140 bool cpu_limited_resolution = false; 141 bool bw_limited_framerate = false; 142 bool cpu_limited_framerate = false; 143 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationreason 144 QualityLimitationReason quality_limitation_reason = 145 QualityLimitationReason::kNone; 146 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationdurations 147 std::map<QualityLimitationReason, int64_t> quality_limitation_durations_ms; 148 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationresolutionchanges 149 uint32_t quality_limitation_resolution_changes = 0; 150 // Total number of times resolution as been requested to be changed due to 151 // CPU/quality adaptation. 152 int number_of_cpu_adapt_changes = 0; 153 int number_of_quality_adapt_changes = 0; 154 bool has_entered_low_resolution = false; 155 std::map<uint32_t, StreamStats> substreams; 156 webrtc::VideoContentType content_type = 157 webrtc::VideoContentType::UNSPECIFIED; 158 uint32_t frames_sent = 0; 159 uint32_t huge_frames_sent = 0; 160 std::optional<bool> power_efficient_encoder; 161 }; 162 163 struct Config { 164 public: 165 Config() = delete; 166 Config(Config&&); 167 explicit Config(Transport* send_transport); 168 169 Config& operator=(Config&&); 170 Config& operator=(const Config&) = delete; 171 172 ~Config(); 173 174 // Mostly used by tests. Avoid creating copies if you can. 175 Config Copy() const { return Config(*this); } 176 177 std::string ToString() const; 178 179 RtpConfig rtp; 180 181 VideoStreamEncoderSettings encoder_settings; 182 183 // Time interval between RTCP report for video 184 int rtcp_report_interval_ms = 1000; 185 186 // Transport for outgoing packets. 187 Transport* send_transport = nullptr; 188 189 // Expected delay needed by the renderer, i.e. the frame will be delivered 190 // this many milliseconds, if possible, earlier than expected render time. 191 // Only valid if `local_renderer` is set. 192 int render_delay_ms = 0; 193 194 // Target delay in milliseconds. A positive value indicates this stream is 195 // used for streaming instead of a real-time call. 196 int target_delay_ms = 0; 197 198 // True if the stream should be suspended when the available bitrate fall 199 // below the minimum configured bitrate. If this variable is false, the 200 // stream may send at a rate higher than the estimated available bitrate. 201 bool suspend_below_min_bitrate = false; 202 203 // Enables periodic bandwidth probing in application-limited region. 204 bool periodic_alr_bandwidth_probing = false; 205 206 // An optional custom frame encryptor that allows the entire frame to be 207 // encrypted in whatever way the caller chooses. This is not required by 208 // default. 209 scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor; 210 211 // An optional encoder selector provided by the user. 212 // Overrides VideoEncoderFactory::GetEncoderSelector(). 213 // Owned by RtpSenderBase. 214 VideoEncoderFactory::EncoderSelectorInterface* encoder_selector = nullptr; 215 216 // Per PeerConnection cryptography options. 217 CryptoOptions crypto_options; 218 219 scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer; 220 221 private: 222 // Access to the copy constructor is private to force use of the Copy() 223 // method for those exceptional cases where we do use it. 224 Config(const Config&); 225 }; 226 227 // Starts stream activity. 228 // When a stream is active, it can receive, process and deliver packets. 229 virtual void Start() = 0; 230 231 // Stops stream activity. 232 // When a stream is stopped, it can't receive, process or deliver packets. 233 virtual void Stop() = 0; 234 235 // Accessor for determining if the stream is active. This is an inexpensive 236 // call that must be made on the same thread as `Start()` and `Stop()` methods 237 // are called on and will return `true` iff activity has been started 238 // via `Start()`. 239 virtual bool started() = 0; 240 241 // If the resource is overusing, the VideoSendStream will try to reduce 242 // resolution or frame rate until no resource is overusing. 243 // TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor 244 // is moved to Call this method could be deleted altogether in favor of 245 // Call-level APIs only. 246 virtual void AddAdaptationResource(scoped_refptr<Resource> resource) = 0; 247 virtual std::vector<scoped_refptr<Resource>> GetAdaptationResources() = 0; 248 249 virtual void SetSource( 250 VideoSourceInterface<webrtc::VideoFrame>* source, 251 const DegradationPreference& degradation_preference) = 0; 252 253 // Set which streams to send. Must have at least as many SSRCs as configured 254 // in the config. Encoder settings are passed on to the encoder instance along 255 // with the VideoStream settings. 256 virtual void ReconfigureVideoEncoder(VideoEncoderConfig config) = 0; 257 258 virtual void ReconfigureVideoEncoder(VideoEncoderConfig config, 259 SetParametersCallback callback) = 0; 260 261 virtual Stats GetStats() = 0; 262 263 // TODO: webrtc:40644448 - Make this pure virtual. 264 virtual void SetStats(const Stats& stats) { RTC_CHECK_NOTREACHED(); } 265 266 // Sets the list of CSRCs to be included in every packet. 267 virtual void SetCsrcs(ArrayView<const uint32_t> csrcs) = 0; 268 269 virtual void GenerateKeyFrame(const std::vector<std::string>& rids) = 0; 270 271 protected: 272 virtual ~VideoSendStream() {} 273 }; 274 275 } // namespace webrtc 276 277 #endif // CALL_VIDEO_SEND_STREAM_H_