webrtc_video_engine.h (38510B)
1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 12 #define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 13 14 #include <stddef.h> 15 16 #include <cstdint> 17 #include <functional> 18 #include <map> 19 #include <memory> 20 #include <optional> 21 #include <set> 22 #include <string> 23 #include <utility> 24 #include <vector> 25 26 #include "absl/functional/any_invocable.h" 27 #include "absl/strings/string_view.h" 28 #include "api/crypto/crypto_options.h" 29 #include "api/crypto/frame_decryptor_interface.h" 30 #include "api/crypto/frame_encryptor_interface.h" 31 #include "api/environment/environment.h" 32 #include "api/field_trials_view.h" 33 #include "api/frame_transformer_interface.h" 34 #include "api/media_types.h" 35 #include "api/rtc_error.h" 36 #include "api/rtp_headers.h" 37 #include "api/rtp_parameters.h" 38 #include "api/rtp_sender_interface.h" 39 #include "api/scoped_refptr.h" 40 #include "api/sequence_checker.h" 41 #include "api/task_queue/pending_task_safety_flag.h" 42 #include "api/task_queue/task_queue_base.h" 43 #include "api/transport/bitrate_settings.h" 44 #include "api/transport/rtp/rtp_source.h" 45 #include "api/video/recordable_encoded_frame.h" 46 #include "api/video/video_bitrate_allocator_factory.h" 47 #include "api/video/video_frame.h" 48 #include "api/video/video_sink_interface.h" 49 #include "api/video/video_source_interface.h" 50 #include "api/video/video_stream_encoder_settings.h" 51 #include "api/video_codecs/sdp_video_format.h" 52 #include "api/video_codecs/video_encoder_factory.h" 53 #include "call/call.h" 54 #include "call/flexfec_receive_stream.h" 55 #include "call/rtp_config.h" 56 #include "call/video_receive_stream.h" 57 #include "call/video_send_stream.h" 58 #include "media/base/codec.h" 59 #include "media/base/media_channel.h" 60 #include "media/base/media_channel_impl.h" 61 #include "media/base/media_config.h" 62 #include "media/base/media_engine.h" 63 #include "media/base/stream_params.h" 64 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" 65 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 66 #include "rtc_base/checks.h" 67 #include "rtc_base/network/sent_packet.h" 68 #include "rtc_base/network_route.h" 69 #include "rtc_base/synchronization/mutex.h" 70 #include "rtc_base/system/no_unique_address.h" 71 #include "rtc_base/thread_annotations.h" 72 #include "video/config/video_encoder_config.h" 73 74 namespace webrtc { 75 class VideoDecoderFactory; 76 class VideoEncoderFactory; 77 } // namespace webrtc 78 79 namespace webrtc { 80 81 // Public for testing. 82 // Inputs StreamStats for all types of substreams (kMedia, kRtx, kFlexfec) and 83 // merges any non-kMedia substream stats object into its referenced kMedia-type 84 // substream. The resulting substreams are all kMedia. This means, for example, 85 // that packet and byte counters of RTX and FlexFEC streams are accounted for in 86 // the relevant RTP media stream's stats. This makes the resulting StreamStats 87 // objects ready to be turned into "outbound-rtp" stats objects for GetStats() 88 // which does not create separate stream stats objects for complementary 89 // streams. 90 std::map<uint32_t, VideoSendStream::StreamStats> 91 MergeInfoAboutOutboundRtpSubstreamsForTesting( 92 const std::map<uint32_t, VideoSendStream::StreamStats>& substreams); 93 94 // WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667). 95 class WebRtcVideoEngine : public VideoEngineInterface { 96 public: 97 // These video codec factories represents all video codecs, i.e. both software 98 // and external hardware codecs. 99 WebRtcVideoEngine(std::unique_ptr<VideoEncoderFactory> video_encoder_factory, 100 std::unique_ptr<VideoDecoderFactory> video_decoder_factory, 101 const FieldTrialsView& trials); 102 103 ~WebRtcVideoEngine() override; 104 105 std::unique_ptr<VideoMediaSendChannelInterface> CreateSendChannel( 106 const Environment& env, 107 Call* call, 108 const MediaConfig& config, 109 const VideoOptions& options, 110 const CryptoOptions& crypto_options, 111 VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) override; 112 std::unique_ptr<VideoMediaReceiveChannelInterface> CreateReceiveChannel( 113 const Environment& env, 114 Call* call, 115 const MediaConfig& config, 116 const VideoOptions& options, 117 const CryptoOptions& crypto_options) override; 118 119 // TODO: https://issues.webrtc.org/360058654 - remove Legacy functions. 120 std::vector<Codec> LegacySendCodecs() const override { 121 return LegacySendCodecs(true); 122 } 123 std::vector<Codec> LegacyRecvCodecs() const override { 124 return LegacyRecvCodecs(true); 125 } 126 std::vector<Codec> LegacySendCodecs(bool include_rtx) const override; 127 std::vector<Codec> LegacyRecvCodecs(bool include_rtx) const override; 128 129 std::vector<RtpHeaderExtensionCapability> GetRtpHeaderExtensions( 130 /* optional field trials from PeerConnection that override those from 131 PeerConnectionFactory */ 132 const webrtc::FieldTrialsView* field_trials) const override; 133 134 private: 135 const std::unique_ptr<VideoDecoderFactory> decoder_factory_; 136 const std::unique_ptr<VideoEncoderFactory> encoder_factory_; 137 const std::unique_ptr<VideoBitrateAllocatorFactory> 138 bitrate_allocator_factory_; 139 const FieldTrialsView& trials_; // from PeerConnectionFactory 140 }; 141 142 struct VideoCodecSettings { 143 explicit VideoCodecSettings(const Codec& codec); 144 145 // Checks if all members of |*this| are equal to the corresponding members 146 // of `other`. 147 bool operator==(const VideoCodecSettings& other) const; 148 bool operator!=(const VideoCodecSettings& other) const; 149 150 // Checks if all members of `a`, except `flexfec_payload_type`, are equal 151 // to the corresponding members of `b`. 152 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a, 153 const VideoCodecSettings& b); 154 155 Codec codec; 156 UlpfecConfig ulpfec; 157 int flexfec_payload_type; // -1 if absent. 158 int rtx_payload_type; // -1 if absent. 159 std::optional<int> rtx_time; 160 }; 161 162 class WebRtcVideoSendChannel : public MediaChannelUtil, 163 public VideoMediaSendChannelInterface, 164 public EncoderSwitchRequestCallback { 165 public: 166 WebRtcVideoSendChannel( 167 const Environment& env, 168 Call* call, 169 const MediaConfig& config, 170 const VideoOptions& options, 171 const CryptoOptions& crypto_options, 172 VideoEncoderFactory* encoder_factory, 173 VideoBitrateAllocatorFactory* bitrate_allocator_factory); 174 ~WebRtcVideoSendChannel() override; 175 176 MediaType media_type() const override { return MediaType::VIDEO; } 177 // Type manipulations 178 VideoMediaSendChannelInterface* AsVideoSendChannel() override { return this; } 179 VoiceMediaSendChannelInterface* AsVoiceSendChannel() override { 180 RTC_CHECK_NOTREACHED(); 181 return nullptr; 182 } 183 // Functions imported from MediaChannelUtil 184 bool HasNetworkInterface() const override { 185 return MediaChannelUtil::HasNetworkInterface(); 186 } 187 void SetExtmapAllowMixed(bool extmap_allow_mixed) override { 188 MediaChannelUtil::SetExtmapAllowMixed(extmap_allow_mixed); 189 } 190 bool ExtmapAllowMixed() const override { 191 return MediaChannelUtil::ExtmapAllowMixed(); 192 } 193 194 // Common functions between sender and receiver 195 void SetInterface(MediaChannelNetworkInterface* iface) override; 196 // VideoMediaSendChannelInterface implementation 197 bool SetSenderParameters(const VideoSenderParameters& params) override; 198 RTCError SetRtpSendParameters(uint32_t ssrc, 199 const RtpParameters& parameters, 200 SetParametersCallback callback) override; 201 RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; 202 std::optional<Codec> GetSendCodec() const override; 203 bool SetSend(bool send) override; 204 bool SetVideoSend(uint32_t ssrc, 205 const VideoOptions* options, 206 VideoSourceInterface<VideoFrame>* source) override; 207 bool AddSendStream(const StreamParams& sp) override; 208 bool RemoveSendStream(uint32_t ssrc) override; 209 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override; 210 bool GetStats(VideoMediaSendInfo* info) override; 211 212 void OnPacketSent(const SentPacketInfo& sent_packet) override; 213 void OnReadyToSend(bool ready) override; 214 void OnNetworkRouteChanged(absl::string_view transport_name, 215 const NetworkRoute& network_route) override; 216 217 // Set a frame encryptor to a particular ssrc that will intercept all 218 // outgoing video frames and attempt to encrypt them and forward the result 219 // to the packetizer. 220 void SetFrameEncryptor( 221 uint32_t ssrc, 222 scoped_refptr<FrameEncryptorInterface> frame_encryptor) override; 223 224 // note: The encoder_selector object must remain valid for the lifetime of the 225 // MediaChannel, unless replaced. 226 void SetEncoderSelector( 227 uint32_t ssrc, 228 VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) override; 229 230 void SetSendCodecChangedCallback( 231 absl::AnyInvocable<void()> callback) override { 232 send_codec_changed_callback_ = std::move(callback); 233 } 234 235 void SetSsrcListChangedCallback( 236 absl::AnyInvocable<void(const std::set<uint32_t>&)> callback) override { 237 ssrc_list_changed_callback_ = std::move(callback); 238 } 239 240 // Implemented for VideoMediaChannelTest. 241 bool sending() const { 242 RTC_DCHECK_RUN_ON(&thread_checker_); 243 return sending_; 244 } 245 246 // AdaptReason is used for expressing why a WebRtcVideoSendStream request 247 // a lower input frame size than the currently configured camera input frame 248 // size. There can be more than one reason OR:ed together. 249 enum AdaptReason { 250 ADAPTREASON_NONE = 0, 251 ADAPTREASON_CPU = 1, 252 ADAPTREASON_BANDWIDTH = 2, 253 }; 254 255 // Implements EncoderSwitchRequestCallback. 256 void RequestEncoderFallback() override; 257 void RequestEncoderSwitch(const SdpVideoFormat& format, 258 bool allow_default_fallback) override; 259 260 void GenerateSendKeyFrame(uint32_t ssrc, 261 const std::vector<std::string>& rids) override; 262 263 void SetEncoderToPacketizerFrameTransformer( 264 uint32_t ssrc, 265 scoped_refptr<FrameTransformerInterface> frame_transformer) override; 266 // Information queries to support SetReceiverFeedbackParameters 267 RtcpMode SendCodecRtcpMode() const override { 268 RTC_DCHECK_RUN_ON(&thread_checker_); 269 return send_params_.rtcp.reduced_size ? RtcpMode::kReducedSize 270 : RtcpMode::kCompound; 271 } 272 273 bool SendCodecHasLntf() const override { 274 RTC_DCHECK_RUN_ON(&thread_checker_); 275 if (!send_codec()) { 276 return false; 277 } 278 return HasLntf(send_codec()->codec); 279 } 280 bool SendCodecHasNack() const override { 281 RTC_DCHECK_RUN_ON(&thread_checker_); 282 if (!send_codec()) { 283 return false; 284 } 285 return HasNack(send_codec()->codec); 286 } 287 std::optional<int> SendCodecRtxTime() const override { 288 RTC_DCHECK_RUN_ON(&thread_checker_); 289 if (!send_codec()) { 290 return std::nullopt; 291 } 292 return send_codec()->rtx_time; 293 } 294 295 private: 296 struct ChangedSenderParameters { 297 // These optionals are unset if not changed. 298 std::optional<VideoCodecSettings> send_codec; 299 std::optional<std::vector<VideoCodecSettings>> negotiated_codecs; 300 std::optional<std::vector<VideoCodecSettings>> send_codecs; 301 std::optional<std::vector<RtpExtension>> rtp_header_extensions; 302 std::optional<std::string> mid; 303 std::optional<bool> extmap_allow_mixed; 304 std::optional<int> max_bandwidth_bps; 305 std::optional<bool> conference_mode; 306 std::optional<RtcpMode> rtcp_mode; 307 }; 308 309 bool GetChangedSenderParameters(const VideoSenderParameters& params, 310 ChangedSenderParameters* changed_params) const 311 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 312 bool ApplyChangedParams(const ChangedSenderParameters& changed_params); 313 bool ValidateSendSsrcAvailability(const StreamParams& sp) const 314 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 315 316 // Wrapper for the sender part. 317 class WebRtcVideoSendStream { 318 public: 319 WebRtcVideoSendStream( 320 const Environment& env, 321 Call* call, 322 const StreamParams& sp, 323 VideoSendStream::Config config, 324 const VideoOptions& options, 325 bool enable_cpu_overuse_detection, 326 int max_bitrate_bps, 327 const std::optional<VideoCodecSettings>& codec_settings, 328 const std::vector<VideoCodecSettings>& codec_settings_list, 329 const std::optional<std::vector<RtpExtension>>& rtp_extensions, 330 const VideoSenderParameters& send_params); 331 ~WebRtcVideoSendStream(); 332 333 void SetSenderParameters(const ChangedSenderParameters& send_params); 334 RTCError SetRtpParameters(const RtpParameters& parameters, 335 SetParametersCallback callback); 336 RtpParameters GetRtpParameters() const; 337 338 void SetFrameEncryptor( 339 scoped_refptr<FrameEncryptorInterface> frame_encryptor); 340 341 bool SetVideoSend(const VideoOptions* options, 342 VideoSourceInterface<VideoFrame>* source); 343 344 // note: The encoder_selector object must remain valid for the lifetime of 345 // the MediaChannel, unless replaced. 346 void SetEncoderSelector( 347 VideoEncoderFactory::EncoderSelectorInterface* encoder_selector); 348 349 void SetSend(bool send); 350 351 const std::vector<uint32_t>& GetSsrcs() const; 352 // Returns per ssrc VideoSenderInfos. Useful for simulcast scenario. 353 std::vector<VideoSenderInfo> GetPerLayerVideoSenderInfos(bool log_stats); 354 // Aggregates per ssrc VideoSenderInfos to single VideoSenderInfo for 355 // legacy reasons. Used in old GetStats API and track stats. 356 VideoSenderInfo GetAggregatedVideoSenderInfo( 357 const std::vector<VideoSenderInfo>& infos) const; 358 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info); 359 360 void SetEncoderToPacketizerFrameTransformer( 361 scoped_refptr<FrameTransformerInterface> frame_transformer); 362 void GenerateKeyFrame(const std::vector<std::string>& rids); 363 364 private: 365 // Parameters needed to reconstruct the underlying stream. 366 // VideoSendStream doesn't support setting a lot of options on the 367 // fly, so when those need to be changed we tear down and reconstruct with 368 // similar parameters depending on which options changed etc. 369 struct VideoSendStreamParameters { 370 VideoSendStreamParameters( 371 VideoSendStream::Config config, 372 const VideoOptions& options, 373 int max_bitrate_bps, 374 const std::optional<VideoCodecSettings>& codec_settings, 375 const std::vector<VideoCodecSettings>& codec_settings_list); 376 VideoSendStream::Config config; 377 VideoOptions options; 378 int max_bitrate_bps; 379 bool conference_mode; 380 std::optional<VideoCodecSettings> codec_settings; 381 std::vector<VideoCodecSettings> codec_settings_list; 382 // Sent resolutions + bitrates etc. by the underlying VideoSendStream, 383 // typically changes when setting a new resolution or reconfiguring 384 // bitrates. 385 VideoEncoderConfig encoder_config; 386 }; 387 388 scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> 389 ConfigureVideoEncoderSettings(const Codec& codec); 390 void SetCodec(const VideoCodecSettings& codec, 391 const std::vector<VideoCodecSettings>& codec_settings_list); 392 void RecreateWebRtcStream(); 393 VideoEncoderConfig CreateVideoEncoderConfig(const Codec& codec) const; 394 void ReconfigureEncoder(SetParametersCallback callback); 395 396 // Calls Start or Stop according to whether or not `sending_` is true. 397 void UpdateSendState(); 398 399 DegradationPreference GetDegradationPreference() const 400 RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_); 401 402 const Environment env_; 403 RTC_NO_UNIQUE_ADDRESS SequenceChecker thread_checker_; 404 TaskQueueBase* const worker_thread_; 405 const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_); 406 const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_); 407 Call* const call_; 408 const bool enable_cpu_overuse_detection_; 409 VideoSourceInterface<VideoFrame>* source_ RTC_GUARDED_BY(&thread_checker_); 410 411 VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_); 412 413 // Contains settings that are the same for all streams in the MediaChannel, 414 // such as codecs, header extensions, and the global bitrate limit for the 415 // entire channel. 416 VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_); 417 // Contains settings that are unique for each stream, such as max_bitrate. 418 // Does *not* contain codecs, however. 419 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_. 420 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only 421 // one stream per MediaChannel. 422 RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_); 423 424 bool sending_ RTC_GUARDED_BY(&thread_checker_); 425 426 // TODO(asapersson): investigate why setting 427 // DegrationPreferences::MAINTAIN_RESOLUTION isn't sufficient to disable 428 // downscaling everywhere in the pipeline. 429 const bool disable_automatic_resize_; 430 }; 431 432 void Construct(Call* call, WebRtcVideoEngine* engine); 433 434 // Get all codecs that are compatible with the receiver. 435 std::vector<VideoCodecSettings> SelectSendVideoCodecs( 436 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const 437 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 438 439 void FillSenderStats(VideoMediaSendInfo* info, bool log_stats) 440 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 441 void FillBandwidthEstimationStats(const Call::Stats& stats, 442 VideoMediaInfo* info) 443 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 444 void FillSendCodecStats(VideoMediaSendInfo* video_media_info) 445 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 446 447 // Accessor function for send_codec_. Introduced in order to ensure 448 // that a receive channel does not touch the send codec directly. 449 // Can go away once these are different classes. 450 // TODO(bugs.webrtc.org/13931): Remove this function 451 std::optional<VideoCodecSettings>& send_codec() { return send_codec_; } 452 const std::optional<VideoCodecSettings>& send_codec() const { 453 return send_codec_; 454 } 455 456 const Environment env_; 457 TaskQueueBase* const worker_thread_; 458 ScopedTaskSafety task_safety_; 459 RTC_NO_UNIQUE_ADDRESS SequenceChecker network_thread_checker_{ 460 SequenceChecker::kDetached}; 461 RTC_NO_UNIQUE_ADDRESS SequenceChecker thread_checker_; 462 463 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); 464 bool sending_ RTC_GUARDED_BY(thread_checker_); 465 bool receiving_ RTC_GUARDED_BY(&thread_checker_); 466 Call* const call_; 467 468 VideoSinkInterface<VideoFrame>* default_sink_ RTC_GUARDED_BY(thread_checker_); 469 470 // Delay for unsignaled streams, which may be set before the stream exists. 471 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0; 472 473 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_); 474 475 // Using primary-ssrc (first ssrc) as key. 476 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_ 477 RTC_GUARDED_BY(thread_checker_); 478 // When the channel and demuxer get reconfigured, there is a window of time 479 // where we have to be prepared for packets arriving based on the old demuxer 480 // criteria because the streams live on the worker thread and the demuxer 481 // lives on the network thread. Because packets are posted from the network 482 // thread to the worker thread, they can still be in-flight when streams are 483 // reconfgured. This can happen when `demuxer_criteria_id_` and 484 // `demuxer_criteria_completed_id_` don't match. During this time, we do not 485 // want to create unsignalled receive streams and should instead drop the 486 // packets. E.g: 487 // * If RemoveRecvStream(old_ssrc) was recently called, there may be packets 488 // in-flight for that ssrc. This happens when a receiver becomes inactive. 489 // * If we go from one to many m= sections, the demuxer may change from 490 // forwarding all packets to only forwarding the configured ssrcs, so there 491 // is a risk of receiving ssrcs for other, recently added m= sections. 492 uint32_t demuxer_criteria_id_ RTC_GUARDED_BY(thread_checker_) = 0; 493 uint32_t demuxer_criteria_completed_id_ RTC_GUARDED_BY(thread_checker_) = 0; 494 std::optional<int64_t> last_unsignalled_ssrc_creation_time_ms_ 495 RTC_GUARDED_BY(thread_checker_); 496 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_); 497 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_); 498 499 std::optional<VideoCodecSettings> send_codec_ RTC_GUARDED_BY(thread_checker_); 500 std::vector<VideoCodecSettings> negotiated_codecs_ 501 RTC_GUARDED_BY(thread_checker_); 502 std::vector<VideoCodecSettings> send_codecs_ RTC_GUARDED_BY(thread_checker_); 503 504 std::vector<RtpExtension> send_rtp_extensions_ 505 RTC_GUARDED_BY(thread_checker_); 506 507 VideoEncoderFactory* const encoder_factory_ RTC_GUARDED_BY(thread_checker_); 508 VideoBitrateAllocatorFactory* const bitrate_allocator_factory_ 509 RTC_GUARDED_BY(thread_checker_); 510 // See reason for keeping track of the FlexFEC payload type separately in 511 // comment in WebRtcVideoChannel::ChangedReceiverParameters. 512 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_); 513 BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_); 514 // TODO(deadbeef): Don't duplicate information between 515 // send_params/recv_params, rtp_extensions, options, etc. 516 VideoSenderParameters send_params_ RTC_GUARDED_BY(thread_checker_); 517 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_); 518 VideoReceiverParameters recv_params_ RTC_GUARDED_BY(thread_checker_); 519 int64_t last_send_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); 520 int64_t last_receive_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); 521 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_); 522 // This is a stream param that comes from the remote description, but wasn't 523 // signaled with any a=ssrc lines. It holds information that was signaled 524 // before the unsignaled receive stream is created when the first packet is 525 // received. 526 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_); 527 // Per peer connection crypto options that last for the lifetime of the peer 528 // connection. 529 const CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_); 530 531 // Optional frame transformer set on unsignaled streams. 532 scoped_refptr<FrameTransformerInterface> unsignaled_frame_transformer_ 533 RTC_GUARDED_BY(thread_checker_); 534 535 // RTP parameters that need to be set when creating a video receive stream. 536 // Only used in Receiver mode - in Both mode, it reads those things from the 537 // codec. 538 VideoReceiveStreamInterface::Config::Rtp rtp_config_; 539 540 // Callback invoked whenever the send codec changes. 541 // TODO(bugs.webrtc.org/13931): Remove again when coupling isn't needed. 542 absl::AnyInvocable<void()> send_codec_changed_callback_; 543 // Callback invoked whenever the list of SSRCs changes. 544 absl::AnyInvocable<void(const std::set<uint32_t>&)> 545 ssrc_list_changed_callback_; 546 }; 547 548 class WebRtcVideoReceiveChannel : public MediaChannelUtil, 549 public VideoMediaReceiveChannelInterface { 550 public: 551 WebRtcVideoReceiveChannel(const Environment& env, 552 Call* call, 553 const MediaConfig& config, 554 const VideoOptions& options, 555 const CryptoOptions& crypto_options, 556 VideoDecoderFactory* decoder_factory); 557 ~WebRtcVideoReceiveChannel() override; 558 559 public: 560 MediaType media_type() const override { return MediaType::VIDEO; } 561 VideoMediaReceiveChannelInterface* AsVideoReceiveChannel() override { 562 return this; 563 } 564 VoiceMediaReceiveChannelInterface* AsVoiceReceiveChannel() override { 565 RTC_CHECK_NOTREACHED(); 566 return nullptr; 567 } 568 569 // Common functions between sender and receiver 570 void SetInterface(MediaChannelNetworkInterface* iface) override; 571 // VideoMediaReceiveChannelInterface implementation 572 bool SetReceiverParameters(const VideoReceiverParameters& params) override; 573 RtpParameters GetRtpReceiverParameters(uint32_t ssrc) const override; 574 RtpParameters GetDefaultRtpReceiveParameters() const override; 575 void SetReceive(bool receive) override; 576 bool AddRecvStream(const StreamParams& sp) override; 577 bool AddDefaultRecvStreamForTesting(const StreamParams& sp) override { 578 // Invokes private AddRecvStream variant function 579 return AddRecvStream(sp, true); 580 } 581 bool RemoveRecvStream(uint32_t ssrc) override; 582 void ResetUnsignaledRecvStream() override; 583 std::optional<uint32_t> GetUnsignaledSsrc() const override; 584 void OnDemuxerCriteriaUpdatePending() override; 585 void OnDemuxerCriteriaUpdateComplete() override; 586 bool SetSink(uint32_t ssrc, VideoSinkInterface<VideoFrame>* sink) override; 587 void SetDefaultSink(VideoSinkInterface<VideoFrame>* sink) override; 588 bool GetStats(VideoMediaReceiveInfo* info) override; 589 void OnPacketReceived(const RtpPacketReceived& packet) override; 590 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; 591 592 std::optional<int> GetBaseMinimumPlayoutDelayMs(uint32_t ssrc) const override; 593 594 // Choose one of the available SSRCs (or default if none) as the current 595 // receiver report SSRC. 596 void ChooseReceiverReportSsrc(const std::set<uint32_t>& choices) override; 597 598 // E2E Encrypted Video Frame API 599 // Set a frame decryptor to a particular ssrc that will intercept all 600 // incoming video frames and attempt to decrypt them before forwarding the 601 // result. 602 void SetFrameDecryptor( 603 uint32_t ssrc, 604 scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; 605 void SetRecordableEncodedFrameCallback( 606 uint32_t ssrc, 607 std::function<void(const RecordableEncodedFrame&)> callback) override; 608 void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; 609 void RequestRecvKeyFrame(uint32_t ssrc) override; 610 void SetDepacketizerToDecoderFrameTransformer( 611 uint32_t ssrc, 612 scoped_refptr<FrameTransformerInterface> frame_transformer) override; 613 std::vector<RtpSource> GetSources(uint32_t ssrc) const override; 614 615 void SetReceiverFeedbackParameters(bool lntf_enabled, 616 bool nack_enabled, 617 RtcpMode rtcp_mode, 618 std::optional<int> rtx_time) override; 619 620 private: 621 class WebRtcVideoReceiveStream; 622 struct ChangedReceiverParameters { 623 // These optionals are unset if not changed. 624 std::optional<std::vector<VideoCodecSettings>> codec_settings; 625 std::optional<std::vector<RtpExtension>> rtp_header_extensions; 626 // Keep track of the FlexFEC payload type separately from `codec_settings`. 627 // This allows us to recreate the FlexfecReceiveStream separately from the 628 // VideoReceiveStreamInterface when the FlexFEC payload type is changed. 629 std::optional<int> flexfec_payload_type; 630 }; 631 632 // Finds VideoReceiveStreamInterface corresponding to ssrc. Aware of 633 // unsignalled ssrc handling. 634 WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc) 635 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 636 637 void ProcessReceivedPacket(RtpPacketReceived packet) 638 RTC_RUN_ON(thread_checker_); 639 640 // Expected to be invoked once per packet that belongs to this channel that 641 // can not be demuxed. 642 // Returns true if a new default stream has been created. 643 bool MaybeCreateDefaultReceiveStream(const RtpPacketReceived& parsed_packet) 644 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 645 void ReCreateDefaultReceiveStream(uint32_t ssrc, 646 std::optional<uint32_t> rtx_ssrc); 647 // Add a receive stream. Used for testing. 648 bool AddRecvStream(const StreamParams& sp, bool default_stream); 649 650 void ConfigureReceiverRtp(VideoReceiveStreamInterface::Config* config, 651 FlexfecReceiveStream::Config* flexfec_config, 652 const StreamParams& sp) const 653 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 654 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const 655 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 656 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream) 657 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 658 659 // Called when the local ssrc changes. Sets `rtcp_receiver_report_ssrc_` and 660 // updates the receive streams. 661 void SetReceiverReportSsrc(uint32_t ssrc) RTC_RUN_ON(&thread_checker_); 662 663 // Wrapper for the receiver part, contains configs etc. that are needed to 664 // reconstruct the underlying VideoReceiveStreamInterface. 665 class WebRtcVideoReceiveStream : public VideoSinkInterface<VideoFrame> { 666 public: 667 WebRtcVideoReceiveStream( 668 const Environment& env, 669 Call* call, 670 const StreamParams& sp, 671 VideoReceiveStreamInterface::Config config, 672 bool default_stream, 673 const std::vector<VideoCodecSettings>& recv_codecs, 674 const FlexfecReceiveStream::Config& flexfec_config); 675 ~WebRtcVideoReceiveStream(); 676 677 VideoReceiveStreamInterface& stream(); 678 // Return value may be nullptr. 679 FlexfecReceiveStream* flexfec_stream(); 680 681 const std::vector<uint32_t>& GetSsrcs() const; 682 683 std::vector<RtpSource> GetSources(); 684 685 // Does not return codecs, nor header extensions, they are filled by the 686 // owning WebRtcVideoChannel. 687 RtpParameters GetRtpParameters() const; 688 689 // TODO(deadbeef): Move these feedback parameters into the recv parameters. 690 void SetFeedbackParameters(bool lntf_enabled, 691 bool nack_enabled, 692 RtcpMode rtcp_mode, 693 std::optional<int> rtx_time); 694 void SetReceiverParameters(const ChangedReceiverParameters& recv_params); 695 696 void OnFrame(const VideoFrame& frame) override; 697 bool IsDefaultStream() const; 698 699 void SetFrameDecryptor( 700 scoped_refptr<FrameDecryptorInterface> frame_decryptor); 701 702 bool SetBaseMinimumPlayoutDelayMs(int delay_ms); 703 704 int GetBaseMinimumPlayoutDelayMs() const; 705 706 void SetSink(VideoSinkInterface<VideoFrame>* sink); 707 708 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); 709 710 void SetRecordableEncodedFrameCallback( 711 std::function<void(const RecordableEncodedFrame&)> callback); 712 void ClearRecordableEncodedFrameCallback(); 713 void GenerateKeyFrame(); 714 715 void SetDepacketizerToDecoderFrameTransformer( 716 scoped_refptr<FrameTransformerInterface> frame_transformer); 717 718 void SetLocalSsrc(uint32_t local_ssrc); 719 void UpdateRtxSsrc(uint32_t ssrc); 720 void StartReceiveStream(); 721 void StopReceiveStream(); 722 723 private: 724 // Attempts to reconfigure an already existing `flexfec_stream_`, create 725 // one if the configuration is now complete or remove a flexfec stream 726 // when disabled. 727 void SetFlexFecPayload(int payload_type); 728 729 void RecreateReceiveStream(); 730 void CreateReceiveStream(); 731 732 // Applies a new receive codecs configration to `config_`. Returns true 733 // if the internal stream needs to be reconstructed, or false if no changes 734 // were applied. 735 bool ReconfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs); 736 737 const Environment env_; 738 Call* const call_; 739 const StreamParams stream_params_; 740 741 // Both `stream_` and `flexfec_stream_` are managed by `this`. They are 742 // destroyed by calling call_->DestroyVideoReceiveStream and 743 // call_->DestroyFlexfecReceiveStream, respectively. 744 VideoReceiveStreamInterface* stream_; 745 const bool default_stream_; 746 VideoReceiveStreamInterface::Config config_; 747 FlexfecReceiveStream::Config flexfec_config_; 748 FlexfecReceiveStream* flexfec_stream_; 749 750 Mutex sink_lock_; 751 VideoSinkInterface<VideoFrame>* sink_ RTC_GUARDED_BY(sink_lock_); 752 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_); 753 // Start NTP time is estimated as current remote NTP time (estimated from 754 // RTCP) minus the elapsed time, as soon as remote NTP time is available. 755 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_); 756 757 RTC_NO_UNIQUE_ADDRESS SequenceChecker thread_checker_; 758 bool receiving_ RTC_GUARDED_BY(&thread_checker_); 759 }; 760 bool GetChangedReceiverParameters(const VideoReceiverParameters& params, 761 ChangedReceiverParameters* changed_params) 762 const RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 763 764 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_ 765 RTC_GUARDED_BY(thread_checker_); 766 void FillReceiverStats(VideoMediaReceiveInfo* info, bool log_stats) 767 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 768 void FillReceiveCodecStats(VideoMediaReceiveInfo* video_media_info) 769 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 770 771 StreamParams unsignaled_stream_params() { 772 RTC_DCHECK_RUN_ON(&thread_checker_); 773 return unsignaled_stream_params_; 774 } 775 // Variables. 776 const Environment env_; 777 TaskQueueBase* const worker_thread_; 778 ScopedTaskSafety task_safety_; 779 RTC_NO_UNIQUE_ADDRESS SequenceChecker network_thread_checker_{ 780 SequenceChecker::kDetached}; 781 RTC_NO_UNIQUE_ADDRESS SequenceChecker thread_checker_; 782 783 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); 784 bool receiving_ RTC_GUARDED_BY(&thread_checker_); 785 Call* const call_; 786 787 VideoSinkInterface<VideoFrame>* default_sink_ RTC_GUARDED_BY(thread_checker_); 788 789 // Delay for unsignaled streams, which may be set before the stream exists. 790 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0; 791 792 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_); 793 794 // When the channel and demuxer get reconfigured, there is a window of time 795 // where we have to be prepared for packets arriving based on the old demuxer 796 // criteria because the streams live on the worker thread and the demuxer 797 // lives on the network thread. Because packets are posted from the network 798 // thread to the worker thread, they can still be in-flight when streams are 799 // reconfgured. This can happen when `demuxer_criteria_id_` and 800 // `demuxer_criteria_completed_id_` don't match. During this time, we do not 801 // want to create unsignalled receive streams and should instead drop the 802 // packets. E.g: 803 // * If RemoveRecvStream(old_ssrc) was recently called, there may be packets 804 // in-flight for that ssrc. This happens when a receiver becomes inactive. 805 // * If we go from one to many m= sections, the demuxer may change from 806 // forwarding all packets to only forwarding the configured ssrcs, so there 807 // is a risk of receiving ssrcs for other, recently added m= sections. 808 uint32_t demuxer_criteria_id_ RTC_GUARDED_BY(thread_checker_) = 0; 809 uint32_t demuxer_criteria_completed_id_ RTC_GUARDED_BY(thread_checker_) = 0; 810 std::optional<int64_t> last_unsignalled_ssrc_creation_time_ms_ 811 RTC_GUARDED_BY(thread_checker_); 812 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_); 813 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_); 814 815 std::optional<VideoCodecSettings> send_codec_ RTC_GUARDED_BY(thread_checker_); 816 std::vector<VideoCodecSettings> negotiated_codecs_ 817 RTC_GUARDED_BY(thread_checker_); 818 819 std::vector<RtpExtension> send_rtp_extensions_ 820 RTC_GUARDED_BY(thread_checker_); 821 822 VideoDecoderFactory* const decoder_factory_ RTC_GUARDED_BY(thread_checker_); 823 std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_); 824 RtpHeaderExtensionMap recv_rtp_extension_map_ RTC_GUARDED_BY(thread_checker_); 825 std::vector<RtpExtension> recv_rtp_extensions_ 826 RTC_GUARDED_BY(thread_checker_); 827 // See reason for keeping track of the FlexFEC payload type separately in 828 // comment in WebRtcVideoChannel::ChangedReceiverParameters. 829 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_); 830 BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_); 831 // TODO(deadbeef): Don't duplicate information between 832 // send_params/recv_params, rtp_extensions, options, etc. 833 VideoSenderParameters send_params_ RTC_GUARDED_BY(thread_checker_); 834 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_); 835 VideoReceiverParameters recv_params_ RTC_GUARDED_BY(thread_checker_); 836 int64_t last_receive_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); 837 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_); 838 // This is a stream param that comes from the remote description, but wasn't 839 // signaled with any a=ssrc lines. It holds information that was signaled 840 // before the unsignaled receive stream is created when the first packet is 841 // received. 842 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_); 843 // Per peer connection crypto options that last for the lifetime of the peer 844 // connection. 845 const CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_); 846 847 // Optional frame transformer set on unsignaled streams. 848 scoped_refptr<FrameTransformerInterface> unsignaled_frame_transformer_ 849 RTC_GUARDED_BY(thread_checker_); 850 851 // RTP parameters that need to be set when creating a video receive stream. 852 // Only used in Receiver mode - in Both mode, it reads those things from the 853 // codec. 854 VideoReceiveStreamInterface::Config::Rtp rtp_config_; 855 856 // Callback invoked whenever the send codec changes. 857 // TODO(bugs.webrtc.org/13931): Remove again when coupling isn't needed. 858 absl::AnyInvocable<void()> send_codec_changed_callback_; 859 // Callback invoked whenever the list of SSRCs changes. 860 absl::AnyInvocable<void(const std::set<uint32_t>&)> 861 ssrc_list_changed_callback_; 862 863 const int receive_buffer_size_; 864 }; 865 866 // Keeping the old name "WebRtcVideoChannel" around because some external 867 // customers are using WebRtcVideoChannel::AdaptReason 868 // TODO(bugs.webrtc.org/15216): Move this enum to an interface class and 869 // delete this workaround. 870 class WebRtcVideoChannel : public WebRtcVideoSendChannel { 871 public: 872 // Make all the values of AdaptReason available as 873 // WebRtcVideoChannel::ADAPT_xxx. 874 using WebRtcVideoSendChannel::AdaptReason; 875 }; 876 877 } // namespace webrtc 878 879 #endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_