simulcast_encoder_adapter.h (8138B)
1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 * 10 */ 11 12 #ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 13 #define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 14 15 #include <atomic> 16 #include <cstddef> 17 #include <cstdint> 18 #include <list> 19 #include <memory> 20 #include <optional> 21 #include <vector> 22 23 #include "absl/base/nullability.h" 24 #include "api/environment/environment.h" 25 #include "api/fec_controller_override.h" 26 #include "api/sequence_checker.h" 27 #include "api/units/timestamp.h" 28 #include "api/video/encoded_image.h" 29 #include "api/video/video_frame.h" 30 #include "api/video/video_frame_type.h" 31 #include "api/video_codecs/sdp_video_format.h" 32 #include "api/video_codecs/video_codec.h" 33 #include "api/video_codecs/video_encoder.h" 34 #include "api/video_codecs/video_encoder_factory.h" 35 #include "common_video/framerate_controller.h" 36 #include "modules/video_coding/include/video_codec_interface.h" 37 #include "rtc_base/experiments/encoder_info_settings.h" 38 #include "rtc_base/system/no_unique_address.h" 39 #include "rtc_base/system/rtc_export.h" 40 41 namespace webrtc { 42 43 // SimulcastEncoderAdapter implements simulcast support by creating multiple 44 // VideoEncoder instances with the given VideoEncoderFactory. 45 // The object is created and destroyed on the worker thread, but all public 46 // interfaces should be called from the encoder task queue. 47 class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { 48 public: 49 // `primary_factory` produces the first-choice encoders to use. 50 // `fallback_factory`, if non-null, is used to create fallback encoder that 51 // will be used if InitEncode() fails for the primary encoder. 52 SimulcastEncoderAdapter(const Environment& env, 53 VideoEncoderFactory* absl_nonnull primary_factory, 54 VideoEncoderFactory* absl_nullable fallback_factory, 55 const SdpVideoFormat& format); 56 57 ~SimulcastEncoderAdapter() override; 58 59 // Implements VideoEncoder. 60 void SetFecControllerOverride( 61 FecControllerOverride* fec_controller_override) override; 62 int Release() override; 63 int InitEncode(const VideoCodec* codec_settings, 64 const VideoEncoder::Settings& settings) override; 65 int Encode(const VideoFrame& input_image, 66 const std::vector<VideoFrameType>* frame_types) override; 67 int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; 68 void SetRates(const RateControlParameters& parameters) override; 69 void OnPacketLossRateUpdate(float packet_loss_rate) override; 70 void OnRttUpdate(int64_t rtt_ms) override; 71 void OnLossNotification(const LossNotification& loss_notification) override; 72 73 EncoderInfo GetEncoderInfo() const override; 74 75 protected: 76 void DestroyStoredEncoders(); 77 78 private: 79 class EncoderContext { 80 public: 81 EncoderContext(std::unique_ptr<VideoEncoder> encoder, 82 bool prefer_temporal_support, 83 VideoEncoder::EncoderInfo primary_info, 84 VideoEncoder::EncoderInfo fallback_info, 85 SdpVideoFormat video_format); 86 EncoderContext& operator=(EncoderContext&&) = delete; 87 88 VideoEncoder& encoder() { return *encoder_; } 89 bool prefer_temporal_support() { return prefer_temporal_support_; } 90 void Release(); 91 92 const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; } 93 94 const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; } 95 96 const SdpVideoFormat& video_format() { return video_format_; } 97 98 private: 99 std::unique_ptr<VideoEncoder> encoder_; 100 bool prefer_temporal_support_; 101 const VideoEncoder::EncoderInfo primary_info_; 102 const VideoEncoder::EncoderInfo fallback_info_; 103 const SdpVideoFormat video_format_; 104 }; 105 106 class StreamContext : public EncodedImageCallback { 107 public: 108 StreamContext(SimulcastEncoderAdapter* parent, 109 std::unique_ptr<EncoderContext> encoder_context, 110 std::unique_ptr<FramerateController> framerate_controller, 111 int stream_idx, 112 uint16_t width, 113 uint16_t height, 114 bool send_stream); 115 StreamContext(StreamContext&& rhs); 116 StreamContext& operator=(StreamContext&&) = delete; 117 ~StreamContext() override; 118 119 Result OnEncodedImage( 120 const EncodedImage& encoded_image, 121 const CodecSpecificInfo* codec_specific_info) override; 122 void OnDroppedFrame(DropReason reason) override; 123 124 VideoEncoder& encoder() { return encoder_context_->encoder(); } 125 const VideoEncoder& encoder() const { return encoder_context_->encoder(); } 126 int stream_idx() const { return stream_idx_; } 127 uint16_t width() const { return width_; } 128 uint16_t height() const { return height_; } 129 bool is_keyframe_needed() const { 130 return !is_paused_ && is_keyframe_needed_; 131 } 132 void set_is_keyframe_needed() { is_keyframe_needed_ = true; } 133 bool is_paused() const { return is_paused_; } 134 void set_is_paused(bool is_paused) { is_paused_ = is_paused; } 135 std::optional<double> target_fps() const { 136 return framerate_controller_ == nullptr 137 ? std::nullopt 138 : std::optional<double>( 139 framerate_controller_->GetMaxFramerate()); 140 } 141 142 std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&; 143 void OnKeyframe(Timestamp timestamp); 144 bool ShouldDropFrame(Timestamp timestamp); 145 146 private: 147 SimulcastEncoderAdapter* const parent_; 148 std::unique_ptr<EncoderContext> encoder_context_; 149 std::unique_ptr<FramerateController> framerate_controller_; 150 const int stream_idx_; 151 const uint16_t width_; 152 const uint16_t height_; 153 bool is_keyframe_needed_; 154 bool is_paused_; 155 }; 156 157 bool Initialized() const; 158 159 // This method creates encoder. May reuse previously created encoders from 160 // `cached_encoder_contexts_`. It's const because it's used from 161 // const GetEncoderInfo(). 162 std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext( 163 bool is_lowest_quality_stream, 164 std::optional<int> stream_idx) const; 165 166 VideoCodec MakeStreamCodec(const VideoCodec& codec, 167 int stream_idx, 168 uint32_t start_bitrate_kbps, 169 bool is_lowest_quality_stream, 170 bool is_highest_quality_stream); 171 172 EncodedImageCallback::Result OnEncodedImage( 173 size_t stream_idx, 174 const EncodedImage& encoded_image, 175 const CodecSpecificInfo* codec_specific_info); 176 177 void OnDroppedFrame(size_t stream_idx); 178 179 void OverrideFromFieldTrial(VideoEncoder::EncoderInfo* info) const; 180 181 const Environment env_; 182 std::atomic<int> inited_; 183 VideoEncoderFactory* const primary_encoder_factory_; 184 VideoEncoderFactory* const fallback_encoder_factory_; 185 const SdpVideoFormat video_format_; 186 VideoCodec codec_; 187 int total_streams_count_; 188 bool bypass_mode_; 189 std::vector<StreamContext> stream_contexts_; 190 EncodedImageCallback* encoded_complete_callback_; 191 192 // Used for checking the single-threaded access of the encoder interface. 193 RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_; 194 195 // Store previously created and released encoders , so they don't have to be 196 // recreated. Remaining encoders are destroyed by the destructor. 197 // Marked as `mutable` becuase we may need to temporarily create encoder in 198 // GetEncoderInfo(), which is const. 199 mutable std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_; 200 201 const bool boost_base_layer_quality_; 202 const bool prefer_temporal_support_on_base_layer_; 203 const bool per_layer_pli_; 204 const bool drop_unaligned_resolution_; 205 206 const SimulcastEncoderAdapterEncoderInfoSettings encoder_info_override_; 207 }; 208 209 } // namespace webrtc 210 211 #endif // MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_