libvpx_vp9_encoder.h (10332B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 * 10 */ 11 12 #ifndef MODULES_VIDEO_CODING_CODECS_VP9_LIBVPX_VP9_ENCODER_H_ 13 #define MODULES_VIDEO_CODING_CODECS_VP9_LIBVPX_VP9_ENCODER_H_ 14 15 #ifdef RTC_ENABLE_VP9 16 17 #include <array> 18 #include <cstddef> 19 #include <cstdint> 20 #include <memory> 21 #include <optional> 22 #include <vector> 23 24 #include "api/environment/environment.h" 25 #include "api/fec_controller_override.h" 26 #include "api/field_trials_view.h" 27 #include "api/scoped_refptr.h" 28 #include "api/video/encoded_image.h" 29 #include "api/video/video_bitrate_allocation.h" 30 #include "api/video/video_frame.h" 31 #include "api/video/video_frame_buffer.h" 32 #include "api/video/video_frame_type.h" 33 #include "api/video_codecs/scalability_mode.h" 34 #include "api/video_codecs/video_codec.h" 35 #include "api/video_codecs/video_encoder.h" 36 #include "api/video_codecs/vp9_profile.h" 37 #include "modules/video_coding/codecs/interface/libvpx_interface.h" 38 #include "modules/video_coding/codecs/vp9/include/vp9.h" 39 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" 40 #include "modules/video_coding/include/video_codec_interface.h" 41 #include "modules/video_coding/svc/scalable_video_controller.h" 42 #include "modules/video_coding/svc/simulcast_to_svc_converter.h" 43 #include "modules/video_coding/utility/frame_sampler.h" 44 #include "modules/video_coding/utility/framerate_controller_deprecated.h" 45 #include "rtc_base/containers/flat_map.h" 46 #include "rtc_base/experiments/encoder_info_settings.h" 47 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" 48 #include "third_party/libvpx/source/libvpx/vpx/vpx_codec.h" 49 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h" 50 #include "third_party/libvpx/source/libvpx/vpx/vpx_image.h" 51 52 namespace webrtc { 53 54 class LibvpxVp9Encoder : public VideoEncoder { 55 public: 56 LibvpxVp9Encoder(const Environment& env, 57 Vp9EncoderSettings settings, 58 std::unique_ptr<LibvpxInterface> interface); 59 60 ~LibvpxVp9Encoder() override; 61 62 void SetFecControllerOverride( 63 FecControllerOverride* fec_controller_override) override; 64 65 int Release() override; 66 67 int InitEncode(const VideoCodec* codec_settings, 68 const Settings& settings) override; 69 70 int Encode(const VideoFrame& input_image, 71 const std::vector<VideoFrameType>* frame_types) override; 72 73 int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; 74 75 void SetRates(const RateControlParameters& parameters) override; 76 77 EncoderInfo GetEncoderInfo() const override; 78 79 private: 80 // Determine number of encoder threads to use. 81 int NumberOfThreads(int width, int height, int number_of_cores); 82 83 // Call encoder initialize function and set control settings. 84 int InitAndSetControlSettings(); 85 86 // Update frame size for codec. 87 int UpdateCodecFrameSize(const VideoFrame& input_image); 88 89 bool PopulateCodecSpecific(CodecSpecificInfo* codec_specific, 90 std::optional<int>* spatial_idx, 91 std::optional<int>* temporal_idx, 92 const vpx_codec_cx_pkt& pkt); 93 void FillReferenceIndices(const vpx_codec_cx_pkt& pkt, 94 size_t pic_num, 95 bool inter_layer_predicted, 96 CodecSpecificInfoVP9* vp9_info); 97 void UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt, size_t pic_num); 98 vpx_svc_ref_frame_config_t SetReferences(bool is_key_pic, 99 int first_active_spatial_layer_id); 100 101 bool ExplicitlyConfiguredSpatialLayers() const; 102 bool SetSvcRates(const VideoBitrateAllocation& bitrate_allocation); 103 104 // Adjust sclaing factors assuming that the top active SVC layer 105 // will be the input resolution. 106 void AdjustScalingFactorsForTopActiveLayer(); 107 108 // Configures which spatial layers libvpx should encode according to 109 // configuration provided by svc_controller_. 110 void EnableSpatialLayer(int sid); 111 void DisableSpatialLayer(int sid); 112 void SetActiveSpatialLayers(); 113 114 void GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt); 115 116 // Callback function for outputting packets per spatial layer. 117 static void EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt, 118 void* user_data); 119 120 void DeliverBufferedFrame(bool end_of_picture); 121 122 bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp); 123 124 // Determine maximum target for Intra frames 125 // 126 // Input: 127 // - optimal_buffer_size : Optimal buffer size 128 // Return Value : Max target size for Intra frames represented as 129 // percentage of the per frame bandwidth 130 uint32_t MaxIntraTarget(uint32_t optimal_buffer_size); 131 132 size_t SteadyStateSize(int sid, int tid); 133 134 void MaybeRewrapRawWithFormat(const vpx_img_fmt fmt, 135 unsigned int width, 136 unsigned int height); 137 // Prepares `raw_` to reference image data of `buffer`, or of mapped or scaled 138 // versions of `buffer`. Returns the buffer that got referenced as a result, 139 // allowing the caller to keep a reference to it until after encoding has 140 // finished. On failure to convert the buffer, null is returned. 141 scoped_refptr<VideoFrameBuffer> PrepareBufferForProfile0( 142 scoped_refptr<VideoFrameBuffer> buffer); 143 144 const Environment env_; 145 const std::unique_ptr<LibvpxInterface> libvpx_; 146 EncodedImage encoded_image_; 147 CodecSpecificInfo codec_specific_; 148 EncodedImageCallback* encoded_complete_callback_; 149 VideoCodec codec_; 150 const VP9Profile profile_; 151 bool inited_; 152 int64_t timestamp_; 153 uint32_t rc_max_intra_target_; 154 vpx_codec_ctx_t* encoder_; 155 vpx_codec_enc_cfg_t* config_; 156 vpx_image_t* raw_; 157 vpx_svc_extra_cfg_t svc_params_; 158 const VideoFrame* input_image_; 159 GofInfoVP9 gof_; // Contains each frame's temporal information for 160 // non-flexible mode. 161 bool force_key_frame_; 162 size_t pics_since_key_; 163 uint8_t num_temporal_layers_; 164 uint8_t num_spatial_layers_; // Number of configured SLs 165 uint8_t num_active_spatial_layers_; // Number of actively encoded SLs 166 uint8_t first_active_layer_; 167 uint8_t last_active_layer_; 168 bool layer_deactivation_requires_key_frame_; 169 bool is_svc_; 170 InterLayerPredMode inter_layer_pred_; 171 const bool trusted_rate_controller_; 172 vpx_svc_frame_drop_t svc_drop_frame_; 173 bool first_frame_in_picture_; 174 VideoBitrateAllocation current_bitrate_allocation_; 175 bool ss_info_needed_; 176 bool force_all_active_layers_; 177 uint8_t num_cores_; 178 179 const bool enable_svc_for_simulcast_; 180 std::optional<SimulcastToSvcConverter> simulcast_to_svc_converter_; 181 182 std::unique_ptr<ScalableVideoController> svc_controller_; 183 std::optional<ScalabilityMode> scalability_mode_; 184 std::vector<FramerateControllerDeprecated> framerate_controller_; 185 186 // Used for flexible mode. 187 bool is_flexible_mode_; 188 struct RefFrameBuffer { 189 bool operator==(const RefFrameBuffer& o) { 190 return pic_num == o.pic_num && spatial_layer_id == o.spatial_layer_id && 191 temporal_layer_id == o.temporal_layer_id; 192 } 193 194 size_t pic_num = 0; 195 int spatial_layer_id = 0; 196 int temporal_layer_id = 0; 197 }; 198 std::array<RefFrameBuffer, kNumVp9Buffers> ref_buf_; 199 std::vector<ScalableVideoController::LayerFrameConfig> layer_frames_; 200 201 FramerateControllerDeprecated variable_framerate_controller_; 202 203 // Original scaling factors for all configured layers active and inactive. 204 // `svc_config_` stores factors ignoring top inactive layers. 205 std::vector<int> scaling_factors_num_, scaling_factors_den_; 206 207 const struct QualityScalerExperiment { 208 int low_qp; 209 int high_qp; 210 bool enabled; 211 } quality_scaler_experiment_; 212 static QualityScalerExperiment ParseQualityScalerConfig( 213 const FieldTrialsView& trials); 214 215 // Flags that can affect speed vs quality tradeoff, and are configureable per 216 // resolution ranges. 217 struct PerformanceFlags { 218 // If false, a lookup will be made in `settings_by_resolution` base on the 219 // highest currently active resolution, and the overall speed then set to 220 // to the `base_layer_speed` matching that entry. 221 // If true, each active resolution will have it's speed and deblock_mode set 222 // based on it resolution, and the high layer speed configured for non 223 // base temporal layer frames. 224 bool use_per_layer_speed = false; 225 226 struct ParameterSet { 227 int base_layer_speed = -1; // Speed setting for TL0. 228 int high_layer_speed = -1; // Speed setting for TL1-TL3. 229 // 0 = deblock all temporal layers (TL) 230 // 1 = disable deblock for top-most TL 231 // 2 = disable deblock for all TLs 232 int deblock_mode = 0; 233 bool allow_denoising = true; 234 }; 235 // Map from min pixel count to settings for that resolution and above. 236 // E.g. if you want some settings A if below wvga (640x360) and some other 237 // setting B at wvga and above, you'd use map {{0, A}, {230400, B}}. 238 flat_map<int, ParameterSet> settings_by_resolution; 239 }; 240 // Performance flags, ordered by `min_pixel_count`. 241 const PerformanceFlags performance_flags_; 242 // Caching of of `speed_configs_`, where index i maps to the resolution as 243 // specified in `codec_.spatialLayer[i]`. 244 std::vector<PerformanceFlags::ParameterSet> 245 performance_flags_by_spatial_index_; 246 void UpdatePerformanceFlags(); 247 static PerformanceFlags ParsePerformanceFlagsFromTrials( 248 const FieldTrialsView& trials); 249 static PerformanceFlags GetDefaultPerformanceFlags(); 250 251 int num_steady_state_frames_; 252 // Only set config when this flag is set. 253 bool config_changed_; 254 255 const LibvpxVp9EncoderInfoSettings encoder_info_override_; 256 257 // Determine whether the frame should be sampled for PSNR. 258 FrameSampler psnr_frame_sampler_; 259 // TODO(webrtc:388070060): Remove after rollout. 260 const bool calculate_psnr_; 261 }; 262 263 } // namespace webrtc 264 265 #endif // RTC_ENABLE_VP9 266 267 #endif // MODULES_VIDEO_CODING_CODECS_VP9_LIBVPX_VP9_ENCODER_H_