video_codec_initializer.cc (16942B)
1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/video_coding/include/video_codec_initializer.h" 12 13 #include <algorithm> 14 #include <cstdint> 15 #include <cstring> 16 #include <optional> 17 #include <vector> 18 19 #include "api/field_trials_view.h" 20 #include "api/scoped_refptr.h" 21 #include "api/units/data_rate.h" 22 #include "api/video/video_codec_constants.h" 23 #include "api/video/video_codec_type.h" 24 #include "api/video_codecs/scalability_mode.h" 25 #include "api/video_codecs/simulcast_stream.h" 26 #include "api/video_codecs/spatial_layer.h" 27 #include "api/video_codecs/video_codec.h" 28 #include "api/video_codecs/video_encoder.h" 29 #include "modules/video_coding/codecs/av1/av1_svc_config.h" 30 #include "modules/video_coding/codecs/vp8/vp8_scalability.h" 31 #include "modules/video_coding/codecs/vp9/svc_config.h" 32 #include "modules/video_coding/include/video_coding_defines.h" 33 #include "modules/video_coding/svc/scalability_mode_util.h" 34 #include "rtc_base/checks.h" 35 #include "rtc_base/experiments/min_video_bitrate_experiment.h" 36 #include "rtc_base/logging.h" 37 #include "rtc_base/numerics/safe_conversions.h" 38 #include "video/config/video_encoder_config.h" 39 40 namespace webrtc { 41 namespace { 42 43 constexpr ScalabilityMode kH265SupportedScalabilityModes[] = { 44 ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3}; 45 46 bool H265SupportsScalabilityMode(ScalabilityMode scalability_mode) { 47 for (const auto& entry : kH265SupportedScalabilityModes) { 48 if (entry == scalability_mode) { 49 return true; 50 } 51 } 52 return false; 53 } 54 55 } // namespace 56 57 // TODO(sprang): Split this up and separate the codec specific parts. 58 VideoCodec VideoCodecInitializer::SetupCodec( 59 const FieldTrialsView& field_trials, 60 const VideoEncoderConfig& config, 61 const std::vector<VideoStream>& streams) { 62 static const int kEncoderMinBitrateKbps = 30; 63 RTC_DCHECK(!streams.empty()); 64 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); 65 66 VideoCodec video_codec; 67 video_codec.codecType = config.codec_type; 68 69 switch (config.content_type) { 70 case VideoEncoderConfig::ContentType::kRealtimeVideo: 71 video_codec.mode = VideoCodecMode::kRealtimeVideo; 72 break; 73 case VideoEncoderConfig::ContentType::kScreen: 74 video_codec.mode = VideoCodecMode::kScreensharing; 75 break; 76 } 77 78 video_codec.legacy_conference_mode = 79 config.content_type == VideoEncoderConfig::ContentType::kScreen && 80 config.legacy_conference_mode; 81 82 video_codec.SetFrameDropEnabled(config.frame_drop_enabled); 83 video_codec.numberOfSimulcastStreams = 84 static_cast<unsigned char>(streams.size()); 85 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; 86 bool codec_active = false; 87 // Active configuration might not be fully copied to `streams` for SVC yet. 88 // Therefore the `config` is checked here. 89 for (const VideoStream& stream : config.simulcast_layers) { 90 if (stream.active) { 91 codec_active = true; 92 break; 93 } 94 } 95 // Set active for the entire video codec for the non simulcast case. 96 video_codec.active = codec_active; 97 if (video_codec.minBitrate < kEncoderMinBitrateKbps) 98 video_codec.minBitrate = kEncoderMinBitrateKbps; 99 video_codec.timing_frame_thresholds = { 100 .delay_ms = kDefaultTimingFramesDelayMs, 101 .outlier_ratio_percent = kDefaultOutlierFrameSizePercent}; 102 RTC_DCHECK_LE(streams.size(), kMaxSimulcastStreams); 103 104 int max_framerate = 0; 105 106 std::optional<ScalabilityMode> scalability_mode = streams[0].scalability_mode; 107 for (size_t i = 0; i < streams.size(); ++i) { 108 SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; 109 RTC_DCHECK_GT(streams[i].width, 0); 110 RTC_DCHECK_GT(streams[i].height, 0); 111 RTC_DCHECK_GT(streams[i].max_framerate, 0); 112 RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); 113 RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); 114 RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); 115 RTC_DCHECK_GE(streams[i].max_qp, 0); 116 117 sim_stream->width = static_cast<uint16_t>(streams[i].width); 118 sim_stream->height = static_cast<uint16_t>(streams[i].height); 119 sim_stream->maxFramerate = streams[i].max_framerate; 120 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; 121 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; 122 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; 123 sim_stream->qpMax = streams[i].max_qp; 124 sim_stream->format = config.GetSimulcastVideoFormat(i); 125 126 int num_temporal_layers = 127 streams[i].scalability_mode.has_value() 128 ? ScalabilityModeToNumTemporalLayers(*streams[i].scalability_mode) 129 : streams[i].num_temporal_layers.value_or(1); 130 131 sim_stream->numberOfTemporalLayers = 132 static_cast<unsigned char>(num_temporal_layers); 133 sim_stream->active = streams[i].active; 134 135 video_codec.width = 136 std::max(video_codec.width, static_cast<uint16_t>(streams[i].width)); 137 video_codec.height = 138 std::max(video_codec.height, static_cast<uint16_t>(streams[i].height)); 139 video_codec.minBitrate = 140 std::min(static_cast<uint16_t>(video_codec.minBitrate), 141 static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000)); 142 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; 143 video_codec.qpMax = std::max(video_codec.qpMax, 144 static_cast<unsigned int>(streams[i].max_qp)); 145 max_framerate = std::max(max_framerate, streams[i].max_framerate); 146 147 // TODO(bugs.webrtc.org/11607): Since scalability mode is a top-level 148 // setting on VideoCodec, setting it makes sense only if it is the same for 149 // all active simulcast streams. 150 if (streams[i].active && 151 streams[0].scalability_mode != streams[i].scalability_mode) { 152 scalability_mode.reset(); 153 // For VP8, top-level scalability mode doesn't matter, since configuration 154 // is based on the per-simulcast stream configuration of temporal layers. 155 if (video_codec.codecType != kVideoCodecVP8) { 156 RTC_LOG(LS_WARNING) << "Inconsistent scalability modes configured."; 157 } 158 } 159 } 160 161 if (scalability_mode.has_value()) { 162 video_codec.SetScalabilityMode(*scalability_mode); 163 } 164 165 if (video_codec.maxBitrate == 0) { 166 // Unset max bitrate -> cap to one bit per pixel. 167 video_codec.maxBitrate = 168 (video_codec.width * video_codec.height * video_codec.maxFramerate) / 169 1000; 170 } 171 if (video_codec.maxBitrate < kEncoderMinBitrateKbps) 172 video_codec.maxBitrate = kEncoderMinBitrateKbps; 173 174 video_codec.maxFramerate = max_framerate; 175 video_codec.spatialLayers[0] = {}; 176 video_codec.spatialLayers[0].width = video_codec.width; 177 video_codec.spatialLayers[0].height = video_codec.height; 178 video_codec.spatialLayers[0].maxFramerate = max_framerate; 179 video_codec.spatialLayers[0].numberOfTemporalLayers = 180 streams[0].scalability_mode.has_value() 181 ? ScalabilityModeToNumTemporalLayers(*streams[0].scalability_mode) 182 : streams[0].num_temporal_layers.value_or(1); 183 184 // Set codec specific options 185 if (config.encoder_specific_settings) 186 config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec); 187 188 switch (video_codec.codecType) { 189 case kVideoCodecVP8: { 190 if (!config.encoder_specific_settings) { 191 *video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings(); 192 } 193 194 // Validate specified scalability modes. If some layer has an unsupported 195 // mode, store it as the top-level scalability mode, which will make 196 // InitEncode fail with an appropriate error. 197 for (const auto& stream : streams) { 198 if (stream.scalability_mode.has_value() && 199 !VP8SupportsScalabilityMode(*stream.scalability_mode)) { 200 RTC_LOG(LS_WARNING) 201 << "Invalid scalability mode for VP8: " 202 << ScalabilityModeToString(*stream.scalability_mode); 203 video_codec.SetScalabilityMode(*stream.scalability_mode); 204 break; 205 } 206 } 207 video_codec.VP8()->numberOfTemporalLayers = 208 streams.back().scalability_mode.has_value() 209 ? ScalabilityModeToNumTemporalLayers( 210 *streams.back().scalability_mode) 211 : streams.back().num_temporal_layers.value_or( 212 video_codec.VP8()->numberOfTemporalLayers); 213 214 RTC_DCHECK_GE(video_codec.VP8()->numberOfTemporalLayers, 1); 215 RTC_DCHECK_LE(video_codec.VP8()->numberOfTemporalLayers, 216 kMaxTemporalStreams); 217 218 break; 219 } 220 case kVideoCodecVP9: { 221 // When the SvcRateAllocator is used, "active" is controlled by 222 // `SpatialLayer::active` instead. 223 if (video_codec.numberOfSimulcastStreams <= 1) { 224 video_codec.simulcastStream[0].active = codec_active; 225 } 226 227 if (!config.encoder_specific_settings) { 228 *video_codec.VP9() = VideoEncoder::GetDefaultVp9Settings(); 229 } 230 231 video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>( 232 streams.back().num_temporal_layers.value_or( 233 video_codec.VP9()->numberOfTemporalLayers)); 234 RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1); 235 RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers, 236 kMaxTemporalStreams); 237 238 RTC_DCHECK(config.spatial_layers.empty() || 239 config.spatial_layers.size() == 240 video_codec.VP9()->numberOfSpatialLayers); 241 242 std::vector<SpatialLayer> spatial_layers; 243 if (!config.spatial_layers.empty()) { 244 // Layering is set explicitly. 245 spatial_layers = config.spatial_layers; 246 } else if (video_codec.GetScalabilityMode().has_value()) { 247 // Layering is set via scalability mode. 248 spatial_layers = GetVp9SvcConfig(video_codec); 249 } else { 250 size_t first_active_layer = 0; 251 for (size_t spatial_idx = 0; 252 spatial_idx < config.simulcast_layers.size(); ++spatial_idx) { 253 if (config.simulcast_layers[spatial_idx].active) { 254 first_active_layer = spatial_idx; 255 break; 256 } 257 } 258 259 spatial_layers = GetSvcConfig( 260 video_codec.width, video_codec.height, video_codec.maxFramerate, 261 first_active_layer, video_codec.VP9()->numberOfSpatialLayers, 262 video_codec.VP9()->numberOfTemporalLayers, 263 video_codec.mode == VideoCodecMode::kScreensharing); 264 265 // If there was no request for spatial layering, don't limit bitrate 266 // of single spatial layer. 267 const bool no_spatial_layering = 268 video_codec.VP9()->numberOfSpatialLayers <= 1; 269 if (no_spatial_layering) { 270 // Use codec's bitrate limits. 271 spatial_layers.back().minBitrate = video_codec.minBitrate; 272 spatial_layers.back().targetBitrate = video_codec.maxBitrate; 273 spatial_layers.back().maxBitrate = video_codec.maxBitrate; 274 } 275 276 for (size_t spatial_idx = first_active_layer; 277 spatial_idx < config.simulcast_layers.size() && 278 spatial_idx < spatial_layers.size() + first_active_layer; 279 ++spatial_idx) { 280 spatial_layers[spatial_idx - first_active_layer].active = 281 config.simulcast_layers[spatial_idx].active; 282 } 283 } 284 285 RTC_DCHECK(!spatial_layers.empty()); 286 for (size_t i = 0; i < spatial_layers.size(); ++i) { 287 video_codec.spatialLayers[i] = spatial_layers[i]; 288 } 289 290 // The top spatial layer dimensions may not be equal to the input 291 // resolution because of the rounding or explicit configuration. 292 // This difference must be propagated to the stream configuration. 293 video_codec.width = spatial_layers.back().width; 294 video_codec.height = spatial_layers.back().height; 295 // Only propagate if we're not doing simulcast. Simulcast is assumed not 296 // to have multiple spatial layers, if we wanted to support simulcast+SVC 297 // combos we would need to calculate unique spatial layers per simulcast 298 // layer, but VideoCodec is not capable of expressing per-simulcastStream 299 // spatialLayers. 300 if (video_codec.numberOfSimulcastStreams == 1) { 301 video_codec.simulcastStream[0].width = spatial_layers.back().width; 302 video_codec.simulcastStream[0].height = spatial_layers.back().height; 303 } 304 305 // Update layering settings. 306 video_codec.VP9()->numberOfSpatialLayers = 307 static_cast<unsigned char>(spatial_layers.size()); 308 RTC_DCHECK_GE(video_codec.VP9()->numberOfSpatialLayers, 1); 309 RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers, 310 kMaxSpatialLayers); 311 312 video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>( 313 spatial_layers.back().numberOfTemporalLayers); 314 RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1); 315 RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers, 316 kMaxTemporalStreams); 317 318 break; 319 } 320 case kVideoCodecAV1: 321 if (SetAv1SvcConfig(video_codec, 322 /*num_temporal_layers=*/ 323 streams.back().num_temporal_layers.value_or(1), 324 /*num_spatial_layers=*/ 325 std::max<int>(config.spatial_layers.size(), 1))) { 326 // If min bitrate is set via RtpEncodingParameters, use this value on 327 // lowest spatial layer. 328 if (!config.simulcast_layers.empty() && 329 config.simulcast_layers[0].min_bitrate_bps > 0) { 330 video_codec.spatialLayers[0].minBitrate = std::min( 331 config.simulcast_layers[0].min_bitrate_bps / 1000, 332 static_cast<int>(video_codec.spatialLayers[0].targetBitrate)); 333 } 334 for (size_t i = 0; i < config.spatial_layers.size(); ++i) { 335 video_codec.spatialLayers[i].active = config.spatial_layers[i].active; 336 } 337 } else { 338 RTC_LOG(LS_WARNING) << "Failed to configure svc bitrates for av1."; 339 } 340 break; 341 case kVideoCodecH264: { 342 RTC_CHECK(!config.encoder_specific_settings); 343 344 *video_codec.H264() = VideoEncoder::GetDefaultH264Settings(); 345 video_codec.H264()->numberOfTemporalLayers = static_cast<unsigned char>( 346 streams.back().num_temporal_layers.value_or( 347 video_codec.H264()->numberOfTemporalLayers)); 348 RTC_DCHECK_GE(video_codec.H264()->numberOfTemporalLayers, 1); 349 RTC_DCHECK_LE(video_codec.H264()->numberOfTemporalLayers, 350 kMaxTemporalStreams); 351 break; 352 } 353 case kVideoCodecH265: 354 RTC_DCHECK(!config.encoder_specific_settings) << "No encoder-specific " 355 "settings for H.265."; 356 357 // Validate specified scalability modes. If some layer has an unsupported 358 // mode, store it as the top-level scalability mode, which will make 359 // InitEncode fail with an appropriate error. 360 for (const auto& stream : streams) { 361 if (stream.scalability_mode.has_value() && 362 !H265SupportsScalabilityMode(*stream.scalability_mode)) { 363 RTC_LOG(LS_WARNING) 364 << "Invalid scalability mode for H.265: " 365 << ScalabilityModeToString(*stream.scalability_mode); 366 video_codec.SetScalabilityMode(*stream.scalability_mode); 367 break; 368 } 369 } 370 video_codec.spatialLayers[0].minBitrate = video_codec.minBitrate; 371 video_codec.spatialLayers[0].targetBitrate = video_codec.maxBitrate; 372 video_codec.spatialLayers[0].maxBitrate = video_codec.maxBitrate; 373 video_codec.spatialLayers[0].active = codec_active; 374 break; 375 default: 376 // TODO(pbos): Support encoder_settings codec-agnostically. 377 RTC_DCHECK(!config.encoder_specific_settings) 378 << "Encoder-specific settings for codec type not wired up."; 379 break; 380 } 381 382 const std::optional<DataRate> experimental_min_bitrate = 383 GetExperimentalMinVideoBitrate(field_trials, video_codec.codecType); 384 if (experimental_min_bitrate) { 385 const int experimental_min_bitrate_kbps = 386 saturated_cast<int>(experimental_min_bitrate->kbps()); 387 video_codec.minBitrate = experimental_min_bitrate_kbps; 388 video_codec.simulcastStream[0].minBitrate = experimental_min_bitrate_kbps; 389 if (video_codec.codecType == kVideoCodecVP9 || 390 #ifdef RTC_ENABLE_H265 391 video_codec.codecType == kVideoCodecH265 || 392 #endif 393 video_codec.codecType == kVideoCodecAV1) { 394 video_codec.spatialLayers[0].minBitrate = experimental_min_bitrate_kbps; 395 } 396 } 397 398 return video_codec; 399 } 400 401 } // namespace webrtc