video_encoder_software_fallback_wrapper.cc (20122B)
1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "api/video_codecs/video_encoder_software_fallback_wrapper.h" 12 13 #include <cstdint> 14 #include <cstdio> 15 #include <memory> 16 #include <numeric> 17 #include <optional> 18 #include <string> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/strings/match.h" 23 #include "api/environment/environment.h" 24 #include "api/fec_controller_override.h" 25 #include "api/field_trials_view.h" 26 #include "api/scoped_refptr.h" 27 #include "api/video/video_codec_type.h" 28 #include "api/video/video_frame.h" 29 #include "api/video/video_frame_buffer.h" 30 #include "api/video/video_frame_type.h" 31 #include "api/video_codecs/video_codec.h" 32 #include "api/video_codecs/video_encoder.h" 33 #include "modules/video_coding/include/video_error_codes.h" 34 #include "modules/video_coding/include/video_error_codes_utils.h" 35 #include "modules/video_coding/utility/simulcast_utility.h" 36 #include "rtc_base/checks.h" 37 #include "rtc_base/experiments/field_trial_parser.h" 38 #include "rtc_base/logging.h" 39 40 namespace webrtc { 41 42 namespace { 43 44 // If forced fallback is allowed, either: 45 // 46 // 1) The forced fallback is requested if the resolution is less than or equal 47 // to `max_pixels_`. The resolution is allowed to be scaled down to 48 // `min_pixels_`. 49 // 50 // 2) The forced fallback is requested if temporal support is preferred and the 51 // SW fallback supports temporal layers while the HW encoder does not. 52 53 struct ForcedFallbackParams { 54 public: 55 bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const { 56 if (!enable_resolution_based_switch || 57 codec.width * codec.height > max_pixels) { 58 return false; 59 } 60 61 if (vp8_specific_resolution_switch && 62 (codec.codecType != kVideoCodecVP8 || 63 codec.numberOfSimulcastStreams > 1)) { 64 return false; 65 } 66 67 return true; 68 } 69 70 bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const { 71 return enable_temporal_based_switch && 72 SimulcastUtility::NumberOfTemporalLayers(codec, 0) != 1; 73 } 74 75 bool enable_temporal_based_switch = false; 76 bool enable_resolution_based_switch = false; 77 bool vp8_specific_resolution_switch = false; 78 int min_pixels = kDefaultMinPixelsPerFrame; 79 int max_pixels = 320 * 240; 80 }; 81 82 const char kVp8ForceFallbackEncoderFieldTrial[] = 83 "WebRTC-VP8-Forced-Fallback-Encoder-v2"; 84 85 std::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials( 86 const FieldTrialsView& field_trials, 87 const VideoEncoder& main_encoder) { 88 // Ignore WebRTC-VP8-Forced-Fallback-Encoder-v2 if 89 // WebRTC-Video-EncoderFallbackSettings is present. 90 FieldTrialOptional<int> resolution_threshold_px("resolution_threshold_px"); 91 ParseFieldTrial({&resolution_threshold_px}, 92 field_trials.Lookup("WebRTC-Video-EncoderFallbackSettings")); 93 if (resolution_threshold_px) { 94 ForcedFallbackParams params; 95 params.enable_resolution_based_switch = true; 96 params.max_pixels = resolution_threshold_px.Value(); 97 return params; 98 } 99 100 const std::string field_trial = 101 field_trials.Lookup(kVp8ForceFallbackEncoderFieldTrial); 102 if (!absl::StartsWith(field_trial, "Enabled")) { 103 return std::nullopt; 104 } 105 106 int max_pixels_lower_bound = 107 main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1; 108 109 ForcedFallbackParams params; 110 params.enable_resolution_based_switch = true; 111 112 int min_bps = 0; 113 if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", ¶ms.min_pixels, 114 ¶ms.max_pixels, &min_bps) != 3) { 115 RTC_LOG(LS_WARNING) 116 << "Invalid number of forced fallback parameters provided."; 117 return std::nullopt; 118 } else if (params.min_pixels <= 0 || 119 params.max_pixels < max_pixels_lower_bound || 120 params.max_pixels < params.min_pixels || min_bps <= 0) { 121 RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided."; 122 return std::nullopt; 123 } 124 125 params.vp8_specific_resolution_switch = true; 126 return params; 127 } 128 129 std::optional<ForcedFallbackParams> GetForcedFallbackParams( 130 const FieldTrialsView& field_trials, 131 bool prefer_temporal_support, 132 const VideoEncoder& main_encoder) { 133 std::optional<ForcedFallbackParams> params = 134 ParseFallbackParamsFromFieldTrials(field_trials, main_encoder); 135 if (prefer_temporal_support) { 136 if (!params.has_value()) { 137 params.emplace(); 138 } 139 params->enable_temporal_based_switch = true; 140 } 141 return params; 142 } 143 144 class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder { 145 public: 146 VideoEncoderSoftwareFallbackWrapper(const FieldTrialsView& field_trials, 147 std::unique_ptr<VideoEncoder> sw_encoder, 148 std::unique_ptr<VideoEncoder> hw_encoder, 149 bool prefer_temporal_support); 150 ~VideoEncoderSoftwareFallbackWrapper() override; 151 152 void SetFecControllerOverride( 153 FecControllerOverride* fec_controller_override) override; 154 155 int32_t InitEncode(const VideoCodec* codec_settings, 156 const VideoEncoder::Settings& settings) override; 157 158 int32_t RegisterEncodeCompleteCallback( 159 EncodedImageCallback* callback) override; 160 161 int32_t Release() override; 162 163 int32_t Encode(const VideoFrame& frame, 164 const std::vector<VideoFrameType>* frame_types) override; 165 166 void OnPacketLossRateUpdate(float packet_loss_rate) override; 167 168 void OnRttUpdate(int64_t rtt_ms) override; 169 170 void OnLossNotification(const LossNotification& loss_notification) override; 171 172 void SetRates(const RateControlParameters& parameters) override; 173 174 EncoderInfo GetEncoderInfo() const override; 175 176 private: 177 bool InitFallbackEncoder(bool is_forced); 178 bool TryInitForcedFallbackEncoder(); 179 bool IsFallbackActive() const; 180 181 VideoEncoder* current_encoder() { 182 switch (encoder_state_) { 183 case EncoderState::kUninitialized: 184 RTC_LOG(LS_WARNING) 185 << "Trying to access encoder in uninitialized fallback wrapper."; 186 // Return main encoder to preserve previous behavior. 187 [[fallthrough]]; 188 case EncoderState::kMainEncoderUsed: 189 return encoder_.get(); 190 case EncoderState::kFallbackDueToFailure: 191 case EncoderState::kForcedFallback: 192 return fallback_encoder_.get(); 193 } 194 RTC_CHECK_NOTREACHED(); 195 } 196 197 // Updates encoder with last observed parameters, such as callbacks, rates, 198 // etc. 199 void PrimeEncoder(VideoEncoder* encoder) const; 200 201 // Settings used in the last InitEncode call and used if a dynamic fallback to 202 // software is required. 203 VideoCodec codec_settings_; 204 std::optional<VideoEncoder::Settings> encoder_settings_; 205 206 // The last rate control settings, if set. 207 std::optional<RateControlParameters> rate_control_parameters_; 208 209 // The last channel parameters set. 210 std::optional<float> packet_loss_; 211 std::optional<int64_t> rtt_; 212 std::optional<LossNotification> loss_notification_; 213 214 enum class EncoderState { 215 kUninitialized, 216 kMainEncoderUsed, 217 kFallbackDueToFailure, 218 kForcedFallback 219 }; 220 221 EncoderState encoder_state_; 222 const std::unique_ptr<VideoEncoder> encoder_; 223 const std::unique_ptr<VideoEncoder> fallback_encoder_; 224 225 EncodedImageCallback* callback_; 226 227 const std::optional<ForcedFallbackParams> fallback_params_; 228 int32_t EncodeWithMainEncoder(const VideoFrame& frame, 229 const std::vector<VideoFrameType>* frame_types); 230 }; 231 232 VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( 233 const FieldTrialsView& field_trials, 234 std::unique_ptr<VideoEncoder> sw_encoder, 235 std::unique_ptr<VideoEncoder> hw_encoder, 236 bool prefer_temporal_support) 237 : encoder_state_(EncoderState::kUninitialized), 238 encoder_(std::move(hw_encoder)), 239 fallback_encoder_(std::move(sw_encoder)), 240 callback_(nullptr), 241 fallback_params_(GetForcedFallbackParams(field_trials, 242 prefer_temporal_support, 243 *encoder_)) { 244 RTC_DCHECK(fallback_encoder_); 245 } 246 247 VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() = 248 default; 249 250 void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder( 251 VideoEncoder* encoder) const { 252 RTC_DCHECK(encoder); 253 // Replay callback, rates, and channel parameters. 254 if (callback_) { 255 encoder->RegisterEncodeCompleteCallback(callback_); 256 } 257 if (rate_control_parameters_) { 258 encoder->SetRates(*rate_control_parameters_); 259 } 260 if (rtt_.has_value()) { 261 encoder->OnRttUpdate(rtt_.value()); 262 } 263 if (packet_loss_.has_value()) { 264 encoder->OnPacketLossRateUpdate(packet_loss_.value()); 265 } 266 267 if (loss_notification_.has_value()) { 268 encoder->OnLossNotification(loss_notification_.value()); 269 } 270 } 271 272 bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) { 273 RTC_LOG(LS_WARNING) << "[VESFW] " << __func__ 274 << "(is_forced=" << (is_forced ? "true" : "false") << ")"; 275 276 RTC_DCHECK(encoder_settings_.has_value()); 277 const int ret = fallback_encoder_->InitEncode(&codec_settings_, 278 encoder_settings_.value()); 279 280 if (ret != WEBRTC_VIDEO_CODEC_OK) { 281 RTC_LOG(LS_ERROR) 282 << "[VESFW] software-encoder fallback initialization failed with" 283 << " error code: " << WebRtcVideoCodecErrorToString(ret); 284 fallback_encoder_->Release(); 285 return false; 286 } 287 288 if (encoder_state_ == EncoderState::kMainEncoderUsed) { 289 // Since we're switching to the fallback encoder, Release the real encoder. 290 // It may be re-initialized via InitEncode later, and it will continue to 291 // get Set calls for rates and channel parameters in the meantime. 292 encoder_->Release(); 293 } 294 295 if (is_forced) { 296 encoder_state_ = EncoderState::kForcedFallback; 297 } else { 298 encoder_state_ = EncoderState::kFallbackDueToFailure; 299 } 300 301 return true; 302 } 303 304 void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride( 305 FecControllerOverride* fec_controller_override) { 306 // It is important that only one of those would ever interact with the 307 // `fec_controller_override` at a given time. This is the responsibility 308 // of `this` to maintain. 309 310 encoder_->SetFecControllerOverride(fec_controller_override); 311 fallback_encoder_->SetFecControllerOverride(fec_controller_override); 312 } 313 314 int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( 315 const VideoCodec* codec_settings, 316 const VideoEncoder::Settings& settings) { 317 RTC_LOG(LS_INFO) << "[VESFW] " << __func__ 318 << "(codec=" << codec_settings->ToString() 319 << ", settings={number_of_cores: " 320 << settings.number_of_cores 321 << ", max_payload_size: " << settings.max_payload_size 322 << "})"; 323 // Store settings, in case we need to dynamically switch to the fallback 324 // encoder after a failed Encode call. 325 codec_settings_ = *codec_settings; 326 encoder_settings_ = settings; 327 // Clear stored rate/channel parameters. 328 rate_control_parameters_ = std::nullopt; 329 330 RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized) 331 << "InitEncode() should never be called on an active instance!"; 332 333 // Try to init forced software codec if it should be used. 334 if (TryInitForcedFallbackEncoder()) { 335 PrimeEncoder(current_encoder()); 336 return WEBRTC_VIDEO_CODEC_OK; 337 } 338 339 int32_t ret = encoder_->InitEncode(codec_settings, settings); 340 if (ret == WEBRTC_VIDEO_CODEC_OK) { 341 encoder_state_ = EncoderState::kMainEncoderUsed; 342 PrimeEncoder(current_encoder()); 343 return ret; 344 } 345 if (ret == WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED) { 346 return ret; 347 } 348 RTC_LOG(LS_WARNING) << "[VESFW] Hardware encoder initialization failed with" 349 << " error code: " << WebRtcVideoCodecErrorToString(ret); 350 351 // Try to instantiate software codec. 352 if (InitFallbackEncoder(/*is_forced=*/false)) { 353 PrimeEncoder(current_encoder()); 354 return WEBRTC_VIDEO_CODEC_OK; 355 } 356 357 // Software encoder failed too, use original return code. 358 RTC_LOG(LS_WARNING) 359 << "[VESFW] Software fallback encoder initialization also failed."; 360 encoder_state_ = EncoderState::kUninitialized; 361 return ret; 362 } 363 364 int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback( 365 EncodedImageCallback* callback) { 366 callback_ = callback; 367 return current_encoder()->RegisterEncodeCompleteCallback(callback); 368 } 369 370 int32_t VideoEncoderSoftwareFallbackWrapper::Release() { 371 if (encoder_state_ == EncoderState::kUninitialized) { 372 return WEBRTC_VIDEO_CODEC_OK; 373 } 374 int32_t ret = current_encoder()->Release(); 375 encoder_state_ = EncoderState::kUninitialized; 376 return ret; 377 } 378 379 int32_t VideoEncoderSoftwareFallbackWrapper::Encode( 380 const VideoFrame& frame, 381 const std::vector<VideoFrameType>* frame_types) { 382 switch (encoder_state_) { 383 case EncoderState::kUninitialized: 384 return WEBRTC_VIDEO_CODEC_ERROR; 385 case EncoderState::kMainEncoderUsed: { 386 return EncodeWithMainEncoder(frame, frame_types); 387 } 388 case EncoderState::kFallbackDueToFailure: 389 case EncoderState::kForcedFallback: 390 return fallback_encoder_->Encode(frame, frame_types); 391 } 392 RTC_CHECK_NOTREACHED(); 393 } 394 395 int32_t VideoEncoderSoftwareFallbackWrapper::EncodeWithMainEncoder( 396 const VideoFrame& frame, 397 const std::vector<VideoFrameType>* frame_types) { 398 int32_t ret = encoder_->Encode(frame, frame_types); 399 // If requested, try a software fallback. 400 bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); 401 if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) { 402 // Start using the fallback with this frame. 403 PrimeEncoder(current_encoder()); 404 if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative && 405 fallback_encoder_->GetEncoderInfo().supports_native_handle) { 406 return fallback_encoder_->Encode(frame, frame_types); 407 } else { 408 RTC_LOG(LS_INFO) << "Fallback encoder does not support native handle - " 409 "converting frame to I420"; 410 scoped_refptr<I420BufferInterface> src_buffer = 411 frame.video_frame_buffer()->ToI420(); 412 if (!src_buffer) { 413 RTC_LOG(LS_ERROR) << "Failed to convert from to I420"; 414 return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; 415 } 416 scoped_refptr<VideoFrameBuffer> dst_buffer = 417 src_buffer->Scale(codec_settings_.width, codec_settings_.height); 418 if (!dst_buffer) { 419 RTC_LOG(LS_ERROR) << "Failed to scale video frame."; 420 return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; 421 } 422 VideoFrame scaled_frame = frame; 423 scaled_frame.set_video_frame_buffer(dst_buffer); 424 scaled_frame.set_update_rect( 425 VideoFrame::UpdateRect{.offset_x = 0, 426 .offset_y = 0, 427 .width = scaled_frame.width(), 428 .height = scaled_frame.height()}); 429 return fallback_encoder_->Encode(scaled_frame, frame_types); 430 } 431 } 432 // Fallback encoder failed too, return original error code. 433 return ret; 434 } 435 436 void VideoEncoderSoftwareFallbackWrapper::SetRates( 437 const RateControlParameters& parameters) { 438 rate_control_parameters_ = parameters; 439 return current_encoder()->SetRates(parameters); 440 } 441 442 void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate( 443 float packet_loss_rate) { 444 packet_loss_ = packet_loss_rate; 445 current_encoder()->OnPacketLossRateUpdate(packet_loss_rate); 446 } 447 448 void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) { 449 rtt_ = rtt_ms; 450 current_encoder()->OnRttUpdate(rtt_ms); 451 } 452 453 void VideoEncoderSoftwareFallbackWrapper::OnLossNotification( 454 const LossNotification& loss_notification) { 455 loss_notification_ = loss_notification; 456 current_encoder()->OnLossNotification(loss_notification); 457 } 458 459 VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() 460 const { 461 EncoderInfo fallback_encoder_info = fallback_encoder_->GetEncoderInfo(); 462 EncoderInfo default_encoder_info = encoder_->GetEncoderInfo(); 463 464 EncoderInfo info = 465 IsFallbackActive() ? fallback_encoder_info : default_encoder_info; 466 467 info.requested_resolution_alignment = 468 std::lcm(fallback_encoder_info.requested_resolution_alignment, 469 default_encoder_info.requested_resolution_alignment); 470 info.apply_alignment_to_all_simulcast_layers = 471 fallback_encoder_info.apply_alignment_to_all_simulcast_layers || 472 default_encoder_info.apply_alignment_to_all_simulcast_layers; 473 474 if (fallback_params_ && fallback_params_->vp8_specific_resolution_switch) { 475 info.scaling_settings.min_pixels_per_frame = fallback_params_->min_pixels; 476 } 477 478 return info; 479 } 480 481 bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const { 482 return encoder_state_ == EncoderState::kForcedFallback || 483 encoder_state_ == EncoderState::kFallbackDueToFailure; 484 } 485 486 bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() { 487 if (!fallback_params_) { 488 return false; 489 } 490 491 RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized); 492 493 if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) { 494 // Settings valid, try to instantiate software codec. 495 RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: " 496 << codec_settings_.width << "x" << codec_settings_.height; 497 return InitFallbackEncoder(/*is_forced=*/true); 498 } 499 500 if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) { 501 // First init main encoder to see if that supports temporal layers. 502 if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) == 503 WEBRTC_VIDEO_CODEC_OK) { 504 encoder_state_ = EncoderState::kMainEncoderUsed; 505 } 506 507 if (encoder_state_ == EncoderState::kMainEncoderUsed && 508 encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) { 509 // Primary encoder already supports temporal layers, use that instead. 510 return true; 511 } 512 513 // Try to initialize fallback and check if it supports temporal layers. 514 if (fallback_encoder_->InitEncode(&codec_settings_, 515 encoder_settings_.value()) == 516 WEBRTC_VIDEO_CODEC_OK) { 517 if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) { 518 // Fallback encoder available and supports temporal layers, use it! 519 if (encoder_state_ == EncoderState::kMainEncoderUsed) { 520 // Main encoder initialized but does not support temporal layers, 521 // release it again. 522 encoder_->Release(); 523 } 524 encoder_state_ = EncoderState::kForcedFallback; 525 RTC_LOG(LS_INFO) 526 << "Forced switch to SW encoder due to temporal support."; 527 return true; 528 } else { 529 // Fallback encoder intialization succeeded, but it does not support 530 // temporal layers either - release it. 531 fallback_encoder_->Release(); 532 } 533 } 534 535 if (encoder_state_ == EncoderState::kMainEncoderUsed) { 536 // Main encoder already initialized - make use of it. 537 RTC_LOG(LS_INFO) 538 << "Cannot fall back for temporal support since fallback that " 539 "supports is not available. Using main encoder instead."; 540 return true; 541 } 542 } 543 544 // Neither forced fallback mode supported. 545 return false; 546 } 547 548 } // namespace 549 550 std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper( 551 const Environment& env, 552 std::unique_ptr<VideoEncoder> sw_fallback_encoder, 553 std::unique_ptr<VideoEncoder> hw_encoder, 554 bool prefer_temporal_support) { 555 return std::make_unique<VideoEncoderSoftwareFallbackWrapper>( 556 env.field_trials(), std::move(sw_fallback_encoder), std::move(hw_encoder), 557 prefer_temporal_support); 558 } 559 560 } // namespace webrtc