tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

video_encoder_software_fallback_wrapper.cc (20122B)


      1 /*
      2 *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
      3 *
      4 *  Use of this source code is governed by a BSD-style license
      5 *  that can be found in the LICENSE file in the root of the source
      6 *  tree. An additional intellectual property rights grant can be found
      7 *  in the file PATENTS.  All contributing project authors may
      8 *  be found in the AUTHORS file in the root of the source tree.
      9 */
     10 
     11 #include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
     12 
     13 #include <cstdint>
     14 #include <cstdio>
     15 #include <memory>
     16 #include <numeric>
     17 #include <optional>
     18 #include <string>
     19 #include <utility>
     20 #include <vector>
     21 
     22 #include "absl/strings/match.h"
     23 #include "api/environment/environment.h"
     24 #include "api/fec_controller_override.h"
     25 #include "api/field_trials_view.h"
     26 #include "api/scoped_refptr.h"
     27 #include "api/video/video_codec_type.h"
     28 #include "api/video/video_frame.h"
     29 #include "api/video/video_frame_buffer.h"
     30 #include "api/video/video_frame_type.h"
     31 #include "api/video_codecs/video_codec.h"
     32 #include "api/video_codecs/video_encoder.h"
     33 #include "modules/video_coding/include/video_error_codes.h"
     34 #include "modules/video_coding/include/video_error_codes_utils.h"
     35 #include "modules/video_coding/utility/simulcast_utility.h"
     36 #include "rtc_base/checks.h"
     37 #include "rtc_base/experiments/field_trial_parser.h"
     38 #include "rtc_base/logging.h"
     39 
     40 namespace webrtc {
     41 
     42 namespace {
     43 
     44 // If forced fallback is allowed, either:
     45 //
     46 // 1) The forced fallback is requested if the resolution is less than or equal
     47 //    to `max_pixels_`. The resolution is allowed to be scaled down to
     48 //    `min_pixels_`.
     49 //
     50 // 2) The forced fallback is requested if temporal support is preferred and the
     51 //    SW fallback supports temporal layers while the HW encoder does not.
     52 
     53 struct ForcedFallbackParams {
     54 public:
     55  bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const {
     56    if (!enable_resolution_based_switch ||
     57        codec.width * codec.height > max_pixels) {
     58      return false;
     59    }
     60 
     61    if (vp8_specific_resolution_switch &&
     62        (codec.codecType != kVideoCodecVP8 ||
     63         codec.numberOfSimulcastStreams > 1)) {
     64      return false;
     65    }
     66 
     67    return true;
     68  }
     69 
     70  bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const {
     71    return enable_temporal_based_switch &&
     72           SimulcastUtility::NumberOfTemporalLayers(codec, 0) != 1;
     73  }
     74 
     75  bool enable_temporal_based_switch = false;
     76  bool enable_resolution_based_switch = false;
     77  bool vp8_specific_resolution_switch = false;
     78  int min_pixels = kDefaultMinPixelsPerFrame;
     79  int max_pixels = 320 * 240;
     80 };
     81 
     82 const char kVp8ForceFallbackEncoderFieldTrial[] =
     83    "WebRTC-VP8-Forced-Fallback-Encoder-v2";
     84 
     85 std::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials(
     86    const FieldTrialsView& field_trials,
     87    const VideoEncoder& main_encoder) {
     88  // Ignore WebRTC-VP8-Forced-Fallback-Encoder-v2 if
     89  // WebRTC-Video-EncoderFallbackSettings is present.
     90  FieldTrialOptional<int> resolution_threshold_px("resolution_threshold_px");
     91  ParseFieldTrial({&resolution_threshold_px},
     92                  field_trials.Lookup("WebRTC-Video-EncoderFallbackSettings"));
     93  if (resolution_threshold_px) {
     94    ForcedFallbackParams params;
     95    params.enable_resolution_based_switch = true;
     96    params.max_pixels = resolution_threshold_px.Value();
     97    return params;
     98  }
     99 
    100  const std::string field_trial =
    101      field_trials.Lookup(kVp8ForceFallbackEncoderFieldTrial);
    102  if (!absl::StartsWith(field_trial, "Enabled")) {
    103    return std::nullopt;
    104  }
    105 
    106  int max_pixels_lower_bound =
    107      main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1;
    108 
    109  ForcedFallbackParams params;
    110  params.enable_resolution_based_switch = true;
    111 
    112  int min_bps = 0;
    113  if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", &params.min_pixels,
    114             &params.max_pixels, &min_bps) != 3) {
    115    RTC_LOG(LS_WARNING)
    116        << "Invalid number of forced fallback parameters provided.";
    117    return std::nullopt;
    118  } else if (params.min_pixels <= 0 ||
    119             params.max_pixels < max_pixels_lower_bound ||
    120             params.max_pixels < params.min_pixels || min_bps <= 0) {
    121    RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided.";
    122    return std::nullopt;
    123  }
    124 
    125  params.vp8_specific_resolution_switch = true;
    126  return params;
    127 }
    128 
    129 std::optional<ForcedFallbackParams> GetForcedFallbackParams(
    130    const FieldTrialsView& field_trials,
    131    bool prefer_temporal_support,
    132    const VideoEncoder& main_encoder) {
    133  std::optional<ForcedFallbackParams> params =
    134      ParseFallbackParamsFromFieldTrials(field_trials, main_encoder);
    135  if (prefer_temporal_support) {
    136    if (!params.has_value()) {
    137      params.emplace();
    138    }
    139    params->enable_temporal_based_switch = true;
    140  }
    141  return params;
    142 }
    143 
    144 class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
    145 public:
    146  VideoEncoderSoftwareFallbackWrapper(const FieldTrialsView& field_trials,
    147                                      std::unique_ptr<VideoEncoder> sw_encoder,
    148                                      std::unique_ptr<VideoEncoder> hw_encoder,
    149                                      bool prefer_temporal_support);
    150  ~VideoEncoderSoftwareFallbackWrapper() override;
    151 
    152  void SetFecControllerOverride(
    153      FecControllerOverride* fec_controller_override) override;
    154 
    155  int32_t InitEncode(const VideoCodec* codec_settings,
    156                     const VideoEncoder::Settings& settings) override;
    157 
    158  int32_t RegisterEncodeCompleteCallback(
    159      EncodedImageCallback* callback) override;
    160 
    161  int32_t Release() override;
    162 
    163  int32_t Encode(const VideoFrame& frame,
    164                 const std::vector<VideoFrameType>* frame_types) override;
    165 
    166  void OnPacketLossRateUpdate(float packet_loss_rate) override;
    167 
    168  void OnRttUpdate(int64_t rtt_ms) override;
    169 
    170  void OnLossNotification(const LossNotification& loss_notification) override;
    171 
    172  void SetRates(const RateControlParameters& parameters) override;
    173 
    174  EncoderInfo GetEncoderInfo() const override;
    175 
    176 private:
    177  bool InitFallbackEncoder(bool is_forced);
    178  bool TryInitForcedFallbackEncoder();
    179  bool IsFallbackActive() const;
    180 
    181  VideoEncoder* current_encoder() {
    182    switch (encoder_state_) {
    183      case EncoderState::kUninitialized:
    184        RTC_LOG(LS_WARNING)
    185            << "Trying to access encoder in uninitialized fallback wrapper.";
    186        // Return main encoder to preserve previous behavior.
    187        [[fallthrough]];
    188      case EncoderState::kMainEncoderUsed:
    189        return encoder_.get();
    190      case EncoderState::kFallbackDueToFailure:
    191      case EncoderState::kForcedFallback:
    192        return fallback_encoder_.get();
    193    }
    194    RTC_CHECK_NOTREACHED();
    195  }
    196 
    197  // Updates encoder with last observed parameters, such as callbacks, rates,
    198  // etc.
    199  void PrimeEncoder(VideoEncoder* encoder) const;
    200 
    201  // Settings used in the last InitEncode call and used if a dynamic fallback to
    202  // software is required.
    203  VideoCodec codec_settings_;
    204  std::optional<VideoEncoder::Settings> encoder_settings_;
    205 
    206  // The last rate control settings, if set.
    207  std::optional<RateControlParameters> rate_control_parameters_;
    208 
    209  // The last channel parameters set.
    210  std::optional<float> packet_loss_;
    211  std::optional<int64_t> rtt_;
    212  std::optional<LossNotification> loss_notification_;
    213 
    214  enum class EncoderState {
    215    kUninitialized,
    216    kMainEncoderUsed,
    217    kFallbackDueToFailure,
    218    kForcedFallback
    219  };
    220 
    221  EncoderState encoder_state_;
    222  const std::unique_ptr<VideoEncoder> encoder_;
    223  const std::unique_ptr<VideoEncoder> fallback_encoder_;
    224 
    225  EncodedImageCallback* callback_;
    226 
    227  const std::optional<ForcedFallbackParams> fallback_params_;
    228  int32_t EncodeWithMainEncoder(const VideoFrame& frame,
    229                                const std::vector<VideoFrameType>* frame_types);
    230 };
    231 
    232 VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
    233    const FieldTrialsView& field_trials,
    234    std::unique_ptr<VideoEncoder> sw_encoder,
    235    std::unique_ptr<VideoEncoder> hw_encoder,
    236    bool prefer_temporal_support)
    237    : encoder_state_(EncoderState::kUninitialized),
    238      encoder_(std::move(hw_encoder)),
    239      fallback_encoder_(std::move(sw_encoder)),
    240      callback_(nullptr),
    241      fallback_params_(GetForcedFallbackParams(field_trials,
    242                                               prefer_temporal_support,
    243                                               *encoder_)) {
    244  RTC_DCHECK(fallback_encoder_);
    245 }
    246 
    247 VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() =
    248    default;
    249 
    250 void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder(
    251    VideoEncoder* encoder) const {
    252  RTC_DCHECK(encoder);
    253  // Replay callback, rates, and channel parameters.
    254  if (callback_) {
    255    encoder->RegisterEncodeCompleteCallback(callback_);
    256  }
    257  if (rate_control_parameters_) {
    258    encoder->SetRates(*rate_control_parameters_);
    259  }
    260  if (rtt_.has_value()) {
    261    encoder->OnRttUpdate(rtt_.value());
    262  }
    263  if (packet_loss_.has_value()) {
    264    encoder->OnPacketLossRateUpdate(packet_loss_.value());
    265  }
    266 
    267  if (loss_notification_.has_value()) {
    268    encoder->OnLossNotification(loss_notification_.value());
    269  }
    270 }
    271 
    272 bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) {
    273  RTC_LOG(LS_WARNING) << "[VESFW] " << __func__
    274                      << "(is_forced=" << (is_forced ? "true" : "false") << ")";
    275 
    276  RTC_DCHECK(encoder_settings_.has_value());
    277  const int ret = fallback_encoder_->InitEncode(&codec_settings_,
    278                                                encoder_settings_.value());
    279 
    280  if (ret != WEBRTC_VIDEO_CODEC_OK) {
    281    RTC_LOG(LS_ERROR)
    282        << "[VESFW] software-encoder fallback initialization failed with"
    283        << " error code: " << WebRtcVideoCodecErrorToString(ret);
    284    fallback_encoder_->Release();
    285    return false;
    286  }
    287 
    288  if (encoder_state_ == EncoderState::kMainEncoderUsed) {
    289    // Since we're switching to the fallback encoder, Release the real encoder.
    290    // It may be re-initialized via InitEncode later, and it will continue to
    291    // get Set calls for rates and channel parameters in the meantime.
    292    encoder_->Release();
    293  }
    294 
    295  if (is_forced) {
    296    encoder_state_ = EncoderState::kForcedFallback;
    297  } else {
    298    encoder_state_ = EncoderState::kFallbackDueToFailure;
    299  }
    300 
    301  return true;
    302 }
    303 
    304 void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride(
    305    FecControllerOverride* fec_controller_override) {
    306  // It is important that only one of those would ever interact with the
    307  // `fec_controller_override` at a given time. This is the responsibility
    308  // of `this` to maintain.
    309 
    310  encoder_->SetFecControllerOverride(fec_controller_override);
    311  fallback_encoder_->SetFecControllerOverride(fec_controller_override);
    312 }
    313 
    314 int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
    315    const VideoCodec* codec_settings,
    316    const VideoEncoder::Settings& settings) {
    317  RTC_LOG(LS_INFO) << "[VESFW] " << __func__
    318                   << "(codec=" << codec_settings->ToString()
    319                   << ", settings={number_of_cores: "
    320                   << settings.number_of_cores
    321                   << ", max_payload_size: " << settings.max_payload_size
    322                   << "})";
    323  // Store settings, in case we need to dynamically switch to the fallback
    324  // encoder after a failed Encode call.
    325  codec_settings_ = *codec_settings;
    326  encoder_settings_ = settings;
    327  // Clear stored rate/channel parameters.
    328  rate_control_parameters_ = std::nullopt;
    329 
    330  RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized)
    331      << "InitEncode() should never be called on an active instance!";
    332 
    333  // Try to init forced software codec if it should be used.
    334  if (TryInitForcedFallbackEncoder()) {
    335    PrimeEncoder(current_encoder());
    336    return WEBRTC_VIDEO_CODEC_OK;
    337  }
    338 
    339  int32_t ret = encoder_->InitEncode(codec_settings, settings);
    340  if (ret == WEBRTC_VIDEO_CODEC_OK) {
    341    encoder_state_ = EncoderState::kMainEncoderUsed;
    342    PrimeEncoder(current_encoder());
    343    return ret;
    344  }
    345  if (ret == WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED) {
    346    return ret;
    347  }
    348  RTC_LOG(LS_WARNING) << "[VESFW] Hardware encoder initialization failed with"
    349                      << " error code: " << WebRtcVideoCodecErrorToString(ret);
    350 
    351  // Try to instantiate software codec.
    352  if (InitFallbackEncoder(/*is_forced=*/false)) {
    353    PrimeEncoder(current_encoder());
    354    return WEBRTC_VIDEO_CODEC_OK;
    355  }
    356 
    357  // Software encoder failed too, use original return code.
    358  RTC_LOG(LS_WARNING)
    359      << "[VESFW] Software fallback encoder initialization also failed.";
    360  encoder_state_ = EncoderState::kUninitialized;
    361  return ret;
    362 }
    363 
    364 int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
    365    EncodedImageCallback* callback) {
    366  callback_ = callback;
    367  return current_encoder()->RegisterEncodeCompleteCallback(callback);
    368 }
    369 
    370 int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
    371  if (encoder_state_ == EncoderState::kUninitialized) {
    372    return WEBRTC_VIDEO_CODEC_OK;
    373  }
    374  int32_t ret = current_encoder()->Release();
    375  encoder_state_ = EncoderState::kUninitialized;
    376  return ret;
    377 }
    378 
    379 int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
    380    const VideoFrame& frame,
    381    const std::vector<VideoFrameType>* frame_types) {
    382  switch (encoder_state_) {
    383    case EncoderState::kUninitialized:
    384      return WEBRTC_VIDEO_CODEC_ERROR;
    385    case EncoderState::kMainEncoderUsed: {
    386      return EncodeWithMainEncoder(frame, frame_types);
    387    }
    388    case EncoderState::kFallbackDueToFailure:
    389    case EncoderState::kForcedFallback:
    390      return fallback_encoder_->Encode(frame, frame_types);
    391  }
    392  RTC_CHECK_NOTREACHED();
    393 }
    394 
    395 int32_t VideoEncoderSoftwareFallbackWrapper::EncodeWithMainEncoder(
    396    const VideoFrame& frame,
    397    const std::vector<VideoFrameType>* frame_types) {
    398  int32_t ret = encoder_->Encode(frame, frame_types);
    399  // If requested, try a software fallback.
    400  bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
    401  if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) {
    402    // Start using the fallback with this frame.
    403    PrimeEncoder(current_encoder());
    404    if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative &&
    405        fallback_encoder_->GetEncoderInfo().supports_native_handle) {
    406      return fallback_encoder_->Encode(frame, frame_types);
    407    } else {
    408      RTC_LOG(LS_INFO) << "Fallback encoder does not support native handle - "
    409                          "converting frame to I420";
    410      scoped_refptr<I420BufferInterface> src_buffer =
    411          frame.video_frame_buffer()->ToI420();
    412      if (!src_buffer) {
    413        RTC_LOG(LS_ERROR) << "Failed to convert from to I420";
    414        return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
    415      }
    416      scoped_refptr<VideoFrameBuffer> dst_buffer =
    417          src_buffer->Scale(codec_settings_.width, codec_settings_.height);
    418      if (!dst_buffer) {
    419        RTC_LOG(LS_ERROR) << "Failed to scale video frame.";
    420        return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
    421      }
    422      VideoFrame scaled_frame = frame;
    423      scaled_frame.set_video_frame_buffer(dst_buffer);
    424      scaled_frame.set_update_rect(
    425          VideoFrame::UpdateRect{.offset_x = 0,
    426                                 .offset_y = 0,
    427                                 .width = scaled_frame.width(),
    428                                 .height = scaled_frame.height()});
    429      return fallback_encoder_->Encode(scaled_frame, frame_types);
    430    }
    431  }
    432  // Fallback encoder failed too, return original error code.
    433  return ret;
    434 }
    435 
    436 void VideoEncoderSoftwareFallbackWrapper::SetRates(
    437    const RateControlParameters& parameters) {
    438  rate_control_parameters_ = parameters;
    439  return current_encoder()->SetRates(parameters);
    440 }
    441 
    442 void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate(
    443    float packet_loss_rate) {
    444  packet_loss_ = packet_loss_rate;
    445  current_encoder()->OnPacketLossRateUpdate(packet_loss_rate);
    446 }
    447 
    448 void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) {
    449  rtt_ = rtt_ms;
    450  current_encoder()->OnRttUpdate(rtt_ms);
    451 }
    452 
    453 void VideoEncoderSoftwareFallbackWrapper::OnLossNotification(
    454    const LossNotification& loss_notification) {
    455  loss_notification_ = loss_notification;
    456  current_encoder()->OnLossNotification(loss_notification);
    457 }
    458 
    459 VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
    460    const {
    461  EncoderInfo fallback_encoder_info = fallback_encoder_->GetEncoderInfo();
    462  EncoderInfo default_encoder_info = encoder_->GetEncoderInfo();
    463 
    464  EncoderInfo info =
    465      IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
    466 
    467  info.requested_resolution_alignment =
    468      std::lcm(fallback_encoder_info.requested_resolution_alignment,
    469               default_encoder_info.requested_resolution_alignment);
    470  info.apply_alignment_to_all_simulcast_layers =
    471      fallback_encoder_info.apply_alignment_to_all_simulcast_layers ||
    472      default_encoder_info.apply_alignment_to_all_simulcast_layers;
    473 
    474  if (fallback_params_ && fallback_params_->vp8_specific_resolution_switch) {
    475    info.scaling_settings.min_pixels_per_frame = fallback_params_->min_pixels;
    476  }
    477 
    478  return info;
    479 }
    480 
    481 bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const {
    482  return encoder_state_ == EncoderState::kForcedFallback ||
    483         encoder_state_ == EncoderState::kFallbackDueToFailure;
    484 }
    485 
    486 bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() {
    487  if (!fallback_params_) {
    488    return false;
    489  }
    490 
    491  RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized);
    492 
    493  if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) {
    494    // Settings valid, try to instantiate software codec.
    495    RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
    496                     << codec_settings_.width << "x" << codec_settings_.height;
    497    return InitFallbackEncoder(/*is_forced=*/true);
    498  }
    499 
    500  if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) {
    501    // First init main encoder to see if that supports temporal layers.
    502    if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) ==
    503        WEBRTC_VIDEO_CODEC_OK) {
    504      encoder_state_ = EncoderState::kMainEncoderUsed;
    505    }
    506 
    507    if (encoder_state_ == EncoderState::kMainEncoderUsed &&
    508        encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) {
    509      // Primary encoder already supports temporal layers, use that instead.
    510      return true;
    511    }
    512 
    513    // Try to initialize fallback and check if it supports temporal layers.
    514    if (fallback_encoder_->InitEncode(&codec_settings_,
    515                                      encoder_settings_.value()) ==
    516        WEBRTC_VIDEO_CODEC_OK) {
    517      if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) {
    518        // Fallback encoder available and supports temporal layers, use it!
    519        if (encoder_state_ == EncoderState::kMainEncoderUsed) {
    520          // Main encoder initialized but does not support temporal layers,
    521          // release it again.
    522          encoder_->Release();
    523        }
    524        encoder_state_ = EncoderState::kForcedFallback;
    525        RTC_LOG(LS_INFO)
    526            << "Forced switch to SW encoder due to temporal support.";
    527        return true;
    528      } else {
    529        // Fallback encoder intialization succeeded, but it does not support
    530        // temporal layers either - release it.
    531        fallback_encoder_->Release();
    532      }
    533    }
    534 
    535    if (encoder_state_ == EncoderState::kMainEncoderUsed) {
    536      // Main encoder already initialized - make use of it.
    537      RTC_LOG(LS_INFO)
    538          << "Cannot fall back for temporal support since fallback that "
    539             "supports is not available. Using main encoder instead.";
    540      return true;
    541    }
    542  }
    543 
    544  // Neither forced fallback mode supported.
    545  return false;
    546 }
    547 
    548 }  // namespace
    549 
    550 std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper(
    551    const Environment& env,
    552    std::unique_ptr<VideoEncoder> sw_fallback_encoder,
    553    std::unique_ptr<VideoEncoder> hw_encoder,
    554    bool prefer_temporal_support) {
    555  return std::make_unique<VideoEncoderSoftwareFallbackWrapper>(
    556      env.field_trials(), std::move(sw_fallback_encoder), std::move(hw_encoder),
    557      prefer_temporal_support);
    558 }
    559 
    560 }  // namespace webrtc