tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

VideoStreamFactory.cpp (13661B)

      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
      6 
      7 #include "VideoStreamFactory.h"
      8 
      9 #include <stdint.h>
     10 
     11 #include <algorithm>
     12 #include <cmath>
     13 #include <limits>
     14 #include <vector>
     15 
     16 #include "GMPUtils.h"
     17 #include "VideoConduit.h"
     18 #include "common/browser_logging/CSFLog.h"
     19 #include "mozilla/Assertions.h"
     20 #include "mozilla/gfx/Point.h"
     21 #include "video/config/video_encoder_config.h"
     22 
     23 template <class t>
     24 void ConstrainPreservingAspectRatio(uint16_t aMaxWidth, uint16_t aMaxHeight,
     25                                    t* aWidth, t* aHeight) {
     26  if (((*aWidth) <= aMaxWidth) && ((*aHeight) <= aMaxHeight)) {
     27    return;
     28  }
     29 
     30  if ((*aWidth) * aMaxHeight > aMaxWidth * (*aHeight)) {
     31    (*aHeight) = aMaxWidth * (*aHeight) / (*aWidth);
     32    (*aWidth) = aMaxWidth;
     33  } else {
     34    (*aWidth) = aMaxHeight * (*aWidth) / (*aHeight);
     35    (*aHeight) = aMaxHeight;
     36  }
     37 }
     38 
     39 namespace mozilla {
     40 
     41 #ifdef LOGTAG
     42 #  undef LOGTAG
     43 #endif
     44 #define LOGTAG "WebrtcVideoSessionConduit"
     45 
     46 #define DEFAULT_VIDEO_MAX_FRAMERATE 30u
     47 
     48 #define MB_OF(w, h) \
     49  ((unsigned int)(((((w) + 15) >> 4)) * ((unsigned int)(((h) + 15) >> 4))))
     50 // For now, try to set the max rates well above the knee in the curve.
     51 // Chosen somewhat arbitrarily; it's hard to find good data oriented for
     52 // realtime interactive/talking-head recording.  These rates assume
     53 // 30fps.
     54 
     55 // XXX Populate this based on a pref (which we should consider sorting because
     56 // people won't assume they need to).
     57 static constexpr VideoStreamFactory::ResolutionAndBitrateLimits
     58    kResolutionAndBitrateLimits[] = {
     59        // clang-format off
     60  {MB_OF(1920, 1200), KBPS(1500), KBPS(2000), KBPS(10000)}, // >HD (3K, 4K, etc)
     61  {MB_OF(1280, 720), KBPS(1200), KBPS(1500), KBPS(5000)}, // HD ~1080-1200
     62  {MB_OF(800, 480), KBPS(200), KBPS(800), KBPS(2500)}, // HD ~720
     63  {MB_OF(480, 270), KBPS(150), KBPS(500), KBPS(2000)}, // WVGA
     64  {std::max(MB_OF(400, 240), MB_OF(352, 288)), KBPS(125), KBPS(300), KBPS(1300)}, // VGA
     65  {MB_OF(176, 144), KBPS(100), KBPS(150), KBPS(500)}, // WQVGA, CIF
     66  {0 , KBPS(40), KBPS(80), KBPS(250)} // QCIF and below
     67        // clang-format on
     68 };
     69 
     70 auto VideoStreamFactory::GetLimitsFor(gfx::IntSize aSize, int aCapBps /* = 0 */)
     71    -> ResolutionAndBitrateLimits {
     72  // max bandwidth should be proportional (not linearly!) to resolution, and
     73  // proportional (perhaps linearly, or close) to current frame rate.
     74  int fs = MB_OF(aSize.width, aSize.height);
     75 
     76  for (const auto& resAndLimits : kResolutionAndBitrateLimits) {
     77    if (fs >= resAndLimits.resolution_in_mb &&
     78        // pick the highest range where at least start rate is within cap
     79        // (or if we're at the end of the array).
     80        (aCapBps == 0 || resAndLimits.start_bitrate_bps <= aCapBps ||
     81         resAndLimits.resolution_in_mb == 0)) {
     82      return resAndLimits;
     83    }
     84  }
     85 
     86  MOZ_CRASH("Loop should have handled fallback");
     87 }
     88 
     89 /**
     90 * Function to set the encoding bitrate limits based on incoming frame size and
     91 * rate
     92 * @param size: dimensions of the frame
     93 * @param min: minimum bitrate in bps
     94 * @param start: bitrate in bps that the encoder should start with
     95 * @param cap: user-enforced max bitrate, or 0
     96 * @param pref_cap: cap enforced by prefs
     97 * @param negotiated_cap: cap negotiated through SDP
     98 * @param aVideoStream stream to apply bitrates to
     99 */
    100 static void SelectBitrates(gfx::IntSize size, int min, int start, int cap,
    101                           int pref_cap, int negotiated_cap,
    102                           webrtc::VideoStream& aVideoStream) {
    103  int& out_min = aVideoStream.min_bitrate_bps;
    104  int& out_start = aVideoStream.target_bitrate_bps;
    105  int& out_max = aVideoStream.max_bitrate_bps;
    106 
    107  VideoStreamFactory::ResolutionAndBitrateLimits resAndLimits =
    108      VideoStreamFactory::GetLimitsFor(size);
    109  out_min = MinIgnoreZero(resAndLimits.min_bitrate_bps, cap);
    110  out_start = MinIgnoreZero(resAndLimits.start_bitrate_bps, cap);
    111  out_max = MinIgnoreZero(resAndLimits.max_bitrate_bps, cap);
    112 
    113  // Note: negotiated_cap is the max transport bitrate - it applies to
    114  // a single codec encoding, but should also apply to the sum of all
    115  // simulcast layers in this encoding! So sum(layers.maxBitrate) <=
    116  // negotiated_cap
    117  // Note that out_max already has had pref_cap applied to it
    118  out_max = MinIgnoreZero(negotiated_cap, out_max);
    119  out_min = std::min(out_min, out_max);
    120  out_start = std::min(out_start, out_max);
    121 
    122  if (min && min > out_min) {
    123    out_min = min;
    124  }
    125  // If we try to set a minimum bitrate that is too low, ViE will reject it.
    126  out_min = std::max(kViEMinCodecBitrate_bps, out_min);
    127  out_max = std::max(kViEMinCodecBitrate_bps, out_max);
    128  if (start && start > out_start) {
    129    out_start = start;
    130  }
    131 
    132  // Ensure that min <= start <= max
    133  if (out_min > out_max) {
    134    out_min = out_max;
    135  }
    136  out_start = std::clamp(out_start, out_min, out_max);
    137 
    138  MOZ_ASSERT(pref_cap == 0 || out_max <= pref_cap);
    139 }
    140 
    141 void VideoStreamFactory::SelectResolutionAndMaxFramerate(
    142    gfx::IntSize aSize, const VideoCodecConfig::Encoding& aEncoding,
    143    webrtc::VideoStream& aVideoStream) {
    144  MOZ_ASSERT(aSize.width > 0);
    145  MOZ_ASSERT(aSize.height > 0);
    146  MOZ_ASSERT(aEncoding.constraints.scaleDownBy >= 1.0);
    147  gfx::IntSize newSize(0, 0);
    148 
    149  newSize = CalculateScaledResolution(aSize, aEncoding.constraints.scaleDownBy);
    150 
    151  if (newSize.width == 0 || newSize.height == 0) {
    152    aVideoStream.width = aVideoStream.height = 0;
    153    return;
    154  }
    155 
    156  uint16_t max_width = mCodecConfig.mEncodingConstraints.maxWidth;
    157  uint16_t max_height = mCodecConfig.mEncodingConstraints.maxHeight;
    158  if (max_width || max_height) {
    159    max_width = max_width ? max_width : UINT16_MAX;
    160    max_height = max_height ? max_height : UINT16_MAX;
    161    ConstrainPreservingAspectRatio(max_width, max_height, &newSize.width,
    162                                   &newSize.height);
    163  }
    164 
    165  MOZ_ASSERT(newSize.width > 0);
    166  MOZ_ASSERT(newSize.height > 0);
    167  aVideoStream.width = newSize.width;
    168  aVideoStream.height = newSize.height;
    169  SelectMaxFramerateForAllStreams(newSize);
    170 
    171  CSFLogInfo(LOGTAG, "%s Input frame %ux%u, RID %s scaling to %zux%zu",
    172             __FUNCTION__, aSize.width, aSize.height, aEncoding.rid.c_str(),
    173             aVideoStream.width, aVideoStream.height);
    174 
    175  // mMaxFramerateForAllStreams is based on codec-wide stuff like fmtp, and
    176  // hard-coded limits based on the source resolution.
    177  // mCodecConfig.mEncodingConstraints.maxFps does not take the hard-coded
    178  // limits into account, so we have mMaxFramerateForAllStreams which
    179  // incorporates those. Per-encoding max framerate is based on parameters
    180  // from JS, and maybe rid
    181  unsigned int max_framerate = SelectFrameRate(
    182      mMaxFramerateForAllStreams, {aVideoStream.width, aVideoStream.height});
    183  max_framerate = std::min(
    184      WebrtcVideoConduit::ToLibwebrtcMaxFramerate(aEncoding.constraints.maxFps),
    185      max_framerate);
    186  if (max_framerate >= std::numeric_limits<int>::max()) {
    187    // If nothing has specified any kind of limit (uncommon), pick something
    188    // reasonable.
    189    max_framerate = DEFAULT_VIDEO_MAX_FRAMERATE;
    190  }
    191  aVideoStream.max_framerate = static_cast<int>(max_framerate);
    192 }
    193 
    194 std::vector<webrtc::VideoStream> VideoStreamFactory::CreateEncoderStreams(
    195    const webrtc::FieldTrialsView& field_trials, int aWidth, int aHeight,
    196    const webrtc::VideoEncoderConfig& aConfig) {
    197  mEncodeQueue->AssertOnCurrentThread();
    198  const size_t streamCount = aConfig.number_of_streams;
    199 
    200  MOZ_RELEASE_ASSERT(streamCount >= 1, "Should request at least one stream");
    201  MOZ_RELEASE_ASSERT(streamCount <= aConfig.simulcast_layers.size());
    202 
    203  std::vector<webrtc::VideoStream> streams;
    204  streams.reserve(streamCount);
    205 
    206  for (size_t idx = 0; idx < streamCount; ++idx) {
    207    webrtc::VideoStream video_stream = aConfig.simulcast_layers[idx];
    208    const auto& encoding = mCodecConfig.mEncodings[idx];
    209    MOZ_ASSERT(video_stream.active == encoding.active);
    210 
    211    SelectResolutionAndMaxFramerate({aWidth, aHeight}, encoding, video_stream);
    212 
    213    CSFLogInfo(
    214        LOGTAG,
    215        "%s Stream %zu with RID %s scaling %dx%d->%zux%zu; scaleDownBy=%.2f).",
    216        __FUNCTION__, idx, encoding.rid.c_str(), aWidth, aHeight,
    217        video_stream.width, video_stream.height,
    218        encoding.constraints.scaleDownBy);
    219 
    220    if (video_stream.width == 0 || video_stream.height == 0) {
    221      CSFLogInfo(LOGTAG, "%s Stream with RID %s ignored: has no resolution.",
    222                 __FUNCTION__, encoding.rid.c_str());
    223    }
    224 
    225    SelectBitrates({video_stream.width, video_stream.height}, mMinBitrate,
    226                   mStartBitrate,
    227                   SaturatingCast<int>(encoding.constraints.maxBr),
    228                   mPrefMaxBitrate, mNegotiatedMaxBitrate, video_stream);
    229 
    230    CSFLogInfo(LOGTAG,
    231               "%s Stream with RID %s maxFps=%d (global max fps = %u), "
    232               "bitrate=[%dkbps, %dkbps, %dkbps]",
    233               __FUNCTION__, encoding.rid.c_str(), video_stream.max_framerate,
    234               (unsigned)mMaxFramerateForAllStreams,
    235               video_stream.min_bitrate_bps / 1000,
    236               video_stream.target_bitrate_bps / 1000,
    237               video_stream.max_bitrate_bps / 1000);
    238 
    239    video_stream.bitrate_priority = aConfig.bitrate_priority;
    240    video_stream.max_qp = kQpMax;
    241 
    242    if (streamCount > 1) {
    243      video_stream.num_temporal_layers = 2;
    244      if (mCodecConfig.mName == "H264") {
    245 #ifdef ANDROID
    246        video_stream.num_temporal_layers = 1;
    247 #else
    248        if (!HaveGMPFor("encode-video"_ns, {"moz-h264-temporal-svc"_ns})) {
    249          video_stream.num_temporal_layers = 1;
    250        }
    251 #endif
    252      }
    253      // XXX Bug 1390215 investigate using more of
    254      // simulcast.cc:GetSimulcastConfig() or our own algorithm to replace it
    255    }
    256 
    257    streams.push_back(video_stream);
    258  }
    259 
    260  MOZ_RELEASE_ASSERT(streams.size(), "Should configure at least one stream");
    261  return streams;
    262 }
    263 
    264 void VideoStreamFactory::SetEncoderInfo(
    265    const webrtc::VideoEncoder::EncoderInfo& aInfo) {
    266  if (!mEncodeQueue) {
    267    mEncodeQueue = Nothing();
    268    mEncodeQueue.emplace(GetCurrentSerialEventTarget());
    269  }
    270  mEncodeQueue->AssertOnCurrentThread();
    271  mRequestedResolutionAlignment =
    272      Some(SaturatingCast<int>(aInfo.requested_resolution_alignment));
    273 }
    274 
    275 gfx::IntSize VideoStreamFactory::CalculateScaledResolution(
    276    gfx::IntSize aSize, double aScaleDownByResolution) {
    277  mEncodeQueue->AssertOnCurrentThread();
    278  // If any adjustments like scaleResolutionDownBy or maxFS are being given
    279  // we want to choose a height and width here to provide for more variety
    280  // in possible resolutions.
    281  int width = aSize.width;
    282  int height = aSize.height;
    283 
    284  if (aScaleDownByResolution > 1) {
    285    width = static_cast<int>(aSize.width / aScaleDownByResolution);
    286    height = static_cast<int>(aSize.height / aScaleDownByResolution);
    287  }
    288 
    289  // Check if we still need to adjust resolution down more due to other
    290  // constraints.
    291  if (mCodecConfig.mEncodingConstraints.maxFs > 0) {
    292    auto currentFs = static_cast<unsigned int>(width * height);
    293    auto maxFs = mCodecConfig.mEncodingConstraints.maxFs * 16 * 16;
    294 
    295    // If our currentFs is greater than maxFs we calculate a width and height
    296    // that will get as close as possible to maxFs and try to maintain aspect
    297    // ratio.
    298    if (currentFs > maxFs) {
    299      if (aSize.width > aSize.height) {  // Landscape
    300        auto aspectRatio = static_cast<double>(aSize.width) / aSize.height;
    301 
    302        height = static_cast<int>(std::sqrt(maxFs / aspectRatio));
    303        width = static_cast<int>(height * aspectRatio);
    304      } else {  // Portrait
    305        auto aspectRatio = static_cast<double>(aSize.height) / aSize.width;
    306 
    307        width = static_cast<int>(std::sqrt(maxFs / aspectRatio));
    308        height = static_cast<int>(width * aspectRatio);
    309      }
    310    }
    311  }
    312 
    313  // Simplest possible adaptation to resolution alignment.
    314  width -= width % *mRequestedResolutionAlignment;
    315  height -= height % *mRequestedResolutionAlignment;
    316 
    317  // Guard against a negative size.
    318  const int minSize = 0;
    319  if (width < minSize || height < minSize) {
    320    width = minSize;
    321    height = minSize;
    322  }
    323 
    324  return gfx::IntSize(width, height);
    325 }
    326 
    327 void VideoStreamFactory::SelectMaxFramerateForAllStreams(gfx::IntSize aSize) {
    328  unsigned int framerate_all_streams =
    329      SelectFrameRate(mMaxFramerateForAllStreams, aSize);
    330  unsigned int maxFrameRate = mMaxFramerateForAllStreams;
    331  if (mMaxFramerateForAllStreams != framerate_all_streams) {
    332    CSFLogDebug(LOGTAG, "%s: framerate changing to %u (from %u)", __FUNCTION__,
    333                framerate_all_streams, maxFrameRate);
    334    mMaxFramerateForAllStreams = framerate_all_streams;
    335  }
    336 }
    337 
    338 unsigned int VideoStreamFactory::SelectFrameRate(unsigned int aOldFramerate,
    339                                                 gfx::IntSize aSize) {
    340  unsigned int new_framerate = aOldFramerate;
    341 
    342  // Limit frame rate based on max-mbps
    343  if (mCodecConfig.mEncodingConstraints.maxMbps) {
    344    unsigned int cur_fs, mb_width, mb_height;
    345 
    346    mb_width = (aSize.width + 15) >> 4;
    347    mb_height = (aSize.height + 15) >> 4;
    348 
    349    cur_fs = mb_width * mb_height;
    350    if (cur_fs > 0) {  // in case no frames have been sent
    351      new_framerate = mCodecConfig.mEncodingConstraints.maxMbps / cur_fs;
    352    }
    353  }
    354 
    355  new_framerate =
    356      std::min(new_framerate, WebrtcVideoConduit::ToLibwebrtcMaxFramerate(
    357                                  mCodecConfig.mEncodingConstraints.maxFps));
    358  return new_framerate;
    359 }
    360 
    361 }  // namespace mozilla