[ tor-browser ].git.dasho

VP8TrackEncoder.cpp (25154B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
      2 /* This Source Code Form is subject to the terms of the Mozilla Public
      3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
      4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
      5 
      6 #include "VP8TrackEncoder.h"
      7 
      8 #include <vpx/vp8cx.h>
      9 #include <vpx/vpx_encoder.h>
     10 
     11 #include "DriftCompensation.h"
     12 #include "ImageConversion.h"
     13 #include "VideoSegment.h"
     14 #include "VideoUtils.h"
     15 #include "WebMWriter.h"
     16 #include "mozilla/ProfilerLabels.h"
     17 #include "mozilla/dom/ImageBitmapBinding.h"
     18 #include "mozilla/dom/ImageUtils.h"
     19 #include "mozilla/gfx/2D.h"
     20 #include "mozilla/media/MediaUtils.h"
     21 #include "prsystem.h"
     22 
     23 namespace mozilla {
     24 
     25 LazyLogModule gVP8TrackEncoderLog("VP8TrackEncoder");
     26 #define VP8LOG(level, msg, ...) \
     27  MOZ_LOG(gVP8TrackEncoderLog, level, (msg, ##__VA_ARGS__))
     28 
     29 constexpr int DEFAULT_BITRATE_BPS = 2500000;
     30 constexpr int DEFAULT_KEYFRAME_INTERVAL_MS = 10000;
     31 constexpr int DYNAMIC_MAXKFDIST_CHECK_INTERVAL = 5;
     32 constexpr float DYNAMIC_MAXKFDIST_DIFFACTOR = 0.4;
     33 constexpr float DYNAMIC_MAXKFDIST_KFINTERVAL_FACTOR = 0.75;
     34 constexpr int I420_STRIDE_ALIGN = 16;
     35 
     36 using namespace mozilla::gfx;
     37 using namespace mozilla::layers;
     38 using namespace mozilla::media;
     39 using namespace mozilla::dom;
     40 
     41 namespace {
     42 
     43 template <int N>
     44 static int Aligned(int aValue) {
     45  if (aValue < N) {
     46    return N;
     47  }
     48 
     49  // The `- 1` avoids overreaching when `aValue % N == 0`.
     50  return (((aValue - 1) / N) + 1) * N;
     51 }
     52 
     53 template <int Alignment>
     54 size_t I420Size(int aWidth, int aHeight) {
     55  int yStride = Aligned<Alignment>(aWidth);
     56  int yHeight = aHeight;
     57  size_t yPlaneSize = yStride * yHeight;
     58 
     59  int uvStride = Aligned<Alignment>((aWidth + 1) / 2);
     60  int uvHeight = (aHeight + 1) / 2;
     61  size_t uvPlaneSize = uvStride * uvHeight;
     62 
     63  return yPlaneSize + uvPlaneSize * 2;
     64 }
     65 
     66 nsresult CreateEncoderConfig(int32_t aWidth, int32_t aHeight,
     67                             uint32_t aVideoBitrate, TrackRate aTrackRate,
     68                             int32_t aMaxKeyFrameDistance,
     69                             vpx_codec_enc_cfg_t* config) {
     70  // Encoder configuration structure.
     71  memset(config, 0, sizeof(vpx_codec_enc_cfg_t));
     72  if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config, 0)) {
     73    VP8LOG(LogLevel::Error, "Failed to get default configuration");
     74    return NS_ERROR_FAILURE;
     75  }
     76 
     77  config->g_w = aWidth;
     78  config->g_h = aHeight;
     79  // TODO: Maybe we should have various aFrameRate bitrate pair for each
     80  // devices? or for different platform
     81 
     82  // rc_target_bitrate needs kbit/s
     83  config->rc_target_bitrate = std::max(
     84      1U, (aVideoBitrate != 0 ? aVideoBitrate : DEFAULT_BITRATE_BPS) / 1000);
     85 
     86  // Setting the time base of the codec
     87  config->g_timebase.num = 1;
     88  config->g_timebase.den = aTrackRate;
     89 
     90  // No error resilience as this is not intended for UDP transports
     91  config->g_error_resilient = 0;
     92 
     93  // Allow some frame lagging for large timeslices (when low latency is not
     94  // needed)
     95  /*std::min(10U, mKeyFrameInterval / 200)*/
     96  config->g_lag_in_frames = 0;
     97 
     98  int32_t number_of_cores = PR_GetNumberOfProcessors();
     99  if (aWidth * aHeight > 1920 * 1080 && number_of_cores >= 8) {
    100    config->g_threads = 4;  // 4 threads for > 1080p.
    101  } else if (aWidth * aHeight > 1280 * 960 && number_of_cores >= 6) {
    102    config->g_threads = 3;  // 3 threads for 1080p.
    103  } else if (aWidth * aHeight > 640 * 480 && number_of_cores >= 3) {
    104    config->g_threads = 2;  // 2 threads for qHD/HD.
    105  } else {
    106    config->g_threads = 1;  // 1 thread for VGA or less
    107  }
    108 
    109  // rate control settings
    110 
    111  // No frame dropping
    112  config->rc_dropframe_thresh = 0;
    113  // Variable bitrate
    114  config->rc_end_usage = VPX_VBR;
    115  // Single pass encoding
    116  config->g_pass = VPX_RC_ONE_PASS;
    117  // ffmpeg doesn't currently support streams that use resize.
    118  // Therefore, for safety, we should turn it off until it does.
    119  config->rc_resize_allowed = 0;
    120  // Allows 100% under target bitrate to compensate for prior overshoot
    121  config->rc_undershoot_pct = 100;
    122  // Allows 15% over target bitrate to compensate for prior undershoot
    123  config->rc_overshoot_pct = 15;
    124  // Tells the decoding application to buffer 500ms before beginning playback
    125  config->rc_buf_initial_sz = 500;
    126  // The decoding application will try to keep 600ms of buffer during playback
    127  config->rc_buf_optimal_sz = 600;
    128  // The decoding application may buffer 1000ms worth of encoded data
    129  config->rc_buf_sz = 1000;
    130 
    131  // We set key frame interval to automatic and try to set kf_max_dist so that
    132  // the encoder chooses to put keyframes slightly more often than
    133  // mKeyFrameInterval (which will encode with VPX_EFLAG_FORCE_KF when reached).
    134  config->kf_mode = VPX_KF_AUTO;
    135  config->kf_max_dist = aMaxKeyFrameDistance;
    136 
    137  return NS_OK;
    138 }
    139 }  // namespace
    140 
    141 VP8TrackEncoder::VP8TrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
    142                                 TrackRate aTrackRate,
    143                                 MediaQueue<EncodedFrame>& aEncodedDataQueue,
    144                                 FrameDroppingMode aFrameDroppingMode,
    145                                 Maybe<float> aKeyFrameIntervalFactor)
    146    : VideoTrackEncoder(std::move(aDriftCompensator), aTrackRate,
    147                        aEncodedDataQueue, aFrameDroppingMode),
    148      mKeyFrameInterval(
    149          TimeDuration::FromMilliseconds(DEFAULT_KEYFRAME_INTERVAL_MS)),
    150      mKeyFrameIntervalFactor(aKeyFrameIntervalFactor.valueOr(
    151          DYNAMIC_MAXKFDIST_KFINTERVAL_FACTOR)) {
    152  MOZ_COUNT_CTOR(VP8TrackEncoder);
    153  CalculateMaxKeyFrameDistance().apply(
    154      [&](auto aKfd) { SetMaxKeyFrameDistance(aKfd); });
    155 }
    156 
    157 VP8TrackEncoder::~VP8TrackEncoder() {
    158  Destroy();
    159  MOZ_COUNT_DTOR(VP8TrackEncoder);
    160 }
    161 
    162 void VP8TrackEncoder::Destroy() {
    163  if (mInitialized) {
    164    vpx_codec_destroy(&mVPXContext);
    165  }
    166 
    167  mInitialized = false;
    168 }
    169 
    170 Maybe<int32_t> VP8TrackEncoder::CalculateMaxKeyFrameDistance(
    171    Maybe<float> aEstimatedFrameRate /* = Nothing() */) const {
    172  if (!aEstimatedFrameRate && mMeanFrameDuration.empty()) {
    173    // Not enough data to make a new calculation.
    174    return Nothing();
    175  }
    176 
    177  // Calculate an estimation of our current framerate
    178  const float estimatedFrameRate = aEstimatedFrameRate.valueOrFrom(
    179      [&] { return 1.0f / mMeanFrameDuration.mean().ToSeconds(); });
    180  // Set a kf_max_dist that should avoid triggering the VPX_EFLAG_FORCE_KF flag
    181  return Some(std::max(
    182      1, static_cast<int32_t>(estimatedFrameRate * mKeyFrameIntervalFactor *
    183                              mKeyFrameInterval.ToSeconds())));
    184 }
    185 
    186 void VP8TrackEncoder::SetMaxKeyFrameDistance(int32_t aMaxKeyFrameDistance) {
    187  if (mInitialized) {
    188    VP8LOG(
    189        LogLevel::Debug,
    190        "%p SetMaxKeyFrameDistance() set kf_max_dist to %d based on estimated "
    191        "framerate %.2ffps keyframe-factor %.2f and keyframe-interval %.2fs",
    192        this, aMaxKeyFrameDistance, 1 / mMeanFrameDuration.mean().ToSeconds(),
    193        mKeyFrameIntervalFactor, mKeyFrameInterval.ToSeconds());
    194    DebugOnly<nsresult> rv =
    195        Reconfigure(mFrameWidth, mFrameHeight, aMaxKeyFrameDistance);
    196    MOZ_ASSERT(
    197        NS_SUCCEEDED(rv),
    198        "Reconfig for new key frame distance with proven size should succeed");
    199  } else {
    200    VP8LOG(LogLevel::Debug, "%p SetMaxKeyFrameDistance() distance=%d", this,
    201           aMaxKeyFrameDistance);
    202    mMaxKeyFrameDistance = Some(aMaxKeyFrameDistance);
    203  }
    204 }
    205 
    206 nsresult VP8TrackEncoder::Init(int32_t aWidth, int32_t aHeight,
    207                               int32_t aDisplayWidth, int32_t aDisplayHeight,
    208                               float aEstimatedFrameRate) {
    209  if (aDisplayWidth < 1 || aDisplayHeight < 1) {
    210    return NS_ERROR_FAILURE;
    211  }
    212 
    213  if (aEstimatedFrameRate <= 0) {
    214    return NS_ERROR_FAILURE;
    215  }
    216 
    217  int32_t maxKeyFrameDistance =
    218      *CalculateMaxKeyFrameDistance(Some(aEstimatedFrameRate));
    219 
    220  nsresult rv = InitInternal(aWidth, aHeight, maxKeyFrameDistance);
    221  NS_ENSURE_SUCCESS(rv, rv);
    222 
    223  MOZ_ASSERT(!mI420Frame);
    224  MOZ_ASSERT(mI420FrameSize == 0);
    225  const size_t neededSize = I420Size<I420_STRIDE_ALIGN>(aWidth, aHeight);
    226  mI420Frame.reset(new (fallible) uint8_t[neededSize]);
    227  mI420FrameSize = mI420Frame ? neededSize : 0;
    228  if (!mI420Frame) {
    229    VP8LOG(LogLevel::Warning, "Allocating I420 frame of size %zu failed",
    230           neededSize);
    231    return NS_ERROR_FAILURE;
    232  }
    233  vpx_img_wrap(&mVPXImageWrapper, VPX_IMG_FMT_I420, aWidth, aHeight,
    234               I420_STRIDE_ALIGN, mI420Frame.get());
    235 
    236  if (!mMetadata) {
    237    mMetadata = MakeAndAddRef<VP8Metadata>();
    238    mMetadata->mWidth = aWidth;
    239    mMetadata->mHeight = aHeight;
    240    mMetadata->mDisplayWidth = aDisplayWidth;
    241    mMetadata->mDisplayHeight = aDisplayHeight;
    242 
    243    VP8LOG(LogLevel::Info,
    244           "%p Init() created metadata. width=%d, height=%d, displayWidth=%d, "
    245           "displayHeight=%d, framerate=%.2f",
    246           this, mMetadata->mWidth, mMetadata->mHeight,
    247           mMetadata->mDisplayWidth, mMetadata->mDisplayHeight,
    248           aEstimatedFrameRate);
    249 
    250    SetInitialized();
    251  }
    252 
    253  return NS_OK;
    254 }
    255 
    256 nsresult VP8TrackEncoder::InitInternal(int32_t aWidth, int32_t aHeight,
    257                                       int32_t aMaxKeyFrameDistance) {
    258  if (aWidth < 1 || aHeight < 1) {
    259    return NS_ERROR_FAILURE;
    260  }
    261 
    262  if (mInitialized) {
    263    MOZ_ASSERT(false);
    264    return NS_ERROR_FAILURE;
    265  }
    266 
    267  VP8LOG(LogLevel::Debug,
    268         "%p InitInternal(). width=%d, height=%d, kf_max_dist=%d", this, aWidth,
    269         aHeight, aMaxKeyFrameDistance);
    270 
    271  // Encoder configuration structure.
    272  vpx_codec_enc_cfg_t config;
    273  nsresult rv = CreateEncoderConfig(aWidth, aHeight, mVideoBitrate, mTrackRate,
    274                                    aMaxKeyFrameDistance, &config);
    275  NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
    276 
    277  vpx_codec_flags_t flags = 0;
    278  flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
    279  if (vpx_codec_enc_init(&mVPXContext, vpx_codec_vp8_cx(), &config, flags)) {
    280    return NS_ERROR_FAILURE;
    281  }
    282 
    283  vpx_codec_control(&mVPXContext, VP8E_SET_STATIC_THRESHOLD, 1);
    284  vpx_codec_control(&mVPXContext, VP8E_SET_CPUUSED, 15);
    285  vpx_codec_control(&mVPXContext, VP8E_SET_TOKEN_PARTITIONS,
    286                    VP8_TWO_TOKENPARTITION);
    287 
    288  mFrameWidth = aWidth;
    289  mFrameHeight = aHeight;
    290  mMaxKeyFrameDistance = Some(aMaxKeyFrameDistance);
    291 
    292  return NS_OK;
    293 }
    294 
    295 nsresult VP8TrackEncoder::Reconfigure(int32_t aWidth, int32_t aHeight,
    296                                      int32_t aMaxKeyFrameDistance) {
    297  if (aWidth <= 0 || aHeight <= 0) {
    298    MOZ_ASSERT(false);
    299    return NS_ERROR_FAILURE;
    300  }
    301 
    302  if (!mInitialized) {
    303    MOZ_ASSERT(false);
    304    return NS_ERROR_FAILURE;
    305  }
    306 
    307  bool needsReInit = aMaxKeyFrameDistance != *mMaxKeyFrameDistance;
    308 
    309  if (aWidth != mFrameWidth || aHeight != mFrameHeight) {
    310    VP8LOG(LogLevel::Info, "Dynamic resolution change (%dx%d -> %dx%d).",
    311           mFrameWidth, mFrameHeight, aWidth, aHeight);
    312    const size_t neededSize = I420Size<I420_STRIDE_ALIGN>(aWidth, aHeight);
    313    if (neededSize > mI420FrameSize) {
    314      needsReInit = true;
    315      mI420Frame.reset(new (fallible) uint8_t[neededSize]);
    316      mI420FrameSize = mI420Frame ? neededSize : 0;
    317    }
    318    if (!mI420Frame) {
    319      VP8LOG(LogLevel::Warning, "Allocating I420 frame of size %zu failed",
    320             neededSize);
    321      return NS_ERROR_FAILURE;
    322    }
    323    vpx_img_wrap(&mVPXImageWrapper, VPX_IMG_FMT_I420, aWidth, aHeight,
    324                 I420_STRIDE_ALIGN, mI420Frame.get());
    325  }
    326 
    327  if (needsReInit) {
    328    Destroy();
    329    mMaxKeyFrameDistance = Some(aMaxKeyFrameDistance);
    330    nsresult rv = InitInternal(aWidth, aHeight, aMaxKeyFrameDistance);
    331    NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
    332    mInitialized = true;
    333    return NS_OK;
    334  }
    335 
    336  // Encoder configuration structure.
    337  vpx_codec_enc_cfg_t config;
    338  nsresult rv = CreateEncoderConfig(aWidth, aHeight, mVideoBitrate, mTrackRate,
    339                                    aMaxKeyFrameDistance, &config);
    340  NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
    341  // Set new configuration
    342  if (vpx_codec_enc_config_set(&mVPXContext, &config) != VPX_CODEC_OK) {
    343    VP8LOG(LogLevel::Error, "Failed to set new configuration");
    344    return NS_ERROR_FAILURE;
    345  }
    346 
    347  mFrameWidth = aWidth;
    348  mFrameHeight = aHeight;
    349 
    350  return NS_OK;
    351 }
    352 
    353 already_AddRefed<TrackMetadataBase> VP8TrackEncoder::GetMetadata() {
    354  AUTO_PROFILER_LABEL("VP8TrackEncoder::GetMetadata", OTHER);
    355 
    356  MOZ_ASSERT(mInitialized);
    357 
    358  if (!mInitialized) {
    359    return nullptr;
    360  }
    361 
    362  MOZ_ASSERT(mMetadata);
    363  return do_AddRef(mMetadata);
    364 }
    365 
    366 Result<RefPtr<EncodedFrame>, nsresult> VP8TrackEncoder::ExtractEncodedData() {
    367  vpx_codec_iter_t iter = nullptr;
    368  EncodedFrame::FrameType frameType = EncodedFrame::VP8_P_FRAME;
    369  auto frameData = MakeRefPtr<EncodedFrame::FrameData>();
    370  const vpx_codec_cx_pkt_t* pkt = nullptr;
    371  while ((pkt = vpx_codec_get_cx_data(&mVPXContext, &iter)) != nullptr) {
    372    switch (pkt->kind) {
    373      case VPX_CODEC_CX_FRAME_PKT: {
    374        // Copy the encoded data from libvpx to frameData
    375        frameData->AppendElements((uint8_t*)pkt->data.frame.buf,
    376                                  pkt->data.frame.sz);
    377        break;
    378      }
    379      default: {
    380        break;
    381      }
    382    }
    383    // End of frame
    384    if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) {
    385      if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
    386        frameType = EncodedFrame::VP8_I_FRAME;
    387      }
    388      break;
    389    }
    390  }
    391 
    392  if (frameData->IsEmpty()) {
    393    return RefPtr<EncodedFrame>(nullptr);
    394  }
    395 
    396  if (!pkt) {
    397    // This check silences a coverity warning about accessing a null pkt below.
    398    return RefPtr<EncodedFrame>(nullptr);
    399  }
    400 
    401  if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
    402    // Update the since-last-keyframe counter, and account for this frame's
    403    // time.
    404    TrackTime frameTime = pkt->data.frame.pts;
    405    DebugOnly<TrackTime> frameDuration = pkt->data.frame.duration;
    406    MOZ_ASSERT(frameTime + frameDuration <= mEncodedTimestamp);
    407    mDurationSinceLastKeyframe =
    408        std::min(mDurationSinceLastKeyframe, mEncodedTimestamp - frameTime);
    409  }
    410 
    411  // Convert the timestamp and duration to Usecs.
    412  media::TimeUnit timestamp = media::TimeUnit(pkt->data.frame.pts, mTrackRate);
    413  if (!timestamp.IsValid()) {
    414    NS_ERROR("Microsecond timestamp overflow");
    415    return Err(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR);
    416  }
    417 
    418  mExtractedDuration += pkt->data.frame.duration;
    419  if (!mExtractedDuration.isValid()) {
    420    NS_ERROR("Duration overflow");
    421    return Err(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR);
    422  }
    423 
    424  media::TimeUnit totalDuration =
    425      media::TimeUnit(mExtractedDuration.value(), mTrackRate);
    426  if (!totalDuration.IsValid()) {
    427    NS_ERROR("Duration overflow");
    428    return Err(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR);
    429  }
    430 
    431  media::TimeUnit duration = totalDuration - mExtractedDurationUs;
    432  if (!duration.IsValid()) {
    433    NS_ERROR("Duration overflow");
    434    return Err(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR);
    435  }
    436 
    437  mExtractedDurationUs = totalDuration;
    438 
    439  VP8LOG(LogLevel::Verbose,
    440         "ExtractEncodedData TimeStamp %.2f, Duration %.2f, FrameType %d",
    441         timestamp.ToSeconds(), duration.ToSeconds(), frameType);
    442 
    443  if (static_cast<int>(totalDuration.ToSeconds()) /
    444          DYNAMIC_MAXKFDIST_CHECK_INTERVAL >
    445      static_cast<int>(mLastKeyFrameDistanceUpdate.ToSeconds()) /
    446          DYNAMIC_MAXKFDIST_CHECK_INTERVAL) {
    447    // The interval has passed since the last keyframe update. Update again.
    448    mLastKeyFrameDistanceUpdate = totalDuration;
    449    const int32_t maxKfDistance =
    450        CalculateMaxKeyFrameDistance().valueOr(*mMaxKeyFrameDistance);
    451    const float diffFactor =
    452        static_cast<float>(maxKfDistance) / *mMaxKeyFrameDistance;
    453    VP8LOG(LogLevel::Debug, "maxKfDistance: %d, factor: %.2f", maxKfDistance,
    454           diffFactor);
    455    if (std::abs(1.0 - diffFactor) > DYNAMIC_MAXKFDIST_DIFFACTOR) {
    456      SetMaxKeyFrameDistance(maxKfDistance);
    457    }
    458  }
    459 
    460  return MakeRefPtr<EncodedFrame>(timestamp, duration.ToMicroseconds(),
    461                                  PR_USEC_PER_SEC, frameType,
    462                                  std::move(frameData));
    463 }
    464 
    465 /**
    466 * Encoding flow in Encode():
    467 * 1: Assert valid state.
    468 * 2: Encode the video chunks in mSourceSegment in a for-loop.
    469 * 2.1: The duration is taken straight from the video chunk's duration.
    470 * 2.2: Setup the video chunk with mVPXImageWrapper by PrepareRawFrame().
    471 * 2.3: Pass frame to vp8 encoder by vpx_codec_encode().
    472 * 2.4: Extract the encoded frame from encoder by ExtractEncodedData().
    473 * 2.5: Set the nextEncodeOperation for the next frame.
    474 * 2.6: If we are not skipping the next frame, add the encoded frame to
    475 *      mEncodedDataQueue. If we are skipping the next frame, extend the encoded
    476 *      frame's duration in the next run of the loop.
    477 * 3. Clear aSegment.
    478 */
    479 nsresult VP8TrackEncoder::Encode(VideoSegment* aSegment) {
    480  MOZ_ASSERT(mInitialized);
    481  MOZ_ASSERT(!IsEncodingComplete());
    482 
    483  AUTO_PROFILER_LABEL("VP8TrackEncoder::Encode", OTHER);
    484 
    485  EncodeOperation nextEncodeOperation = ENCODE_NORMAL_FRAME;
    486 
    487  RefPtr<EncodedFrame> encodedFrame;
    488  for (VideoSegment::ChunkIterator iter(*aSegment); !iter.IsEnded();
    489       iter.Next()) {
    490    VideoChunk& chunk = *iter;
    491 
    492    VP8LOG(LogLevel::Verbose,
    493           "nextEncodeOperation is %d for frame of duration %" PRId64,
    494           nextEncodeOperation, chunk.GetDuration());
    495 
    496    TimeStamp timebase = TimeStamp::Now();
    497 
    498    // Encode frame.
    499    if (nextEncodeOperation != SKIP_FRAME) {
    500      MOZ_ASSERT(!encodedFrame);
    501      nsresult rv = PrepareRawFrame(chunk);
    502      NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
    503 
    504      // Encode the data with VP8 encoder
    505      int flags = 0;
    506      if (nextEncodeOperation == ENCODE_I_FRAME) {
    507        VP8LOG(LogLevel::Warning,
    508               "MediaRecorder lagging behind. Encoding keyframe.");
    509        flags |= VPX_EFLAG_FORCE_KF;
    510      }
    511 
    512      // Sum duration of non-key frames and force keyframe if exceeded the
    513      // given keyframe interval
    514      if (mKeyFrameInterval > TimeDuration::FromSeconds(0)) {
    515        if (media::TimeUnit(mDurationSinceLastKeyframe, mTrackRate)
    516                .ToTimeDuration() >= mKeyFrameInterval) {
    517          VP8LOG(LogLevel::Warning,
    518                 "Reached mKeyFrameInterval without seeing a keyframe. Forcing "
    519                 "one. time: %.2f, interval: %.2f",
    520                 media::TimeUnit(mDurationSinceLastKeyframe, mTrackRate)
    521                     .ToSeconds(),
    522                 mKeyFrameInterval.ToSeconds());
    523          mDurationSinceLastKeyframe = 0;
    524          flags |= VPX_EFLAG_FORCE_KF;
    525        }
    526        mDurationSinceLastKeyframe += chunk.GetDuration();
    527      }
    528 
    529      if (vpx_codec_encode(&mVPXContext, &mVPXImageWrapper, mEncodedTimestamp,
    530                           (unsigned long)chunk.GetDuration(), flags,
    531                           VPX_DL_REALTIME)) {
    532        VP8LOG(LogLevel::Error, "vpx_codec_encode failed to encode the frame.");
    533        return NS_ERROR_FAILURE;
    534      }
    535 
    536      // Move forward the mEncodedTimestamp.
    537      mEncodedTimestamp += chunk.GetDuration();
    538 
    539      // Extract the encoded data from the underlying encoder and push it to
    540      // mEncodedDataQueue.
    541      auto result = ExtractEncodedData();
    542      if (result.isErr()) {
    543        VP8LOG(LogLevel::Error, "ExtractEncodedData failed.");
    544        return NS_ERROR_FAILURE;
    545      }
    546 
    547      MOZ_ASSERT(result.inspect(),
    548                 "We expected a frame here. EOS is handled explicitly later");
    549      encodedFrame = result.unwrap();
    550    } else {
    551      // SKIP_FRAME
    552 
    553      MOZ_DIAGNOSTIC_ASSERT(encodedFrame);
    554 
    555      if (mKeyFrameInterval > TimeDuration::FromSeconds(0)) {
    556        mDurationSinceLastKeyframe += chunk.GetDuration();
    557      }
    558 
    559      // Move forward the mEncodedTimestamp.
    560      mEncodedTimestamp += chunk.GetDuration();
    561 
    562      // Extend the duration of the last encoded frame in mEncodedDataQueue
    563      // because this frame will be skipped.
    564      VP8LOG(LogLevel::Warning,
    565             "MediaRecorder lagging behind. Skipping a frame.");
    566 
    567      mExtractedDuration += chunk.mDuration;
    568      if (!mExtractedDuration.isValid()) {
    569        NS_ERROR("skipped duration overflow");
    570        return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
    571      }
    572 
    573      media::TimeUnit totalDuration =
    574          media::TimeUnit(mExtractedDuration.value(), mTrackRate);
    575      media::TimeUnit skippedDuration = totalDuration - mExtractedDurationUs;
    576      mExtractedDurationUs = totalDuration;
    577      if (!skippedDuration.IsValid()) {
    578        NS_ERROR("skipped duration overflow");
    579        return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
    580      }
    581 
    582      encodedFrame = MakeRefPtr<EncodedFrame>(
    583          encodedFrame->mTime,
    584          encodedFrame->mDuration + skippedDuration.ToMicroseconds(),
    585          encodedFrame->mDurationBase, encodedFrame->mFrameType,
    586          encodedFrame->mFrameData);
    587    }
    588 
    589    mMeanFrameEncodeDuration.insert(TimeStamp::Now() - timebase);
    590    mMeanFrameDuration.insert(
    591        media::TimeUnit(chunk.GetDuration(), mTrackRate).ToTimeDuration());
    592    nextEncodeOperation = GetNextEncodeOperation(
    593        mMeanFrameEncodeDuration.mean(), mMeanFrameDuration.mean());
    594 
    595    if (nextEncodeOperation != SKIP_FRAME) {
    596      // Note that the next operation might be SKIP_FRAME even if there is no
    597      // next frame.
    598      mEncodedDataQueue.Push(encodedFrame.forget());
    599    }
    600  }
    601 
    602  if (encodedFrame) {
    603    // Push now if we ended on a SKIP_FRAME before.
    604    mEncodedDataQueue.Push(encodedFrame.forget());
    605  }
    606 
    607  // Remove the chunks we have processed.
    608  aSegment->Clear();
    609 
    610  if (mEndOfStream) {
    611    // EOS: Extract the remaining frames from the underlying encoder.
    612    VP8LOG(LogLevel::Debug, "mEndOfStream is true");
    613    // No more frames will be encoded. Clearing temporary frames saves some
    614    // memory.
    615    if (mI420Frame) {
    616      mI420Frame = nullptr;
    617      mI420FrameSize = 0;
    618    }
    619    // mMuteFrame must be released before gfx shutdown. We do it now since it
    620    // may be too late when this VP8TrackEncoder gets destroyed.
    621    mMuteFrame = nullptr;
    622    // Bug 1243611, keep calling vpx_codec_encode and vpx_codec_get_cx_data
    623    // until vpx_codec_get_cx_data return null.
    624    while (true) {
    625      if (vpx_codec_encode(&mVPXContext, nullptr, mEncodedTimestamp, 0, 0,
    626                           VPX_DL_REALTIME)) {
    627        return NS_ERROR_FAILURE;
    628      }
    629      auto result = ExtractEncodedData();
    630      if (result.isErr()) {
    631        return NS_ERROR_FAILURE;
    632      }
    633      if (!result.inspect()) {
    634        // Null means end-of-stream.
    635        break;
    636      }
    637      mEncodedDataQueue.Push(result.unwrap().forget());
    638    }
    639    mEncodedDataQueue.Finish();
    640  }
    641 
    642  return NS_OK;
    643 }
    644 
    645 nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk& aChunk) {
    646  gfx::IntSize intrinsicSize = aChunk.mFrame.GetIntrinsicSize();
    647  RefPtr<Image> img;
    648  if (aChunk.mFrame.GetForceBlack() || aChunk.IsNull()) {
    649    if (!mMuteFrame || mMuteFrame->GetSize() != intrinsicSize) {
    650      mMuteFrame = mozilla::VideoFrame::CreateBlackImage(intrinsicSize);
    651    }
    652    if (!mMuteFrame) {
    653      VP8LOG(LogLevel::Warning, "Failed to allocate black image of size %dx%d",
    654             intrinsicSize.width, intrinsicSize.height);
    655      return NS_OK;
    656    }
    657    img = mMuteFrame;
    658  } else {
    659    img = aChunk.mFrame.GetImage();
    660  }
    661 
    662  gfx::IntSize imgSize = img->GetSize();
    663  if (imgSize != IntSize(mFrameWidth, mFrameHeight)) {
    664    nsresult rv =
    665        Reconfigure(imgSize.width, imgSize.height, *mMaxKeyFrameDistance);
    666    NS_ENSURE_SUCCESS(rv, rv);
    667  }
    668 
    669  MOZ_ASSERT(mFrameWidth == imgSize.width);
    670  MOZ_ASSERT(mFrameHeight == imgSize.height);
    671 
    672  nsresult rv = ConvertToI420(img, mVPXImageWrapper.planes[VPX_PLANE_Y],
    673                              mVPXImageWrapper.stride[VPX_PLANE_Y],
    674                              mVPXImageWrapper.planes[VPX_PLANE_U],
    675                              mVPXImageWrapper.stride[VPX_PLANE_U],
    676                              mVPXImageWrapper.planes[VPX_PLANE_V],
    677                              mVPXImageWrapper.stride[VPX_PLANE_V], imgSize);
    678  if (NS_FAILED(rv)) {
    679    VP8LOG(LogLevel::Error, "Converting to I420 failed");
    680    return rv;
    681  }
    682 
    683  return NS_OK;
    684 }
    685 
    686 // These two define value used in GetNextEncodeOperation to determine the
    687 // EncodeOperation for next target frame.
    688 #define I_FRAME_RATIO (0.85)  // Effectively disabled, because perceived quality
    689 #define SKIP_FRAME_RATIO (0.85)
    690 
    691 /**
    692 * Compares the elapsed time from the beginning of GetEncodedTrack and
    693 * the processed frame duration in mSourceSegment
    694 * in order to set the nextEncodeOperation for next target frame.
    695 */
    696 VP8TrackEncoder::EncodeOperation VP8TrackEncoder::GetNextEncodeOperation(
    697    TimeDuration aTimeElapsed, TimeDuration aProcessedDuration) {
    698  if (mFrameDroppingMode == FrameDroppingMode::DISALLOW) {
    699    return ENCODE_NORMAL_FRAME;
    700  }
    701 
    702  if (aTimeElapsed.ToSeconds() >
    703      aProcessedDuration.ToSeconds() * SKIP_FRAME_RATIO) {
    704    // The encoder is too slow.
    705    // We should skip next frame to consume the mSourceSegment.
    706    return SKIP_FRAME;
    707  }
    708 
    709  if (aTimeElapsed.ToSeconds() >
    710      aProcessedDuration.ToSeconds() * I_FRAME_RATIO) {
    711    // The encoder is a little slow.
    712    // We force the encoder to encode an I-frame to accelerate.
    713    return ENCODE_I_FRAME;
    714  }
    715 
    716  return ENCODE_NORMAL_FRAME;
    717 }
    718 
    719 }  // namespace mozilla
    720 
    721 #undef VP8LOG
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE