[ tor-browser ].git.dasho

TrackEncoder.cpp (26530B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
      2 /* This Source Code Form is subject to the terms of the Mozilla Public
      3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
      4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
      5 
      6 #include "TrackEncoder.h"
      7 
      8 #include "AudioChannelFormat.h"
      9 #include "DriftCompensation.h"
     10 #include "MediaTrackGraph.h"
     11 #include "MediaTrackListener.h"
     12 #include "VideoUtils.h"
     13 #include "mozilla/AbstractThread.h"
     14 #include "mozilla/Logging.h"
     15 #include "mozilla/ProfilerLabels.h"
     16 #include "mozilla/RollingMean.h"
     17 
     18 namespace mozilla {
     19 
     20 LazyLogModule gTrackEncoderLog("TrackEncoder");
     21 #define TRACK_LOG(type, msg) MOZ_LOG(gTrackEncoderLog, type, msg)
     22 
     23 constexpr int DEFAULT_CHANNELS = 1;
     24 constexpr int DEFAULT_FRAME_WIDTH = 640;
     25 constexpr int DEFAULT_FRAME_HEIGHT = 480;
     26 constexpr int DEFAULT_FRAME_RATE = 30;
     27 // 10 second threshold if the audio encoder cannot be initialized.
     28 constexpr int AUDIO_INIT_FAILED_DURATION = 10;
     29 // 30 second threshold if the video encoder cannot be initialized.
     30 constexpr int VIDEO_INIT_FAILED_DURATION = 30;
     31 constexpr int FRAMERATE_DETECTION_ROLLING_WINDOW = 3;
     32 constexpr size_t FRAMERATE_DETECTION_MIN_CHUNKS = 5;
     33 constexpr int FRAMERATE_DETECTION_MAX_DURATION_S = 6;
     34 
     35 TrackEncoder::TrackEncoder(TrackRate aTrackRate,
     36                           MediaQueue<EncodedFrame>& aEncodedDataQueue)
     37    : mInitialized(false),
     38      mStarted(false),
     39      mEndOfStream(false),
     40      mCanceled(false),
     41      mInitCounter(0),
     42      mSuspended(false),
     43      mTrackRate(aTrackRate),
     44      mEncodedDataQueue(aEncodedDataQueue) {}
     45 
     46 bool TrackEncoder::IsInitialized() {
     47  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
     48  return mInitialized;
     49 }
     50 
     51 bool TrackEncoder::IsStarted() {
     52  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
     53  return mStarted;
     54 }
     55 
     56 bool TrackEncoder::IsEncodingComplete() const {
     57  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
     58  return mEncodedDataQueue.IsFinished();
     59 }
     60 
     61 void TrackEncoder::SetInitialized() {
     62  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
     63 
     64  if (mInitialized) {
     65    return;
     66  }
     67 
     68  mInitialized = true;
     69 
     70  for (auto& l : mListeners.Clone()) {
     71    l->Initialized(this);
     72  }
     73 }
     74 
     75 void TrackEncoder::SetStarted() {
     76  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
     77 
     78  if (mStarted) {
     79    return;
     80  }
     81 
     82  mStarted = true;
     83 
     84  for (auto& l : mListeners.Clone()) {
     85    l->Started(this);
     86  }
     87 }
     88 
     89 void TrackEncoder::OnError() {
     90  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
     91 
     92  Cancel();
     93 
     94  for (auto& l : mListeners.Clone()) {
     95    l->Error(this);
     96  }
     97 }
     98 
     99 void TrackEncoder::RegisterListener(TrackEncoderListener* aListener) {
    100  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    101  MOZ_ASSERT(!mListeners.Contains(aListener));
    102  mListeners.AppendElement(aListener);
    103 }
    104 
    105 bool TrackEncoder::UnregisterListener(TrackEncoderListener* aListener) {
    106  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    107  return mListeners.RemoveElement(aListener);
    108 }
    109 
    110 void TrackEncoder::SetWorkerThread(AbstractThread* aWorkerThread) {
    111  mWorkerThread = aWorkerThread;
    112 }
    113 
    114 void AudioTrackEncoder::Suspend() {
    115  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    116  TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Suspend(), was %s", this,
    117                             mSuspended ? "suspended" : "live"));
    118 
    119  if (mSuspended) {
    120    return;
    121  }
    122 
    123  mSuspended = true;
    124 }
    125 
    126 void AudioTrackEncoder::Resume() {
    127  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    128  TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Resume(), was %s", this,
    129                             mSuspended ? "suspended" : "live"));
    130 
    131  if (!mSuspended) {
    132    return;
    133  }
    134 
    135  mSuspended = false;
    136 }
    137 
    138 void AudioTrackEncoder::AppendAudioSegment(AudioSegment&& aSegment) {
    139  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    140  AUTO_PROFILER_LABEL("AudioTrackEncoder::AppendAudioSegment", OTHER);
    141  TRACK_LOG(LogLevel::Verbose,
    142            ("[AudioTrackEncoder %p]: AppendAudioSegment() duration=%" PRIu64,
    143             this, aSegment.GetDuration()));
    144 
    145  if (mCanceled) {
    146    return;
    147  }
    148 
    149  if (mEndOfStream) {
    150    return;
    151  }
    152 
    153  TryInit(mOutgoingBuffer, aSegment.GetDuration());
    154 
    155  if (mSuspended) {
    156    return;
    157  }
    158 
    159  SetStarted();
    160  mOutgoingBuffer.AppendFrom(&aSegment);
    161 
    162  if (!mInitialized) {
    163    return;
    164  }
    165 
    166  if (NS_FAILED(Encode(&mOutgoingBuffer))) {
    167    OnError();
    168    return;
    169  }
    170 
    171  MOZ_ASSERT_IF(IsEncodingComplete(), mOutgoingBuffer.IsEmpty());
    172 }
    173 
    174 void AudioTrackEncoder::TryInit(const AudioSegment& aSegment,
    175                                TrackTime aDuration) {
    176  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    177 
    178  if (mInitialized) {
    179    return;
    180  }
    181 
    182  mInitCounter++;
    183  TRACK_LOG(LogLevel::Debug,
    184            ("[AudioTrackEncoder %p]: Inited the audio encoder %d times", this,
    185             mInitCounter));
    186 
    187  for (AudioSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded();
    188       iter.Next()) {
    189    // The number of channels is determined by the first non-null chunk, and
    190    // thus the audio encoder is initialized at this time.
    191    if (iter->IsNull()) {
    192      continue;
    193    }
    194 
    195    nsresult rv = Init(iter->mChannelData.Length());
    196 
    197    if (NS_SUCCEEDED(rv)) {
    198      TRACK_LOG(LogLevel::Info,
    199                ("[AudioTrackEncoder %p]: Successfully initialized!", this));
    200      return;
    201    } else {
    202      TRACK_LOG(
    203          LogLevel::Error,
    204          ("[AudioTrackEncoder %p]: Failed to initialize the encoder!", this));
    205      OnError();
    206      return;
    207    }
    208    break;
    209  }
    210 
    211  mNotInitDuration += aDuration;
    212  if (!mInitialized &&
    213      ((mNotInitDuration - 1) / mTrackRate >= AUDIO_INIT_FAILED_DURATION) &&
    214      mInitCounter > 1) {
    215    // Perform a best effort initialization since we haven't gotten any
    216    // data yet. Motivated by issues like Bug 1336367
    217    TRACK_LOG(LogLevel::Warning,
    218              ("[AudioTrackEncoder]: Initialize failed for %ds. Attempting to "
    219               "init with %d (default) channels!",
    220               AUDIO_INIT_FAILED_DURATION, DEFAULT_CHANNELS));
    221    nsresult rv = Init(DEFAULT_CHANNELS);
    222    if (NS_FAILED(rv)) {
    223      TRACK_LOG(LogLevel::Error,
    224                ("[AudioTrackEncoder %p]: Default-channel-init failed.", this));
    225      OnError();
    226      return;
    227    }
    228  }
    229 }
    230 
    231 void AudioTrackEncoder::Cancel() {
    232  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    233  TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Cancel()", this));
    234  mCanceled = true;
    235  mEndOfStream = true;
    236  mOutgoingBuffer.Clear();
    237  mEncodedDataQueue.Finish();
    238 }
    239 
    240 void AudioTrackEncoder::NotifyEndOfStream() {
    241  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    242  TRACK_LOG(LogLevel::Info,
    243            ("[AudioTrackEncoder %p]: NotifyEndOfStream()", this));
    244 
    245  if (!mCanceled && !mInitialized) {
    246    // If source audio track is completely silent till the end of encoding,
    247    // initialize the encoder with a default channel count.
    248    Init(DEFAULT_CHANNELS);
    249  }
    250 
    251  if (mEndOfStream) {
    252    return;
    253  }
    254 
    255  mEndOfStream = true;
    256 
    257  if (NS_FAILED(Encode(&mOutgoingBuffer))) {
    258    mOutgoingBuffer.Clear();
    259    OnError();
    260  }
    261 
    262  MOZ_ASSERT(mOutgoingBuffer.GetDuration() == 0);
    263 }
    264 
    265 /*static*/
    266 void AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
    267                                            int32_t aDuration,
    268                                            uint32_t aOutputChannels,
    269                                            AudioDataValue* aOutput) {
    270  uint32_t numChannelsToCopy = std::min(
    271      aOutputChannels, static_cast<uint32_t>(aChunk.mChannelData.Length()));
    272  switch (aChunk.mBufferFormat) {
    273    case AUDIO_FORMAT_S16: {
    274      AutoTArray<const int16_t*, 2> array;
    275      array.SetLength(numChannelsToCopy);
    276      for (uint32_t i = 0; i < array.Length(); i++) {
    277        array[i] = static_cast<const int16_t*>(aChunk.mChannelData[i]);
    278      }
    279      InterleaveTrackData(array, aDuration, aOutputChannels, aOutput,
    280                          aChunk.mVolume);
    281      break;
    282    }
    283    case AUDIO_FORMAT_FLOAT32: {
    284      AutoTArray<const float*, 2> array;
    285      array.SetLength(numChannelsToCopy);
    286      for (uint32_t i = 0; i < array.Length(); i++) {
    287        array[i] = static_cast<const float*>(aChunk.mChannelData[i]);
    288      }
    289      InterleaveTrackData(array, aDuration, aOutputChannels, aOutput,
    290                          aChunk.mVolume);
    291      break;
    292    }
    293    case AUDIO_FORMAT_SILENCE: {
    294      MOZ_ASSERT(false, "To implement.");
    295    }
    296  };
    297 }
    298 
    299 /*static*/
    300 void AudioTrackEncoder::DeInterleaveTrackData(AudioDataValue* aInput,
    301                                              int32_t aDuration,
    302                                              int32_t aChannels,
    303                                              AudioDataValue* aOutput) {
    304  for (int32_t i = 0; i < aChannels; ++i) {
    305    for (int32_t j = 0; j < aDuration; ++j) {
    306      aOutput[i * aDuration + j] = aInput[i + j * aChannels];
    307    }
    308  }
    309 }
    310 
    311 size_t AudioTrackEncoder::SizeOfExcludingThis(
    312    mozilla::MallocSizeOf aMallocSizeOf) {
    313  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    314  return mOutgoingBuffer.SizeOfExcludingThis(aMallocSizeOf);
    315 }
    316 
    317 VideoTrackEncoder::VideoTrackEncoder(
    318    RefPtr<DriftCompensator> aDriftCompensator, TrackRate aTrackRate,
    319    MediaQueue<EncodedFrame>& aEncodedDataQueue,
    320    FrameDroppingMode aFrameDroppingMode)
    321    : TrackEncoder(aTrackRate, aEncodedDataQueue),
    322      mDriftCompensator(std::move(aDriftCompensator)),
    323      mEncodedTicks(0),
    324      mVideoBitrate(0),
    325      mFrameDroppingMode(aFrameDroppingMode),
    326      mEnabled(true) {
    327  mLastChunk.mDuration = 0;
    328 }
    329 
    330 void VideoTrackEncoder::Suspend(const TimeStamp& aTime) {
    331  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    332  TRACK_LOG(LogLevel::Info,
    333            ("[VideoTrackEncoder %p]: Suspend() at %.3fs, was %s", this,
    334             mStartTime.IsNull() ? 0.0 : (aTime - mStartTime).ToSeconds(),
    335             mSuspended ? "suspended" : "live"));
    336 
    337  if (mSuspended) {
    338    return;
    339  }
    340 
    341  mSuspended = true;
    342  mSuspendTime = aTime;
    343 }
    344 
    345 void VideoTrackEncoder::Resume(const TimeStamp& aTime) {
    346  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    347 
    348  if (!mSuspended) {
    349    return;
    350  }
    351 
    352  TRACK_LOG(
    353      LogLevel::Info,
    354      ("[VideoTrackEncoder %p]: Resume() after %.3fs, was %s", this,
    355       (aTime - mSuspendTime).ToSeconds(), mSuspended ? "suspended" : "live"));
    356 
    357  mSuspended = false;
    358 
    359  TimeDuration suspendDuration = aTime - mSuspendTime;
    360  if (!mLastChunk.mTimeStamp.IsNull()) {
    361    VideoChunk* nextChunk = mIncomingBuffer.FindChunkContaining(aTime);
    362    MOZ_ASSERT_IF(nextChunk, nextChunk->mTimeStamp <= aTime);
    363    if (nextChunk) {
    364      nextChunk->mTimeStamp = aTime;
    365    }
    366    mLastChunk.mTimeStamp += suspendDuration;
    367  }
    368  if (!mStartTime.IsNull()) {
    369    mStartTime += suspendDuration;
    370  }
    371 
    372  mSuspendTime = TimeStamp();
    373 }
    374 
    375 void VideoTrackEncoder::Disable(const TimeStamp& aTime) {
    376  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    377  TRACK_LOG(LogLevel::Debug, ("[VideoTrackEncoder %p]: Disable()", this));
    378 
    379  if (mStartTime.IsNull()) {
    380    // We haven't started yet. No need to touch future frames.
    381    mEnabled = false;
    382    return;
    383  }
    384 
    385  // Advancing currentTime to process any frames in mIncomingBuffer between
    386  // mCurrentTime and aTime.
    387  AdvanceCurrentTime(aTime);
    388  if (!mLastChunk.mTimeStamp.IsNull()) {
    389    // Insert a black frame at t=aTime into mIncomingBuffer, to trigger the
    390    // shift to black at the right moment.
    391    VideoSegment tempSegment;
    392    tempSegment.AppendFrom(&mIncomingBuffer);
    393    mIncomingBuffer.AppendFrame(do_AddRef(mLastChunk.mFrame.GetImage()),
    394                                mLastChunk.mFrame.GetIntrinsicSize(),
    395                                mLastChunk.mFrame.GetPrincipalHandle(), true,
    396                                aTime);
    397    mIncomingBuffer.AppendFrom(&tempSegment);
    398  }
    399  mEnabled = false;
    400 }
    401 
    402 void VideoTrackEncoder::Enable(const TimeStamp& aTime) {
    403  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    404  TRACK_LOG(LogLevel::Debug, ("[VideoTrackEncoder %p]: Enable()", this));
    405 
    406  if (mStartTime.IsNull()) {
    407    // We haven't started yet. No need to touch future frames.
    408    mEnabled = true;
    409    return;
    410  }
    411 
    412  // Advancing currentTime to process any frames in mIncomingBuffer between
    413  // mCurrentTime and aTime.
    414  AdvanceCurrentTime(aTime);
    415  if (!mLastChunk.mTimeStamp.IsNull()) {
    416    // Insert a real frame at t=aTime into mIncomingBuffer, to trigger the
    417    // shift from black at the right moment.
    418    VideoSegment tempSegment;
    419    tempSegment.AppendFrom(&mIncomingBuffer);
    420    mIncomingBuffer.AppendFrame(do_AddRef(mLastChunk.mFrame.GetImage()),
    421                                mLastChunk.mFrame.GetIntrinsicSize(),
    422                                mLastChunk.mFrame.GetPrincipalHandle(),
    423                                mLastChunk.mFrame.GetForceBlack(), aTime);
    424    mIncomingBuffer.AppendFrom(&tempSegment);
    425  }
    426  mEnabled = true;
    427 }
    428 
    429 void VideoTrackEncoder::AppendVideoSegment(VideoSegment&& aSegment) {
    430  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    431  TRACK_LOG(LogLevel::Verbose,
    432            ("[VideoTrackEncoder %p]: AppendVideoSegment()", this));
    433 
    434  if (mCanceled) {
    435    return;
    436  }
    437 
    438  if (mEndOfStream) {
    439    return;
    440  }
    441 
    442  for (VideoSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded();
    443       iter.Next()) {
    444    if (iter->IsNull()) {
    445      // A null image was sent. This is a signal from the source that we should
    446      // clear any images buffered in the future.
    447      mIncomingBuffer.Clear();
    448      continue;  // Don't append iter, as it is null.
    449    }
    450    if (VideoChunk* c = mIncomingBuffer.GetLastChunk()) {
    451      if (iter->mTimeStamp < c->mTimeStamp) {
    452        // Time went backwards. This can happen when a MediaDecoder seeks.
    453        // We need to handle this by removing any frames buffered in the future
    454        // and start over at iter->mTimeStamp.
    455        mIncomingBuffer.Clear();
    456      }
    457    }
    458    SetStarted();
    459    mIncomingBuffer.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
    460                                iter->mFrame.GetIntrinsicSize(),
    461                                iter->mFrame.GetPrincipalHandle(),
    462                                iter->mFrame.GetForceBlack(), iter->mTimeStamp);
    463  }
    464  aSegment.Clear();
    465 }
    466 
    467 void VideoTrackEncoder::Init(const VideoSegment& aSegment,
    468                             const TimeStamp& aTime,
    469                             size_t aFrameRateDetectionMinChunks) {
    470  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    471  MOZ_ASSERT(!aTime.IsNull());
    472 
    473  if (mInitialized) {
    474    return;
    475  }
    476 
    477  mInitCounter++;
    478  TRACK_LOG(LogLevel::Debug,
    479            ("[VideoTrackEncoder %p]: Init the video encoder %d times", this,
    480             mInitCounter));
    481 
    482  Maybe<float> framerate;
    483  if (!aSegment.IsEmpty()) {
    484    // The number of whole frames, i.e., with known duration.
    485    size_t frameCount = 0;
    486    RollingMean<TimeDuration, TimeDuration> meanDuration(
    487        FRAMERATE_DETECTION_ROLLING_WINDOW);
    488    VideoSegment::ConstChunkIterator iter(aSegment);
    489    TimeStamp previousChunkTime = iter->mTimeStamp;
    490    iter.Next();
    491    for (; !iter.IsEnded(); iter.Next(), ++frameCount) {
    492      meanDuration.insert(iter->mTimeStamp - previousChunkTime);
    493      previousChunkTime = iter->mTimeStamp;
    494    }
    495    TRACK_LOG(LogLevel::Debug, ("[VideoTrackEncoder %p]: Init() frameCount=%zu",
    496                                this, frameCount));
    497    if (frameCount >= aFrameRateDetectionMinChunks) {
    498      if (meanDuration.empty()) {
    499        // No whole frames available, use aTime as end time.
    500        framerate = Some(1.0f / (aTime - mStartTime).ToSeconds());
    501      } else {
    502        // We want some frames for estimating the framerate.
    503        framerate = Some(1.0f / meanDuration.mean().ToSeconds());
    504      }
    505    } else if ((aTime - mStartTime).ToSeconds() >
    506               FRAMERATE_DETECTION_MAX_DURATION_S) {
    507      // Instead of failing init after the fail-timeout, we fallback to a very
    508      // low rate.
    509      framerate = Some(static_cast<float>(frameCount) /
    510                       (aTime - mStartTime).ToSeconds());
    511    }
    512  }
    513 
    514  if (framerate) {
    515    for (VideoSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded();
    516         iter.Next()) {
    517      if (iter->IsNull()) {
    518        continue;
    519      }
    520 
    521      gfx::IntSize imgsize = iter->mFrame.GetImage()->GetSize();
    522      gfx::IntSize intrinsicSize = iter->mFrame.GetIntrinsicSize();
    523      nsresult rv = Init(imgsize.width, imgsize.height, intrinsicSize.width,
    524                         intrinsicSize.height, *framerate);
    525 
    526      if (NS_SUCCEEDED(rv)) {
    527        TRACK_LOG(LogLevel::Info,
    528                  ("[VideoTrackEncoder %p]: Successfully initialized!", this));
    529        return;
    530      }
    531 
    532      TRACK_LOG(
    533          LogLevel::Error,
    534          ("[VideoTrackEncoder %p]: Failed to initialize the encoder!", this));
    535      OnError();
    536      break;
    537    }
    538  }
    539 
    540  if (((aTime - mStartTime).ToSeconds() > VIDEO_INIT_FAILED_DURATION) &&
    541      mInitCounter > 1) {
    542    TRACK_LOG(LogLevel::Warning,
    543              ("[VideoTrackEncoder %p]: No successful init for %ds.", this,
    544               VIDEO_INIT_FAILED_DURATION));
    545    OnError();
    546    return;
    547  }
    548 }
    549 
    550 void VideoTrackEncoder::Cancel() {
    551  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    552  TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder %p]: Cancel()", this));
    553  mCanceled = true;
    554  mEndOfStream = true;
    555  mIncomingBuffer.Clear();
    556  mOutgoingBuffer.Clear();
    557  mLastChunk.SetNull(0);
    558  mEncodedDataQueue.Finish();
    559 }
    560 
    561 void VideoTrackEncoder::NotifyEndOfStream() {
    562  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    563 
    564  if (mCanceled) {
    565    return;
    566  }
    567 
    568  if (mEndOfStream) {
    569    // We have already been notified.
    570    return;
    571  }
    572 
    573  mEndOfStream = true;
    574  TRACK_LOG(LogLevel::Info,
    575            ("[VideoTrackEncoder %p]: NotifyEndOfStream()", this));
    576 
    577  if (!mLastChunk.IsNull()) {
    578    RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
    579    const TimeStamp now = TimeStamp::Now();
    580    TimeStamp currentTime = mSuspended ? mSuspendTime : mCurrentTime;
    581    currentTime = mDriftCompensator->GetVideoTime(now, currentTime);
    582    TimeDuration absoluteEndTime = currentTime - mStartTime;
    583    CheckedInt64 duration =
    584        UsecsToFrames(absoluteEndTime.ToMicroseconds(), mTrackRate) -
    585        mEncodedTicks;
    586    if (duration.isValid() && duration.value() > 0) {
    587      mEncodedTicks += duration.value();
    588      TRACK_LOG(LogLevel::Debug,
    589                ("[VideoTrackEncoder %p]: Appending last video frame %p at pos "
    590                 "%.3fs, "
    591                 "track-end=%.3fs",
    592                 this, lastImage.get(),
    593                 (mLastChunk.mTimeStamp - mStartTime).ToSeconds(),
    594                 absoluteEndTime.ToSeconds()));
    595      mOutgoingBuffer.AppendFrame(
    596          lastImage.forget(), mLastChunk.mFrame.GetIntrinsicSize(),
    597          PRINCIPAL_HANDLE_NONE, mLastChunk.mFrame.GetForceBlack() || !mEnabled,
    598          mLastChunk.mTimeStamp);
    599      mOutgoingBuffer.ExtendLastFrameBy(duration.value());
    600    }
    601 
    602    if (!mInitialized) {
    603      // Try to init without waiting for an accurate framerate.
    604      Init(mOutgoingBuffer, currentTime, 0);
    605    }
    606  }
    607 
    608  if (mCanceled) {
    609    // Previous Init failed and we got canceled. Nothing to do here.
    610    return;
    611  }
    612 
    613  mIncomingBuffer.Clear();
    614  mLastChunk.SetNull(0);
    615 
    616  if (NS_WARN_IF(!mInitialized)) {
    617    // Still not initialized. There was probably no real frame at all, perhaps
    618    // by muting. Initialize the encoder with default frame width, frame
    619    // height, and frame rate.
    620    Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT, DEFAULT_FRAME_WIDTH,
    621         DEFAULT_FRAME_HEIGHT, DEFAULT_FRAME_RATE);
    622  }
    623 
    624  if (NS_FAILED(Encode(&mOutgoingBuffer))) {
    625    OnError();
    626  }
    627 
    628  MOZ_ASSERT(mOutgoingBuffer.IsEmpty());
    629 }
    630 
    631 void VideoTrackEncoder::SetStartOffset(const TimeStamp& aStartOffset) {
    632  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    633  MOZ_ASSERT(mCurrentTime.IsNull());
    634  TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder %p]: SetStartOffset()", this));
    635  mStartTime = aStartOffset;
    636  mCurrentTime = aStartOffset;
    637 }
    638 
    639 void VideoTrackEncoder::AdvanceCurrentTime(const TimeStamp& aTime) {
    640  AUTO_PROFILER_LABEL("VideoTrackEncoder::AdvanceCurrentTime", OTHER);
    641 
    642  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    643  MOZ_ASSERT(!mStartTime.IsNull());
    644  MOZ_ASSERT(!mCurrentTime.IsNull());
    645 
    646  if (mCanceled) {
    647    return;
    648  }
    649 
    650  if (mEndOfStream) {
    651    return;
    652  }
    653 
    654  if (mSuspended) {
    655    TRACK_LOG(
    656        LogLevel::Verbose,
    657        ("[VideoTrackEncoder %p]: AdvanceCurrentTime() suspended at %.3fs",
    658         this, (mCurrentTime - mStartTime).ToSeconds()));
    659    mCurrentTime = aTime;
    660    mIncomingBuffer.ForgetUpToTime(mCurrentTime);
    661    return;
    662  }
    663 
    664  TRACK_LOG(LogLevel::Verbose,
    665            ("[VideoTrackEncoder %p]: AdvanceCurrentTime() to %.3fs", this,
    666             (aTime - mStartTime).ToSeconds()));
    667 
    668  // Grab frames within the currentTime range from the incoming buffer.
    669  VideoSegment tempSegment;
    670  {
    671    VideoChunk* previousChunk = &mLastChunk;
    672    auto appendDupes = [&](const TimeStamp& aUpTo) {
    673      while ((aUpTo - previousChunk->mTimeStamp).ToSeconds() > 1.0) {
    674        // We encode at least one frame per second, even if there are none
    675        // flowing.
    676        previousChunk->mTimeStamp += TimeDuration::FromSeconds(1.0);
    677        tempSegment.AppendFrame(
    678            do_AddRef(previousChunk->mFrame.GetImage()),
    679            previousChunk->mFrame.GetIntrinsicSize(),
    680            previousChunk->mFrame.GetPrincipalHandle(),
    681            previousChunk->mFrame.GetForceBlack() || !mEnabled,
    682            previousChunk->mTimeStamp);
    683        TRACK_LOG(
    684            LogLevel::Verbose,
    685            ("[VideoTrackEncoder %p]: Duplicating video frame (%p) at pos %.3f",
    686             this, previousChunk->mFrame.GetImage(),
    687             (previousChunk->mTimeStamp - mStartTime).ToSeconds()));
    688      }
    689    };
    690    for (VideoSegment::ChunkIterator iter(mIncomingBuffer); !iter.IsEnded();
    691         iter.Next()) {
    692      MOZ_ASSERT(!iter->IsNull());
    693      if (!previousChunk->IsNull() &&
    694          iter->mTimeStamp <= previousChunk->mTimeStamp) {
    695        // This frame starts earlier than previousChunk. Skip.
    696        continue;
    697      }
    698      if (iter->mTimeStamp >= aTime) {
    699        // This frame starts in the future. Stop.
    700        break;
    701      }
    702      if (!previousChunk->IsNull()) {
    703        appendDupes(iter->mTimeStamp);
    704      }
    705      tempSegment.AppendFrame(
    706          do_AddRef(iter->mFrame.GetImage()), iter->mFrame.GetIntrinsicSize(),
    707          iter->mFrame.GetPrincipalHandle(),
    708          iter->mFrame.GetForceBlack() || !mEnabled, iter->mTimeStamp);
    709      TRACK_LOG(LogLevel::Verbose,
    710                ("[VideoTrackEncoder %p]: Taking video frame (%p) at pos %.3f",
    711                 this, iter->mFrame.GetImage(),
    712                 (iter->mTimeStamp - mStartTime).ToSeconds()));
    713      previousChunk = &*iter;
    714    }
    715    if (!previousChunk->IsNull()) {
    716      appendDupes(aTime);
    717    }
    718  }
    719  mCurrentTime = aTime;
    720  mIncomingBuffer.ForgetUpToTime(mCurrentTime);
    721 
    722  // Convert tempSegment timestamps to durations and add chunks with known
    723  // duration to mOutgoingBuffer.
    724  const TimeStamp now = TimeStamp::Now();
    725  for (VideoSegment::ConstChunkIterator iter(tempSegment); !iter.IsEnded();
    726       iter.Next()) {
    727    VideoChunk chunk = *iter;
    728 
    729    if (mLastChunk.mTimeStamp.IsNull()) {
    730      // This is the first real chunk in the track. Make it start at the
    731      // beginning of the track.
    732      MOZ_ASSERT(!iter->mTimeStamp.IsNull());
    733 
    734      TRACK_LOG(
    735          LogLevel::Verbose,
    736          ("[VideoTrackEncoder %p]: Got the first video frame (%p) at pos %.3f "
    737           "(moving it to beginning)",
    738           this, iter->mFrame.GetImage(),
    739           (iter->mTimeStamp - mStartTime).ToSeconds()));
    740 
    741      mLastChunk = *iter;
    742      mLastChunk.mTimeStamp = mStartTime;
    743      continue;
    744    }
    745 
    746    MOZ_ASSERT(!mLastChunk.IsNull());
    747    MOZ_ASSERT(!chunk.IsNull());
    748 
    749    TimeDuration absoluteEndTime =
    750        mDriftCompensator->GetVideoTime(now, chunk.mTimeStamp) - mStartTime;
    751    TRACK_LOG(LogLevel::Verbose,
    752              ("[VideoTrackEncoder %p]: Appending video frame %p, at pos %.3fs "
    753               "until %.3fs",
    754               this, mLastChunk.mFrame.GetImage(),
    755               (mDriftCompensator->GetVideoTime(now, mLastChunk.mTimeStamp) -
    756                mStartTime)
    757                   .ToSeconds(),
    758               absoluteEndTime.ToSeconds()));
    759    CheckedInt64 duration =
    760        UsecsToFrames(absoluteEndTime.ToMicroseconds(), mTrackRate) -
    761        mEncodedTicks;
    762    if (!duration.isValid()) {
    763      NS_ERROR("Duration overflow");
    764      return;
    765    }
    766 
    767    if (duration.value() <= 0) {
    768      // A frame either started before the last frame (can happen when
    769      // multiple frames are added before SetStartOffset), or
    770      // two frames were so close together that they ended up at the same
    771      // position. We handle both cases by ignoring the previous frame.
    772 
    773      TRACK_LOG(LogLevel::Verbose,
    774                ("[VideoTrackEncoder %p]: Duration from frame %p to frame %p "
    775                 "is %" PRId64 ". Ignoring %p",
    776                 this, mLastChunk.mFrame.GetImage(), iter->mFrame.GetImage(),
    777                 duration.value(), mLastChunk.mFrame.GetImage()));
    778 
    779      TimeStamp t = mLastChunk.mTimeStamp;
    780      mLastChunk = *iter;
    781      mLastChunk.mTimeStamp = t;
    782      continue;
    783    }
    784 
    785    mEncodedTicks += duration.value();
    786    mOutgoingBuffer.AppendFrame(
    787        do_AddRef(mLastChunk.mFrame.GetImage()),
    788        mLastChunk.mFrame.GetIntrinsicSize(), PRINCIPAL_HANDLE_NONE,
    789        mLastChunk.mFrame.GetForceBlack() || !mEnabled, mLastChunk.mTimeStamp);
    790    mOutgoingBuffer.ExtendLastFrameBy(duration.value());
    791    mLastChunk = chunk;
    792  }
    793 
    794  if (mOutgoingBuffer.IsEmpty()) {
    795    return;
    796  }
    797 
    798  Init(mOutgoingBuffer, mCurrentTime, FRAMERATE_DETECTION_MIN_CHUNKS);
    799 
    800  if (!mInitialized) {
    801    return;
    802  }
    803 
    804  if (NS_FAILED(Encode(&mOutgoingBuffer))) {
    805    OnError();
    806    return;
    807  }
    808 
    809  MOZ_ASSERT(mOutgoingBuffer.IsEmpty());
    810 }
    811 
    812 size_t VideoTrackEncoder::SizeOfExcludingThis(
    813    mozilla::MallocSizeOf aMallocSizeOf) {
    814  MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
    815  return mIncomingBuffer.SizeOfExcludingThis(aMallocSizeOf) +
    816         mOutgoingBuffer.SizeOfExcludingThis(aMallocSizeOf);
    817 }
    818 
    819 }  // namespace mozilla
    820 
    821 #undef TRACK_LOG
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE