tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

MediaSourceDecoder.cpp (14824B)

      1 /* -*- mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 #include "MediaSourceDecoder.h"
      7 
      8 #include <algorithm>
      9 
     10 #include "ExternalEngineStateMachine.h"
     11 #include "MediaDecoder.h"
     12 #include "MediaDecoderStateMachine.h"
     13 #include "MediaShutdownManager.h"
     14 #include "MediaSource.h"
     15 #include "MediaSourceDemuxer.h"
     16 #include "MediaSourceUtils.h"
     17 #include "SourceBuffer.h"
     18 #include "SourceBufferList.h"
     19 #include "VideoUtils.h"
     20 #include "base/process_util.h"
     21 #include "mozilla/Logging.h"
     22 
     23 extern mozilla::LogModule* GetMediaSourceLog();
     24 
     25 #define MSE_DEBUG(arg, ...)                                              \
     26  DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, "::%s: " arg, \
     27            __func__, ##__VA_ARGS__)
     28 #define MSE_DEBUGV(arg, ...)                                               \
     29  DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, "::%s: " arg, \
     30            __func__, ##__VA_ARGS__)
     31 
     32 using namespace mozilla::media;
     33 
     34 namespace mozilla {
     35 
     36 MediaSourceDecoder::MediaSourceDecoder(MediaDecoderInit& aInit)
     37    : MediaDecoder(aInit), mMediaSource(nullptr), mEnded(false) {
     38  mExplicitDuration.emplace(UnspecifiedNaN<double>());
     39 }
     40 
     41 MediaDecoderStateMachineBase* MediaSourceDecoder::CreateStateMachine(
     42    bool aDisableExternalEngine) {
     43  MOZ_ASSERT(NS_IsMainThread());
     44  // if `mDemuxer` already exists, that means we're in the process of recreating
     45  // the state machine. The track buffers are tied to the demuxer so we would
     46  // need to reuse it.
     47  if (!mDemuxer) {
     48    mDemuxer = new MediaSourceDemuxer(AbstractMainThread());
     49  }
     50  MediaFormatReaderInit init;
     51  init.mVideoFrameContainer = GetVideoFrameContainer();
     52  init.mKnowsCompositor = GetCompositor();
     53  init.mCrashHelper = GetOwner()->CreateGMPCrashHelper();
     54  init.mFrameStats = mFrameStats;
     55  init.mMediaDecoderOwnerID = mOwner;
     56  static Atomic<uint32_t> sTrackingIdCounter(0);
     57  init.mTrackingId.emplace(TrackingId::Source::MSEDecoder, sTrackingIdCounter++,
     58                           TrackingId::TrackAcrossProcesses::Yes);
     59  mReader = new MediaFormatReader(init, mDemuxer);
     60 #ifdef MOZ_WMF_CDM
     61  // ExternalEngineStateMachine is primarily used for encrypted playback when
     62  // the key system is supported via the WMF-based CDM. However, we cannot
     63  // currently determine the purpose of the playback, so we will always start
     64  // with ExternalEngineStateMachine. If this is not the case, we will switch
     65  // back to MediaDecoderStateMachine. The following outlines different
     66  // scenarios:
     67  // 1) Playback is non-encrypted or media format is not supported
     68  //    An internal error NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR
     69  //    will be received, resulting in a switch to another state machine.
     70  // 2) Playback is encrypted but the media key is not yet set
     71  //   2-1) If the CDMProxy is not WMF-based CDM when setting the media key,
     72  //        An internal error NS_ERROR_DOM_MEDIA_CDM_PROXY_NOT_SUPPORTED_ERR
     73  //        will be received, causing a switch to another state machine.
     74  //   2-2) If the CDMProxy is WMF-based CDM when setting the media key,
     75  //        There will be no error, and ExternalEngineStateMachine will operate.
     76  // 3) Playback is encrypted and the media key is already set
     77  //   3-1) If the CDMProxy is not WMF-based CDM,
     78  //        An internal error NS_ERROR_DOM_MEDIA_CDM_PROXY_NOT_SUPPORTED_ERR
     79  //        will be received, resulting in a switch to another state machine.
     80  //   3-2) If the CDMProxy is WMF-based CDM,
     81  //        There will be no error, and ExternalEngineStateMachine will operate.
     82  // Additionally, for testing purposes, non-encrypted playback can be performed
     83  // via ExternalEngineStateMachine as well by modifying the preference value.
     84  bool isCDMNotSupported =
     85      !!mOwner->GetCDMProxy() && !mOwner->GetCDMProxy()->AsWMFCDMProxy();
     86  if (StaticPrefs::media_wmf_media_engine_enabled() && !isCDMNotSupported &&
     87      !aDisableExternalEngine) {
     88    return new ExternalEngineStateMachine(this, mReader);
     89  }
     90 #endif
     91  return new MediaDecoderStateMachine(this, mReader);
     92 }
     93 
     94 nsresult MediaSourceDecoder::Load(nsIPrincipal* aPrincipal) {
     95  MOZ_ASSERT(NS_IsMainThread());
     96  MOZ_ASSERT(!GetStateMachine());
     97 
     98  mPrincipal = aPrincipal;
     99 
    100  nsresult rv = MediaShutdownManager::Instance().Register(this);
    101  if (NS_WARN_IF(NS_FAILED(rv))) {
    102    return rv;
    103  }
    104  return CreateAndInitStateMachine(!mEnded);
    105 }
    106 
    107 template <typename IntervalType>
    108 IntervalType MediaSourceDecoder::GetSeekableImpl() {
    109  MOZ_ASSERT(NS_IsMainThread());
    110  if (!mMediaSource) {
    111    NS_WARNING("MediaSource element isn't attached");
    112    return IntervalType();
    113  }
    114 
    115  TimeIntervals seekable;
    116  double duration = mMediaSource->Duration();
    117  if (std::isnan(duration)) {
    118    // Return empty range.
    119  } else if (duration > 0 && std::isinf(duration)) {
    120    media::TimeIntervals buffered = GetBuffered();
    121 
    122    // 1. If live seekable range is not empty:
    123    if (mMediaSource->HasLiveSeekableRange()) {
    124      // 1. Let union ranges be the union of live seekable range and the
    125      // HTMLMediaElement.buffered attribute.
    126      TimeRanges unionRanges =
    127          media::TimeRanges(buffered) + mMediaSource->LiveSeekableRange();
    128      // 2. Return a single range with a start time equal to the earliest start
    129      // time in union ranges and an end time equal to the highest end time in
    130      // union ranges and abort these steps.
    131      if constexpr (std::is_same<IntervalType, TimeRanges>::value) {
    132        TimeRanges seekableRange = media::TimeRanges(
    133            TimeRange(unionRanges.GetStart(), unionRanges.GetEnd()));
    134        return seekableRange;
    135      } else {
    136        MOZ_RELEASE_ASSERT(false);
    137      }
    138    }
    139 
    140    if (!buffered.IsEmpty()) {
    141      seekable += media::TimeInterval(TimeUnit::Zero(), buffered.GetEnd());
    142    }
    143  } else {
    144    if constexpr (std::is_same<IntervalType, TimeRanges>::value) {
    145      // Common case: seekable in entire range of the media.
    146      return TimeRanges(TimeRange(0, duration));
    147    } else if constexpr (std::is_same<IntervalType, TimeIntervals>::value) {
    148      seekable += media::TimeInterval(TimeUnit::Zero(),
    149                                      mDuration.match(DurationToTimeUnit()));
    150    } else {
    151      MOZ_RELEASE_ASSERT(false);
    152    }
    153  }
    154  MSE_DEBUG("ranges=%s", DumpTimeRanges(seekable).get());
    155  return IntervalType(seekable);
    156 }
    157 
    158 media::TimeIntervals MediaSourceDecoder::GetSeekable() {
    159  return GetSeekableImpl<media::TimeIntervals>();
    160 }
    161 
    162 media::TimeRanges MediaSourceDecoder::GetSeekableTimeRanges() {
    163  return GetSeekableImpl<media::TimeRanges>();
    164 }
    165 
    166 media::TimeIntervals MediaSourceDecoder::GetBuffered() {
    167  MOZ_ASSERT(NS_IsMainThread());
    168 
    169  if (!mMediaSource) {
    170    NS_WARNING("MediaSource element isn't attached");
    171    return media::TimeIntervals::Invalid();
    172  }
    173  dom::SourceBufferList* sourceBuffers = mMediaSource->ActiveSourceBuffers();
    174  if (!sourceBuffers) {
    175    // Media source object is shutting down.
    176    return TimeIntervals();
    177  }
    178  TimeUnit highestEndTime;
    179  nsTArray<media::TimeIntervals> activeRanges;
    180  media::TimeIntervals buffered;
    181 
    182  for (uint32_t i = 0; i < sourceBuffers->Length(); i++) {
    183    bool found;
    184    dom::SourceBuffer* sb = sourceBuffers->IndexedGetter(i, found);
    185    MOZ_ASSERT(found);
    186 
    187    activeRanges.AppendElement(sb->GetTimeIntervals());
    188    highestEndTime =
    189        std::max(highestEndTime, activeRanges.LastElement().GetEnd());
    190  }
    191 
    192  buffered += media::TimeInterval(TimeUnit::Zero(), highestEndTime);
    193 
    194  for (auto& range : activeRanges) {
    195    if (mEnded && !range.IsEmpty()) {
    196      // Set the end time on the last range to highestEndTime by adding a
    197      // new range spanning the current end time to highestEndTime, which
    198      // Normalize() will then merge with the old last range.
    199      range += media::TimeInterval(range.GetEnd(), highestEndTime);
    200    }
    201    buffered.Intersection(range);
    202  }
    203 
    204  MSE_DEBUG("ranges=%s", DumpTimeRanges(buffered).get());
    205  return buffered;
    206 }
    207 
    208 void MediaSourceDecoder::Shutdown() {
    209  MOZ_ASSERT(NS_IsMainThread());
    210  MSE_DEBUG("Shutdown");
    211  // Detach first so that TrackBuffers are unused on the main thread when
    212  // shut down on the decode task queue.
    213  if (mMediaSource) {
    214    mMediaSource->Detach();
    215  }
    216  mDemuxer = nullptr;
    217 
    218  MediaDecoder::Shutdown();
    219 }
    220 
    221 void MediaSourceDecoder::AttachMediaSource(dom::MediaSource* aMediaSource) {
    222  MOZ_ASSERT(!mMediaSource && !GetStateMachine() && NS_IsMainThread());
    223  mMediaSource = aMediaSource;
    224  DDLINKCHILD("mediasource", aMediaSource);
    225 }
    226 
    227 void MediaSourceDecoder::DetachMediaSource() {
    228  MOZ_ASSERT(mMediaSource && NS_IsMainThread());
    229  DDUNLINKCHILD(mMediaSource);
    230  mMediaSource = nullptr;
    231 }
    232 
    233 void MediaSourceDecoder::Ended(bool aEnded) {
    234  MOZ_ASSERT(NS_IsMainThread());
    235  if (aEnded) {
    236    // We want the MediaSourceReader to refresh its buffered range as it may
    237    // have been modified (end lined up).
    238    NotifyDataArrived();
    239  }
    240  mEnded = aEnded;
    241  GetStateMachine()->DispatchIsLiveStream(!mEnded);
    242 }
    243 
    244 void MediaSourceDecoder::AddSizeOfResources(ResourceSizes* aSizes) {
    245  MOZ_ASSERT(NS_IsMainThread());
    246  if (GetDemuxer()) {
    247    GetDemuxer()->AddSizeOfResources(aSizes);
    248  }
    249 }
    250 
    251 void MediaSourceDecoder::SetInitialDuration(const TimeUnit& aDuration) {
    252  MOZ_ASSERT(NS_IsMainThread());
    253  // Only use the decoded duration if one wasn't already
    254  // set.
    255  if (!mMediaSource || !std::isnan(ExplicitDuration())) {
    256    return;
    257  }
    258  SetMediaSourceDuration(aDuration);
    259 }
    260 
    261 void MediaSourceDecoder::SetMediaSourceDuration(const TimeUnit& aDuration) {
    262  MOZ_ASSERT(NS_IsMainThread());
    263  MOZ_ASSERT(!IsShutdown());
    264  if (aDuration.IsPositiveOrZero()) {
    265    // Truncate to microsecond resolution for consistency with the
    266    // SourceBuffer.buffered getter.
    267    SetExplicitDuration(aDuration.ToBase(USECS_PER_S).ToSeconds());
    268  } else {
    269    SetExplicitDuration(PositiveInfinity<double>());
    270  }
    271 }
    272 
    273 void MediaSourceDecoder::SetMediaSourceDuration(double aDuration) {
    274  MOZ_ASSERT(NS_IsMainThread());
    275  MOZ_ASSERT(!IsShutdown());
    276  if (aDuration >= 0) {
    277    SetExplicitDuration(aDuration);
    278  } else {
    279    SetExplicitDuration(PositiveInfinity<double>());
    280  }
    281 }
    282 
    283 RefPtr<GenericPromise> MediaSourceDecoder::RequestDebugInfo(
    284    dom::MediaSourceDecoderDebugInfo& aInfo) {
    285  // This should be safe to call off main thead, but there's no such usage at
    286  // time of writing. Can be carefully relaxed if needed.
    287  MOZ_ASSERT(NS_IsMainThread(), "Expects to be called on main thread.");
    288  nsTArray<RefPtr<GenericPromise>> promises;
    289  if (mReader) {
    290    promises.AppendElement(mReader->RequestDebugInfo(aInfo.mReader));
    291  }
    292  if (mDemuxer) {
    293    promises.AppendElement(mDemuxer->GetDebugInfo(aInfo.mDemuxer));
    294  }
    295  return GenericPromise::All(GetCurrentSerialEventTarget(), promises)
    296      ->Then(
    297          GetCurrentSerialEventTarget(), __func__,
    298          []() { return GenericPromise::CreateAndResolve(true, __func__); },
    299          [] {
    300            return GenericPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
    301          });
    302 }
    303 
    304 double MediaSourceDecoder::GetDuration() {
    305  MOZ_ASSERT(NS_IsMainThread());
    306  return ExplicitDuration();
    307 }
    308 
    309 MediaDecoderOwner::NextFrameStatus
    310 MediaSourceDecoder::NextFrameBufferedStatus() {
    311  MOZ_ASSERT(NS_IsMainThread());
    312 
    313  if (!mMediaSource ||
    314      mMediaSource->ReadyState() == dom::MediaSourceReadyState::Closed) {
    315    return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE;
    316  }
    317 
    318  // Next frame hasn't been decoded yet.
    319  // Use the buffered range to consider if we have the next frame available.
    320  auto currentPosition = CurrentPosition();
    321  TimeIntervals buffered = GetBuffered();
    322  buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
    323  TimeInterval interval(
    324      currentPosition, currentPosition + DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED);
    325  return buffered.ContainsWithStrictEnd(ClampIntervalToEnd(interval))
    326             ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE
    327             : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE;
    328 }
    329 
    330 bool MediaSourceDecoder::CanPlayThroughImpl() {
    331  MOZ_ASSERT(NS_IsMainThread());
    332 
    333  if (NextFrameBufferedStatus() == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE) {
    334    return false;
    335  }
    336 
    337  if (std::isnan(mMediaSource->Duration())) {
    338    // Don't have any data yet.
    339    return false;
    340  }
    341  TimeUnit duration = TimeUnit::FromSeconds(mMediaSource->Duration());
    342  auto currentPosition = CurrentPosition();
    343  if (duration <= currentPosition) {
    344    return true;
    345  }
    346  // If we have data up to the mediasource's duration or 3s ahead, we can
    347  // assume that we can play without interruption.
    348  TimeIntervals buffered = GetBuffered();
    349  buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
    350  TimeUnit timeAhead =
    351      std::min(duration, currentPosition + TimeUnit::FromSeconds(3));
    352  TimeInterval interval(currentPosition, timeAhead);
    353  return buffered.ToMicrosecondResolution().ContainsWithStrictEnd(
    354      ClampIntervalToEnd(interval));
    355 }
    356 
    357 TimeInterval MediaSourceDecoder::ClampIntervalToEnd(
    358    const TimeInterval& aInterval) {
    359  MOZ_ASSERT(NS_IsMainThread());
    360 
    361  if (!mEnded) {
    362    return aInterval;
    363  }
    364  TimeUnit duration = mDuration.match(DurationToTimeUnit());
    365  if (duration < aInterval.mStart) {
    366    return aInterval;
    367  }
    368  return TimeInterval(aInterval.mStart, std::min(aInterval.mEnd, duration),
    369                      aInterval.mFuzz);
    370 }
    371 
    372 void MediaSourceDecoder::NotifyInitDataArrived() {
    373  MOZ_ASSERT(NS_IsMainThread());
    374  if (mDemuxer) {
    375    mDemuxer->NotifyInitDataArrived();
    376  }
    377 }
    378 
    379 void MediaSourceDecoder::NotifyDataArrived() {
    380  MOZ_ASSERT(NS_IsMainThread());
    381  MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
    382  NotifyReaderDataArrived();
    383  GetOwner()->DownloadProgressed();
    384 }
    385 
    386 already_AddRefed<nsIPrincipal> MediaSourceDecoder::GetCurrentPrincipal() {
    387  MOZ_ASSERT(NS_IsMainThread());
    388  return do_AddRef(mPrincipal);
    389 }
    390 
    391 bool MediaSourceDecoder::HadCrossOriginRedirects() {
    392  MOZ_ASSERT(NS_IsMainThread());
    393  return false;
    394 }
    395 
    396 #ifdef MOZ_WMF_MEDIA_ENGINE
    397 void MediaSourceDecoder::MetadataLoaded(
    398    UniquePtr<MediaInfo> aInfo, UniquePtr<MetadataTags> aTags,
    399    MediaDecoderEventVisibility aEventVisibility) {
    400  // If the metadata has been loaded before, we don't want to notify throughout
    401  // that again when switching from media engine playback to normal playback.
    402  if (mPendingStatusUpdateForNewlyCreatedStateMachine && mFiredMetadataLoaded) {
    403    MSE_DEBUG(
    404        "Metadata already loaded and being informed by previous state machine");
    405    SetStatusUpdateForNewlyCreatedStateMachineIfNeeded();
    406    return;
    407  }
    408  MediaDecoder::MetadataLoaded(std::move(aInfo), std::move(aTags),
    409                               aEventVisibility);
    410 }
    411 #endif
    412 
    413 #undef MSE_DEBUG
    414 #undef MSE_DEBUGV
    415 
    416 }  // namespace mozilla