tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

DecoderAgent.cpp (18516B)

      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "DecoderAgent.h"
      8 
      9 #include "ImageContainer.h"
     10 #include "MP4Decoder.h"
     11 #include "MediaDataDecoderProxy.h"
     12 #include "PDMFactory.h"
     13 #include "VideoUtils.h"
     14 #include "mozilla/DebugOnly.h"
     15 #include "mozilla/Logging.h"
     16 #include "mozilla/StaticPrefs_media.h"
     17 #include "mozilla/layers/ImageBridgeChild.h"
     18 #include "nsThreadUtils.h"
     19 
     20 extern mozilla::LazyLogModule gWebCodecsLog;
     21 
     22 namespace mozilla {
     23 
     24 #ifdef LOG_INTERNAL
     25 #  undef LOG_INTERNAL
     26 #endif  // LOG_INTERNAL
     27 #define LOG_INTERNAL(level, msg, ...) \
     28  MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
     29 
     30 #ifdef LOG
     31 #  undef LOG
     32 #endif  // LOG
     33 #define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
     34 
     35 #ifdef LOGW
     36 #  undef LOGW
     37 #endif  // LOGE
     38 #define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
     39 
     40 #ifdef LOGE
     41 #  undef LOGE
     42 #endif  // LOGE
     43 #define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
     44 
     45 #ifdef LOGV
     46 #  undef LOGV
     47 #endif  // LOGV
     48 #define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
     49 
     50 DecoderAgent::DecoderAgent(Id aId, UniquePtr<TrackInfo>&& aInfo)
     51    : mId(aId),
     52      mInfo(std::move(aInfo)),
     53      mOwnerThread(GetCurrentSerialEventTarget()),
     54      mPDMFactory(MakeRefPtr<PDMFactory>()),
     55      mImageContainer(MakeAndAddRef<layers::ImageContainer>(
     56          layers::ImageUsageType::WebCodecs,
     57          layers::ImageContainer::ASYNCHRONOUS)),
     58      mDecoder(nullptr),
     59      mState(State::Unconfigured) {
     60  MOZ_ASSERT(mInfo);
     61  MOZ_ASSERT(mOwnerThread);
     62  MOZ_ASSERT(mPDMFactory);
     63  MOZ_ASSERT(mImageContainer);
     64  LOG("DecoderAgent #%d (%p) ctor", mId, this);
     65 }
     66 
     67 DecoderAgent::~DecoderAgent() {
     68  LOG("DecoderAgent #%d (%p) dtor", mId, this);
     69  MOZ_ASSERT(mState == State::Unconfigured, "decoder release in wrong state");
     70  MOZ_ASSERT(!mDecoder, "decoder must be shutdown");
     71 }
     72 
     73 RefPtr<DecoderAgent::ConfigurePromise> DecoderAgent::Configure(
     74    bool aPreferSoftwareDecoder, bool aLowLatency) {
     75  MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
     76  MOZ_ASSERT(mState == State::Unconfigured || mState == State::Error);
     77  MOZ_ASSERT(mConfigurePromise.IsEmpty());
     78  MOZ_ASSERT(!mCreateRequest.Exists());
     79  MOZ_ASSERT(!mInitRequest.Exists());
     80 
     81  if (mState == State::Error) {
     82    LOGE("DecoderAgent #%d (%p) tried to configure in error state", mId, this);
     83    return ConfigurePromise::CreateAndReject(
     84        MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
     85                    "Cannot configure in error state"),
     86        __func__);
     87  }
     88 
     89  MOZ_ASSERT(mState == State::Unconfigured);
     90  MOZ_ASSERT(!mDecoder);
     91  SetState(State::Configuring);
     92 
     93  RefPtr<layers::KnowsCompositor> knowsCompositor =
     94      layers::ImageBridgeChild::GetSingleton();
     95 
     96  auto params = CreateDecoderParams{
     97      *mInfo,
     98      CreateDecoderParams::OptionSet(
     99          aPreferSoftwareDecoder
    100              ? CreateDecoderParams::Option::HardwareDecoderNotAllowed
    101              : CreateDecoderParams::Option::Default),
    102      mInfo->GetType(), mImageContainer, knowsCompositor};
    103  if (aLowLatency) {
    104    params.mOptions += CreateDecoderParams::Option::LowLatency;
    105  }
    106  // MediaChangeMonitor is requested to decode raw AAC in ADTS.
    107  if (MP4Decoder::IsAAC(mInfo->mMimeType)) {
    108    params.mWrappers += media::Wrapper::MediaChangeMonitor;
    109  }
    110  // This should only be used for testing.
    111  if (StaticPrefs::media_test_null_decoder_creation_failure()) {
    112    params.mUseNullDecoder = CreateDecoderParams::UseNullDecoder(true);
    113  }
    114 
    115  // Always even use the pts that were set on the input samples when returning
    116  // decoded video frames.
    117  params.mOptions += CreateDecoderParams::Option::KeepOriginalPts;
    118 
    119  LOG("DecoderAgent #%d (%p) is creating a decoder (mime: %s) - PreferSW: %s, "
    120      "low-latency: %s, create-decoder-params: %s",
    121      mId, this, mInfo->mMimeType.get(), aPreferSoftwareDecoder ? "yes" : "no",
    122      aLowLatency ? "yes" : "no", params.ToString().get());
    123 
    124  RefPtr<ConfigurePromise> p = mConfigurePromise.Ensure(__func__);
    125 
    126  mPDMFactory->CreateDecoder(params)
    127      ->Then(
    128          mOwnerThread, __func__,
    129          [self = RefPtr{this}](RefPtr<MediaDataDecoder>&& aDecoder) {
    130            self->mCreateRequest.Complete();
    131 
    132            // If DecoderAgent has been shut down, shut the created decoder down
    133            // and return.
    134            if (!self->mShutdownWhileCreationPromise.IsEmpty()) {
    135              MOZ_ASSERT(self->mState == State::ShuttingDown);
    136              MOZ_ASSERT(self->mConfigurePromise.IsEmpty(),
    137                         "configuration should have been rejected");
    138 
    139              LOGW(
    140                  "DecoderAgent #%d (%p) has been shut down. We need to shut "
    141                  "the newly created decoder down",
    142                  self->mId, self.get());
    143              aDecoder->Shutdown()->Then(
    144                  self->mOwnerThread, __func__,
    145                  [self](const ShutdownPromise::ResolveOrRejectValue& aValue) {
    146                    MOZ_ASSERT(self->mState == State::ShuttingDown);
    147 
    148                    LOGW(
    149                        "DecoderAgent #%d (%p), newly created decoder shutdown "
    150                        "has been %s",
    151                        self->mId, self.get(),
    152                        aValue.IsResolve() ? "resolved" : "rejected");
    153 
    154                    self->SetState(State::Unconfigured);
    155 
    156                    self->mShutdownWhileCreationPromise.ResolveOrReject(
    157                        aValue, __func__);
    158                  });
    159              return;
    160            }
    161 
    162            self->mDecoder = new MediaDataDecoderProxy(
    163                aDecoder.forget(),
    164                CreateMediaDecodeTaskQueue("DecoderAgent TaskQueue"));
    165            LOG("DecoderAgent #%d (%p) has created a decoder, now initialize "
    166                "it",
    167                self->mId, self.get());
    168            self->mDecoder->Init()
    169                ->Then(
    170                    self->mOwnerThread, __func__,
    171                    [self](const TrackInfo::TrackType aTrackType) {
    172                      self->mInitRequest.Complete();
    173                      LOG("DecoderAgent #%d (%p) has initialized the decoder",
    174                          self->mId, self.get());
    175                      MOZ_ASSERT(aTrackType == self->mInfo->GetType());
    176                      self->SetState(State::Configured);
    177                      self->mConfigurePromise.Resolve(true, __func__);
    178                    },
    179                    [self](const MediaResult& aError) {
    180                      self->mInitRequest.Complete();
    181                      LOGE(
    182                          "DecoderAgent #%d (%p) failed to initialize the "
    183                          "decoder",
    184                          self->mId, self.get());
    185                      self->SetState(State::Error);
    186                      self->mConfigurePromise.Reject(aError, __func__);
    187                    })
    188                ->Track(self->mInitRequest);
    189          },
    190          [self = RefPtr{this}](const MediaResult& aError) {
    191            self->mCreateRequest.Complete();
    192            LOGE("DecoderAgent #%d (%p) failed to create a decoder", self->mId,
    193                 self.get());
    194 
    195            // If DecoderAgent has been shut down, we need to resolve the
    196            // shutdown promise.
    197            if (!self->mShutdownWhileCreationPromise.IsEmpty()) {
    198              MOZ_ASSERT(self->mState == State::ShuttingDown);
    199              MOZ_ASSERT(self->mConfigurePromise.IsEmpty(),
    200                         "configuration should have been rejected");
    201 
    202              LOGW(
    203                  "DecoderAgent #%d (%p) has been shut down. Resolve the "
    204                  "shutdown promise right away since decoder creation failed",
    205                  self->mId, self.get());
    206 
    207              self->SetState(State::Unconfigured);
    208              self->mShutdownWhileCreationPromise.Resolve(true, __func__);
    209              return;
    210            }
    211 
    212            self->SetState(State::Error);
    213            self->mConfigurePromise.Reject(aError, __func__);
    214          })
    215      ->Track(mCreateRequest);
    216 
    217  return p;
    218 }
    219 
    220 RefPtr<ShutdownPromise> DecoderAgent::Shutdown() {
    221  MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
    222 
    223  LOG("DecoderAgent #%d (%p), shutdown in %s state", mId, this,
    224      EnumValueToString(mState));
    225 
    226  MOZ_ASSERT(mShutdownWhileCreationPromise.IsEmpty(),
    227             "Shutdown while shutting down is prohibited");
    228 
    229  auto r =
    230      MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled by decoder shutdown");
    231 
    232  // If the decoder creation has not been completed yet, wait until the decoder
    233  // being created has been shut down.
    234  if (mCreateRequest.Exists()) {
    235    MOZ_ASSERT(!mInitRequest.Exists());
    236    MOZ_ASSERT(!mConfigurePromise.IsEmpty());
    237    MOZ_ASSERT(!mDecoder);
    238    MOZ_ASSERT(mState == State::Configuring);
    239 
    240    LOGW(
    241        "DecoderAgent #%d (%p) shutdown while the decoder-creation for "
    242        "configuration is in flight. Reject the configuration now and defer "
    243        "the shutdown until the created decoder has been shut down",
    244        mId, this);
    245 
    246    // Reject the configuration in flight.
    247    mConfigurePromise.Reject(r, __func__);
    248 
    249    // Get the promise that will be resolved when the decoder being created has
    250    // been destroyed.
    251    SetState(State::ShuttingDown);
    252    return mShutdownWhileCreationPromise.Ensure(__func__);
    253  }
    254 
    255  // If decoder creation has been completed but failed, no decoder is set.
    256  if (!mDecoder) {
    257    LOG("DecoderAgent #%d (%p) shutdown without an active decoder", mId, this);
    258    MOZ_ASSERT(mState == State::Error);
    259    MOZ_ASSERT(!mInitRequest.Exists());
    260    MOZ_ASSERT(mConfigurePromise.IsEmpty());
    261    MOZ_ASSERT(!mDecodeRequest.Exists());
    262    MOZ_ASSERT(mDecodePromise.IsEmpty());
    263    MOZ_ASSERT(!mDrainRequest.Exists());
    264    MOZ_ASSERT(!mFlushRequest.Exists());
    265    MOZ_ASSERT(!mDryRequest.Exists());
    266    MOZ_ASSERT(mDryPromise.IsEmpty());
    267    MOZ_ASSERT(mDrainAndFlushPromise.IsEmpty());
    268    // ~DecoderAgent() will ensure that the decoder is shutdown.
    269    SetState(State::Unconfigured);
    270    return ShutdownPromise::CreateAndResolve(true, __func__);
    271  }
    272 
    273  // If decoder creation has succeeded, we must have the decoder now.
    274 
    275  // Cancel pending initialization for configuration in flight if any.
    276  mInitRequest.DisconnectIfExists();
    277  mConfigurePromise.RejectIfExists(r, __func__);
    278 
    279  // Cancel decode in flight if any.
    280  mDecodeRequest.DisconnectIfExists();
    281  mDecodePromise.RejectIfExists(r, __func__);
    282 
    283  // Cancel flush-out in flight if any.
    284  mDrainRequest.DisconnectIfExists();
    285  mFlushRequest.DisconnectIfExists();
    286  mDryRequest.DisconnectIfExists();
    287  mDryPromise.RejectIfExists(r, __func__);
    288  mDrainAndFlushPromise.RejectIfExists(r, __func__);
    289  mDryData.Clear();
    290  mDrainAndFlushData.Clear();
    291 
    292  SetState(State::Unconfigured);
    293 
    294  RefPtr<MediaDataDecoder> decoder = std::move(mDecoder);
    295  return decoder->Shutdown();
    296 }
    297 
    298 RefPtr<DecoderAgent::DecodePromise> DecoderAgent::Decode(
    299    MediaRawData* aSample) {
    300  MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
    301  MOZ_ASSERT(aSample);
    302  MOZ_ASSERT(mState == State::Configured || mState == State::Error);
    303  MOZ_ASSERT(mDecodePromise.IsEmpty());
    304  MOZ_ASSERT(!mDecodeRequest.Exists());
    305 
    306  if (mState == State::Error) {
    307    LOGE("DecoderAgent #%d (%p) tried to decode in error state", mId, this);
    308    return DecodePromise::CreateAndReject(
    309        MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    310                    "Cannot decode in error state"),
    311        __func__);
    312  }
    313 
    314  MOZ_ASSERT(mState == State::Configured);
    315  MOZ_ASSERT(mDecoder);
    316  SetState(State::Decoding);
    317 
    318  RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
    319 
    320  mDecoder->Decode(aSample)
    321      ->Then(
    322          mOwnerThread, __func__,
    323          [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) {
    324            self->mDecodeRequest.Complete();
    325            LOGV("DecoderAgent #%d (%p) decode successfully", self->mId,
    326                 self.get());
    327            self->SetState(State::Configured);
    328            self->mDecodePromise.Resolve(std::move(aData), __func__);
    329          },
    330          [self = RefPtr{this}](const MediaResult& aError) {
    331            self->mDecodeRequest.Complete();
    332            LOGV("DecoderAgent #%d (%p) failed to decode", self->mId,
    333                 self.get());
    334            self->SetState(State::Error);
    335            self->mDecodePromise.Reject(aError, __func__);
    336          })
    337      ->Track(mDecodeRequest);
    338 
    339  return p;
    340 }
    341 
    342 RefPtr<DecoderAgent::DecodePromise> DecoderAgent::DrainAndFlush() {
    343  MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
    344  MOZ_ASSERT(mState == State::Configured || mState == State::Error);
    345  MOZ_ASSERT(mDrainAndFlushPromise.IsEmpty());
    346  MOZ_ASSERT(mDrainAndFlushData.IsEmpty());
    347  MOZ_ASSERT(!mDryRequest.Exists());
    348  MOZ_ASSERT(mDryPromise.IsEmpty());
    349  MOZ_ASSERT(mDryData.IsEmpty());
    350  MOZ_ASSERT(!mDrainRequest.Exists());
    351  MOZ_ASSERT(!mFlushRequest.Exists());
    352 
    353  if (mState == State::Error) {
    354    LOGE("DecoderAgent #%d (%p) tried to flush-out in error state", mId, this);
    355    return DecodePromise::CreateAndReject(
    356        MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    357                    "Cannot flush in error state"),
    358        __func__);
    359  }
    360 
    361  MOZ_ASSERT(mState == State::Configured);
    362  MOZ_ASSERT(mDecoder);
    363  SetState(State::Flushing);
    364 
    365  RefPtr<DecoderAgent::DecodePromise> p =
    366      mDrainAndFlushPromise.Ensure(__func__);
    367 
    368  Dry()
    369      ->Then(
    370          mOwnerThread, __func__,
    371          [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) {
    372            self->mDryRequest.Complete();
    373            LOG("DecoderAgent #%d (%p) has dried the decoder. Now flushing the "
    374                "decoder",
    375                self->mId, self.get());
    376            MOZ_ASSERT(self->mDrainAndFlushData.IsEmpty());
    377            self->mDrainAndFlushData.AppendElements(std::move(aData));
    378            self->mDecoder->Flush()
    379                ->Then(
    380                    self->mOwnerThread, __func__,
    381                    [self](const bool /* aUnUsed */) {
    382                      self->mFlushRequest.Complete();
    383                      LOG("DecoderAgent #%d (%p) has flushed the decoder",
    384                          self->mId, self.get());
    385                      self->SetState(State::Configured);
    386                      self->mDrainAndFlushPromise.Resolve(
    387                          std::move(self->mDrainAndFlushData), __func__);
    388                    },
    389                    [self](const MediaResult& aError) {
    390                      self->mFlushRequest.Complete();
    391                      LOGE("DecoderAgent #%d (%p) failed to flush the decoder",
    392                           self->mId, self.get());
    393                      self->SetState(State::Error);
    394                      self->mDrainAndFlushData.Clear();
    395                      self->mDrainAndFlushPromise.Reject(aError, __func__);
    396                    })
    397                ->Track(self->mFlushRequest);
    398          },
    399          [self = RefPtr{this}](const MediaResult& aError) {
    400            self->mDryRequest.Complete();
    401            LOGE("DecoderAgent #%d (%p) failed to dry the decoder", self->mId,
    402                 self.get());
    403            self->SetState(State::Error);
    404            self->mDrainAndFlushPromise.Reject(aError, __func__);
    405          })
    406      ->Track(mDryRequest);
    407 
    408  return p;
    409 }
    410 
    411 RefPtr<DecoderAgent::DecodePromise> DecoderAgent::Dry() {
    412  MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
    413  MOZ_ASSERT(mState == State::Flushing);
    414  MOZ_ASSERT(mDryPromise.IsEmpty());
    415  MOZ_ASSERT(!mDryRequest.Exists());
    416  MOZ_ASSERT(mDryData.IsEmpty());
    417  MOZ_ASSERT(mDecoder);
    418 
    419  RefPtr<DecodePromise> p = mDryPromise.Ensure(__func__);
    420  DrainUntilDry();
    421  return p;
    422 }
    423 
    424 void DecoderAgent::DrainUntilDry() {
    425  MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
    426  MOZ_ASSERT(mState == State::Flushing);
    427  MOZ_ASSERT(!mDryPromise.IsEmpty());
    428  MOZ_ASSERT(!mDrainRequest.Exists());
    429  MOZ_ASSERT(mDecoder);
    430 
    431  LOG("DecoderAgent #%d (%p) is drainng the decoder", mId, this);
    432  mDecoder->Drain()
    433      ->Then(
    434          mOwnerThread, __func__,
    435          [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) {
    436            self->mDrainRequest.Complete();
    437 
    438            if (aData.IsEmpty()) {
    439              LOG("DecoderAgent #%d (%p) is dry now", self->mId, self.get());
    440              self->mDryPromise.Resolve(std::move(self->mDryData), __func__);
    441              return;
    442            }
    443 
    444            LOG("DecoderAgent #%d (%p) drained %zu decoded data. Keep draining "
    445                "until dry",
    446                self->mId, self.get(), aData.Length());
    447            self->mDryData.AppendElements(std::move(aData));
    448            self->DrainUntilDry();
    449          },
    450          [self = RefPtr{this}](const MediaResult& aError) {
    451            self->mDrainRequest.Complete();
    452 
    453            LOGE("DecoderAgent %p failed to drain decoder", self.get());
    454            self->mDryData.Clear();
    455            self->mDryPromise.Reject(aError, __func__);
    456          })
    457      ->Track(mDrainRequest);
    458 }
    459 
    460 void DecoderAgent::SetState(State aState) {
    461  MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
    462 
    463  auto validateStateTransition = [](State aOldState, State aNewState) {
    464    switch (aOldState) {
    465      case State::Unconfigured:
    466        return aNewState == State::Configuring;
    467      case State::Configuring:
    468        return aNewState == State::Configured || aNewState == State::Error ||
    469               aNewState == State::Unconfigured ||
    470               aNewState == State::ShuttingDown;
    471      case State::Configured:
    472        return aNewState == State::Unconfigured ||
    473               aNewState == State::Decoding || aNewState == State::Flushing;
    474      case State::Decoding:
    475      case State::Flushing:
    476        return aNewState == State::Configured || aNewState == State::Error ||
    477               aNewState == State::Unconfigured;
    478      case State::ShuttingDown:
    479        return aNewState == State::Unconfigured;
    480      case State::Error:
    481        return aNewState == State::Unconfigured;
    482      default:
    483        break;
    484    }
    485    MOZ_ASSERT_UNREACHABLE("Unhandled state transition");
    486    return false;
    487  };
    488 
    489  DebugOnly<bool> isValid = validateStateTransition(mState, aState);
    490  MOZ_ASSERT(isValid);
    491  LOG("DecoderAgent #%d (%p) state change: %s -> %s", mId, this,
    492      EnumValueToString(mState), EnumValueToString(aState));
    493  mState = aState;
    494 }
    495 
    496 #undef LOG
    497 #undef LOGW
    498 #undef LOGE
    499 #undef LOGV
    500 #undef LOG_INTERNAL
    501 
    502 }  // namespace mozilla