tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

GMPVideoDecoder.cpp (16419B)

      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "GMPVideoDecoder.h"
      8 
      9 #include "AnnexB.h"
     10 #include "GMPDecoderModule.h"
     11 #include "GMPLog.h"
     12 #include "GMPUtils.h"
     13 #include "GMPVideoHost.h"
     14 #include "H264.h"
     15 #include "MP4Decoder.h"
     16 #include "MediaData.h"
     17 #include "VPXDecoder.h"
     18 #include "VideoUtils.h"
     19 #include "mozilla/EndianUtils.h"
     20 #include "mozilla/StaticPrefs_media.h"
     21 #include "nsServiceManagerUtils.h"
     22 #include "prsystem.h"
     23 
     24 namespace mozilla {
     25 
     26 GMPVideoDecoderParams::GMPVideoDecoderParams(const CreateDecoderParams& aParams)
     27    : mConfig(aParams.VideoConfig()),
     28      mImageContainer(aParams.mImageContainer),
     29      mCrashHelper(aParams.mCrashHelper),
     30      mKnowsCompositor(aParams.mKnowsCompositor),
     31      mTrackingId(aParams.mTrackingId) {}
     32 
     33 nsCString GMPVideoDecoder::GetCodecName() const {
     34  if (MP4Decoder::IsH264(mConfig.mMimeType)) {
     35    return "h264"_ns;
     36  } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
     37    return "vp8"_ns;
     38  } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
     39    return "vp9"_ns;
     40  }
     41  return "unknown"_ns;
     42 }
     43 
     44 void GMPVideoDecoder::Decoded(GMPVideoi420Frame* aDecodedFrame) {
     45  GMP_LOG_DEBUG("GMPVideoDecoder::Decoded");
     46 
     47  GMPUniquePtr<GMPVideoi420Frame> decodedFrame(aDecodedFrame);
     48  MOZ_ASSERT(IsOnGMPThread());
     49 
     50  VideoData::YCbCrBuffer b;
     51  for (int i = 0; i < kGMPNumOfPlanes; ++i) {
     52    b.mPlanes[i].mData = decodedFrame->Buffer(GMPPlaneType(i));
     53    b.mPlanes[i].mStride = decodedFrame->Stride(GMPPlaneType(i));
     54    if (i == kGMPYPlane) {
     55      b.mPlanes[i].mWidth = decodedFrame->Width();
     56      b.mPlanes[i].mHeight = decodedFrame->Height();
     57    } else {
     58      b.mPlanes[i].mWidth = (decodedFrame->Width() + 1) / 2;
     59      b.mPlanes[i].mHeight = (decodedFrame->Height() + 1) / 2;
     60    }
     61    b.mPlanes[i].mSkip = 0;
     62  }
     63 
     64  b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
     65  b.mYUVColorSpace =
     66      DefaultColorSpace({decodedFrame->Width(), decodedFrame->Height()});
     67 
     68  UniquePtr<SampleMetadata> sampleData;
     69  if (auto entryHandle = mSamples.Lookup(decodedFrame->Timestamp())) {
     70    sampleData = std::move(entryHandle.Data());
     71    entryHandle.Remove();
     72  } else {
     73    GMP_LOG_DEBUG(
     74        "GMPVideoDecoder::Decoded(this=%p) missing sample metadata for "
     75        "time %" PRIu64,
     76        this, decodedFrame->Timestamp());
     77    if (mSamples.IsEmpty()) {
     78      // If we have no remaining samples in the table, then we have processed
     79      // all outstanding decode requests.
     80      ProcessReorderQueue(mDecodePromise, __func__);
     81    }
     82    return;
     83  }
     84 
     85  MOZ_ASSERT(sampleData);
     86 
     87  gfx::IntRect pictureRegion(0, 0, decodedFrame->Width(),
     88                             decodedFrame->Height());
     89  Result<already_AddRefed<VideoData>, MediaResult> r =
     90      VideoData::CreateAndCopyData(
     91          mConfig, mImageContainer, sampleData->mOffset,
     92          media::TimeUnit::FromMicroseconds(decodedFrame->UpdatedTimestamp()),
     93          media::TimeUnit::FromMicroseconds(decodedFrame->Duration()), b,
     94          sampleData->mKeyframe, media::TimeUnit::FromMicroseconds(-1),
     95          pictureRegion, mKnowsCompositor);
     96  if (r.isErr()) {
     97    mReorderQueue.Clear();
     98    mUnorderedData.Clear();
     99    mSamples.Clear();
    100    mDecodePromise.RejectIfExists(r.unwrapErr(), __func__);
    101    return;
    102  }
    103 
    104  RefPtr<VideoData> v = r.unwrap();
    105  MOZ_ASSERT(v);
    106  mPerformanceRecorder.Record(
    107      static_cast<int64_t>(decodedFrame->Timestamp()),
    108      [&](DecodeStage& aStage) {
    109        aStage.SetImageFormat(DecodeStage::YUV420P);
    110        aStage.SetResolution(decodedFrame->Width(), decodedFrame->Height());
    111        aStage.SetYUVColorSpace(b.mYUVColorSpace);
    112        aStage.SetColorDepth(b.mColorDepth);
    113        aStage.SetColorRange(b.mColorRange);
    114        aStage.SetStartTimeAndEndTime(v->mTime.ToMicroseconds(),
    115                                      v->GetEndTime().ToMicroseconds());
    116      });
    117 
    118  if (mReorderFrames) {
    119    mReorderQueue.Push(std::move(v));
    120  } else {
    121    mUnorderedData.AppendElement(std::move(v));
    122  }
    123 
    124  if (mSamples.IsEmpty()) {
    125    // If we have no remaining samples in the table, then we have processed
    126    // all outstanding decode requests.
    127    ProcessReorderQueue(mDecodePromise, __func__);
    128  }
    129 }
    130 
    131 void GMPVideoDecoder::ReceivedDecodedReferenceFrame(const uint64_t aPictureId) {
    132  GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedReferenceFrame");
    133  MOZ_ASSERT(IsOnGMPThread());
    134 }
    135 
    136 void GMPVideoDecoder::ReceivedDecodedFrame(const uint64_t aPictureId) {
    137  GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedFrame");
    138  MOZ_ASSERT(IsOnGMPThread());
    139 }
    140 
    141 void GMPVideoDecoder::InputDataExhausted() {
    142  GMP_LOG_DEBUG("GMPVideoDecoder::InputDataExhausted");
    143  MOZ_ASSERT(IsOnGMPThread());
    144  mSamples.Clear();
    145  ProcessReorderQueue(mDecodePromise, __func__);
    146 }
    147 
    148 void GMPVideoDecoder::DrainComplete() {
    149  GMP_LOG_DEBUG("GMPVideoDecoder::DrainComplete");
    150  MOZ_ASSERT(IsOnGMPThread());
    151  mSamples.Clear();
    152 
    153  if (mDrainPromise.IsEmpty()) {
    154    return;
    155  }
    156 
    157  DecodedData results;
    158  if (mReorderFrames) {
    159    results.SetCapacity(mReorderQueue.Length());
    160    while (!mReorderQueue.IsEmpty()) {
    161      results.AppendElement(mReorderQueue.Pop());
    162    }
    163  } else {
    164    results = std::move(mUnorderedData);
    165  }
    166 
    167  mDrainPromise.Resolve(std::move(results), __func__);
    168 }
    169 
    170 void GMPVideoDecoder::ResetComplete() {
    171  GMP_LOG_DEBUG("GMPVideoDecoder::ResetComplete");
    172  MOZ_ASSERT(IsOnGMPThread());
    173  mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
    174  mFlushPromise.ResolveIfExists(true, __func__);
    175 }
    176 
    177 void GMPVideoDecoder::Error(GMPErr aErr) {
    178  GMP_LOG_DEBUG("GMPVideoDecoder::Error");
    179  MOZ_ASSERT(IsOnGMPThread());
    180  Teardown(ToMediaResult(aErr, "Error GMP callback"_ns), __func__);
    181 }
    182 
    183 void GMPVideoDecoder::Terminated() {
    184  GMP_LOG_DEBUG("GMPVideoDecoder::Terminated");
    185  MOZ_ASSERT(IsOnGMPThread());
    186  Teardown(
    187      MediaResult(NS_ERROR_DOM_MEDIA_ABORT_ERR, "Terminated GMP callback"_ns),
    188      __func__);
    189 }
    190 
    191 void GMPVideoDecoder::ProcessReorderQueue(
    192    MozPromiseHolder<DecodePromise>& aPromise, StaticString aMethodName) {
    193  if (aPromise.IsEmpty()) {
    194    return;
    195  }
    196 
    197  if (!mReorderFrames) {
    198    aPromise.Resolve(std::move(mUnorderedData), aMethodName);
    199    return;
    200  }
    201 
    202  DecodedData results;
    203  size_t availableFrames = mReorderQueue.Length();
    204  if (availableFrames > mMaxRefFrames) {
    205    size_t resolvedFrames = availableFrames - mMaxRefFrames;
    206    results.SetCapacity(resolvedFrames);
    207    do {
    208      results.AppendElement(mReorderQueue.Pop());
    209    } while (--resolvedFrames > 0);
    210  }
    211 
    212  aPromise.Resolve(std::move(results), aMethodName);
    213 }
    214 
    215 GMPVideoDecoder::GMPVideoDecoder(const GMPVideoDecoderParams& aParams)
    216    : mConfig(aParams.mConfig),
    217      mGMP(nullptr),
    218      mHost(nullptr),
    219      mConvertNALUnitLengths(false),
    220      mCrashHelper(aParams.mCrashHelper),
    221      mImageContainer(aParams.mImageContainer),
    222      mKnowsCompositor(aParams.mKnowsCompositor),
    223      mTrackingId(aParams.mTrackingId),
    224      mCanDecodeBatch(StaticPrefs::media_gmp_decoder_decode_batch()),
    225      mReorderFrames(StaticPrefs::media_gmp_decoder_reorder_frames()) {}
    226 
    227 void GMPVideoDecoder::InitTags(nsTArray<nsCString>& aTags) {
    228  if (MP4Decoder::IsH264(mConfig.mMimeType)) {
    229    aTags.AppendElement("h264"_ns);
    230  } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
    231    aTags.AppendElement("vp8"_ns);
    232  } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
    233    aTags.AppendElement("vp9"_ns);
    234  }
    235 }
    236 
    237 nsCString GMPVideoDecoder::GetNodeId() { return SHARED_GMP_DECODING_NODE_ID; }
    238 
    239 GMPUniquePtr<GMPVideoEncodedFrame> GMPVideoDecoder::CreateFrame(
    240    MediaRawData* aSample) {
    241  GMPVideoFrame* ftmp = nullptr;
    242  GMPErr err = mHost->CreateFrame(kGMPEncodedVideoFrame, &ftmp);
    243  if (GMP_FAILED(err)) {
    244    return nullptr;
    245  }
    246 
    247  GMPUniquePtr<GMPVideoEncodedFrame> frame(
    248      static_cast<GMPVideoEncodedFrame*>(ftmp));
    249  err = frame->CreateEmptyFrame(aSample->Size());
    250  if (GMP_FAILED(err)) {
    251    return nullptr;
    252  }
    253 
    254  memcpy(frame->Buffer(), aSample->Data(), frame->Size());
    255 
    256  // Convert 4-byte NAL unit lengths to host-endian 4-byte buffer lengths to
    257  // suit the GMP API.
    258  if (mConvertNALUnitLengths) {
    259    const int kNALLengthSize = 4;
    260    uint8_t* buf = frame->Buffer();
    261    while (buf < frame->Buffer() + frame->Size() - kNALLengthSize) {
    262      uint32_t length = BigEndian::readUint32(buf) + kNALLengthSize;
    263      *reinterpret_cast<uint32_t*>(buf) = length;
    264      buf += length;
    265    }
    266  }
    267 
    268  frame->SetBufferType(GMP_BufferLength32);
    269 
    270  frame->SetEncodedWidth(mConfig.mDisplay.width);
    271  frame->SetEncodedHeight(mConfig.mDisplay.height);
    272  frame->SetTimeStamp(aSample->mTime.ToMicroseconds());
    273  frame->SetCompleteFrame(true);
    274  frame->SetDuration(aSample->mDuration.ToMicroseconds());
    275  frame->SetFrameType(aSample->mKeyframe ? kGMPKeyFrame : kGMPDeltaFrame);
    276 
    277  return frame;
    278 }
    279 
    280 const VideoInfo& GMPVideoDecoder::GetConfig() const { return mConfig; }
    281 
    282 void GMPVideoDecoder::GMPInitDone(GMPVideoDecoderProxy* aGMP,
    283                                  GMPVideoHost* aHost) {
    284  MOZ_ASSERT(IsOnGMPThread());
    285 
    286  if (!aGMP) {
    287    mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    288    return;
    289  }
    290  MOZ_ASSERT(aHost);
    291 
    292  if (mInitPromise.IsEmpty()) {
    293    // GMP must have been shutdown while we were waiting for Init operation
    294    // to complete.
    295    aGMP->Close();
    296    return;
    297  }
    298 
    299  bool isOpenH264 = aGMP->GetPluginType() == GMPPluginType::OpenH264;
    300 
    301  GMPVideoCodec codec;
    302  memset(&codec, 0, sizeof(codec));
    303 
    304  codec.mGMPApiVersion = kGMPVersion34;
    305  nsTArray<uint8_t> codecSpecific;
    306  if (MP4Decoder::IsH264(mConfig.mMimeType)) {
    307    codec.mCodecType = kGMPVideoCodecH264;
    308    codecSpecific.AppendElement(0);  // mPacketizationMode.
    309    codecSpecific.AppendElements(mConfig.mExtraData->Elements(),
    310                                 mConfig.mExtraData->Length());
    311    // OpenH264 expects pseudo-AVCC, but others must be passed
    312    // AnnexB for H264.
    313    mConvertToAnnexB = !isOpenH264;
    314    mMaxRefFrames = H264::ComputeMaxRefFrames(mConfig.mExtraData);
    315  } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
    316    codec.mCodecType = kGMPVideoCodecVP8;
    317  } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
    318    codec.mCodecType = kGMPVideoCodecVP9;
    319  } else {
    320    // Unrecognized mime type
    321    aGMP->Close();
    322    mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    323    return;
    324  }
    325  codec.mWidth = mConfig.mImage.width;
    326  codec.mHeight = mConfig.mImage.height;
    327  codec.mUseThreadedDecode = StaticPrefs::media_gmp_decoder_multithreaded();
    328  codec.mLogLevel = GetGMPLibraryLogLevel();
    329 
    330  nsresult rv =
    331      aGMP->InitDecode(codec, codecSpecific, this, PR_GetNumberOfProcessors());
    332  if (NS_FAILED(rv)) {
    333    aGMP->Close();
    334    mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    335    return;
    336  }
    337 
    338  mGMP = aGMP;
    339  mHost = aHost;
    340 
    341  // GMP implementations have interpreted the meaning of GMP_BufferLength32
    342  // differently.  The OpenH264 GMP expects GMP_BufferLength32 to behave as
    343  // specified in the GMP API, where each buffer is prefixed by a 32-bit
    344  // host-endian buffer length that includes the size of the buffer length
    345  // field.  Other existing GMPs currently expect GMP_BufferLength32 (when
    346  // combined with kGMPVideoCodecH264) to mean "like AVCC but restricted to
    347  // 4-byte NAL lengths" (i.e. buffer lengths are specified in big-endian
    348  // and do not include the length of the buffer length field.
    349  mConvertNALUnitLengths = isOpenH264;
    350 
    351  mInitPromise.Resolve(TrackInfo::kVideoTrack, __func__);
    352 }
    353 
    354 RefPtr<MediaDataDecoder::InitPromise> GMPVideoDecoder::Init() {
    355  MOZ_ASSERT(IsOnGMPThread());
    356 
    357  mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1");
    358  MOZ_ASSERT(mMPS);
    359 
    360  RefPtr<InitPromise> promise(mInitPromise.Ensure(__func__));
    361 
    362  nsTArray<nsCString> tags;
    363  InitTags(tags);
    364  UniquePtr<GetGMPVideoDecoderCallback> callback(new GMPInitDoneCallback(this));
    365  if (NS_FAILED(mMPS->GetGMPVideoDecoder(mCrashHelper, &tags, GetNodeId(),
    366                                         std::move(callback)))) {
    367    mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    368  }
    369 
    370  return promise;
    371 }
    372 
    373 RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Decode(
    374    MediaRawData* aSample) {
    375  MOZ_ASSERT(IsOnGMPThread());
    376 
    377  RefPtr<MediaRawData> sample(aSample);
    378  if (!mGMP) {
    379    return DecodePromise::CreateAndReject(
    380        MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    381                    RESULT_DETAIL("mGMP not initialized")),
    382        __func__);
    383  }
    384 
    385  if (mTrackingId) {
    386    MediaInfoFlag flag = MediaInfoFlag::None;
    387    flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
    388                                : MediaInfoFlag::NonKeyFrame);
    389    if (mGMP->GetPluginType() == GMPPluginType::OpenH264) {
    390      flag |= MediaInfoFlag::SoftwareDecoding;
    391    }
    392    if (MP4Decoder::IsH264(mConfig.mMimeType)) {
    393      flag |= MediaInfoFlag::VIDEO_H264;
    394    } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
    395      flag |= MediaInfoFlag::VIDEO_VP8;
    396    } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
    397      flag |= MediaInfoFlag::VIDEO_VP9;
    398    }
    399    mPerformanceRecorder.Start(aSample->mTime.ToMicroseconds(),
    400                               "GMPVideoDecoder"_ns, *mTrackingId, flag);
    401  }
    402 
    403  GMPUniquePtr<GMPVideoEncodedFrame> frame = CreateFrame(sample);
    404  if (!frame) {
    405    return DecodePromise::CreateAndReject(
    406        MediaResult(NS_ERROR_OUT_OF_MEMORY,
    407                    RESULT_DETAIL("CreateFrame returned null")),
    408        __func__);
    409  }
    410 
    411  uint64_t frameTimestamp = frame->TimeStamp();
    412  RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
    413  nsTArray<uint8_t> info;  // No codec specific per-frame info to pass.
    414  nsresult rv = mGMP->Decode(std::move(frame), false, info, 0);
    415  if (NS_FAILED(rv)) {
    416    mDecodePromise.Reject(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
    417                                      RESULT_DETAIL("mGMP->Decode:%" PRIx32,
    418                                                    static_cast<uint32_t>(rv))),
    419                          __func__);
    420  }
    421 
    422  // If we have multiple outstanding frames, we need to track which offset
    423  // belongs to which frame. During seek, it is possible to get the same frame
    424  // requested twice, if the old frame is still outstanding. We will simply drop
    425  // the extra decoded frame and request more input if the last outstanding.
    426  mSamples.WithEntryHandle(frameTimestamp, [&](auto entryHandle) {
    427    auto sampleData = MakeUnique<SampleMetadata>(sample);
    428    entryHandle.InsertOrUpdate(std::move(sampleData));
    429  });
    430 
    431  return p;
    432 }
    433 
    434 RefPtr<MediaDataDecoder::FlushPromise> GMPVideoDecoder::Flush() {
    435  MOZ_ASSERT(IsOnGMPThread());
    436 
    437  mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    438  mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    439 
    440  RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__);
    441  if (!mGMP || NS_FAILED(mGMP->Reset())) {
    442    // Abort the flush.
    443    mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
    444    mFlushPromise.Resolve(true, __func__);
    445  }
    446  return p;
    447 }
    448 
    449 RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Drain() {
    450  MOZ_ASSERT(IsOnGMPThread());
    451 
    452  MOZ_ASSERT(mDecodePromise.IsEmpty(), "Must wait for decoding to complete");
    453 
    454  RefPtr<DecodePromise> p = mDrainPromise.Ensure(__func__);
    455  if (!mGMP || NS_FAILED(mGMP->Drain())) {
    456    mDrainPromise.Resolve(DecodedData(), __func__);
    457  }
    458 
    459  return p;
    460 }
    461 
    462 void GMPVideoDecoder::Teardown(const MediaResult& aResult,
    463                               StaticString aCallSite) {
    464  MOZ_ASSERT(IsOnGMPThread());
    465 
    466  // Ensure we are kept alive at least until we return.
    467  RefPtr<GMPVideoDecoder> self(this);
    468 
    469  mInitPromise.RejectIfExists(aResult, aCallSite);
    470  mDecodePromise.RejectIfExists(aResult, aCallSite);
    471 
    472  if (mGMP) {
    473    // Note this unblocks flush and drain operations waiting for callbacks.
    474    mGMP->Close();
    475    mGMP = nullptr;
    476  } else {
    477    mFlushPromise.RejectIfExists(aResult, aCallSite);
    478    mDrainPromise.RejectIfExists(aResult, aCallSite);
    479  }
    480 
    481  mHost = nullptr;
    482 }
    483 
    484 RefPtr<ShutdownPromise> GMPVideoDecoder::Shutdown() {
    485  MOZ_ASSERT(IsOnGMPThread());
    486  Teardown(MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Shutdown"_ns), __func__);
    487  return ShutdownPromise::CreateAndResolve(true, __func__);
    488 }
    489 
    490 }  // namespace mozilla