tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

RemoteVideoDecoder.cpp (12517B)

      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 #include "RemoteVideoDecoder.h"
      7 
      8 #include "mozilla/layers/ImageDataSerializer.h"
      9 
     10 #ifdef MOZ_AV1
     11 #  include "AOMDecoder.h"
     12 #  include "DAV1DDecoder.h"
     13 #endif
     14 #ifdef XP_WIN
     15 #  include "WMFDecoderModule.h"
     16 #endif
     17 #include "GPUVideoImage.h"
     18 #include "ImageContainer.h"  // for PlanarYCbCrData and BufferRecycleBin
     19 #include "MediaDataDecoderProxy.h"
     20 #include "MediaInfo.h"
     21 #include "PDMFactory.h"
     22 #include "RemoteCDMParent.h"
     23 #include "RemoteImageHolder.h"
     24 #include "RemoteMediaManagerParent.h"
     25 #include "mozilla/StaticPrefs_media.h"
     26 #include "mozilla/layers/ImageClient.h"
     27 #include "mozilla/layers/TextureClient.h"
     28 #include "mozilla/layers/VideoBridgeChild.h"
     29 #ifdef MOZ_WIDGET_ANDROID
     30 #  include "mozilla/layers/VideoBridgeParent.h"
     31 #endif
     32 
     33 namespace mozilla {
     34 
     35 using namespace layers;  // for PlanarYCbCrData and BufferRecycleBin
     36 using namespace ipc;
     37 using namespace gfx;
     38 
     39 layers::TextureForwarder* KnowsCompositorVideo::GetTextureForwarder() {
     40  auto* vbc = VideoBridgeChild::GetSingleton();
     41  return (vbc && vbc->CanSend()) ? vbc : nullptr;
     42 }
     43 layers::LayersIPCActor* KnowsCompositorVideo::GetLayersIPCActor() {
     44  return GetTextureForwarder();
     45 }
     46 
     47 /* static */ already_AddRefed<KnowsCompositorVideo>
     48 KnowsCompositorVideo::TryCreateForIdentifier(
     49    const layers::TextureFactoryIdentifier& aIdentifier) {
     50  VideoBridgeChild* child = VideoBridgeChild::GetSingleton();
     51  if (!child) {
     52    return nullptr;
     53  }
     54 
     55  RefPtr<KnowsCompositorVideo> knowsCompositor = new KnowsCompositorVideo();
     56  knowsCompositor->IdentifyTextureHost(aIdentifier);
     57  return knowsCompositor.forget();
     58 }
     59 
     60 RemoteVideoDecoderChild::RemoteVideoDecoderChild(RemoteMediaIn aLocation)
     61    : RemoteDecoderChild(aLocation), mBufferRecycleBin(new BufferRecycleBin) {}
     62 
     63 MediaResult RemoteVideoDecoderChild::ProcessOutput(
     64    DecodedOutputIPDL&& aDecodedData) {
     65  AssertOnManagerThread();
     66  MOZ_ASSERT(aDecodedData.type() == DecodedOutputIPDL::TArrayOfRemoteVideoData);
     67 
     68  nsTArray<RemoteVideoData>& arrayData =
     69      aDecodedData.get_ArrayOfRemoteVideoData()->Array();
     70 
     71  for (auto&& data : arrayData) {
     72    if (data.image().IsEmpty()) {
     73      // This is a NullData object.
     74      mDecodedData.AppendElement(MakeRefPtr<NullData>(
     75          data.base().offset(), data.base().time(), data.base().duration()));
     76      continue;
     77    }
     78    RefPtr<Image> image = data.image().TransferToImage(mBufferRecycleBin);
     79 
     80    RefPtr<VideoData> video = VideoData::CreateFromImage(
     81        data.display(), data.base().offset(), data.base().time(),
     82        data.base().duration(), image, data.base().keyframe(),
     83        data.base().timecode());
     84 
     85    if (!video) {
     86      // OOM
     87      return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__);
     88    }
     89    mDecodedData.AppendElement(std::move(video));
     90  }
     91  return NS_OK;
     92 }
     93 
     94 MediaResult RemoteVideoDecoderChild::InitIPDL(
     95    const VideoInfo& aVideoInfo, float aFramerate,
     96    const CreateDecoderParams::OptionSet& aOptions,
     97    Maybe<layers::TextureFactoryIdentifier> aIdentifier,
     98    const Maybe<uint64_t>& aMediaEngineId, const Maybe<TrackingId>& aTrackingId,
     99    PRemoteCDMActor* aCDM) {
    100  MOZ_ASSERT_IF(mLocation == RemoteMediaIn::GpuProcess, aIdentifier);
    101 
    102  RefPtr<RemoteMediaManagerChild> manager =
    103      RemoteMediaManagerChild::GetSingleton(mLocation);
    104 
    105  // The manager isn't available because RemoteMediaManagerChild has been
    106  // initialized with null end points and we don't want to decode video on RDD
    107  // process anymore. Return false here so that we can fallback to other PDMs.
    108  if (!manager) {
    109    return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    110                       RESULT_DETAIL("RemoteMediaManager is not available."));
    111  }
    112 
    113  if (!manager->CanSend()) {
    114    if (mLocation == RemoteMediaIn::GpuProcess) {
    115      // The manager doesn't support sending messages because we've just crashed
    116      // and are working on reinitialization. Don't initialize mIPDLSelfRef and
    117      // leave us in an error state. We'll then immediately reject the promise
    118      // when Init() is called and the caller can try again. Hopefully by then
    119      // the new manager is ready, or we've notified the caller of it being no
    120      // longer available. If not, then the cycle repeats until we're ready.
    121      return NS_OK;
    122    }
    123 
    124    return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    125                       RESULT_DETAIL("RemoteMediaManager unable to send."));
    126  }
    127 
    128  // If we are given a remote CDM, we need to make sure that it has been remoted
    129  // into the same process as the decoder.
    130  PRemoteCDMChild* cdm = nullptr;
    131  if (aCDM) {
    132    if (aCDM->GetLocation() != mLocation) {
    133      return MediaResult(
    134          NS_ERROR_DOM_MEDIA_FATAL_ERR,
    135          RESULT_DETAIL("PRemoteCDMActor is not in same process."));
    136    }
    137 
    138    cdm = aCDM->AsPRemoteCDMChild();
    139    if (!cdm) {
    140      return MediaResult(
    141          NS_ERROR_DOM_MEDIA_FATAL_ERR,
    142          RESULT_DETAIL("PRemoteCDMActor is not PRemoteCDMChild."));
    143    }
    144  }
    145 
    146  mIPDLSelfRef = this;
    147  VideoDecoderInfoIPDL decoderInfo(aVideoInfo, aFramerate);
    148  MOZ_ALWAYS_TRUE(manager->SendPRemoteDecoderConstructor(
    149      this, decoderInfo, aOptions, aIdentifier, aMediaEngineId, aTrackingId,
    150      cdm));
    151 
    152  return NS_OK;
    153 }
    154 
    155 RemoteVideoDecoderParent::RemoteVideoDecoderParent(
    156    RemoteMediaManagerParent* aParent, const VideoInfo& aVideoInfo,
    157    float aFramerate, const CreateDecoderParams::OptionSet& aOptions,
    158    const Maybe<layers::TextureFactoryIdentifier>& aIdentifier,
    159    nsISerialEventTarget* aManagerThread, TaskQueue* aDecodeTaskQueue,
    160    const Maybe<uint64_t>& aMediaEngineId, Maybe<TrackingId> aTrackingId,
    161    RemoteCDMParent* aCDM)
    162    : RemoteDecoderParent(aParent, aOptions, aManagerThread, aDecodeTaskQueue,
    163                          aMediaEngineId, std::move(aTrackingId), aCDM),
    164      mVideoInfo(aVideoInfo),
    165      mFramerate(aFramerate) {
    166  if (aIdentifier) {
    167    // Check to see if we have a direct PVideoBridge connection to the
    168    // destination process specified in aIdentifier, and create a
    169    // KnowsCompositor representing that connection if so. If this fails, then
    170    // we fall back to returning the decoded frames directly via Output().
    171    mKnowsCompositor =
    172        KnowsCompositorVideo::TryCreateForIdentifier(*aIdentifier);
    173  }
    174 }
    175 
    176 IPCResult RemoteVideoDecoderParent::RecvConstruct(
    177    ConstructResolver&& aResolver) {
    178  auto imageContainer = MakeRefPtr<layers::ImageContainer>(
    179      layers::ImageUsageType::RemoteVideoDecoder,
    180      layers::ImageContainer::SYNCHRONOUS);
    181  if (mKnowsCompositor && XRE_IsRDDProcess()) {
    182    // Ensure to allocate recycle allocator
    183    imageContainer->EnsureRecycleAllocatorForRDD(mKnowsCompositor);
    184  }
    185  auto params = CreateDecoderParams{
    186      mVideoInfo,
    187      mKnowsCompositor,
    188      imageContainer,
    189      static_cast<PRemoteCDMActor*>(mCDM.get()),
    190      CreateDecoderParams::VideoFrameRate(mFramerate),
    191      mOptions,
    192      CreateDecoderParams::WrapperSet({/* No wrapper */}),
    193      mMediaEngineId,
    194      mTrackingId,
    195      mCDM,
    196  };
    197 
    198  mParent->EnsurePDMFactory().CreateDecoder(params)->Then(
    199      GetCurrentSerialEventTarget(), __func__,
    200      [resolver = std::move(aResolver), self = RefPtr{this}](
    201          PlatformDecoderModule::CreateDecoderPromise::ResolveOrRejectValue&&
    202              aValue) {
    203        if (aValue.IsReject()) {
    204          resolver(aValue.RejectValue());
    205          return;
    206        }
    207        MOZ_ASSERT(aValue.ResolveValue());
    208        self->mDecoder =
    209            new MediaDataDecoderProxy(aValue.ResolveValue().forget(),
    210                                      do_AddRef(self->mDecodeTaskQueue.get()));
    211        resolver(NS_OK);
    212      });
    213  return IPC_OK();
    214 }
    215 
    216 MediaResult RemoteVideoDecoderParent::ProcessDecodedData(
    217    MediaDataDecoder::DecodedData&& aData, DecodedOutputIPDL& aDecodedData) {
    218  MOZ_ASSERT(OnManagerThread());
    219 
    220  // If the video decoder bridge has shut down, stop.
    221  if (mKnowsCompositor && !mKnowsCompositor->GetTextureForwarder()) {
    222    aDecodedData = MakeRefPtr<ArrayOfRemoteVideoData>();
    223    return NS_OK;
    224  }
    225 
    226  nsTArray<RemoteVideoData> array;
    227 
    228  for (const auto& data : aData) {
    229    MOZ_ASSERT(data->mType == MediaData::Type::VIDEO_DATA ||
    230                   data->mType == MediaData::Type::NULL_DATA,
    231               "Can only decode videos using RemoteDecoderParent!");
    232    if (data->mType == MediaData::Type::NULL_DATA) {
    233      RemoteVideoData output(
    234          MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode,
    235                        data->mDuration, data->mKeyframe),
    236          IntSize(), RemoteImageHolder(), -1);
    237 
    238      array.AppendElement(std::move(output));
    239      continue;
    240    }
    241    VideoData* video = static_cast<VideoData*>(data.get());
    242 
    243    MOZ_ASSERT(video->mImage,
    244               "Decoded video must output a layer::Image to "
    245               "be used with RemoteDecoderParent");
    246 
    247    RefPtr<TextureClient> texture;
    248    SurfaceDescriptor sd;
    249    IntSize size;
    250    bool needStorage = false;
    251 
    252    YUVColorSpace YUVColorSpace = gfx::YUVColorSpace::Default;
    253    ColorSpace2 colorPrimaries = gfx::ColorSpace2::UNKNOWN;
    254    TransferFunction transferFunction = gfx::TransferFunction::BT709;
    255    ColorRange colorRange = gfx::ColorRange::LIMITED;
    256 
    257    if (mKnowsCompositor) {
    258      texture = video->mImage->GetTextureClient(mKnowsCompositor);
    259 
    260      if (!texture) {
    261        texture = ImageClient::CreateTextureClientForImage(video->mImage,
    262                                                           mKnowsCompositor);
    263      }
    264 
    265      if (texture) {
    266        if (!texture->IsAddedToCompositableClient()) {
    267          texture->InitIPDLActor(mKnowsCompositor, mParent->GetContentId());
    268          texture->SetAddedToCompositableClient();
    269        }
    270        needStorage = true;
    271        SurfaceDescriptorRemoteDecoder remoteSD;
    272        texture->GetSurfaceDescriptorRemoteDecoder(&remoteSD);
    273        sd = remoteSD;
    274        size = texture->GetSize();
    275      }
    276    }
    277 
    278    // If failed to create a GPU accelerated surface descriptor, fall back to
    279    // copying frames via shmem.
    280    if (!IsSurfaceDescriptorValid(sd)) {
    281      needStorage = false;
    282      PlanarYCbCrImage* image = video->mImage->AsPlanarYCbCrImage();
    283      if (!image) {
    284        return MediaResult(NS_ERROR_UNEXPECTED,
    285                           "Expected Planar YCbCr image in "
    286                           "RemoteVideoDecoderParent::ProcessDecodedData");
    287      }
    288      YUVColorSpace = image->GetData()->mYUVColorSpace;
    289      colorPrimaries = image->GetData()->mColorPrimaries;
    290      transferFunction = image->GetData()->mTransferFunction;
    291      colorRange = image->GetData()->mColorRange;
    292 
    293      SurfaceDescriptorBuffer sdBuffer;
    294      nsresult rv = image->BuildSurfaceDescriptorBuffer(
    295          sdBuffer, Image::BuildSdbFlags::Default, [&](uint32_t aBufferSize) {
    296            ShmemBuffer buffer = AllocateBuffer(aBufferSize);
    297            if (buffer.Valid()) {
    298              return MemoryOrShmem(std::move(buffer.Get()));
    299            }
    300            return MemoryOrShmem();
    301          });
    302 
    303      if (NS_WARN_IF(NS_FAILED(rv))) {
    304        if (sdBuffer.data().type() == MemoryOrShmem::TShmem) {
    305          DeallocShmem(sdBuffer.data().get_Shmem());
    306        }
    307        return rv;
    308      }
    309 
    310      sd = sdBuffer;
    311      size = image->GetSize();
    312    }
    313 
    314    if (needStorage) {
    315      MOZ_ASSERT(sd.type() != SurfaceDescriptor::TSurfaceDescriptorBuffer);
    316      mParent->StoreImage(static_cast<const SurfaceDescriptorGPUVideo&>(sd),
    317                          video->mImage, texture);
    318    }
    319 
    320    RemoteVideoData output(
    321        MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode,
    322                      data->mDuration, data->mKeyframe),
    323        video->mDisplay,
    324        RemoteImageHolder(
    325            mParent,
    326            XRE_IsGPUProcess()
    327                ? VideoBridgeSource::GpuProcess
    328                : (XRE_IsRDDProcess()
    329                       ? VideoBridgeSource::RddProcess
    330                       : VideoBridgeSource::MFMediaEngineCDMProcess),
    331            size, video->mImage->GetColorDepth(), sd, YUVColorSpace,
    332            colorPrimaries, transferFunction, colorRange),
    333        video->mFrameID);
    334 
    335    array.AppendElement(std::move(output));
    336  }
    337 
    338  aDecodedData = MakeRefPtr<ArrayOfRemoteVideoData>(std::move(array));
    339 
    340  return NS_OK;
    341 }
    342 
    343 }  // namespace mozilla