tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

VideoFrameContainer.cpp (9384B)

      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
      5 * You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "VideoFrameContainer.h"
      8 
      9 #include "mozilla/Logging.h"
     10 
     11 #ifdef MOZ_WIDGET_ANDROID
     12 #  include "GLImages.h"  // for SurfaceTextureImage
     13 #endif
     14 #include "MediaDecoderOwner.h"
     15 #include "mozilla/AbstractThread.h"
     16 #include "mozilla/gfx/BuildConstants.h"
     17 #include "mozilla/gfx/gfxVars.h"
     18 
     19 using namespace mozilla::layers;
     20 
     21 mozilla::LazyLogModule gVideoFrameContainer("VideoFrameContainer");
     22 
     23 namespace mozilla {
     24 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
     25 
     26 VideoFrameContainer::VideoFrameContainer(
     27    MediaDecoderOwner* aOwner, already_AddRefed<ImageContainer> aContainer)
     28    : mOwner(aOwner),
     29      mImageContainer(aContainer),
     30      mMutex("nsVideoFrameContainer"),
     31      mFrameID(0),
     32      mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE),
     33      mFrameIDForPendingPrincipalHandle(0),
     34      mMainThread(aOwner->AbstractMainThread()),
     35      mSupportsOnly8BitImage(kIsAndroid &&
     36                             !gfx::gfxVars::AllowGLNorm16Textures()) {
     37  NS_ASSERTION(aOwner, "aOwner must not be null");
     38  NS_ASSERTION(mImageContainer, "aContainer must not be null");
     39 }
     40 
     41 VideoFrameContainer::~VideoFrameContainer() = default;
     42 
     43 PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle() {
     44  MutexAutoLock lock(mMutex);
     45  return GetLastPrincipalHandleLocked();
     46 }
     47 
     48 PrincipalHandle VideoFrameContainer::GetLastPrincipalHandleLocked() {
     49  return mLastPrincipalHandle;
     50 }
     51 
     52 void VideoFrameContainer::UpdatePrincipalHandleForFrameID(
     53    const PrincipalHandle& aPrincipalHandle,
     54    const ImageContainer::FrameID& aFrameID) {
     55  MutexAutoLock lock(mMutex);
     56  UpdatePrincipalHandleForFrameIDLocked(aPrincipalHandle, aFrameID);
     57 }
     58 
     59 void VideoFrameContainer::UpdatePrincipalHandleForFrameIDLocked(
     60    const PrincipalHandle& aPrincipalHandle,
     61    const ImageContainer::FrameID& aFrameID) {
     62  if (mPendingPrincipalHandle == aPrincipalHandle) {
     63    return;
     64  }
     65  mPendingPrincipalHandle = aPrincipalHandle;
     66  mFrameIDForPendingPrincipalHandle = aFrameID;
     67 }
     68 
     69 #ifdef MOZ_WIDGET_ANDROID
     70 static void NotifySetCurrent(Image* aImage) {
     71  if (aImage) {
     72    MOZ_LOG_FMT(gVideoFrameContainer, LogLevel::Debug,
     73                "NotifySetCurrent, serial={}", aImage->GetSerial());
     74    aImage->OnSetCurrent();
     75  }
     76 }
     77 #endif
     78 
     79 void VideoFrameContainer::SetCurrentFrame(
     80    const gfx::IntSize& aIntrinsicSize, Image* aImage,
     81    const TimeStamp& aTargetTime, const media::TimeUnit& aProcessingDuration,
     82    const media::TimeUnit& aMediaTime) {
     83  MOZ_LOG_FMT(
     84      gVideoFrameContainer, LogLevel::Debug,
     85      "SetCurrentFrame, processing duration={}us,pts={}",
     86      aProcessingDuration.IsValid() ? aProcessingDuration.ToMicroseconds() : -1,
     87      aMediaTime.ToString());
     88 #ifdef MOZ_WIDGET_ANDROID
     89  NotifySetCurrent(aImage);
     90 #endif
     91  AutoTArray<ImageContainer::NonOwningImage, 1> imageList;
     92  if (aImage) {
     93    imageList.AppendElement(ImageContainer::NonOwningImage(
     94        aImage, aTargetTime, ++mFrameID, 0, aProcessingDuration, aMediaTime));
     95  }
     96  MutexAutoLock lock(mMutex);
     97  SetCurrentFramesLocked(aIntrinsicSize, imageList);
     98 }
     99 
    100 void VideoFrameContainer::SetCurrentFrames(
    101    const gfx::IntSize& aIntrinsicSize,
    102    const nsTArray<ImageContainer::NonOwningImage>& aImages) {
    103  MOZ_LOG_FMT(gVideoFrameContainer, LogLevel::Debug,
    104              "SetCurrentFrames ({} images)", aImages.Length());
    105 #ifdef MOZ_WIDGET_ANDROID
    106  // When there are multiple frames, only the last one is effective
    107  // (see bug 1299068 comment 4). Here I just count on VideoSink and VideoOutput
    108  // to send one frame at a time and warn if not.
    109  (void)NS_WARN_IF(aImages.Length() > 1);
    110  for (auto& image : aImages) {
    111    NotifySetCurrent(image.mImage);
    112  }
    113 #endif
    114  MutexAutoLock lock(mMutex);
    115  SetCurrentFramesLocked(aIntrinsicSize, aImages);
    116 }
    117 
    118 #ifdef DEBUG
    119 static bool Is8BitImage(const ImageContainer::NonOwningImage& aFrame) {
    120  return aFrame.mImage->GetColorDepth() == gfx::ColorDepth::COLOR_8;
    121 }
    122 #endif
    123 
    124 void VideoFrameContainer::SetCurrentFramesLocked(
    125    const gfx::IntSize& aIntrinsicSize,
    126    const nsTArray<ImageContainer::NonOwningImage>& aImages) {
    127  MOZ_LOG_FMT(gVideoFrameContainer, LogLevel::Debug,
    128              "SetCurrentFramesLocked ({} images", aImages.Length());
    129  mMutex.AssertCurrentThreadOwns();
    130 
    131  MOZ_ASSERT(!SupportsOnly8BitImage() ||
    132                 std::all_of(aImages.begin(), aImages.end(), Is8BitImage),
    133             "Images should be 8-bit");
    134 
    135  if (auto size = Some(aIntrinsicSize); size != mIntrinsicSize) {
    136    mIntrinsicSize = size;
    137    mMainThread->Dispatch(NS_NewRunnableFunction(
    138        "IntrinsicSizeChanged", [this, self = RefPtr(this), size]() {
    139          mMainThreadState.mNewIntrinsicSize = size;
    140        }));
    141  }
    142 
    143  gfx::IntSize oldFrameSize = mImageContainer->GetCurrentSize();
    144 
    145  // When using the OMX decoder, destruction of the current image can indirectly
    146  //  block on main thread I/O. If we let this happen while holding onto
    147  //  |mImageContainer|'s lock, then when the main thread then tries to
    148  //  composite it can then block on |mImageContainer|'s lock, causing a
    149  //  deadlock. We use this hack to defer the destruction of the current image
    150  //  until it is safe.
    151  nsTArray<ImageContainer::OwningImage> oldImages;
    152  mImageContainer->GetCurrentImages(&oldImages);
    153 
    154  PrincipalHandle principalHandle = PRINCIPAL_HANDLE_NONE;
    155  if (mPendingPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
    156      (aImages.IsEmpty() ||
    157       aImages[0].mFrameID >= mFrameIDForPendingPrincipalHandle)) {
    158    // There are no FrameIDs prior to `mFrameIDForPendingPrincipalHandle`
    159    // in the new set of images.
    160    // This means that the old principal handle has been flushed out and we
    161    // can notify our video element about this change.
    162    principalHandle = mPendingPrincipalHandle;
    163    mLastPrincipalHandle = mPendingPrincipalHandle;
    164    mPendingPrincipalHandle = PRINCIPAL_HANDLE_NONE;
    165    mFrameIDForPendingPrincipalHandle = 0;
    166  }
    167 
    168  if (aImages.IsEmpty()) {
    169    mImageContainer->ClearImagesInHost(layers::ClearImagesType::All);
    170  } else {
    171    mImageContainer->SetCurrentImages(aImages);
    172  }
    173  gfx::IntSize newFrameSize = mImageContainer->GetCurrentSize();
    174  bool imageSizeChanged = (oldFrameSize != newFrameSize);
    175 
    176  if (principalHandle != PRINCIPAL_HANDLE_NONE || imageSizeChanged) {
    177    RefPtr<VideoFrameContainer> self = this;
    178    mMainThread->Dispatch(NS_NewRunnableFunction(
    179        "PrincipalHandleOrImageSizeChanged",
    180        [this, self, principalHandle, imageSizeChanged]() {
    181          mMainThreadState.mImageSizeChanged = imageSizeChanged;
    182          if (mOwner && principalHandle != PRINCIPAL_HANDLE_NONE) {
    183            mOwner->PrincipalHandleChangedForVideoFrameContainer(
    184                this, principalHandle);
    185          }
    186        }));
    187  }
    188 }
    189 
    190 void VideoFrameContainer::ClearFutureFrames(TimeStamp aNow) {
    191  MutexAutoLock lock(mMutex);
    192 
    193  MOZ_LOG_FMT(gVideoFrameContainer, LogLevel::Debug, "ClearFutureFrame");
    194  // See comment in SetCurrentFrame for the reasoning behind
    195  // using a kungFuDeathGrip here.
    196  AutoTArray<ImageContainer::OwningImage, 10> kungFuDeathGrip;
    197  mImageContainer->GetCurrentImages(&kungFuDeathGrip);
    198 
    199  if (!kungFuDeathGrip.IsEmpty()) {
    200    AutoTArray<ImageContainer::NonOwningImage, 1> currentFrame;
    201    const ImageContainer::OwningImage* img = &kungFuDeathGrip[0];
    202    // Find the current image in case there are several.
    203    for (const auto& image : kungFuDeathGrip) {
    204      if (image.mTimeStamp > aNow) {
    205        break;
    206      }
    207      img = &image;
    208    }
    209    currentFrame.AppendElement(ImageContainer::NonOwningImage(
    210        img->mImage, img->mTimeStamp, img->mFrameID, img->mProducerID,
    211        img->mProcessingDuration, img->mMediaTime, img->mWebrtcCaptureTime,
    212        img->mWebrtcReceiveTime, img->mRtpTimestamp));
    213    mImageContainer->SetCurrentImages(currentFrame);
    214  }
    215 }
    216 
    217 void VideoFrameContainer::ClearCachedResources() {
    218  MutexAutoLock lock(mMutex);
    219  mImageContainer->ClearCachedResources();
    220 }
    221 
    222 void VideoFrameContainer::ClearImagesInHost(layers::ClearImagesType aType) {
    223  MutexAutoLock lock(mMutex);
    224  mImageContainer->ClearImagesInHost(aType);
    225 }
    226 
    227 ImageContainer* VideoFrameContainer::GetImageContainer() {
    228  // Note - you'll need the lock to manipulate this.  The pointer is not
    229  // modified from multiple threads, just the data pointed to by it.
    230  return mImageContainer;
    231 }
    232 
    233 double VideoFrameContainer::GetFrameDelay() {
    234  MutexAutoLock lock(mMutex);
    235  return mImageContainer->GetPaintDelay().ToSeconds();
    236 }
    237 
    238 void VideoFrameContainer::InvalidateWithFlags(uint32_t aFlags) {
    239  NS_ASSERTION(NS_IsMainThread(), "Must call on main thread");
    240 
    241  if (!mOwner) {
    242    // Owner has been destroyed
    243    return;
    244  }
    245 
    246  MediaDecoderOwner::ImageSizeChanged imageSizeChanged{
    247      mMainThreadState.mImageSizeChanged};
    248  mMainThreadState.mImageSizeChanged = false;
    249 
    250  auto newIntrinsicSize = std::move(mMainThreadState.mNewIntrinsicSize);
    251 
    252  MediaDecoderOwner::ForceInvalidate forceInvalidate{
    253      (aFlags & INVALIDATE_FORCE) != 0};
    254  mOwner->Invalidate(imageSizeChanged, newIntrinsicSize, forceInvalidate);
    255 }
    256 
    257 }  // namespace mozilla
    258 
    259 #undef NS_DispatchToMainThread