tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

VideoFrame.h (10389B)

      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #ifndef mozilla_dom_VideoFrame_h
      8 #define mozilla_dom_VideoFrame_h
      9 
     10 #include "js/TypeDecls.h"
     11 #include "mozilla/ErrorResult.h"
     12 #include "mozilla/Span.h"
     13 #include "mozilla/WeakPtr.h"
     14 #include "mozilla/dom/BindingDeclarations.h"
     15 #include "mozilla/dom/BufferSourceBindingFwd.h"
     16 #include "mozilla/dom/TypedArray.h"
     17 #include "mozilla/dom/VideoColorSpaceBinding.h"
     18 #include "mozilla/dom/WebCodecsUtils.h"
     19 #include "mozilla/gfx/Point.h"
     20 #include "mozilla/gfx/Rect.h"
     21 #include "mozilla/media/MediaUtils.h"
     22 #include "nsCycleCollectionParticipant.h"
     23 #include "nsTArrayForwardDeclare.h"
     24 #include "nsWrapperCache.h"
     25 
     26 class nsIGlobalObject;
     27 class nsIURI;
     28 
     29 namespace mozilla {
     30 
     31 namespace layers {
     32 class Image;
     33 }  // namespace layers
     34 
     35 namespace dom {
     36 
     37 class DOMRectReadOnly;
     38 class HTMLCanvasElement;
     39 class HTMLImageElement;
     40 class HTMLVideoElement;
     41 class ImageBitmap;
     42 class OffscreenCanvas;
     43 class Promise;
     44 class SVGImageElement;
     45 class StructuredCloneHolder;
     46 class VideoColorSpace;
     47 class VideoFrame;
     48 enum class PredefinedColorSpace : uint8_t;
     49 enum class VideoPixelFormat : uint8_t;
     50 struct VideoFrameBufferInit;
     51 struct VideoFrameCopyToOptions;
     52 struct VideoFrameInit;
     53 
     54 }  // namespace dom
     55 
     56 namespace webgpu {
     57 class ExternalTexture;
     58 }  // namespace webgpu
     59 
     60 }  // namespace mozilla
     61 
     62 namespace mozilla::dom {
     63 
     64 struct VideoFrameData {
     65  VideoFrameData(layers::Image* aImage, const Maybe<VideoPixelFormat>& aFormat,
     66                 gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
     67                 Maybe<uint64_t> aDuration, int64_t aTimestamp,
     68                 const VideoColorSpaceInternal& aColorSpace);
     69  VideoFrameData(const VideoFrameData& aData) = default;
     70 
     71  const RefPtr<layers::Image> mImage;
     72  const Maybe<VideoPixelFormat> mFormat;
     73  const gfx::IntRect mVisibleRect;
     74  const gfx::IntSize mDisplaySize;
     75  const Maybe<uint64_t> mDuration;
     76  const int64_t mTimestamp;
     77  const VideoColorSpaceInternal mColorSpace;
     78 };
     79 
     80 struct VideoFrameSerializedData : VideoFrameData {
     81  VideoFrameSerializedData(const VideoFrameData& aData,
     82                           gfx::IntSize aCodedSize);
     83 
     84  const gfx::IntSize mCodedSize;
     85 };
     86 
     87 class VideoFrame final : public nsISupports,
     88                         public nsWrapperCache,
     89                         public media::ShutdownConsumer {
     90 public:
     91  NS_DECL_CYCLE_COLLECTING_ISUPPORTS
     92  NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame)
     93 
     94 public:
     95  VideoFrame(nsIGlobalObject* aParent, const RefPtr<layers::Image>& aImage,
     96             const Maybe<VideoPixelFormat>& aFormat, gfx::IntSize aCodedSize,
     97             gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
     98             const Maybe<uint64_t>& aDuration, int64_t aTimestamp,
     99             const VideoColorSpaceInternal& aColorSpace);
    100  VideoFrame(nsIGlobalObject* aParent, const VideoFrameSerializedData& aData);
    101  VideoFrame(const VideoFrame& aOther);
    102 
    103 protected:
    104  ~VideoFrame();
    105 
    106 public:
    107  nsIGlobalObject* GetParentObject() const;
    108 
    109  JSObject* WrapObject(JSContext* aCx,
    110                       JS::Handle<JSObject*> aGivenProto) override;
    111 
    112  static bool PrefEnabled(JSContext* aCx, JSObject* aObj = nullptr);
    113 
    114  static already_AddRefed<VideoFrame> Constructor(
    115      const GlobalObject& aGlobal, HTMLImageElement& aImageElement,
    116      const VideoFrameInit& aInit, ErrorResult& aRv);
    117  static already_AddRefed<VideoFrame> Constructor(
    118      const GlobalObject& aGlobal, SVGImageElement& aSVGImageElement,
    119      const VideoFrameInit& aInit, ErrorResult& aRv);
    120  static already_AddRefed<VideoFrame> Constructor(
    121      const GlobalObject& aGlobal, HTMLCanvasElement& aCanvasElement,
    122      const VideoFrameInit& aInit, ErrorResult& aRv);
    123  static already_AddRefed<VideoFrame> Constructor(
    124      const GlobalObject& aGlobal, HTMLVideoElement& aVideoElement,
    125      const VideoFrameInit& aInit, ErrorResult& aRv);
    126  static already_AddRefed<VideoFrame> Constructor(
    127      const GlobalObject& aGlobal, OffscreenCanvas& aOffscreenCanvas,
    128      const VideoFrameInit& aInit, ErrorResult& aRv);
    129  static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal,
    130                                                  ImageBitmap& aImageBitmap,
    131                                                  const VideoFrameInit& aInit,
    132                                                  ErrorResult& aRv);
    133  static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal,
    134                                                  VideoFrame& aVideoFrame,
    135                                                  const VideoFrameInit& aInit,
    136                                                  ErrorResult& aRv);
    137  static already_AddRefed<VideoFrame> Constructor(
    138      const GlobalObject& aGlobal, const ArrayBufferView& aBufferView,
    139      const VideoFrameBufferInit& aInit, ErrorResult& aRv);
    140  static already_AddRefed<VideoFrame> Constructor(
    141      const GlobalObject& aGlobal, const ArrayBuffer& aBuffer,
    142      const VideoFrameBufferInit& aInit, ErrorResult& aRv);
    143 
    144  Nullable<VideoPixelFormat> GetFormat() const;
    145 
    146  uint32_t CodedWidth() const;
    147 
    148  uint32_t CodedHeight() const;
    149 
    150  already_AddRefed<DOMRectReadOnly> GetCodedRect() const;
    151 
    152  already_AddRefed<DOMRectReadOnly> GetVisibleRect() const;
    153 
    154  uint32_t DisplayWidth() const;
    155 
    156  uint32_t DisplayHeight() const;
    157 
    158  Nullable<uint64_t> GetDuration() const;
    159 
    160  int64_t Timestamp() const;
    161 
    162  already_AddRefed<VideoColorSpace> ColorSpace() const;
    163 
    164  uint32_t AllocationSize(const VideoFrameCopyToOptions& aOptions,
    165                          ErrorResult& aRv);
    166 
    167  already_AddRefed<Promise> CopyTo(const AllowSharedBufferSource& aDestination,
    168                                   const VideoFrameCopyToOptions& aOptions,
    169                                   ErrorResult& aRv);
    170 
    171  already_AddRefed<VideoFrame> Clone(ErrorResult& aRv) const;
    172 
    173  void Close();
    174  bool IsClosed() const;
    175  void OnShutdown() override;
    176 
    177  // [Serializable] implementations: {Read, Write}StructuredClone
    178  static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
    179                                       JSStructuredCloneReader* aReader,
    180                                       const VideoFrameSerializedData& aData);
    181 
    182  bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
    183                            StructuredCloneHolder* aHolder) const;
    184 
    185  // [Transferable] implementations: Transfer, FromTransferred
    186  using TransferredData = VideoFrameSerializedData;
    187 
    188  UniquePtr<TransferredData> Transfer();
    189 
    190  static already_AddRefed<VideoFrame> FromTransferred(nsIGlobalObject* aGlobal,
    191                                                      TransferredData* aData);
    192 
    193  // Native only methods.
    194  const gfx::IntSize& NativeCodedSize() const { return mCodedSize; }
    195  const gfx::IntSize& NativeDisplaySize() const { return mDisplaySize; }
    196  const gfx::IntRect& NativeVisibleRect() const { return mVisibleRect; }
    197  already_AddRefed<layers::Image> GetImage() const;
    198 
    199  // Track a WebGPU ExternalTexture as being imported from this video frame.
    200  // This ensures it will be correctly expired when the video frame is closed.
    201  void TrackWebGPUExternalTexture(
    202      WeakPtr<webgpu::ExternalTexture> aExternalTexture);
    203 
    204  nsCString ToString() const;
    205 
    206 public:
    207  // A VideoPixelFormat wrapper providing utilities for VideoFrame.
    208  class Format final {
    209   public:
    210    explicit Format(const VideoPixelFormat& aFormat);
    211    ~Format() = default;
    212    const VideoPixelFormat& PixelFormat() const;
    213    gfx::SurfaceFormat ToSurfaceFormat() const;
    214    void MakeOpaque();
    215 
    216    // TODO: Assign unique value for each plane?
    217    // The value indicates the order of the plane in format.
    218    enum class Plane : uint8_t { Y = 0, RGBA = Y, U = 1, UV = U, V = 2, A = 3 };
    219    nsTArray<Plane> Planes() const;
    220    const char* PlaneName(const Plane& aPlane) const;
    221    uint32_t SampleBytes(const Plane& aPlane) const;
    222    gfx::IntSize SampleSize(const Plane& aPlane) const;
    223    bool IsValidSize(const gfx::IntSize& aSize) const;
    224    size_t ByteCount(const gfx::IntSize& aSize) const;
    225 
    226   private:
    227    bool IsYUV() const;
    228    VideoPixelFormat mFormat;
    229  };
    230 
    231 private:
    232  // VideoFrame can run on either main thread or worker thread.
    233  void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(VideoFrame); }
    234 
    235  already_AddRefed<VideoFrame> ConvertToRGBFrame(
    236      const VideoPixelFormat& aFormat, const PredefinedColorSpace& aColorSpace);
    237 
    238  VideoFrameData GetVideoFrameData() const;
    239 
    240  // Below helpers are used to automatically release the holding Resource if
    241  // VideoFrame is never Close()d by the users.
    242  void StartAutoClose();
    243  void StopAutoClose();
    244  void CloseIfNeeded();
    245 
    246  // A class representing the VideoFrame's data.
    247  class Resource final {
    248   public:
    249    Resource(const RefPtr<layers::Image>& aImage, Maybe<Format>&& aFormat);
    250    Resource(const Resource& aOther);
    251    ~Resource() = default;
    252    Maybe<VideoPixelFormat> TryPixelFormat() const;
    253    uint32_t Stride(const Format::Plane& aPlane) const;
    254    bool CopyTo(const Format::Plane& aPlane, const gfx::IntRect& aRect,
    255                Span<uint8_t>&& aPlaneDest, size_t aDestinationStride) const;
    256 
    257    const RefPtr<layers::Image> mImage;
    258    // Nothing() if mImage is not in VideoPixelFormat
    259    const Maybe<Format> mFormat;
    260  };
    261 
    262  nsCOMPtr<nsIGlobalObject> mParent;
    263 
    264  // Use Maybe instead of UniquePtr to allow copy ctor.
    265  // The mResource's existence is used as the [[Detached]] for [Transferable].
    266  Maybe<const Resource> mResource;  // Nothing() after `Close()`d
    267 
    268  // TODO: Replace this by mResource->mImage->GetSize()?
    269  gfx::IntSize mCodedSize;
    270  gfx::IntRect mVisibleRect;
    271  gfx::IntSize mDisplaySize;
    272 
    273  Maybe<uint64_t> mDuration;
    274  int64_t mTimestamp;
    275  VideoColorSpaceInternal mColorSpace;
    276 
    277  // The following are used to help monitoring mResource release.
    278  RefPtr<media::ShutdownWatcher> mShutdownWatcher = nullptr;
    279 
    280  // WebGPU external textures that were imported from this video frame. We must
    281  // call `Expire()` on them when the video frame is closed.
    282  nsTArray<WeakPtr<webgpu::ExternalTexture>> mWebGPUExternalTextures;
    283 };
    284 
    285 }  // namespace mozilla::dom
    286 
    287 #endif  // mozilla_dom_VideoFrame_h