tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

EncoderTemplate.h (12347B)

      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #ifndef mozilla_dom_EncoderTemplate_h
      8 #define mozilla_dom_EncoderTemplate_h
      9 
     10 #include <queue>
     11 
     12 #include "EncoderAgent.h"
     13 #include "MediaData.h"
     14 #include "SimpleMap.h"
     15 #include "WebCodecsUtils.h"
     16 #include "mozilla/DOMEventTargetHelper.h"
     17 #include "mozilla/MozPromise.h"
     18 #include "mozilla/RefPtr.h"
     19 #include "mozilla/Result.h"
     20 #include "mozilla/UniquePtr.h"
     21 #include "mozilla/dom/AudioEncoderBinding.h"
     22 #include "mozilla/dom/VideoEncoderBinding.h"
     23 #include "mozilla/dom/WorkerRef.h"
     24 #include "mozilla/media/MediaUtils.h"
     25 #include "nsStringFwd.h"
     26 
     27 namespace mozilla::dom {
     28 
     29 class WebCodecsErrorCallback;
     30 class Promise;
     31 enum class CodecState : uint8_t;
     32 
     33 using Id = size_t;
     34 
     35 template <typename EncoderType>
     36 class EncoderTemplate : public DOMEventTargetHelper {
     37  using Self = EncoderTemplate<EncoderType>;
     38  using ConfigType = typename EncoderType::ConfigType;
     39  using ConfigTypeInternal = typename EncoderType::ConfigTypeInternal;
     40  using OutputConfigType = typename EncoderType::OutputConfigType;
     41  using InputType = typename EncoderType::InputType;
     42  using InputTypeInternal = typename EncoderType::InputTypeInternal;
     43  using OutputType = typename EncoderType::OutputType;
     44  using OutputCallbackType = typename EncoderType::OutputCallbackType;
     45 
     46  /* ControlMessage classes */
     47 protected:
     48  class ConfigureMessage;
     49  class EncodeMessage;
     50  class FlushMessage;
     51 
     52  class ControlMessage {
     53   public:
     54    NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ControlMessage)
     55    explicit ControlMessage(Id aConfigureId);
     56    virtual void Cancel() = 0;
     57    virtual bool IsProcessing() = 0;
     58 
     59    virtual nsCString ToString() const = 0;
     60    virtual RefPtr<ConfigureMessage> AsConfigureMessage() { return nullptr; }
     61    virtual RefPtr<EncodeMessage> AsEncodeMessage() { return nullptr; }
     62    virtual RefPtr<FlushMessage> AsFlushMessage() { return nullptr; }
     63 
     64    // For logging purposes
     65    const WebCodecsId mConfigureId;
     66    const WebCodecsId mMessageId;
     67 
     68   protected:
     69    virtual ~ControlMessage() = default;
     70  };
     71 
     72  class ConfigureMessage final
     73      : public ControlMessage,
     74        public MessageRequestHolder<EncoderAgent::ConfigurePromise> {
     75   public:
     76    ConfigureMessage(Id aConfigureId,
     77                     const RefPtr<ConfigTypeInternal>& aConfig);
     78    virtual void Cancel() override { Disconnect(); }
     79    virtual bool IsProcessing() override { return Exists(); };
     80    virtual RefPtr<ConfigureMessage> AsConfigureMessage() override {
     81      return this;
     82    }
     83    RefPtr<ConfigTypeInternal> Config() { return mConfig; }
     84    nsCString ToString() const override {
     85      nsCString rv;
     86      rv.AppendPrintf("ConfigureMessage(#%zu): %s", this->mMessageId,
     87                      mConfig ? mConfig->ToString().get() : "null cfg");
     88      return rv;
     89    }
     90 
     91   private:
     92    const RefPtr<ConfigTypeInternal> mConfig;
     93  };
     94 
     95  // This stores batch encoding requests, created by grouping multiple encode()
     96  // calls. See PushEncodeRequest() for further details.
     97  class EncodeMessage final
     98      : public ControlMessage,
     99        public MessageRequestHolder<EncoderAgent::EncodePromise> {
    100   public:
    101    EncodeMessage(WebCodecsId aConfigureId,
    102                  already_AddRefed<InputTypeInternal> aData,
    103                  Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing());
    104    nsCString ToString() const override {
    105      nsCString rv;
    106      rv.AppendPrintf(
    107          "EncodeMessage(#%zu, #%zu): %zu frames (%zu kfs, %zu held)",
    108          this->mConfigureId, this->mMessageId, mFrames, mKeyFrames,
    109          mData.Length());
    110      return rv;
    111    }
    112    bool IsValid() const { return !mHasEmptyData && !mData.IsEmpty(); }
    113    size_t BatchSize() const { return mData.Length(); }
    114    void PushData(already_AddRefed<InputTypeInternal> aData,
    115                  Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing()) {
    116      mFrames += 1;
    117      RefPtr<InputTypeInternal> data = aData;
    118      if (!data) {
    119        mHasEmptyData = true;
    120      }
    121      MOZ_ASSERT_IF(aOptions.isSome() && aOptions->mKeyFrame, data->mKeyframe);
    122      mKeyFrames += data->mKeyframe ? 1 : 0;
    123      mData.AppendElement(data.forget());
    124    }
    125    nsTArray<RefPtr<MediaData>>&& TakeData() { return std::move(mData); }
    126    virtual void Cancel() override { Disconnect(); }
    127    virtual bool IsProcessing() override { return Exists(); };
    128    virtual RefPtr<EncodeMessage> AsEncodeMessage() override { return this; }
    129 
    130   private:
    131    // Stores data in MediaData rather than InputTypeInternal, as
    132    // MediaDataEncoder::EncodeBatch expects an array of MediaData.
    133    nsTArray<RefPtr<MediaData>> mData;
    134    size_t mFrames = 0;
    135    size_t mKeyFrames = 0;
    136    bool mHasEmptyData = false;
    137  };
    138 
    139  class FlushMessage final
    140      : public ControlMessage,
    141        public MessageRequestHolder<EncoderAgent::EncodePromise> {
    142   public:
    143    explicit FlushMessage(WebCodecsId aConfigureId);
    144    virtual void Cancel() override { Disconnect(); }
    145    virtual bool IsProcessing() override { return Exists(); };
    146    virtual RefPtr<FlushMessage> AsFlushMessage() override { return this; }
    147 
    148    nsCString ToString() const override {
    149      nsCString rv;
    150      rv.AppendPrintf("FlushMessage(#%zu, #%zu)", this->mConfigureId,
    151                      this->mMessageId);
    152      return rv;
    153    }
    154  };
    155 
    156 protected:
    157  EncoderTemplate(nsIGlobalObject* aGlobalObject,
    158                  RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
    159                  RefPtr<OutputCallbackType>&& aOutputCallback);
    160 
    161  virtual ~EncoderTemplate() = default;
    162 
    163  /* WebCodecs interfaces */
    164 public:
    165  IMPL_EVENT_HANDLER(dequeue)
    166 
    167  void StartBlockingMessageQueue();
    168  void StopBlockingMessageQueue();
    169 
    170  MOZ_CAN_RUN_SCRIPT
    171  void OutputEncodedData(const nsTArray<RefPtr<MediaRawData>>&& aData);
    172 
    173  CodecState State() const { return mState; };
    174 
    175  uint32_t EncodeQueueSize() const { return mEncodeQueueSize; };
    176 
    177  MOZ_CAN_RUN_SCRIPT
    178  void Configure(const ConfigType& aConfig, ErrorResult& aRv);
    179 
    180  void EncodeAudioData(InputType& aInput, ErrorResult& aRv);
    181  void EncodeVideoFrame(InputType& aInput,
    182                        const VideoEncoderEncodeOptions& aOptions,
    183                        ErrorResult& aRv);
    184 
    185  already_AddRefed<Promise> Flush(ErrorResult& aRv);
    186 
    187  void Reset(ErrorResult& aRv);
    188 
    189  MOZ_CAN_RUN_SCRIPT
    190  void Close(ErrorResult& aRv);
    191 
    192  /* Type conversion functions for the Encoder implementation */
    193 protected:
    194  virtual RefPtr<OutputType> EncodedDataToOutputType(
    195      nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData) = 0;
    196  virtual void EncoderConfigToDecoderConfig(
    197      JSContext* aCx, const RefPtr<MediaRawData>& aData,
    198      const ConfigTypeInternal& aSrcConfig,
    199      OutputConfigType& aDestConfig) const = 0;
    200 
    201  /* Internal member variables and functions */
    202 protected:
    203  // EncoderTemplate can run on either main thread or worker thread.
    204  void AssertIsOnOwningThread() const {
    205    NS_ASSERT_OWNINGTHREAD(EncoderTemplate);
    206  }
    207 
    208  Result<Ok, nsresult> ResetInternal(const nsresult& aResult);
    209  MOZ_CAN_RUN_SCRIPT
    210  Result<Ok, nsresult> CloseInternalWithAbort();
    211  MOZ_CAN_RUN_SCRIPT
    212  void CloseInternal(const nsresult& aResult);
    213 
    214  MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult);
    215 
    216  MOZ_CAN_RUN_SCRIPT void OutputEncodedVideoData(
    217      const nsTArray<RefPtr<MediaRawData>>&& aData);
    218  MOZ_CAN_RUN_SCRIPT void OutputEncodedAudioData(
    219      const nsTArray<RefPtr<MediaRawData>>&& aData);
    220 
    221  void ScheduleDequeueEvent();
    222  nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType);
    223 
    224  void SchedulePromiseResolveOrReject(already_AddRefed<Promise> aPromise,
    225                                      const nsresult& aResult);
    226 
    227  void ProcessControlMessageQueue();
    228  void CancelPendingControlMessagesAndFlushPromises(const nsresult& aResult);
    229 
    230  template <typename Func>
    231  void QueueATask(const char* aName, Func&& aSteps);
    232 
    233  MessageProcessedResult ProcessConfigureMessage(
    234      RefPtr<ConfigureMessage> aMessage);
    235 
    236  MessageProcessedResult ProcessEncodeMessage(RefPtr<EncodeMessage> aMessage);
    237 
    238  MessageProcessedResult ProcessFlushMessage(RefPtr<FlushMessage> aMessage);
    239 
    240  void Configure(RefPtr<ConfigureMessage> aMessage);
    241  void Reconfigure(RefPtr<ConfigureMessage> aMessage);
    242 
    243  // Returns true when mAgent can be created.
    244  bool CreateEncoderAgent(WebCodecsId aId, RefPtr<ConfigTypeInternal> aConfig);
    245  void DestroyEncoderAgentIfAny();
    246 
    247  void PushEncodeRequest(
    248      WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
    249      Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing());
    250 
    251  // Constant in practice, only set in ctor.
    252  RefPtr<WebCodecsErrorCallback> mErrorCallback;
    253  RefPtr<OutputCallbackType> mOutputCallback;
    254 
    255  CodecState mState;
    256 
    257  bool mMessageQueueBlocked;
    258  std::queue<RefPtr<ControlMessage>> mControlMessageQueue;
    259  RefPtr<ControlMessage> mProcessingMessage;
    260 
    261  // When a flush request is initiated, a promise is created and stored in
    262  // mPendingFlushPromises until it is settled in the task delivering the flush
    263  // result or Reset() is called before the promise is settled.
    264  SimpleMap<int64_t, RefPtr<Promise>> mPendingFlushPromises;
    265 
    266  uint32_t mEncodeQueueSize;
    267  bool mDequeueEventScheduled;
    268 
    269  // A unique id tracking the ConfigureMessage and will be used as the
    270  // EncoderAgent's Id.
    271  uint32_t mLatestConfigureId;
    272  // Tracking how many encoded data has been enqueued and this number will be
    273  // used as the EncodeMessage's Id.
    274  size_t mEncodeCounter;
    275  // Tracking how many flush request has been enqueued and this number will be
    276  // used as the FlushMessage's Id.
    277  size_t mFlushCounter;
    278 
    279  // EncoderAgent will be created the first time "configure" is being
    280  // processed, and will be destroyed when "reset" is called. If another
    281  // "configure" is called, either it's possible to reconfigure the underlying
    282  // encoder without tearing everything down (e.g. a bitrate change), or it's
    283  // not possible, and the current encoder will be destroyed and a new one
    284  // create. In both cases, the encoder is implicitely flushed before the
    285  // configuration change. See CanReconfigure on the
    286  // {Audio,Video}EncoderConfigInternal
    287  RefPtr<EncoderAgent> mAgent;
    288  RefPtr<ConfigTypeInternal> mActiveConfig;
    289  // This is true when a configure call has just been processed, and it's
    290  // necessary to pass the new decoding configuration when the callback is
    291  // called. Read and modified on owner thread only.
    292  bool mOutputNewDecoderConfig = false;
    293 
    294  // Used to add a nsIAsyncShutdownBlocker on main thread to block
    295  // xpcom-shutdown before the underlying MediaDataEncoder is created. The
    296  // blocker will be held until the underlying MediaDataEncoder has been shut
    297  // down. This blocker guarantees RemoteEncoderManagerChild's thread, where
    298  // the underlying RemoteMediaDataEncoder is on, outlives the
    299  // RemoteMediaDataEncoder since the thread releasing, which happens on main
    300  // thread when getting a xpcom-shutdown signal, is blocked by the added
    301  // blocker. As a result, RemoteMediaDataEncoder can safely work on worker
    302  // thread with a holding blocker (otherwise, if RemoteEncoderManagerChild
    303  // releases its thread on main thread before RemoteMediaDataEncoder's
    304  // Shutdown() task run on worker thread, RemoteMediaDataEncoder has no
    305  // thread to run).
    306  UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker;
    307 
    308  // Held to make sure the dispatched tasks can be done before worker is going
    309  // away. As long as this worker-ref is held somewhere, the tasks dispatched
    310  // to the worker can be executed (otherwise the tasks would be canceled).
    311  // This ref should be activated as long as the underlying MediaDataEncoder
    312  // is alive, and should keep alive until mShutdownBlocker is dropped, so all
    313  // MediaDataEncoder's tasks and mShutdownBlocker-releasing task can be
    314  // executed.
    315  // TODO: Use StrongWorkerRef instead if this is always used in the same
    316  // thread?
    317  RefPtr<ThreadSafeWorkerRef> mWorkerRef;
    318  uint64_t mPacketsOutput = 0;
    319 
    320  AsyncDurationTracker mAsyncDurationTracker;
    321 };
    322 
    323 }  // namespace mozilla::dom
    324 
    325 #endif  // mozilla_dom_EncoderTemplate_h