tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

WebrtcGmpVideoCodec.h (17324B)

      1 /*
      2 *  Copyright (c) 2012, The WebRTC project authors. All rights reserved.
      3 *
      4 *  Redistribution and use in source and binary forms, with or without
      5 *  modification, are permitted provided that the following conditions are
      6 *  met:
      7 *
      8 *    * Redistributions of source code must retain the above copyright
      9 *      notice, this list of conditions and the following disclaimer.
     10 *
     11 *    * Redistributions in binary form must reproduce the above copyright
     12 *      notice, this list of conditions and the following disclaimer in
     13 *      the documentation and/or other materials provided with the
     14 *      distribution.
     15 *
     16 *    * Neither the name of Google nor the names of its contributors may
     17 *      be used to endorse or promote products derived from this software
     18 *      without specific prior written permission.
     19 *
     20 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     21 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     22 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     23 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     24 *  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     25 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     26 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     27 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     28 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     29 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     30 *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     31 */
     32 
     33 #ifndef WEBRTCGMPVIDEOCODEC_H_
     34 #define WEBRTCGMPVIDEOCODEC_H_
     35 
     36 #include <string>
     37 
     38 #include "GMPVideoDecoderProxy.h"
     39 #include "GMPVideoEncoderProxy.h"
     40 #include "MediaConduitInterface.h"
     41 #include "PerformanceRecorder.h"
     42 #include "VideoConduit.h"
     43 #include "api/video/video_frame_type.h"
     44 #include "common_video/h264/h264_bitstream_parser.h"
     45 #include "gmp-video-host.h"
     46 #include "jsapi/PeerConnectionImpl.h"
     47 #include "modules/video_coding/include/video_error_codes.h"
     48 #include "modules/video_coding/svc/scalable_video_controller.h"
     49 #include "mozIGeckoMediaPluginService.h"
     50 #include "mozilla/EventTargetCapability.h"
     51 #include "mozilla/Mutex.h"
     52 #include "mozilla/glean/DomMediaWebrtcMetrics.h"
     53 #include "nsTArray.h"
     54 #include "nsThreadUtils.h"
     55 
     56 namespace mozilla::detail {
     57 struct InputImageData {
     58  uint64_t gmp_timestamp_us = 0;
     59  int64_t ntp_timestamp_ms = 0;
     60  int64_t timestamp_us = 0;
     61  uint32_t rtp_timestamp = 0;
     62  webrtc::ScalableVideoController::LayerFrameConfig frame_config;
     63 };
     64 }  // namespace mozilla::detail
     65 
     66 // webrtc::CodecSpecificInfo has members that are not always memmovable, which
     67 // is required by nsTArray/AutoTArray by default. Use move constructors where
     68 // necessary.
     69 template <>
     70 struct nsTArray_RelocationStrategy<mozilla::detail::InputImageData> {
     71  using IID = mozilla::detail::InputImageData;
     72  // This logic follows MemMoveAnnotation.h.
     73  using Type =
     74      std::conditional_t<(std::is_trivially_move_constructible_v<IID> ||
     75                          (!std::is_move_constructible_v<IID> &&
     76                           std::is_trivially_copy_constructible_v<IID>)) &&
     77                             std::is_trivially_destructible_v<IID>,
     78                         nsTArray_RelocateUsingMemutils,
     79                         nsTArray_RelocateUsingMoveConstructor<IID>>;
     80 };
     81 
     82 namespace mozilla {
     83 
     84 static void NotifyGmpInitDone(const std::string& aPCHandle, int32_t aResult,
     85                              const std::string& aError = "") {
     86  if (!NS_IsMainThread()) {
     87    MOZ_ALWAYS_SUCCEEDS(GetMainThreadSerialEventTarget()->Dispatch(
     88        NS_NewRunnableFunction(__func__, [aPCHandle, aResult, aError] {
     89          NotifyGmpInitDone(aPCHandle, aResult, aError);
     90        })));
     91    return;
     92  }
     93 
     94  glean::webrtc::gmp_init_success
     95      .EnumGet(static_cast<glean::webrtc::GmpInitSuccessLabel>(
     96          aResult == WEBRTC_VIDEO_CODEC_OK))
     97      .Add();
     98  if (aResult == WEBRTC_VIDEO_CODEC_OK) {
     99    // Might be useful to notify the PeerConnection about successful init
    100    // someday.
    101    return;
    102  }
    103 
    104  PeerConnectionWrapper wrapper(aPCHandle);
    105  if (wrapper.impl()) {
    106    wrapper.impl()->OnMediaError(aError);
    107  }
    108 }
    109 
    110 // Hold a frame for later decode
    111 class GMPDecodeData {
    112 public:
    113  GMPDecodeData(const webrtc::EncodedImage& aInputImage, bool aMissingFrames,
    114                int64_t aRenderTimeMs)
    115      : mImage(aInputImage),
    116        mMissingFrames(aMissingFrames),
    117        mRenderTimeMs(aRenderTimeMs) {
    118    // We want to use this for queuing, and the calling code recycles the
    119    // buffer on return from Decode()
    120    MOZ_RELEASE_ASSERT(aInputImage.size() <
    121                       (std::numeric_limits<size_t>::max() >> 1));
    122  }
    123 
    124  ~GMPDecodeData() = default;
    125 
    126  const webrtc::EncodedImage mImage;
    127  const bool mMissingFrames;
    128  const int64_t mRenderTimeMs;
    129 };
    130 
    131 class RefCountedWebrtcVideoEncoder {
    132 public:
    133  NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING
    134 
    135  // Implement sort of WebrtcVideoEncoder interface and support refcounting.
    136  // (We cannot use |Release|, since that's needed for nsRefPtr)
    137  virtual int32_t InitEncode(
    138      const webrtc::VideoCodec* aCodecSettings,
    139      const webrtc::VideoEncoder::Settings& aSettings) = 0;
    140 
    141  virtual int32_t Encode(
    142      const webrtc::VideoFrame& aInputImage,
    143      const std::vector<webrtc::VideoFrameType>* aFrameTypes) = 0;
    144 
    145  virtual int32_t RegisterEncodeCompleteCallback(
    146      webrtc::EncodedImageCallback* aCallback) = 0;
    147 
    148  virtual int32_t Shutdown() = 0;
    149 
    150  virtual int32_t SetRates(
    151      const webrtc::VideoEncoder::RateControlParameters& aParameters) = 0;
    152 
    153  virtual MediaEventSource<uint64_t>* InitPluginEvent() = 0;
    154 
    155  virtual MediaEventSource<uint64_t>* ReleasePluginEvent() = 0;
    156 
    157  virtual WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const = 0;
    158 
    159 protected:
    160  virtual ~RefCountedWebrtcVideoEncoder() = default;
    161 };
    162 
    163 class WebrtcGmpVideoEncoder final : public GMPVideoEncoderCallbackProxy,
    164                                    public RefCountedWebrtcVideoEncoder {
    165 public:
    166  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoEncoder, final);
    167 
    168  WebrtcGmpVideoEncoder(const webrtc::SdpVideoFormat& aFormat,
    169                        std::string aPCHandle);
    170 
    171  // Implement VideoEncoder interface, sort of.
    172  // (We cannot use |Release|, since that's needed for nsRefPtr)
    173  int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings,
    174                     const webrtc::VideoEncoder::Settings& aSettings) override;
    175 
    176  int32_t Encode(
    177      const webrtc::VideoFrame& aInputImage,
    178      const std::vector<webrtc::VideoFrameType>* aFrameTypes) override;
    179 
    180  int32_t RegisterEncodeCompleteCallback(
    181      webrtc::EncodedImageCallback* aCallback) override;
    182 
    183  int32_t Shutdown() override;
    184 
    185  int32_t SetRates(
    186      const webrtc::VideoEncoder::RateControlParameters& aParameters) override;
    187 
    188  WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const override;
    189 
    190  MediaEventSource<uint64_t>* InitPluginEvent() override {
    191    return &mInitPluginEvent;
    192  }
    193 
    194  MediaEventSource<uint64_t>* ReleasePluginEvent() override {
    195    return &mReleasePluginEvent;
    196  }
    197 
    198  // GMPVideoEncoderCallback virtual functions.
    199  void Terminated() override;
    200 
    201  void Encoded(GMPVideoEncodedFrame* aEncodedFrame,
    202               const nsTArray<uint8_t>& aCodecSpecificInfo) override;
    203 
    204  void Dropped(uint64_t aTimestamp) override;
    205 
    206  void Error(GMPErr aError) override {}
    207 
    208 private:
    209  virtual ~WebrtcGmpVideoEncoder();
    210 
    211  void InitEncode_g(const GMPVideoCodec& aCodecParams, int32_t aNumberOfCores,
    212                    uint32_t aMaxPayloadSize);
    213  int32_t GmpInitDone_g(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost,
    214                        const GMPVideoCodec& aCodecParams,
    215                        std::string* aErrorOut);
    216  int32_t GmpInitDone_g(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost,
    217                        std::string* aErrorOut);
    218  int32_t InitEncoderForSize(unsigned short aWidth, unsigned short aHeight,
    219                             std::string* aErrorOut);
    220  void Close_g();
    221 
    222  class InitDoneCallback final : public GetGMPVideoEncoderCallback {
    223   public:
    224    InitDoneCallback(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder,
    225                     const GMPVideoCodec& aCodecParams)
    226        : mEncoder(aEncoder), mCodecParams(aCodecParams) {}
    227 
    228    void Done(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost) override {
    229      std::string errorOut;
    230      int32_t result =
    231          mEncoder->GmpInitDone_g(aGMP, aHost, mCodecParams, &errorOut);
    232      NotifyGmpInitDone(mEncoder->mPCHandle, result, errorOut);
    233    }
    234 
    235   private:
    236    const RefPtr<WebrtcGmpVideoEncoder> mEncoder;
    237    const GMPVideoCodec mCodecParams;
    238  };
    239 
    240  void Encode_g(const webrtc::VideoFrame& aInputImage,
    241                std::vector<webrtc::VideoFrameType> aFrameTypes);
    242  void RegetEncoderForResolutionChange(uint32_t aWidth, uint32_t aHeight);
    243 
    244  class InitDoneForResolutionChangeCallback final
    245      : public GetGMPVideoEncoderCallback {
    246   public:
    247    InitDoneForResolutionChangeCallback(
    248        const RefPtr<WebrtcGmpVideoEncoder>& aEncoder, uint32_t aWidth,
    249        uint32_t aHeight)
    250        : mEncoder(aEncoder), mWidth(aWidth), mHeight(aHeight) {}
    251 
    252    void Done(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost) override {
    253      std::string errorOut;
    254      int32_t result = mEncoder->GmpInitDone_g(aGMP, aHost, &errorOut);
    255      if (result != WEBRTC_VIDEO_CODEC_OK) {
    256        NotifyGmpInitDone(mEncoder->mPCHandle, result, errorOut);
    257        return;
    258      }
    259 
    260      result = mEncoder->InitEncoderForSize(mWidth, mHeight, &errorOut);
    261      NotifyGmpInitDone(mEncoder->mPCHandle, result, errorOut);
    262    }
    263 
    264   private:
    265    const RefPtr<WebrtcGmpVideoEncoder> mEncoder;
    266    const uint32_t mWidth;
    267    const uint32_t mHeight;
    268  };
    269 
    270  int32_t SetRates_g(uint32_t aOldBitRateKbps, uint32_t aNewBitRateKbps,
    271                     Maybe<double> aFrameRate);
    272 
    273  nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
    274  nsCOMPtr<nsIThread> mGMPThread;
    275  GMPVideoEncoderProxy* mGMP;
    276  Maybe<EventTargetCapability<nsISerialEventTarget>> mEncodeQueue;
    277  // Used to handle a race where Release() is called while init is in progress
    278  bool mInitting;
    279  uint32_t mConfiguredBitrateKbps MOZ_GUARDED_BY(mEncodeQueue);
    280  GMPVideoHost* mHost;
    281  GMPVideoCodec mCodecParams{};
    282  uint32_t mMaxPayloadSize;
    283  bool mNeedKeyframe;
    284  int mSyncLayerCap;
    285  const webrtc::CodecParameterMap mFormatParams;
    286  webrtc::H264BitstreamParser mH264BitstreamParser;
    287  std::unique_ptr<webrtc::ScalableVideoController> mSvcController;
    288  Mutex mCallbackMutex;
    289  webrtc::EncodedImageCallback* mCallback MOZ_GUARDED_BY(mCallbackMutex);
    290  Maybe<uint64_t> mCachedPluginId;
    291  const std::string mPCHandle;
    292 
    293  static constexpr size_t kMaxImagesInFlight = 1;
    294  // Map rtp time -> input image data
    295  AutoTArray<detail::InputImageData, kMaxImagesInFlight> mInputImageMap;
    296 
    297  MediaEventProducer<uint64_t> mInitPluginEvent;
    298  MediaEventProducer<uint64_t> mReleasePluginEvent;
    299 };
    300 
    301 // Basically a strong ref to a RefCountedWebrtcVideoEncoder, that also
    302 // translates from Release() to RefCountedWebrtcVideoEncoder::Shutdown(),
    303 // since we need RefCountedWebrtcVideoEncoder::Release() for managing the
    304 // refcount. The webrtc.org code gets one of these, so it doesn't unilaterally
    305 // delete the "real" encoder.
    306 class WebrtcVideoEncoderProxy final : public WebrtcVideoEncoder {
    307 public:
    308  explicit WebrtcVideoEncoderProxy(
    309      RefPtr<RefCountedWebrtcVideoEncoder> aEncoder)
    310      : mEncoderImpl(std::move(aEncoder)) {}
    311 
    312  virtual ~WebrtcVideoEncoderProxy() {
    313    RegisterEncodeCompleteCallback(nullptr);
    314  }
    315 
    316  MediaEventSource<uint64_t>* InitPluginEvent() override {
    317    return mEncoderImpl->InitPluginEvent();
    318  }
    319 
    320  MediaEventSource<uint64_t>* ReleasePluginEvent() override {
    321    return mEncoderImpl->ReleasePluginEvent();
    322  }
    323 
    324  int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings,
    325                     const WebrtcVideoEncoder::Settings& aSettings) override {
    326    return mEncoderImpl->InitEncode(aCodecSettings, aSettings);
    327  }
    328 
    329  int32_t Encode(
    330      const webrtc::VideoFrame& aInputImage,
    331      const std::vector<webrtc::VideoFrameType>* aFrameTypes) override {
    332    return mEncoderImpl->Encode(aInputImage, aFrameTypes);
    333  }
    334 
    335  int32_t RegisterEncodeCompleteCallback(
    336      webrtc::EncodedImageCallback* aCallback) override {
    337    return mEncoderImpl->RegisterEncodeCompleteCallback(aCallback);
    338  }
    339 
    340  int32_t Release() override { return mEncoderImpl->Shutdown(); }
    341 
    342  void SetRates(const RateControlParameters& aParameters) override {
    343    mEncoderImpl->SetRates(aParameters);
    344  }
    345 
    346  EncoderInfo GetEncoderInfo() const override {
    347    return mEncoderImpl->GetEncoderInfo();
    348  }
    349 
    350 private:
    351  const RefPtr<RefCountedWebrtcVideoEncoder> mEncoderImpl;
    352 };
    353 
    354 class WebrtcGmpVideoDecoder final : public GMPVideoDecoderCallbackProxy {
    355 public:
    356  WebrtcGmpVideoDecoder(std::string aPCHandle, TrackingId aTrackingId);
    357  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoDecoder, final);
    358 
    359  // Implement VideoEncoder interface, sort of.
    360  // (We cannot use |Release|, since that's needed for nsRefPtr)
    361  virtual bool Configure(const webrtc::VideoDecoder::Settings& settings);
    362  virtual int32_t Decode(const webrtc::EncodedImage& aInputImage,
    363                         bool aMissingFrames, int64_t aRenderTimeMs);
    364  virtual int32_t RegisterDecodeCompleteCallback(
    365      webrtc::DecodedImageCallback* aCallback);
    366 
    367  virtual int32_t ReleaseGmp();
    368 
    369  MediaEventSource<uint64_t>* InitPluginEvent() { return &mInitPluginEvent; }
    370 
    371  MediaEventSource<uint64_t>* ReleasePluginEvent() {
    372    return &mReleasePluginEvent;
    373  }
    374 
    375  // GMPVideoDecoderCallbackProxy
    376  void Terminated() override;
    377 
    378  void Decoded(GMPVideoi420Frame* aDecodedFrame) override;
    379 
    380  void ReceivedDecodedReferenceFrame(const uint64_t aPictureId) override {
    381    MOZ_CRASH();
    382  }
    383 
    384  void ReceivedDecodedFrame(const uint64_t aPictureId) override { MOZ_CRASH(); }
    385 
    386  void InputDataExhausted() override {}
    387 
    388  void DrainComplete() override {}
    389 
    390  void ResetComplete() override {}
    391 
    392  void Error(GMPErr aError) override { mDecoderStatus = aError; }
    393 
    394 private:
    395  virtual ~WebrtcGmpVideoDecoder();
    396 
    397  void Configure_g(const webrtc::VideoDecoder::Settings& settings);
    398  int32_t GmpInitDone_g(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost,
    399                        std::string* aErrorOut);
    400  void Close_g();
    401 
    402  class InitDoneCallback final : public GetGMPVideoDecoderCallback {
    403   public:
    404    explicit InitDoneCallback(const RefPtr<WebrtcGmpVideoDecoder>& aDecoder)
    405        : mDecoder(aDecoder) {}
    406 
    407    void Done(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost) override {
    408      std::string errorOut;
    409      int32_t result = mDecoder->GmpInitDone_g(aGMP, aHost, &errorOut);
    410      NotifyGmpInitDone(mDecoder->mPCHandle, result, errorOut);
    411    }
    412 
    413   private:
    414    const RefPtr<WebrtcGmpVideoDecoder> mDecoder;
    415  };
    416 
    417  void Decode_g(UniquePtr<GMPDecodeData>&& aDecodeData);
    418 
    419  nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
    420  nsCOMPtr<nsIThread> mGMPThread;
    421  GMPVideoDecoderProxy* mGMP;  // Addref is held for us
    422  // Used to handle a race where Release() is called while init is in progress
    423  bool mInitting;
    424  // Frames queued for decode while mInitting is true
    425  nsTArray<UniquePtr<GMPDecodeData>> mQueuedFrames;
    426  GMPVideoHost* mHost;
    427  // Protects mCallback
    428  Mutex mCallbackMutex;
    429  webrtc::DecodedImageCallback* mCallback MOZ_GUARDED_BY(mCallbackMutex);
    430  Maybe<uint64_t> mCachedPluginId;
    431  Atomic<GMPErr, ReleaseAcquire> mDecoderStatus;
    432  const std::string mPCHandle;
    433  const TrackingId mTrackingId;
    434  PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder;
    435 
    436  MediaEventProducer<uint64_t> mInitPluginEvent;
    437  MediaEventProducer<uint64_t> mReleasePluginEvent;
    438 };
    439 
    440 // Basically a strong ref to a WebrtcGmpVideoDecoder, that also translates
    441 // from Release() to WebrtcGmpVideoDecoder::ReleaseGmp(), since we need
    442 // WebrtcGmpVideoDecoder::Release() for managing the refcount.
    443 // The webrtc.org code gets one of these, so it doesn't unilaterally delete
    444 // the "real" encoder.
    445 class WebrtcVideoDecoderProxy final : public WebrtcVideoDecoder {
    446 public:
    447  explicit WebrtcVideoDecoderProxy(std::string aPCHandle,
    448                                   TrackingId aTrackingId)
    449      : mDecoderImpl(new WebrtcGmpVideoDecoder(std::move(aPCHandle),
    450                                               std::move(aTrackingId))) {}
    451 
    452  virtual ~WebrtcVideoDecoderProxy() {
    453    RegisterDecodeCompleteCallback(nullptr);
    454  }
    455 
    456  MediaEventSource<uint64_t>* InitPluginEvent() override {
    457    return mDecoderImpl->InitPluginEvent();
    458  }
    459 
    460  MediaEventSource<uint64_t>* ReleasePluginEvent() override {
    461    return mDecoderImpl->ReleasePluginEvent();
    462  }
    463 
    464  bool Configure(const Settings& settings) override {
    465    return mDecoderImpl->Configure(settings);
    466  }
    467 
    468  int32_t Decode(const webrtc::EncodedImage& aInputImage, bool aMissingFrames,
    469                 int64_t aRenderTimeMs) override {
    470    return mDecoderImpl->Decode(aInputImage, aMissingFrames, aRenderTimeMs);
    471  }
    472 
    473  int32_t RegisterDecodeCompleteCallback(
    474      webrtc::DecodedImageCallback* aCallback) override {
    475    return mDecoderImpl->RegisterDecodeCompleteCallback(aCallback);
    476  }
    477 
    478  int32_t Release() override { return mDecoderImpl->ReleaseGmp(); }
    479 
    480 private:
    481  const RefPtr<WebrtcGmpVideoDecoder> mDecoderImpl;
    482 };
    483 
    484 }  // namespace mozilla
    485 
    486 #endif