tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

MediaConduitInterface.h (17134B)

      1 /* This Source Code Form is subject to the terms of the Mozilla Public
      2 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
      3 * You can obtain one at http://mozilla.org/MPL/2.0/. */
      4 
      5 #ifndef MEDIA_CONDUIT_ABSTRACTION_
      6 #define MEDIA_CONDUIT_ABSTRACTION_
      7 
      8 #include <functional>
      9 #include <map>
     10 #include <vector>
     11 
     12 #include "CodecConfig.h"
     13 #include "ImageContainer.h"
     14 #include "MediaConduitControl.h"
     15 #include "MediaConduitErrors.h"
     16 #include "PerformanceRecorder.h"
     17 #include "WebrtcVideoCodecFactory.h"
     18 #include "jsapi/RTCStatsReport.h"
     19 #include "mozilla/MozPromise.h"
     20 #include "mozilla/dom/RTCRtpSourcesBinding.h"
     21 #include "mozilla/media/MediaUtils.h"
     22 #include "nsTArray.h"
     23 #include "transport/mediapacket.h"
     24 
     25 // libwebrtc includes
     26 #include "api/audio/audio_frame.h"
     27 #include "api/call/transport.h"
     28 #include "api/media_stream_interface.h"
     29 #include "api/rtp_headers.h"
     30 #include "api/rtp_parameters.h"
     31 #include "api/transport/rtp/rtp_source.h"
     32 #include "api/video/video_frame_buffer.h"
     33 #include "call/audio_receive_stream.h"
     34 #include "call/audio_send_stream.h"
     35 #include "call/call_basic_stats.h"
     36 #include "call/video_receive_stream.h"
     37 #include "call/video_send_stream.h"
     38 #include "rtc_base/copy_on_write_buffer.h"
     39 
     40 namespace webrtc {
     41 class RtpPacketReceived;
     42 class VideoFrame;
     43 }  // namespace webrtc
     44 
     45 namespace mozilla {
     46 namespace dom {
     47 struct RTCRtpSourceEntry;
     48 }
     49 
     50 enum class MediaSessionConduitLocalDirection : int { kSend, kRecv };
     51 
     52 class VideoSessionConduit;
     53 class AudioSessionConduit;
     54 class WebrtcCallWrapper;
     55 class FrameTransformerProxy;
     56 
     57 /**
     58 * 1. Abstract renderer for video data
     59 * 2. This class acts as abstract interface between the video-engine and
     60 *    video-engine agnostic renderer implementation.
     61 * 3. Concrete implementation of this interface is responsible for
     62 *    processing and/or rendering the obtained raw video frame to appropriate
     63 *    output , say, <video>
     64 */
     65 class VideoRenderer {
     66 protected:
     67  virtual ~VideoRenderer() {}
     68 
     69 public:
     70  /**
     71   * Callback Function reporting decoded frame for processing.
     72   * @param video_frame: reference to decoded video frame
     73   * NOTE: If decoded video frame is passed through buffer , it is the
     74   * responsibility of the concrete implementations of this class to own copy
     75   * of the frame if needed for time longer than scope of this callback.
     76   * Such implementations should be quick in processing the frames and return
     77   * immediately.
     78   */
     79  virtual void RenderVideoFrame(const webrtc::VideoFrame& video_frame) = 0;
     80 
     81  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoRenderer)
     82 };
     83 
     84 /**
     85 * Generic Interface for representing Audio/Video Session
     86 * MediaSession conduit is identified by 2 main components
     87 * 1. Attached Transport Interface (through events) for inbound and outbound RTP
     88 *    transport
     89 * 2. Attached Renderer Interface for rendering media data off the network
     90 * This class hides specifics of Media-Engine implementation from the consumers
     91 * of this interface.
     92 * Also provides codec configuration API for the media sent and recevied
     93 */
     94 class MediaSessionConduit {
     95 protected:
     96  virtual ~MediaSessionConduit() {}
     97 
     98 public:
     99  enum Type { AUDIO, VIDEO };
    100  enum class PacketType { RTP, RTCP };
    101 
    102  static std::string LocalDirectionToString(
    103      const MediaSessionConduitLocalDirection aDirection) {
    104    return aDirection == MediaSessionConduitLocalDirection::kSend ? "send"
    105                                                                  : "receive";
    106  }
    107 
    108  virtual Type type() const = 0;
    109 
    110  // Call thread only
    111  virtual Maybe<int> ActiveSendPayloadType() const = 0;
    112  virtual Maybe<int> ActiveRecvPayloadType() const = 0;
    113 
    114  // Whether transport is currently sending and receiving packets
    115  virtual void SetTransportActive(bool aActive) = 0;
    116 
    117  // Sending packets
    118  virtual MediaEventSourceExc<MediaPacket>& SenderRtpSendEvent() = 0;
    119  virtual MediaEventSourceExc<MediaPacket>& SenderRtcpSendEvent() = 0;
    120  virtual MediaEventSourceExc<MediaPacket>& ReceiverRtcpSendEvent() = 0;
    121 
    122  // Receiving RTP packets
    123  virtual void ConnectReceiverRtpEvent(
    124      MediaEventSourceExc<webrtc::RtpPacketReceived, webrtc::RTPHeader>&
    125          aEvent) = 0;
    126 
    127  virtual void ConnectReceiverRtcpEvent(
    128      MediaEventSourceExc<webrtc::CopyOnWriteBuffer>& aEvent) = 0;
    129 
    130  virtual void ConnectSenderRtcpEvent(
    131      MediaEventSourceExc<webrtc::CopyOnWriteBuffer>& aEvent) = 0;
    132 
    133  // Sts thread only.
    134  virtual Maybe<uint16_t> RtpSendBaseSeqFor(uint32_t aSsrc) const = 0;
    135 
    136  // Any thread.
    137  virtual const dom::RTCStatsTimestampMaker& GetTimestampMaker() const = 0;
    138 
    139  virtual Ssrcs GetLocalSSRCs() const = 0;
    140 
    141  virtual Maybe<Ssrc> GetRemoteSSRC() const = 0;
    142  virtual void UnsetRemoteSSRC(Ssrc aSsrc) = 0;
    143 
    144  virtual void DisableSsrcChanges() = 0;
    145 
    146  virtual bool HasCodecPluginID(uint64_t aPluginID) const = 0;
    147 
    148  // Stuff for driving mute/unmute events
    149  virtual MediaEventSource<void>& RtcpByeEvent() = 0;
    150  virtual MediaEventSource<void>& RtcpTimeoutEvent() = 0;
    151  virtual MediaEventSource<void>& RtpPacketEvent() = 0;
    152 
    153  virtual bool SendRtp(const uint8_t* aData, size_t aLength,
    154                       const webrtc::PacketOptions& aOptions) = 0;
    155  virtual bool SendSenderRtcp(const uint8_t* aData, size_t aLength) = 0;
    156  virtual bool SendReceiverRtcp(const uint8_t* aData, size_t aLength) = 0;
    157 
    158  virtual void DeliverPacket(webrtc::CopyOnWriteBuffer packet,
    159                             PacketType type) = 0;
    160 
    161  virtual RefPtr<GenericPromise> Shutdown() = 0;
    162  // Call thread only. Is set at the end of Shutdown()
    163  virtual bool IsShutdown() const = 0;
    164 
    165  virtual Maybe<RefPtr<AudioSessionConduit>> AsAudioSessionConduit() = 0;
    166  virtual Maybe<RefPtr<VideoSessionConduit>> AsVideoSessionConduit() = 0;
    167 
    168  virtual Maybe<webrtc::CallBasicStats> GetCallStats() const = 0;
    169 
    170  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSessionConduit)
    171 
    172  void GetRtpSources(nsTArray<dom::RTCRtpSourceEntry>& outSources) const;
    173 
    174  virtual void SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) = 0;
    175 
    176  // test-only: inserts fake CSRCs and audio level data.
    177  // NB: fake data is only valid during the current main thread task.
    178  void InsertAudioLevelForContributingSource(const uint32_t aCsrcSource,
    179                                             const int64_t aTimestamp,
    180                                             const uint32_t aRtpTimestamp,
    181                                             const bool aHasAudioLevel,
    182                                             const uint8_t aAudioLevel);
    183 
    184 protected:
    185  virtual const std::vector<webrtc::RtpSource>& GetUpstreamRtpSources()
    186      const = 0;
    187 
    188 private:
    189  void UpdateRtpSources(const std::vector<webrtc::RtpSource>& aSources) const;
    190 
    191  // Marks the cache as having been updated in the current task, and keeps it
    192  // stable until the current task is finished.
    193  void OnSourcesUpdated() const;
    194 
    195  // Accessed only on main thread. This exists for a couple of reasons:
    196  // 1. The webrtc spec says that source stats are updated using a queued task;
    197  //    libwebrtc's internal representation of these stats is updated without
    198  //    any task queueing, which means we need a mainthread-only cache.
    199  // 2. libwebrtc uses its own clock that is not consistent with the one we
    200  // need to use for stats (the so-called JS timestamps), which means we need
    201  // to adjust the timestamps.  Since timestamp adjustment is inexact and will
    202  // not necessarily yield exactly the same result if performed again later, we
    203  // need to avoid performing it more than once for each entry, which means we
    204  // need to remember both the JS timestamp (in dom::RTCRtpSourceEntry) and the
    205  // libwebrtc timestamp (in SourceKey::mLibwebrtcTimestampMs).
    206  class SourceKey {
    207   public:
    208    explicit SourceKey(const webrtc::RtpSource& aSource)
    209        : SourceKey(aSource.timestamp().ms(), aSource.source_id()) {}
    210 
    211    SourceKey(uint32_t aTimestamp, uint32_t aSrc)
    212        : mLibwebrtcTimestampMs(aTimestamp), mSrc(aSrc) {}
    213 
    214    // TODO: Once we support = default for this in our toolchain, do so
    215    auto operator>(const SourceKey& aRhs) const {
    216      if (mLibwebrtcTimestampMs == aRhs.mLibwebrtcTimestampMs) {
    217        return mSrc > aRhs.mSrc;
    218      }
    219      return mLibwebrtcTimestampMs > aRhs.mLibwebrtcTimestampMs;
    220    }
    221 
    222   private:
    223    uint32_t mLibwebrtcTimestampMs;
    224    uint32_t mSrc;
    225  };
    226  mutable std::map<SourceKey, dom::RTCRtpSourceEntry, std::greater<SourceKey>>
    227      mSourcesCache;
    228  // Accessed only on main thread. A flag saying whether mSourcesCache needs
    229  // updating. Ensures that get*Sources() appear stable from javascript
    230  // throughout a main thread task, even though we don't follow the spec to the
    231  // letter (dispatch a task to update the sources).
    232  mutable bool mSourcesUpdateNeeded = true;
    233 };
    234 
    235 class WebrtcSendTransport : public webrtc::Transport {
    236  // WeakRef to the owning conduit
    237  MediaSessionConduit* mConduit;
    238 
    239 public:
    240  explicit WebrtcSendTransport(MediaSessionConduit* aConduit)
    241      : mConduit(aConduit) {}
    242  bool SendRtp(webrtc::ArrayView<const uint8_t> aPacket,
    243               const webrtc::PacketOptions& aOptions) {
    244    return mConduit->SendRtp(aPacket.data(), aPacket.size(), aOptions);
    245  }
    246  bool SendRtcp(webrtc::ArrayView<const uint8_t> aPacket,
    247                const webrtc::PacketOptions& aOptions) {
    248    return mConduit->SendSenderRtcp(aPacket.data(), aPacket.size());
    249  }
    250 };
    251 
    252 class WebrtcReceiveTransport : public webrtc::Transport {
    253  // WeakRef to the owning conduit
    254  MediaSessionConduit* mConduit;
    255 
    256 public:
    257  explicit WebrtcReceiveTransport(MediaSessionConduit* aConduit)
    258      : mConduit(aConduit) {}
    259  bool SendRtp(webrtc::ArrayView<const uint8_t> aPacket,
    260               const webrtc::PacketOptions& aOptions) {
    261    MOZ_CRASH("Unexpected RTP packet");
    262  }
    263  bool SendRtcp(webrtc::ArrayView<const uint8_t> aPacket,
    264                const webrtc::PacketOptions& aOptions) {
    265    return mConduit->SendReceiverRtcp(aPacket.data(), aPacket.size());
    266  }
    267 };
    268 
    269 // Abstract base classes for external encoder/decoder.
    270 
    271 // Interface to help signal PluginIDs
    272 class CodecPluginID {
    273 public:
    274  virtual MediaEventSource<uint64_t>* InitPluginEvent() { return nullptr; }
    275  virtual MediaEventSource<uint64_t>* ReleasePluginEvent() { return nullptr; }
    276  virtual ~CodecPluginID() {}
    277 };
    278 
    279 class VideoEncoder : public CodecPluginID {
    280 public:
    281  virtual ~VideoEncoder() {}
    282 };
    283 
    284 class VideoDecoder : public CodecPluginID {
    285 public:
    286  virtual ~VideoDecoder() {}
    287 };
    288 
    289 /**
    290 * MediaSessionConduit for video
    291 * Refer to the comments on MediaSessionConduit above for overall
    292 * information
    293 */
    294 class VideoSessionConduit : public MediaSessionConduit {
    295 public:
    296  struct Options {
    297    bool mVideoLatencyTestEnable = false;
    298    // All in bps.
    299    int mMinBitrate = 0;
    300    int mStartBitrate = 0;
    301    int mPrefMaxBitrate = 0;
    302    int mMinBitrateEstimate = 0;
    303    bool mDenoising = false;
    304    bool mLockScaling = false;
    305    uint8_t mSpatialLayers = 1;
    306    uint8_t mTemporalLayers = 1;
    307  };
    308 
    309  /**
    310   * Factory function to create and initialize a Video Conduit Session
    311   * @param  webrtc::Call instance shared by paired audio and video
    312   *         media conduits
    313   * @param  aOptions are a number of options, typically from prefs, used to
    314   *         configure the created VideoConduit.
    315   * @param  aPCHandle is a string representing the RTCPeerConnection that is
    316   *         creating this VideoConduit. This is used when reporting GMP plugin
    317   *         crashes.
    318   * @result Concrete VideoSessionConduitObject or nullptr in the case
    319   *         of failure
    320   */
    321  static RefPtr<VideoSessionConduit> Create(
    322      RefPtr<WebrtcCallWrapper> aCall,
    323      nsCOMPtr<nsISerialEventTarget> aStsThread, Options aOptions,
    324      std::string aPCHandle, const TrackingId& aRecvTrackingId);
    325 
    326  enum FrameRequestType {
    327    FrameRequestNone,
    328    FrameRequestFir,
    329    FrameRequestPli,
    330    FrameRequestUnknown
    331  };
    332 
    333  VideoSessionConduit()
    334      : mFrameRequestMethod(FrameRequestNone),
    335        mUsingNackBasic(false),
    336        mUsingTmmbr(false),
    337        mUsingFEC(false) {}
    338 
    339  virtual ~VideoSessionConduit() {}
    340 
    341  Type type() const override { return VIDEO; }
    342 
    343  Maybe<RefPtr<AudioSessionConduit>> AsAudioSessionConduit() override {
    344    return Nothing();
    345  }
    346 
    347  Maybe<RefPtr<VideoSessionConduit>> AsVideoSessionConduit() override {
    348    return Some(RefPtr<VideoSessionConduit>(this));
    349  }
    350 
    351  /**
    352   * Hooks up mControl Mirrors with aControl Canonicals, and sets up
    353   * mWatchManager to react on Mirror changes.
    354   */
    355  virtual void InitControl(VideoConduitControlInterface* aControl) = 0;
    356 
    357  /**
    358   * Function to attach Renderer end-point of the Media-Video conduit.
    359   * @param aRenderer : Reference to the concrete Video renderer implementation
    360   * Note: Multiple invocations of this API shall remove an existing renderer
    361   * and attaches the new to the Conduit.
    362   */
    363  virtual MediaConduitErrorCode AttachRenderer(
    364      RefPtr<mozilla::VideoRenderer> aRenderer) = 0;
    365  virtual void DetachRenderer() = 0;
    366 
    367  virtual void SetTrackSource(webrtc::VideoTrackSourceInterface* aSource) = 0;
    368 
    369  /**
    370   * These methods allow unit tests to double-check that the
    371   * rtcp-fb settings are as expected.
    372   */
    373  FrameRequestType FrameRequestMethod() const { return mFrameRequestMethod; }
    374 
    375  bool UsingNackBasic() const { return mUsingNackBasic; }
    376 
    377  bool UsingTmmbr() const { return mUsingTmmbr; }
    378 
    379  bool UsingFEC() const { return mUsingFEC; }
    380 
    381  virtual bool LockScaling() const = 0;
    382 
    383  virtual Maybe<webrtc::VideoReceiveStreamInterface::Stats> GetReceiverStats()
    384      const = 0;
    385  virtual Maybe<webrtc::VideoSendStream::Stats> GetSenderStats() const = 0;
    386 
    387  virtual void CollectTelemetryData() = 0;
    388 
    389  virtual bool AddFrameHistory(
    390      dom::Sequence<dom::RTCVideoFrameHistoryInternal>* outHistories) const = 0;
    391 
    392  virtual Maybe<Ssrc> GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const = 0;
    393 
    394  virtual Maybe<gfx::IntSize> GetLastResolution() const = 0;
    395  virtual AbstractCanonical<Maybe<gfx::IntSize>>* CanonicalReceivingSize() = 0;
    396 
    397  virtual void RequestKeyFrame(FrameTransformerProxy* aProxy) = 0;
    398  virtual void GenerateKeyFrame(const Maybe<std::string>& aRid,
    399                                FrameTransformerProxy* aProxy) = 0;
    400 
    401 protected:
    402  /* RTCP feedback settings, for unit testing purposes */
    403  FrameRequestType mFrameRequestMethod;
    404  bool mUsingNackBasic;
    405  bool mUsingTmmbr;
    406  bool mUsingFEC;
    407 };
    408 
    409 /**
    410 * MediaSessionConduit for audio
    411 * Refer to the comments on MediaSessionConduit above for overall
    412 * information
    413 */
    414 class AudioSessionConduit : public MediaSessionConduit {
    415 public:
    416  /**
    417   * Factory function to create and initialize an Audio Conduit Session
    418   * @param  webrtc::Call instance shared by paired audio and video
    419   *         media conduits
    420   * @result Concrete AudioSessionConduitObject or nullptr in the case
    421   *         of failure
    422   */
    423  static RefPtr<AudioSessionConduit> Create(
    424      RefPtr<WebrtcCallWrapper> aCall,
    425      nsCOMPtr<nsISerialEventTarget> aStsThread);
    426 
    427  virtual ~AudioSessionConduit() {}
    428 
    429  Type type() const override { return AUDIO; }
    430 
    431  Maybe<RefPtr<AudioSessionConduit>> AsAudioSessionConduit() override {
    432    return Some(this);
    433  }
    434 
    435  Maybe<RefPtr<VideoSessionConduit>> AsVideoSessionConduit() override {
    436    return Nothing();
    437  }
    438 
    439  /**
    440   * Hooks up mControl Mirrors with aControl Canonicals, and sets up
    441   * mWatchManager to react on Mirror changes.
    442   */
    443  virtual void InitControl(AudioConduitControlInterface* aControl) = 0;
    444 
    445  /**
    446   * Function to deliver externally captured audio sample for encoding and
    447   * transport
    448   * @param frame [in]: AudioFrame in upstream's format for forwarding to the
    449   *                    send stream. Ownership is passed along.
    450   * NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can
    451   * be invoked. This ensures the inserted audio-samples can be transmitted by
    452   * the conduit.
    453   */
    454  virtual MediaConduitErrorCode SendAudioFrame(
    455      std::unique_ptr<webrtc::AudioFrame> frame) = 0;
    456 
    457  /**
    458   * Function to grab a decoded audio-sample from the media engine for
    459   * rendering / playout of length 10 milliseconds.
    460   *
    461   * @param samplingFreqHz [in]: Frequency of the sampling for playback in
    462   *                             Hertz (16000, 32000,..)
    463   * @param frame [in/out]: Pointer to an AudioFrame to which audio data will be
    464   *                        copied
    465   * NOTE: This function should be invoked every 10 milliseconds for the best
    466   *       performance
    467   * NOTE: ConfigureRecvMediaCodec() SHOULD be called before this function can
    468   *       be invoked
    469   * This ensures the decoded samples are ready for reading and playout is
    470   * enabled.
    471   */
    472  virtual MediaConduitErrorCode GetAudioFrame(int32_t samplingFreqHz,
    473                                              webrtc::AudioFrame* frame) = 0;
    474 
    475  /**
    476   * Checks if given sampling frequency is supported
    477   * @param freq: Sampling rate (in Hz) to check
    478   */
    479  virtual bool IsSamplingFreqSupported(int freq) const = 0;
    480 
    481  virtual Maybe<webrtc::AudioReceiveStreamInterface::Stats> GetReceiverStats()
    482      const = 0;
    483  virtual Maybe<webrtc::AudioSendStream::Stats> GetSenderStats() const = 0;
    484 };
    485 }  // namespace mozilla
    486 #endif