tor-browser

MediaEngineRemoteVideoSource.h (10298B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_
#define MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_

#include "CamerasChild.h"
#include "DOMMediaStream.h"
#include "MediaEngineSource.h"
#include "MediaTrackGraph.h"
#include "common_video/include/video_frame_buffer_pool.h"
#include "modules/video_capture/video_capture_defines.h"
#include "mozilla/Mutex.h"
#include "mozilla/dom/MediaStreamTrackBinding.h"

namespace webrtc {
using CaptureCapability = VideoCaptureCapability;
}

namespace mozilla {
namespace dom {
enum class VideoResizeModeEnum : uint8_t;
}

// Fitness distance is defined in
// https://w3c.github.io/mediacapture-main/getusermedia.html#dfn-selectsettings

// In contrast, feasibility distance — used in the implementatioon of
// crop_and_scale — effectively rounds width and height up to the nearest
// native width and height before calculating distance (sorta).
//
// Another difference is that if the constraint is required
// (min, 'max', or 'exact'), and the settings dictionary's
// value for the constraint does not satisfy the constraint, the fitness
// distance is positive infinity. Given a continuous space of settings
// dictionaries comprising all discrete combinations of dimension and frame-rate
// related properties, the feasibility distance is still in keeping with the
// constraints algorithm.
enum DistanceCalculation { kFitness, kFeasibility };

/**
* The WebRTC implementation of the MediaEngine interface.
*/
class MediaEngineRemoteVideoSource : public MediaEngineSource,
                                    public camera::FrameRelay {
 ~MediaEngineRemoteVideoSource();

 struct CapabilityCandidate {
   explicit CapabilityCandidate(webrtc::CaptureCapability aCapability,
                                uint32_t aDistance = 0)
       : mCapability(aCapability), mDistance(aDistance) {}

   const webrtc::CaptureCapability mCapability;
   uint32_t mDistance;
 };

 class CapabilityComparator {
  public:
   bool Equals(const CapabilityCandidate& aCandidate,
               const webrtc::CaptureCapability& aCapability) const {
     return aCandidate.mCapability == aCapability;
   }
 };

 bool ChooseCapability(const NormalizedConstraints& aConstraints,
                       const MediaEnginePrefs& aPrefs,
                       webrtc::CaptureCapability& aCapability,
                       const DistanceCalculation aCalculate,
                       const char** aOutBadConstraint);

 uint32_t GetDistance(const webrtc::CaptureCapability& aCandidate,
                      const NormalizedConstraintSet& aConstraints,
                      const DistanceCalculation aCalculate) const;

 uint32_t GetFitnessDistance(
     const webrtc::CaptureCapability& aCandidate,
     const NormalizedConstraintSet& aConstraints) const;

 uint32_t GetFeasibilityDistance(
     const webrtc::CaptureCapability& aCandidate,
     const NormalizedConstraintSet& aConstraints) const;

 static void TrimLessFitCandidates(nsTArray<CapabilityCandidate>& aSet);

public:
 explicit MediaEngineRemoteVideoSource(const MediaDevice* aMediaDevice);

 static already_AddRefed<MediaEngineRemoteVideoSource> CreateFrom(
     const MediaEngineRemoteVideoSource* aSource,
     const MediaDevice* aMediaDevice);

 // ExternalRenderer
 /**
  * Signals that the capture stream has ended
  **/
 void OnCaptureEnded() override;
 int DeliverFrame(uint8_t* aBuffer,
                  const camera::VideoFrameProperties& aProps) override;

 // MediaEngineSource
 nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
                   const MediaEnginePrefs& aPrefs, uint64_t aWindowID,
                   const char** aOutBadConstraint) override;
 nsresult Deallocate() override;
 void SetTrack(const RefPtr<MediaTrack>& aTrack,
               const PrincipalHandle& aPrincipal) override;
 nsresult Start() override;
 nsresult Reconfigure(const dom::MediaTrackConstraints& aConstraints,
                      const MediaEnginePrefs& aPrefs,
                      const char** aOutBadConstraint) override;
 nsresult FocusOnSelectedSource() override;
 nsresult Stop() override;

 nsresult StartCapture(const NormalizedConstraints& aConstraints,
                       const dom::VideoResizeModeEnum& aResizeMode);
 uint32_t GetBestFitnessDistance(
     const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
     const MediaEnginePrefs& aPrefs) const override;
 void GetSettings(dom::MediaTrackSettings& aOutSettings) const override;

 RefPtr<GenericNonExclusivePromise> GetFirstFramePromise() const override {
   return mFirstFramePromise;
 }

 const TrackingId& GetTrackingId() const override;

 static camera::CaptureEngine CaptureEngine(dom::MediaSourceEnum aMediaSource);

 MediaEventSource<void>* CaptureEndedEvent() override {
   return &mCaptureEndedEvent;
 }

 void GetCapabilities(
     dom::MediaTrackCapabilities& aOutCapabilities) const override;

private:
 /**
  * Returns the number of capabilities for the underlying device.
  *
  * Guaranteed to return at least one capability.
  */
 size_t NumCapabilities() const;

 /**
  * Returns the capability with index `aIndex` for our assigned device.
  *
  * It is an error to call this with `aIndex >= NumCapabilities()`.
  *
  * The lifetime of the returned capability is the same as for this source.
  */
 webrtc::CaptureCapability& GetCapability(size_t aIndex) const;

 int mCaptureId = -1;
 const camera::CaptureEngine mCapEngine;  // source of media (cam, screen etc)

 // A tracking id used to uniquely identify the source of video frames.
 // Set under mMutex on the owning thread. Accessed under one of the two.
 TrackingId mTrackingId;

 // Mirror of mTrackingId on the frame-delivering thread (Cameras IPC).
 Maybe<TrackingId> mFrameDeliveringTrackingId;

 // mMutex protects certain members on 3 threads:
 // MediaManager, Cameras IPC and MediaTrackGraph.
 mutable Mutex mMutex MOZ_UNANNOTATED;

 // Current state of this source.
 // Set under mMutex on the owning thread. Accessed under one of the two.
 MediaEngineSourceState mState = kReleased;

 // The source track that we feed video data to.
 // Set under mMutex on the owning thread. Accessed under one of the two.
 RefPtr<SourceMediaTrack> mTrack;

 // The PrincipalHandle that gets attached to the frames we feed to mTrack.
 // Set under mMutex on the owning thread. Accessed under one of the two.
 PrincipalHandle mPrincipal = PRINCIPAL_HANDLE_NONE;

 // Set in Start() and Deallocate() on the owning thread.
 // Accessed in DeliverFrame() on the camera IPC thread, guaranteed to happen
 // after Start() and before the end of Stop().
 RefPtr<layers::ImageContainer> mImageContainer;

 // A buffer pool used to manage the temporary buffer used when rescaling
 // incoming images. Cameras IPC thread only.
 webrtc::VideoFrameBufferPool mRescalingBufferPool;

 // The intrinsic size of the latest received image, before cropping and
 // scaling down. So we can provide a decent guess to settings for desktop
 // sources, since they don't provide capabilities.
 // Set under mMutex on the Cameras IPC thread. Accessed under one of the two.
 gfx::IntSize mIncomingImageSize = gfx::IntSize(0, 0);

 // The intrinsic size of the latest processed image, after cropping and
 // scaling down. So we can update settings on main thread in response to size
 // changes.
 // Set under mMutex on the Cameras IPC thread. Accessed under one of the two.
 gfx::IntSize mScaledImageSize = gfx::IntSize(0, 0);

 struct AtomicBool {
   Atomic<bool> mValue;
 };

 // True when resolution settings have been updated from a real frame's
 // resolution. Threadsafe. Set to false on the owning thread. Set to true on
 // main thread.
 const RefPtr<media::Refcountable<AtomicBool>> mSettingsUpdatedByFrame;

 // The current settings of this source.
 // Note that these may be different from the settings of the underlying device
 // since we scale frames to avoid fingerprinting.
 // Members are main thread only.
 const RefPtr<media::Refcountable<dom::MediaTrackSettings>> mSettings;
 const RefPtr<media::Refcountable<dom::MediaTrackCapabilities>>
     mTrackCapabilities;
 MozPromiseHolder<GenericNonExclusivePromise> mFirstFramePromiseHolder;
 RefPtr<GenericNonExclusivePromise> mFirstFramePromise;

 // The capability currently chosen by constraints of the user of this source,
 // and the distance calculation used when applying them.
 // Set under mMutex on the owning thread. Accessed under one of the two.
 webrtc::CaptureCapability mCapability;
 DistanceCalculation mCalculation;

 // The constraints we're currently operating under, for calculating incoming
 // video resolution.
 Maybe<NormalizedConstraints> mConstraints MOZ_GUARDED_BY(mMutex);

 // Owning thread only.
 UniquePtr<MediaEnginePrefs> mPrefs;

 /**
  * Capabilities that we choose between when applying constraints.
  *
  * This allows for memoization of capabilities as they're requested from the
  * parent process.
  *
  * This is mutable so that the const methods NumCapabilities() and
  * GetCapability() can reset it. Owning thread only.
  */
 mutable nsTArray<UniquePtr<webrtc::CaptureCapability>> mCapabilities;

 /**
  * True if mCapabilities only contains hardcoded capabilities. This can happen
  * if the underlying device is not reporting any capabilities. These can be
  * affected by constraints, so they're evaluated in ChooseCapability() rather
  * than GetCapability().
  *
  * This is mutable so that the const methods NumCapabilities() and
  * GetCapability() can reset it. Owning thread only.
  */
 mutable bool mCapabilitiesAreHardcoded = false;

 const RefPtr<const MediaDevice> mMediaDevice;
 const nsCString mDeviceUUID;
 Maybe<nsString> mFacingMode;
 MediaEventProducer<void> mCaptureEndedEvent;
};

}  // namespace mozilla

#endif /* MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_ */