commit 48829083dd73f04f736c7e79afdecbd925c284b3 parent 32d8190f51de5349a75f5380fdfa25b504b08a3c Author: Cristina Horotan <chorotan@mozilla.com> Date: Wed, 15 Oct 2025 00:23:04 +0300 Revert "Bug 1771789 - Extend webrtc::VideoFrame lifetime a bit to avoid an extra allocation and copy in some cases. r=jib" for causing multiple failures This reverts commit 238c28a469049199a54e92fcf5b9a55a379c5867. Revert "Bug 1771789 - In CamerasParent use one ShmemPool per source. r=jib" This reverts commit 82e8303b6670528eeb75d6af14e6404a6256694a. Revert "Bug 1771789 - Refactor video capture source sharing in CamerasParent. r=jib" This reverts commit e6ac33e70d49b8475756fb7c936bc0052c50077f. Revert "Bug 1771789 - In nsTArray, allow Comparators to compare elements to a type defined by the caller. r=xpcom-reviewers,nika" This reverts commit 0ef502b946e2146156bfee9cc5c3108fd904e806. Revert "Bug 1771789 - In PCameras rename captureId with streamId for consistency. r=jib" This reverts commit e1924d3ed2faf1ba137baf351ba1d72f7cb315c1. Revert "Bug 1771789 - Clean up type mismatches and unused things in CamerasParent. r=jib" This reverts commit 7ab8aaeb3e46c7ebd8c1121a3ed46f9de1f5db7e. Revert "Bug 1771789 - Pass CubebInputStream the same CubebHandle used to enumerate its device id. r=padenot" This reverts commit 7d2f37838ed207ede8b4957937f70e966e8f24f3. Revert "Bug 1771789 - Test audio capture cloning independence. r=jib" This reverts commit 7b5b612e0ffeafd8548c5137e2c68ac182cd471b. Revert "Bug 1771789 - Test video capture cloning independence. r=jib" This reverts commit 1eae11dc2e9dd9a1095ce607068ac366b15b1d7e. Revert "Bug 1771789 - Break out some helpers from MST-resizeMode.https.html. r=jib" This reverts commit 5a1bdf75f78937e54e5dbd5ca4828e0bf1d9656a. Revert "Bug 1771789 - Use "real fake" camera in some mochitests as loopback device does not work with native cloning. r=jib" This reverts commit a34936b64c023e061e418bf53e958024d098a0b2. Revert "Bug 1771789 - Make AudioInputProcessing::NumberOfChannels always valid. r=padenot,webrtc-reviewers,mjf" This reverts commit e8577e2415aa670712abdaa829389f03c2a97800. Revert "Bug 1771789 - Deep-clone track sources for correct settings synchronously. r=jib" This reverts commit 9a636bdf6176241e3b91cab5beb9d6d88cbab413. Revert "Bug 1771789 - Allow re-allocating video device without active camera permission. r=jib" This reverts commit eb61b57b45dd6d7b825ef780dbaa19141215f4f8. Revert "Bug 1771789 - Implement cloning for LocalTrackSource. r=jib" This reverts commit 82b3c45eb63c50f46b1751027798af3384700a4a. Revert "Bug 1771789 - Add DeviceState::mAllocated to avoid Stop and Deallocate of non-allocated sources. r=jib" This reverts commit 760ee59dea28fcf55dbde503fea440ac0408d805. Revert "Bug 1771789 - Break out a sync Initialize from InitializeAsync. r=jib" This reverts commit 55ca6273e38657bc692c3a901f7a437847826151. Revert "Bug 1771789 - Add and use an API for MediaStreamTrackSource cloning. r=jib" This reverts commit f1eba40e5d7bea8a3496be95aa6c13e79f096a3d. Revert "Bug 1771789 - Implement MediaStreamTrack::CloneInternal in a single place with templates. r=jib" This reverts commit 64051cd4c3da83025038a1fb4231f1eefeebf466. Diffstat:
37 files changed, 622 insertions(+), 1819 deletions(-)
diff --git a/dom/media/AudioStreamTrack.cpp b/dom/media/AudioStreamTrack.cpp @@ -40,8 +40,9 @@ void AudioStreamTrack::GetLabel(nsAString& aLabel, CallerType aCallerType) { MediaStreamTrack::GetLabel(aLabel, aCallerType); } -already_AddRefed<MediaStreamTrack> AudioStreamTrack::Clone() { - return MediaStreamTrack::CloneInternal<AudioStreamTrack>(); +already_AddRefed<MediaStreamTrack> AudioStreamTrack::CloneInternal() { + return do_AddRef(new AudioStreamTrack(mWindow, mInputTrack, mSource, + ReadyState(), Muted(), mConstraints)); } } // namespace mozilla::dom diff --git a/dom/media/AudioStreamTrack.h b/dom/media/AudioStreamTrack.h @@ -24,8 +24,6 @@ class AudioStreamTrack : public MediaStreamTrack { : MediaStreamTrack(aWindow, aInputTrack, aSource, aReadyState, aMuted, aConstraints) {} - already_AddRefed<MediaStreamTrack> Clone() override; - AudioStreamTrack* AsAudioStreamTrack() override { return this; } const AudioStreamTrack* AsAudioStreamTrack() const override { return this; } @@ -40,6 +38,9 @@ class AudioStreamTrack : public MediaStreamTrack { void GetKind(nsAString& aKind) override { aKind.AssignLiteral("audio"); } void GetLabel(nsAString& aLabel, CallerType aCallerType) override; + + protected: + already_AddRefed<MediaStreamTrack> CloneInternal() override; }; } // namespace mozilla::dom diff --git a/dom/media/CubebInputStream.cpp b/dom/media/CubebInputStream.cpp @@ -112,17 +112,18 @@ UniquePtr<CubebInputStream> CubebInputStream::Create(cubeb_devid aDeviceId, LOG("Create a cubeb stream %p successfully", inputStream.get()); - UniquePtr<CubebInputStream> stream(new CubebInputStream( - listener.forget(), handle.forget(), std::move(inputStream))); + UniquePtr<CubebInputStream> stream( + new CubebInputStream(listener.forget(), std::move(inputStream))); stream->Init(); return stream; } CubebInputStream::CubebInputStream( already_AddRefed<Listener>&& aListener, - already_AddRefed<CubebUtils::CubebHandle>&& aCubeb, UniquePtr<cubeb_stream, CubebDestroyPolicy>&& aStream) - : mListener(aListener), mCubeb(aCubeb), mStream(std::move(aStream)) { + : mListener(aListener), + mCubeb(CubebUtils::GetCubeb()), + mStream(std::move(aStream)) { MOZ_ASSERT(mListener); MOZ_ASSERT(mStream); } diff --git a/dom/media/CubebInputStream.h b/dom/media/CubebInputStream.h @@ -62,7 +62,6 @@ class CubebInputStream final { void operator()(cubeb_stream* aStream) const; }; CubebInputStream(already_AddRefed<Listener>&& aListener, - already_AddRefed<CubebUtils::CubebHandle>&& aCubeb, UniquePtr<cubeb_stream, CubebDestroyPolicy>&& aStream); void Init(); diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp @@ -211,11 +211,6 @@ struct DeviceState { MOZ_ASSERT(mTrackSource); } - // true if we have allocated mDevice. When not allocated, we may not stop or - // deallocate. - // MainThread only. - bool mAllocated = false; - // true if we have stopped mDevice, this is a terminal state. // MainThread only. bool mStopped = false; @@ -395,29 +390,13 @@ class DeviceListener : public SupportsWeakPtr { * Marks this listener as active and creates the internal device state. */ void Activate(RefPtr<LocalMediaDevice> aDevice, - RefPtr<LocalTrackSource> aTrackSource, bool aStartMuted, - bool aIsAllocated); + RefPtr<LocalTrackSource> aTrackSource, bool aStartMuted); /** * Posts a task to initialize and start the associated device. */ RefPtr<DeviceListenerPromise> InitializeAsync(); - private: - /** - * Initializes synchronously. Must be called on the media thread. - */ - nsresult Initialize(PrincipalHandle aPrincipal, LocalMediaDevice* aDevice, - MediaTrack* aTrack, bool aStartDevice); - - public: - /** - * Synchronously clones this device listener, setting up the device to match - * our current device state asynchronously. Settings, constraints and other - * main thread state starts applying immediately. - */ - already_AddRefed<DeviceListener> Clone() const; - /** * Posts a task to stop the device associated with this DeviceListener and * notifies the associated window listener that a track was stopped. @@ -488,10 +467,6 @@ class DeviceListener : public SupportsWeakPtr { return mDeviceState ? mDeviceState->mDevice.get() : nullptr; } - LocalTrackSource* GetTrackSource() const { - return mDeviceState ? mDeviceState->mTrackSource.get() : nullptr; - } - bool Activated() const { return static_cast<bool>(mDeviceState); } bool Stopped() const { return mStopped; } @@ -589,7 +564,7 @@ class GetUserMediaWindowListener { */ void Activate(RefPtr<DeviceListener> aListener, RefPtr<LocalMediaDevice> aDevice, - RefPtr<LocalTrackSource> aTrackSource, bool aIsAllocated) { + RefPtr<LocalTrackSource> aTrackSource) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aListener); MOZ_ASSERT(!aListener->Activated()); @@ -607,8 +582,7 @@ class GetUserMediaWindowListener { } mInactiveListeners.RemoveElement(aListener); - aListener->Activate(std::move(aDevice), std::move(aTrackSource), muted, - aIsAllocated); + aListener->Activate(std::move(aDevice), std::move(aTrackSource), muted); mActiveListeners.AppendElement(std::move(aListener)); } @@ -835,7 +809,7 @@ class LocalTrackSource : public MediaStreamTrackSource { LocalTrackSource(nsIPrincipal* aPrincipal, const nsString& aLabel, const RefPtr<DeviceListener>& aListener, MediaSourceEnum aSource, MediaTrack* aTrack, - RefPtr<const PeerIdentity> aPeerIdentity, + RefPtr<PeerIdentity> aPeerIdentity, TrackingId aTrackingId = TrackingId()) : MediaStreamTrackSource(aPrincipal, aLabel, std::move(aTrackingId)), mSource(aSource), @@ -889,20 +863,6 @@ class LocalTrackSource : public MediaStreamTrackSource { } } - CloneResult Clone() override { - if (!mListener) { - return {}; - } - RefPtr listener = mListener->Clone(); - MOZ_ASSERT(listener); - if (!listener) { - return {}; - } - - return {.mSource = listener->GetTrackSource(), - .mInputTrack = listener->GetTrackSource()->mTrack}; - } - void Disable() override { if (mListener) { mListener->SetDeviceEnabled(false); @@ -1192,11 +1152,6 @@ const TrackingId& LocalMediaDevice::GetTrackingId() const { return mSource->GetTrackingId(); } -const dom::MediaTrackConstraints& LocalMediaDevice::Constraints() const { - MOZ_ASSERT(MediaManager::IsInMediaThread()); - return mConstraints; -} - // Threadsafe since mKind and mSource are const. NS_IMETHODIMP LocalMediaDevice::GetMediaSource(nsAString& aMediaSource) { @@ -1221,12 +1176,7 @@ nsresult LocalMediaDevice::Allocate(const MediaTrackConstraints& aConstraints, return NS_ERROR_FAILURE; } - nsresult rv = - Source()->Allocate(aConstraints, aPrefs, aWindowID, aOutBadConstraint); - if (NS_SUCCEEDED(rv)) { - mConstraints = aConstraints; - } - return rv; + return Source()->Allocate(aConstraints, aPrefs, aWindowID, aOutBadConstraint); } void LocalMediaDevice::SetTrack(const RefPtr<MediaTrack>& aTrack, @@ -1270,11 +1220,7 @@ nsresult LocalMediaDevice::Reconfigure( } } } - nsresult rv = Source()->Reconfigure(aConstraints, aPrefs, aOutBadConstraint); - if (NS_SUCCEEDED(rv)) { - mConstraints = aConstraints; - } - return rv; + return Source()->Reconfigure(aConstraints, aPrefs, aOutBadConstraint); } nsresult LocalMediaDevice::FocusOnSelectedSource() { @@ -1294,21 +1240,6 @@ nsresult LocalMediaDevice::Deallocate() { return mSource->Deallocate(); } -already_AddRefed<LocalMediaDevice> LocalMediaDevice::Clone() const { - MOZ_ASSERT(NS_IsMainThread()); - auto device = MakeRefPtr<LocalMediaDevice>(mRawDevice, mID, mGroupID, mName); - device->mSource = - mRawDevice->mEngine->CreateSourceFrom(mSource, device->mRawDevice); -#ifdef MOZ_THREAD_SAFETY_OWNERSHIP_CHECKS_SUPPORTED - // The source is normally created on the MediaManager thread. But for cloning, - // it ends up being created on main thread. Make sure its owning event target - // is set properly. - auto* src = device->Source(); - src->_mOwningThread = mSource->_mOwningThread; -#endif - return device.forget(); -} - MediaSourceEnum MediaDevice::GetMediaSource() const { return mMediaSource; } static const MediaTrackConstraints& GetInvariant( @@ -1836,13 +1767,11 @@ void GetUserMediaStreamTask::PrepareDOMStream() { // is freed when the page is invalidated (on navigation or close). if (mAudioDeviceListener) { mWindowListener->Activate(mAudioDeviceListener, mAudioDevice, - std::move(audioTrackSource), - /*aIsAllocated=*/true); + std::move(audioTrackSource)); } if (mVideoDeviceListener) { mWindowListener->Activate(mVideoDeviceListener, mVideoDevice, - std::move(videoTrackSource), - /*aIsAllocated=*/true); + std::move(videoTrackSource)); } // Dispatch to the media thread to ask it to start the sources, because that @@ -4327,7 +4256,7 @@ void DeviceListener::Register(GetUserMediaWindowListener* aListener) { void DeviceListener::Activate(RefPtr<LocalMediaDevice> aDevice, RefPtr<LocalTrackSource> aTrackSource, - bool aStartMuted, bool aIsAllocated) { + bool aStartMuted) { MOZ_ASSERT(NS_IsMainThread(), "Only call on main thread"); LOG("DeviceListener %p activating %s device %p", this, @@ -4353,7 +4282,6 @@ void DeviceListener::Activate(RefPtr<LocalMediaDevice> aDevice, mDeviceState = MakeUnique<DeviceState>( std::move(aDevice), std::move(aTrackSource), offWhileDisabled); mDeviceState->mDeviceMuted = aStartMuted; - mDeviceState->mAllocated = aIsAllocated; if (aStartMuted) { mDeviceState->mTrackSource->Mute(); } @@ -4364,24 +4292,51 @@ DeviceListener::InitializeAsync() { MOZ_ASSERT(NS_IsMainThread(), "Only call on main thread"); MOZ_DIAGNOSTIC_ASSERT(!mStopped); - return InvokeAsync( - MediaManager::Get()->mMediaThread, __func__, - [this, self = RefPtr(this), principal = GetPrincipalHandle(), - device = mDeviceState->mDevice, + return MediaManager::Dispatch<DeviceListenerPromise>( + __func__, + [principal = GetPrincipalHandle(), device = mDeviceState->mDevice, track = mDeviceState->mTrackSource->mTrack, - deviceMuted = mDeviceState->mDeviceMuted] { - nsresult rv = Initialize(principal, device, track, - /*aStartDevice=*/!deviceMuted); - if (NS_SUCCEEDED(rv)) { - return GenericPromise::CreateAndResolve( - true, "DeviceListener::InitializeAsync success"); + deviceMuted = mDeviceState->mDeviceMuted]( + MozPromiseHolder<DeviceListenerPromise>& aHolder) { + auto kind = device->Kind(); + device->SetTrack(track, principal); + nsresult rv = deviceMuted ? NS_OK : device->Start(); + if (kind == MediaDeviceKind::Audioinput || + kind == MediaDeviceKind::Videoinput) { + if ((rv == NS_ERROR_NOT_AVAILABLE && + kind == MediaDeviceKind::Audioinput) || + (NS_FAILED(rv) && kind == MediaDeviceKind::Videoinput)) { + PR_Sleep(200); + rv = device->Start(); + } + if (rv == NS_ERROR_NOT_AVAILABLE && + kind == MediaDeviceKind::Audioinput) { + nsCString log; + log.AssignLiteral("Concurrent mic process limit."); + aHolder.Reject(MakeRefPtr<MediaMgrError>( + MediaMgrError::Name::NotReadableError, + std::move(log)), + __func__); + return; + } + } + if (NS_FAILED(rv)) { + nsCString log; + log.AppendPrintf("Starting %s failed", + dom::GetEnumString(kind).get()); + aHolder.Reject( + MakeRefPtr<MediaMgrError>(MediaMgrError::Name::AbortError, + std::move(log)), + __func__); + return; } - return GenericPromise::CreateAndReject( - rv, "DeviceListener::InitializeAsync failure"); + LOG("started %s device %p", dom::GetEnumString(kind).get(), + device.get()); + aHolder.Resolve(true, __func__); }) ->Then( GetMainThreadSerialEventTarget(), __func__, - [self = RefPtr<DeviceListener>(this), this](bool) { + [self = RefPtr<DeviceListener>(this), this]() { if (mStopped) { // We were shut down during the async init return DeviceListenerPromise::CreateAndResolve(true, __func__); @@ -4396,25 +4351,10 @@ DeviceListener::InitializeAsync() { mDeviceState->mTrackEnabledTime = TimeStamp::Now(); return DeviceListenerPromise::CreateAndResolve(true, __func__); }, - [self = RefPtr<DeviceListener>(this), this](nsresult aRv) { - auto kind = mDeviceState->mDevice->Kind(); - RefPtr<MediaMgrError> err; - if (aRv == NS_ERROR_NOT_AVAILABLE && - kind == MediaDeviceKind::Audioinput) { - nsCString log; - log.AssignLiteral("Concurrent mic process limit."); - err = MakeRefPtr<MediaMgrError>( - MediaMgrError::Name::NotReadableError, std::move(log)); - } else if (NS_FAILED(aRv)) { - nsCString log; - log.AppendPrintf("Starting %s failed", - dom::GetEnumString(kind).get()); - err = MakeRefPtr<MediaMgrError>(MediaMgrError::Name::AbortError, - std::move(log)); - } - + [self = RefPtr<DeviceListener>(this), + this](const RefPtr<MediaMgrError>& aResult) { if (mStopped) { - return DeviceListenerPromise::CreateAndReject(err, __func__); + return DeviceListenerPromise::CreateAndReject(aResult, __func__); } MOZ_DIAGNOSTIC_ASSERT(!mDeviceState->mTrackEnabled); @@ -4422,150 +4362,10 @@ DeviceListener::InitializeAsync() { MOZ_DIAGNOSTIC_ASSERT(!mDeviceState->mStopped); Stop(); - - return DeviceListenerPromise::CreateAndReject(err, __func__); + return DeviceListenerPromise::CreateAndReject(aResult, __func__); }); } -nsresult DeviceListener::Initialize(PrincipalHandle aPrincipal, - LocalMediaDevice* aDevice, - MediaTrack* aTrack, bool aStartDevice) { - MOZ_ASSERT(MediaManager::IsInMediaThread()); - - auto kind = aDevice->Kind(); - aDevice->SetTrack(aTrack, aPrincipal); - nsresult rv = aStartDevice ? aDevice->Start() : NS_OK; - if (kind == MediaDeviceKind::Audioinput || - kind == MediaDeviceKind::Videoinput) { - if ((rv == NS_ERROR_NOT_AVAILABLE && kind == MediaDeviceKind::Audioinput) || - (NS_FAILED(rv) && kind == MediaDeviceKind::Videoinput)) { - PR_Sleep(200); - rv = aDevice->Start(); - } - } - LOG("started %s device %p", dom::GetEnumString(kind).get(), aDevice); - return rv; -} - -already_AddRefed<DeviceListener> DeviceListener::Clone() const { - MOZ_ASSERT(NS_IsMainThread()); - MediaManager* mgr = MediaManager::GetIfExists(); - if (!mgr) { - return nullptr; - } - if (!mWindowListener) { - return nullptr; - } - auto* thisDevice = GetDevice(); - if (!thisDevice) { - return nullptr; - } - - auto* thisTrackSource = GetTrackSource(); - if (!thisTrackSource) { - return nullptr; - } - - // See PrepareDOMStream for how a gUM/gDM track is created. - RefPtr<MediaTrack> track; - MediaTrackGraph* mtg = thisTrackSource->mTrack->Graph(); - if (const auto source = thisDevice->GetMediaSource(); - source == dom::MediaSourceEnum::Microphone) { -#ifdef MOZ_WEBRTC - if (thisDevice->IsFake()) { - track = mtg->CreateSourceTrack(MediaSegment::AUDIO); - } else { - track = AudioProcessingTrack::Create(mtg); - track->Suspend(); // Microphone source resumes in SetTrack - } -#else - track = mtg->CreateSourceTrack(MediaSegment::AUDIO); -#endif - } else if (source == dom::MediaSourceEnum::Camera || - source == dom::MediaSourceEnum::Screen || - source == dom::MediaSourceEnum::Window || - source == dom::MediaSourceEnum::Browser) { - track = mtg->CreateSourceTrack(MediaSegment::VIDEO); - } - - if (!track) { - return nullptr; - } - - RefPtr device = thisDevice->Clone(); - auto listener = MakeRefPtr<DeviceListener>(); - auto trackSource = MakeRefPtr<LocalTrackSource>( - thisTrackSource->GetPrincipal(), thisTrackSource->mLabel, listener, - thisTrackSource->mSource, track, thisTrackSource->mPeerIdentity, - thisTrackSource->mTrackingId); - - LOG("DeviceListener %p registering clone", this); - mWindowListener->Register(listener); - LOG("DeviceListener %p activating clone", this); - mWindowListener->Activate(listener, device, trackSource, - /*aIsAllocated=*/false); - - listener->mDeviceState->mDeviceEnabled = mDeviceState->mDeviceEnabled; - listener->mDeviceState->mDeviceMuted = mDeviceState->mDeviceMuted; - listener->mDeviceState->mTrackEnabled = mDeviceState->mTrackEnabled; - listener->mDeviceState->mTrackEnabledTime = TimeStamp::Now(); - - // We have to do an async operation here, even though Clone() is sync. - // This is fine because JS will not be able to trigger any operation to run - // async on the media thread. - LOG("DeviceListener %p allocating clone device %p async", this, device.get()); - InvokeAsync( - mgr->mMediaThread, __func__, - [thisDevice = RefPtr(thisDevice), device, prefs = mgr->mPrefs, - windowId = mWindowListener->WindowID(), listener, - principal = GetPrincipalHandle(), track, - startDevice = !listener->mDeviceState->mDeviceMuted && - listener->mDeviceState->mDeviceEnabled] { - const char* outBadConstraint{}; - nsresult rv = device->Source()->Allocate( - thisDevice->Constraints(), prefs, windowId, &outBadConstraint); - LOG("Allocated clone device %p. rv=%s", device.get(), - GetStaticErrorName(rv)); - if (NS_FAILED(rv)) { - return GenericPromise::CreateAndReject( - rv, "DeviceListener::Clone failure #1"); - } - rv = listener->Initialize(principal, device, track, startDevice); - if (NS_SUCCEEDED(rv)) { - return GenericPromise::CreateAndResolve( - true, "DeviceListener::Clone success"); - } - return GenericPromise::CreateAndReject( - rv, "DeviceListener::Clone failure #2"); - }) - ->Then(GetMainThreadSerialEventTarget(), __func__, - [listener, device, - trackSource](GenericPromise::ResolveOrRejectValue&& aValue) { - if (aValue.IsReject()) { - // Allocating/initializing failed. Stopping the device listener - // will destroy the MediaStreamTrackSource's MediaTrack, which - // will make the MediaStreamTrack's mTrack MediaTrack auto-end - // due to lack of inputs. This makes the MediaStreamTrack's - // readyState transition to "ended" as expected. - LOG("Allocating clone device %p failed. Stopping.", - device.get()); - listener->Stop(); - return; - } - listener->mDeviceState->mAllocated = true; - if (listener->mDeviceState->mStopped) { - MediaManager::Dispatch(NS_NewRunnableFunction( - "DeviceListener::Clone::Stop", - [device = listener->mDeviceState->mDevice]() { - device->Stop(); - device->Deallocate(); - })); - } - }); - - return listener.forget(); -} - void DeviceListener::Stop() { MOZ_ASSERT(NS_IsMainThread(), "Only call on main thread"); @@ -4587,12 +4387,10 @@ void DeviceListener::Stop() { mDeviceState->mTrackSource->Stop(); - if (mDeviceState->mAllocated) { - MediaManager::Dispatch(NewTaskFrom([device = mDeviceState->mDevice]() { - device->Stop(); - device->Deallocate(); - })); - } + MediaManager::Dispatch(NewTaskFrom([device = mDeviceState->mDevice]() { + device->Stop(); + device->Deallocate(); + })); mWindowListener->ChromeAffectingStateChanged(); } diff --git a/dom/media/MediaManager.h b/dom/media/MediaManager.h @@ -150,11 +150,6 @@ class LocalMediaDevice final : public nsIMediaDevice { nsresult Stop(); nsresult Deallocate(); - /** - * Clones the LocalMediaDevice and sets a cloned source. - */ - already_AddRefed<LocalMediaDevice> Clone() const; - void GetSettings(dom::MediaTrackSettings& aOutSettings); void GetCapabilities(dom::MediaTrackCapabilities& aOutCapabilities); MediaEngineSource* Source(); @@ -169,7 +164,6 @@ class LocalMediaDevice final : public nsIMediaDevice { dom::MediaDeviceKind Kind() const { return mRawDevice->mKind; } bool IsFake() const { return mRawDevice->mIsFake; } const nsString& RawID() { return mRawDevice->mRawID; } - const dom::MediaTrackConstraints& Constraints() const; private: virtual ~LocalMediaDevice() = default; @@ -192,8 +186,6 @@ class LocalMediaDevice final : public nsIMediaDevice { private: RefPtr<MediaEngineSource> mSource; - // Currently applied constraints. Media thread only. - dom::MediaTrackConstraints mConstraints; }; typedef nsRefPtrHashtable<nsUint64HashKey, GetUserMediaWindowListener> diff --git a/dom/media/MediaStreamTrack.cpp b/dom/media/MediaStreamTrack.cpp @@ -46,8 +46,6 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(MediaStreamTrackSource) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPrincipal) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END -auto MediaStreamTrackSource::Clone() -> CloneResult { return {}; } - auto MediaStreamTrackSource::ApplyConstraints( const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType) -> RefPtr<ApplyConstraintsPromise> { @@ -541,6 +539,13 @@ void MediaStreamTrack::RemoveConsumer(MediaStreamTrackConsumer* aConsumer) { } } +already_AddRefed<MediaStreamTrack> MediaStreamTrack::Clone() { + RefPtr<MediaStreamTrack> newTrack = CloneInternal(); + newTrack->SetEnabled(Enabled()); + newTrack->SetMuted(Muted()); + return newTrack.forget(); +} + void MediaStreamTrack::SetReadyState(MediaStreamTrackState aState) { MOZ_ASSERT(!(mReadyState == MediaStreamTrackState::Ended && aState == MediaStreamTrackState::Live), diff --git a/dom/media/MediaStreamTrack.h b/dom/media/MediaStreamTrack.h @@ -128,18 +128,6 @@ class MediaStreamTrackSource : public nsISupports { */ virtual void Destroy() {} - struct CloneResult { - RefPtr<MediaStreamTrackSource> mSource; - RefPtr<mozilla::MediaTrack> mInputTrack; - }; - - /** - * Clone this MediaStreamTrackSource. Cloned sources allow independent track - * settings. Not supported by all source types. A source not supporting - * cloning returns nullptr. - */ - virtual CloneResult Clone(); - /** * Gets the source's MediaSourceEnum for usage by PeerConnections. */ @@ -481,7 +469,7 @@ class MediaStreamTrack : public DOMEventTargetHelper, public SupportsWeakPtr { already_AddRefed<Promise> ApplyConstraints( const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType, ErrorResult& aRv); - virtual already_AddRefed<MediaStreamTrack> Clone() = 0; + already_AddRefed<MediaStreamTrack> Clone(); MediaStreamTrackState ReadyState() { return mReadyState; } IMPL_EVENT_HANDLER(mute) @@ -654,22 +642,7 @@ class MediaStreamTrack : public DOMEventTargetHelper, public SupportsWeakPtr { * Creates a new MediaStreamTrack with the same kind, input track, input * track ID and source as this MediaStreamTrack. */ - - template <typename TrackType> - already_AddRefed<MediaStreamTrack> CloneInternal() { - auto cloneRes = mSource->Clone(); - MOZ_ASSERT(!!cloneRes.mSource == !!cloneRes.mInputTrack); - if (!cloneRes.mSource || !cloneRes.mInputTrack) { - cloneRes.mSource = mSource; - cloneRes.mInputTrack = mInputTrack; - } - auto newTrack = - MakeRefPtr<TrackType>(mWindow, cloneRes.mInputTrack, cloneRes.mSource, - ReadyState(), Muted(), mConstraints); - newTrack->SetEnabled(Enabled()); - newTrack->SetMuted(Muted()); - return newTrack.forget(); - } + virtual already_AddRefed<MediaStreamTrack> CloneInternal() = 0; nsTArray<PrincipalChangeObserver<MediaStreamTrack>*> mPrincipalChangeObservers; diff --git a/dom/media/VideoStreamTrack.cpp b/dom/media/VideoStreamTrack.cpp @@ -74,8 +74,9 @@ void VideoStreamTrack::GetLabel(nsAString& aLabel, CallerType aCallerType) { MediaStreamTrack::GetLabel(aLabel, aCallerType); } -already_AddRefed<MediaStreamTrack> VideoStreamTrack::Clone() { - return MediaStreamTrack::CloneInternal<VideoStreamTrack>(); +already_AddRefed<MediaStreamTrack> VideoStreamTrack::CloneInternal() { + return do_AddRef(new VideoStreamTrack(mWindow, mInputTrack, mSource, + ReadyState(), Muted(), mConstraints)); } } // namespace mozilla::dom diff --git a/dom/media/VideoStreamTrack.h b/dom/media/VideoStreamTrack.h @@ -25,8 +25,6 @@ class VideoStreamTrack : public MediaStreamTrack { bool aMuted = false, const MediaTrackConstraints& aConstraints = MediaTrackConstraints()); - already_AddRefed<MediaStreamTrack> Clone() override; - void Destroy() override; VideoStreamTrack* AsVideoStreamTrack() override { return this; } @@ -47,6 +45,9 @@ class VideoStreamTrack : public MediaStreamTrack { void GetLabel(nsAString& aLabel, CallerType aCallerType) override; + protected: + already_AddRefed<MediaStreamTrack> CloneInternal() override; + private: nsTArray<RefPtr<VideoOutput>> mVideoOutputs; }; diff --git a/dom/media/systemservices/CamerasChild.cpp b/dom/media/systemservices/CamerasChild.cpp @@ -478,32 +478,27 @@ void Shutdown(void) { CamerasSingleton::Thread() = nullptr; } -mozilla::ipc::IPCResult CamerasChild::RecvCaptureEnded( - nsTArray<int>&& aCaptureIds) { +mozilla::ipc::IPCResult CamerasChild::RecvCaptureEnded(const int& capId) { MutexAutoLock lock(mCallbackMutex); - for (int capId : aCaptureIds) { - if (auto* cb = Callback(capId)) { - cb->OnCaptureEnded(); - } else { - LOG(("CaptureEnded called with dead callback")); - } + if (Callback(capId)) { + Callback(capId)->OnCaptureEnded(); + } else { + LOG(("CaptureEnded called with dead callback")); } return IPC_OK(); } mozilla::ipc::IPCResult CamerasChild::RecvDeliverFrame( - const int& aCaptureId, nsTArray<int>&& aStreamIds, - mozilla::ipc::Shmem&& aShmem, const VideoFrameProperties& aProps) { + const int& capId, mozilla::ipc::Shmem&& shmem, + const VideoFrameProperties& prop) { MutexAutoLock lock(mCallbackMutex); - for (const int& streamId : aStreamIds) { - if (auto* cb = Callback(streamId)) { - unsigned char* image = aShmem.get<unsigned char>(); - cb->DeliverFrame(image, aProps); - } else { - LOG(("DeliverFrame called with dead callback")); - } + if (Callback(capId)) { + unsigned char* image = shmem.get<unsigned char>(); + Callback(capId)->DeliverFrame(image, prop); + } else { + LOG(("DeliverFrame called with dead callback")); } - SendReleaseFrame(aCaptureId, std::move(aShmem)); + SendReleaseFrame(std::move(shmem)); return IPC_OK(); } diff --git a/dom/media/systemservices/CamerasChild.h b/dom/media/systemservices/CamerasChild.h @@ -146,12 +146,10 @@ class CamerasChild final : public PCamerasChild { // IPC messages recevied, received on the PBackground thread // these are the actual callbacks with data - mozilla::ipc::IPCResult RecvCaptureEnded( - nsTArray<int>&& aCaptureIds) override; + mozilla::ipc::IPCResult RecvCaptureEnded(const int&) override; mozilla::ipc::IPCResult RecvDeliverFrame( - const int& aCaptureId, nsTArray<int>&& aStreamIds, - mozilla::ipc::Shmem&& aShmem, - const VideoFrameProperties& aProps) override; + const int&, mozilla::ipc::Shmem&&, + const VideoFrameProperties& prop) override; mozilla::ipc::IPCResult RecvDeviceChange() override; diff --git a/dom/media/systemservices/CamerasParent.cpp b/dom/media/systemservices/CamerasParent.cpp @@ -58,6 +58,10 @@ using dom::VideoResizeModeEnum; using media::ShutdownBlockingTicket; namespace camera { +MOZ_RUNINIT std::map<uint32_t, const char*> sDeviceUniqueIDs; +MOZ_RUNINIT std::map<uint32_t, webrtc::VideoCaptureCapability> + sAllRequestedCapabilities; + uint32_t ResolutionFeasibilityDistance(int32_t candidate, int32_t requested) { // The purpose of this function is to find a smallest resolution // which is larger than all requested capabilities. @@ -116,12 +120,6 @@ static StaticRefPtr<nsIThread> sVideoCaptureThread; // objects. Created on IPC background thread, destroyed on main thread on // shutdown. Outlives the CamerasParent instances. static StaticRefPtr<VideoCaptureFactory> sVideoCaptureFactory; -// All live capturers across all CamerasParent instances. The array and its -// members are only modified on the video capture thread. The outermost refcount -// is IPC background thread only. -static StaticRefPtr< - media::Refcountable<nsTArray<std::unique_ptr<AggregateCapturer>>>> - sCapturers; static void ClearCameraDeviceInfo() { ipc::AssertIsOnBackgroundThread(); @@ -199,9 +197,6 @@ MakeAndAddRefVideoCaptureThreadAndSingletons() { sEngines = MakeRefPtr<VideoEngineArray>(); sEngines->AppendElements(CaptureEngine::MaxEngine); - - sCapturers = MakeRefPtr< - media::Refcountable<nsTArray<std::unique_ptr<AggregateCapturer>>>>(); } ++sNumCamerasParents; @@ -220,10 +215,8 @@ static void ReleaseVideoCaptureThreadAndSingletons() { // No other CamerasParent instances alive. Clean up. LOG("Shutting down VideoEngines and the VideoCapture thread"); - MOZ_ALWAYS_SUCCEEDS(sVideoCaptureThread->Dispatch(NS_NewRunnableFunction( - __func__, [engines = RefPtr(sEngines.forget()), - capturers = RefPtr(sCapturers.forget())] { - MOZ_ASSERT(capturers->IsEmpty(), "No capturers expected on shutdown"); + MOZ_ALWAYS_SUCCEEDS(sVideoCaptureThread->Dispatch( + NS_NewRunnableFunction(__func__, [engines = RefPtr(sEngines.forget())] { for (RefPtr<VideoEngine>& engine : *engines) { if (engine) { VideoEngine::Delete(engine); @@ -259,34 +252,43 @@ void CamerasParent::OnDeviceChange() { class DeliverFrameRunnable : public mozilla::Runnable { public: - // Constructor for when no ShmemBuffer (of the right size) was available, so - // keep the frame around until we can allocate one on PBackground (in Run). DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine, - int aCaptureId, nsTArray<int>&& aStreamIds, - const TrackingId& aTrackingId, + uint32_t aStreamId, const TrackingId& aTrackingId, const webrtc::VideoFrame& aFrame, const VideoFrameProperties& aProperties) : Runnable("camera::DeliverFrameRunnable"), mParent(aParent), mCapEngine(aEngine), - mCaptureId(aCaptureId), - mStreamIds(std::move(aStreamIds)), + mStreamId(aStreamId), mTrackingId(aTrackingId), - mBuffer(aFrame), - mProperties(aProperties) {} + mProperties(aProperties), + mResult(0) { + // No ShmemBuffer (of the right size) was available, so make an + // extra buffer here. We have no idea when we are going to run and + // it will be potentially long after the webrtc frame callback has + // returned, so the copy needs to be no later than here. + // We will need to copy this back into a Shmem later on so we prefer + // using ShmemBuffers to avoid the extra copy. + PerformanceRecorder<CopyVideoStage> rec( + "CamerasParent::VideoFrameToAltBuffer"_ns, aTrackingId, aFrame.width(), + aFrame.height()); + mAlternateBuffer.reset(new unsigned char[aProperties.bufferSize()]); + VideoFrameUtils::CopyVideoFrameBuffers(mAlternateBuffer.get(), + aProperties.bufferSize(), aFrame); + rec.Record(); + } DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine, - int aCaptureId, nsTArray<int>&& aStreamIds, - const TrackingId& aTrackingId, ShmemBuffer aBuffer, - VideoFrameProperties& aProperties) + uint32_t aStreamId, const TrackingId& aTrackingId, + ShmemBuffer aBuffer, VideoFrameProperties& aProperties) : Runnable("camera::DeliverFrameRunnable"), mParent(aParent), mCapEngine(aEngine), - mCaptureId(aCaptureId), - mStreamIds(std::move(aStreamIds)), + mStreamId(aStreamId), mTrackingId(aTrackingId), mBuffer(std::move(aBuffer)), - mProperties(aProperties) {} + mProperties(aProperties), + mResult(0) {}; NS_IMETHOD Run() override { // runs on BackgroundEventTarget @@ -294,43 +296,45 @@ class DeliverFrameRunnable : public mozilla::Runnable { mParent->mPBackgroundEventTarget); if (mParent->IsShuttingDown()) { // Communication channel is being torn down + mResult = 0; return NS_OK; } - mParent->DeliverFrameOverIPC(mCapEngine, mCaptureId, mStreamIds, - mTrackingId, std::move(mBuffer), mProperties); + if (!mParent->DeliverFrameOverIPC(mCapEngine, mStreamId, mTrackingId, + std::move(mBuffer), + mAlternateBuffer.get(), mProperties)) { + mResult = -1; + } else { + mResult = 0; + } return NS_OK; } + int GetResult() { return mResult; } + private: const RefPtr<CamerasParent> mParent; const CaptureEngine mCapEngine; - const int mCaptureId; - const nsTArray<int> mStreamIds; + const uint32_t mStreamId; const TrackingId mTrackingId; - Variant<ShmemBuffer, webrtc::VideoFrame> mBuffer; + ShmemBuffer mBuffer; + UniquePtr<unsigned char[]> mAlternateBuffer; const VideoFrameProperties mProperties; + int mResult; }; -int CamerasParent::DeliverFrameOverIPC( - CaptureEngine aCapEngine, int aCaptureId, const Span<const int>& aStreamIds, - const TrackingId& aTrackingId, - Variant<ShmemBuffer, webrtc::VideoFrame>&& aBuffer, - const VideoFrameProperties& aProps) { +int CamerasParent::DeliverFrameOverIPC(CaptureEngine aCapEngine, + uint32_t aStreamId, + const TrackingId& aTrackingId, + ShmemBuffer aBuffer, + unsigned char* aAltBuffer, + const VideoFrameProperties& aProps) { // No ShmemBuffers were available, so construct one now of the right size // and copy into it. That is an extra copy, but we expect this to be // the exceptional case, because we just assured the next call *will* have a // buffer of the right size. - if (!aBuffer.is<ShmemBuffer>()) { + if (aAltBuffer != nullptr) { // Get a shared memory buffer from the pool, at least size big - ShmemBuffer shMemBuff; - { - auto guard = mShmemPools.Lock(); - auto it = guard->find(aCaptureId); - if (it != guard->end()) { - auto& [_, pool] = *it; - shMemBuff = pool.Get(this, aProps.bufferSize()); - } - } + ShmemBuffer shMemBuff = mShmemPool.Get(this, aProps.bufferSize()); if (!shMemBuff.Valid()) { LOG("No usable Video shmem in DeliverFrame (out of buffers?)"); @@ -341,20 +345,18 @@ int CamerasParent::DeliverFrameOverIPC( PerformanceRecorder<CopyVideoStage> rec( "CamerasParent::AltBufferToShmem"_ns, aTrackingId, aProps.width(), aProps.height()); - VideoFrameUtils::CopyVideoFrameBuffers(shMemBuff, - aBuffer.as<webrtc::VideoFrame>()); + // get() and Size() check for proper alignment of the segment + memcpy(shMemBuff.GetBytes(), aAltBuffer, aProps.bufferSize()); rec.Record(); - if (!SendDeliverFrame(aCaptureId, aStreamIds, std::move(shMemBuff.Get()), - aProps)) { + if (!SendDeliverFrame(aStreamId, std::move(shMemBuff.Get()), aProps)) { return -1; } } else { - MOZ_ASSERT(aBuffer.as<ShmemBuffer>().Valid()); + MOZ_ASSERT(aBuffer.Valid()); // ShmemBuffer was available, we're all good. A single copy happened // in the original webrtc callback. - if (!SendDeliverFrame(aCaptureId, aStreamIds, - std::move(aBuffer.as<ShmemBuffer>().Get()), aProps)) { + if (!SendDeliverFrame(aStreamId, std::move(aBuffer.Get()), aProps)) { return -1; } } @@ -362,267 +364,55 @@ int CamerasParent::DeliverFrameOverIPC( return 0; } -bool CamerasParent::IsWindowCapturing(uint64_t aWindowId, - const nsACString& aUniqueId) const { - MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); - for (const auto& capturer : *mCapturers) { - if (capturer->mUniqueId != aUniqueId) { - continue; - } - auto streamsGuard = capturer->mStreams.ConstLock(); - for (const auto& stream : *streamsGuard) { - if (stream->mWindowId == aWindowId) { - return true; - } - } - } - return false; -} - -ShmemBuffer CamerasParent::GetBuffer(int aCaptureId, size_t aSize) { - auto guard = mShmemPools.Lock(); - auto it = guard->find(aCaptureId); - if (it == guard->end()) { - return ShmemBuffer(); - } - auto& [_, pool] = *it; - return pool.GetIfAvailable(aSize); -} - -/*static*/ -std::unique_ptr<AggregateCapturer> AggregateCapturer::Create( - nsISerialEventTarget* aVideoCaptureThread, CaptureEngine aCapEng, - VideoEngine* aEngine, const nsCString& aUniqueId, uint64_t aWindowId, - nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities, - CamerasParent* aParent) { - MOZ_ASSERT(aVideoCaptureThread->IsOnCurrentThread()); - int captureId = aEngine->CreateVideoCapture(aUniqueId.get(), aWindowId); - auto capturer = WrapUnique( - new AggregateCapturer(aVideoCaptureThread, aCapEng, aEngine, aUniqueId, - captureId, std::move(aCapabilities))); - capturer->AddStream(aParent, captureId, aWindowId); - aEngine->WithEntry(captureId, [&](VideoEngine::CaptureEntry& aEntry) -> void { - aEntry.VideoCapture()->SetTrackingId(capturer->mTrackingId.mUniqueInProcId); - aEntry.VideoCapture()->RegisterCaptureDataCallback(capturer.get()); - if (auto* event = aEntry.CaptureEndedEvent()) { - capturer->mCaptureEndedListener = - event->Connect(aVideoCaptureThread, capturer.get(), - &AggregateCapturer::OnCaptureEnded); - } - }); - return capturer; -} - -AggregateCapturer::AggregateCapturer( - nsISerialEventTarget* aVideoCaptureThread, CaptureEngine aCapEng, - VideoEngine* aEngine, const nsCString& aUniqueId, int aCaptureId, - nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities) - : mVideoCaptureThread(aVideoCaptureThread), - mCapEngine(aCapEng), - mEngine(aEngine), - mUniqueId(aUniqueId), - mCaptureId(aCaptureId), - mTrackingId(CaptureEngineToTrackingSourceStr(aCapEng), mCaptureId), - mCapabilities(std::move(aCapabilities)), - mStreams("CallbackHelper::mStreams") { - MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); -} - -AggregateCapturer::~AggregateCapturer() { - MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); - mCaptureEndedListener.DisconnectIfExists(); - mEngine->WithEntry(mCaptureId, [&](VideoEngine::CaptureEntry& aEntry) { - if (auto* cap = aEntry.VideoCapture().get()) { - cap->DeRegisterCaptureDataCallback(this); - cap->StopCaptureIfAllClientsClose(); - } - }); - MOZ_ALWAYS_FALSE(mEngine->ReleaseVideoCapture(mCaptureId)); -} - -void AggregateCapturer::AddStream(CamerasParent* aParent, int aStreamId, - uint64_t aWindowId) { - auto streamsGuard = mStreams.Lock(); -#ifdef DEBUG - for (const auto& stream : *streamsGuard) { - MOZ_ASSERT(stream->mId != aStreamId); - } -#endif - streamsGuard->AppendElement( - new Stream{.mParent = aParent, .mId = aStreamId, .mWindowId = aWindowId}); -} - -auto AggregateCapturer::RemoveStream(int aStreamId) -> RemoveStreamResult { - auto streamsGuard = mStreams.Lock(); - size_t idx = streamsGuard->IndexOf( - aStreamId, 0, - [](const auto& aElem, const auto& aId) { return aElem->mId - aId; }); - if (idx == streamsGuard->NoIndex) { - return {.mNumRemainingStreams = 0, .mNumRemainingStreamsForParent = 0}; - } - CamerasParent* parent = streamsGuard->ElementAt(idx)->mParent; - streamsGuard->RemoveElementAt(idx); - size_t remainingForParent = 0; - for (const auto& s : *streamsGuard) { - if (s->mParent == parent) { - remainingForParent += 1; - } - } - return {.mNumRemainingStreams = streamsGuard->Length(), - .mNumRemainingStreamsForParent = remainingForParent}; -} - -auto AggregateCapturer::RemoveStreamsFor(CamerasParent* aParent) - -> RemoveStreamResult { - auto streamsGuard = mStreams.Lock(); - streamsGuard->RemoveElementsBy( - [&](const auto& aElem) { return aElem->mParent == aParent; }); - return {.mNumRemainingStreams = streamsGuard->Length(), - .mNumRemainingStreamsForParent = 0}; -} - -Maybe<int> AggregateCapturer::CaptureIdFor(int aStreamId) { - auto streamsGuard = mStreams.Lock(); - for (auto& stream : *streamsGuard) { - if (stream->mId == aStreamId) { - return Some(mCaptureId); - } - } - return Nothing(); +ShmemBuffer CamerasParent::GetBuffer(size_t aSize) { + return mShmemPool.GetIfAvailable(aSize); } -void AggregateCapturer::SetConfigurationFor( - int aStreamId, const webrtc::VideoCaptureCapability& aCapability, +void CallbackHelper::SetConfiguration( + const webrtc::VideoCaptureCapability& aCapability, const NormalizedConstraints& aConstraints, - const dom::VideoResizeModeEnum& aResizeMode, bool aStarted) { - auto streamsGuard = mStreams.Lock(); - for (auto& stream : *streamsGuard) { - if (stream->mId == aStreamId) { - stream->mConfiguration = { - .mCapability = aCapability, - .mConstraints = aConstraints, - .mResizeMode = aResizeMode, - }; - stream->mStarted = aStarted; - break; - } - } -} - -webrtc::VideoCaptureCapability AggregateCapturer::CombinedCapability() { - Maybe<webrtc::VideoCaptureCapability> combinedCap; - CamerasParent* someParent{}; - const auto streamsGuard = mStreams.ConstLock(); - for (const auto& stream : *streamsGuard) { - if (!stream->mStarted) { - continue; - } - if (!someParent) { - someParent = stream->mParent; - } - const auto& cap = stream->mConfiguration.mCapability; - if (!combinedCap) { - combinedCap = Some(cap); - continue; - } - auto combinedRes = combinedCap->width * combinedCap->height; - combinedCap->maxFPS = std::max(combinedCap->maxFPS, cap.maxFPS); - if (mCapEngine == CaptureEngine::CameraEngine) { - auto newCombinedRes = cap.width * cap.height; - if (newCombinedRes > combinedRes) { - combinedCap->videoType = cap.videoType; - } - combinedCap->width = std::max(combinedCap->width, cap.width); - combinedCap->height = std::max(combinedCap->height, cap.height); - } - } - if (mCapEngine == CameraEngine) { - const webrtc::VideoCaptureCapability* minDistanceCapability{}; - uint64_t minDistance = UINT64_MAX; - - for (const auto& candidateCapability : mCapabilities) { - if (candidateCapability.videoType != combinedCap->videoType) { - continue; - } - // The first priority is finding a suitable resolution. - // So here we raise the weight of width and height - uint64_t distance = - uint64_t(ResolutionFeasibilityDistance(candidateCapability.width, - combinedCap->width)) + - uint64_t(ResolutionFeasibilityDistance(candidateCapability.height, - combinedCap->height)) + - uint64_t(FeasibilityDistance(candidateCapability.maxFPS, - combinedCap->maxFPS)); - if (distance < minDistance) { - minDistanceCapability = &candidateCapability; - minDistance = distance; - } - } - if (minDistanceCapability) { - combinedCap = Some(*minDistanceCapability); - } - } - return combinedCap.extract(); + const dom::VideoResizeModeEnum& aResizeMode) { + auto c = mConfiguration.Lock(); + c.ref() = Configuration{ + .mCapability = aCapability, + .mConstraints = aConstraints, + .mResizeMode = aResizeMode, + }; } -void AggregateCapturer::OnCaptureEnded() { - MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); - std::multimap<CamerasParent*, int> parentsAndIds; - { - auto streamsGuard = mStreams.Lock(); - for (const auto& stream : *streamsGuard) { - parentsAndIds.insert({stream->mParent, stream->mId}); - } - } +void CallbackHelper::OnCaptureEnded() { + nsIEventTarget* target = mParent->GetBackgroundEventTarget(); - for (auto it = parentsAndIds.begin(); it != parentsAndIds.end();) { - const auto& parent = it->first; - auto nextParentIt = parentsAndIds.upper_bound(parent); - AutoTArray<int, 4> ids; - while (it != nextParentIt) { - const auto& [_, id] = *it; - ids.AppendElement(id); - ++it; - } - nsIEventTarget* target = parent->GetBackgroundEventTarget(); - MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( - __func__, [&] { Unused << parent->SendCaptureEnded(ids); }))); - } + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( + __func__, [&] { Unused << mParent->SendCaptureEnded(mStreamId); }))); } -void AggregateCapturer::OnFrame(const webrtc::VideoFrame& aVideoFrame) { - std::multimap<CamerasParent*, int> parentsAndIds; +void CallbackHelper::OnFrame(const webrtc::VideoFrame& aVideoFrame) { { // Proactively drop frames that would not get processed anyway. - auto streamsGuard = mStreams.Lock(); - - for (auto& stream : *streamsGuard) { - auto& c = stream->mConfiguration; - const double maxFramerate = static_cast<double>( - c.mCapability.maxFPS > 0 ? c.mCapability.maxFPS : 120); - const double desiredFramerate = - c.mResizeMode == VideoResizeModeEnum::Crop_and_scale - ? c.mConstraints.mFrameRate.Get(maxFramerate) - : maxFramerate; - const double targetFramerate = std::clamp(desiredFramerate, 0.01, 120.); - - // Allow 5% higher fps than configured as frame time sampling is timing - // dependent. - const auto minInterval = - media::TimeUnit(1000, static_cast<int64_t>(1050 * targetFramerate)); - const auto frameTime = - media::TimeUnit::FromMicroseconds(aVideoFrame.timestamp_us()); - const auto frameInterval = frameTime - stream->mLastFrameTime; - if (frameInterval < minInterval) { - continue; - } - stream->mLastFrameTime = frameTime; - LOG_VERBOSE("CamerasParent::%s parent=%p, id=%d.", __func__, - stream->mParent, stream->mId); - parentsAndIds.insert({stream->mParent, stream->mId}); + auto c = mConfiguration.Lock(); + + const double maxFramerate = static_cast<double>( + c->mCapability.maxFPS > 0 ? c->mCapability.maxFPS : 120); + const double desiredFramerate = + c->mResizeMode == VideoResizeModeEnum::Crop_and_scale + ? c->mConstraints.mFrameRate.Get(maxFramerate) + : maxFramerate; + const double targetFramerate = std::clamp(desiredFramerate, 0.01, 120.); + + // Allow 5% higher fps than configured as frame time sampling is timing + // dependent. + const auto minInterval = + media::TimeUnit(1000, static_cast<int64_t>(1050 * targetFramerate)); + const auto frameTime = + media::TimeUnit::FromMicroseconds(aVideoFrame.timestamp_us()); + const auto frameInterval = frameTime - mLastFrameTime; + if (frameInterval < minInterval) { + return; } + mLastFrameTime = frameTime; } - + LOG_VERBOSE("CamerasParent(%p)::%s", mParent, __func__); if (profiler_thread_is_being_profiled_for_markers()) { PROFILER_MARKER_UNTYPED( nsPrintfCString("CaptureVideoFrame %dx%d %s %s", aVideoFrame.width(), @@ -632,68 +422,43 @@ void AggregateCapturer::OnFrame(const webrtc::VideoFrame& aVideoFrame) { mTrackingId.ToString().get()), MEDIA_RT); } - RefPtr<DeliverFrameRunnable> runnable = nullptr; // Get frame properties camera::VideoFrameProperties properties; VideoFrameUtils::InitFrameBufferProperties(aVideoFrame, properties); - - for (auto it = parentsAndIds.begin(); it != parentsAndIds.end();) { - const auto& parent = it->first; - auto nextParentIt = parentsAndIds.upper_bound(parent); - AutoTArray<int, 4> ids; - while (it != nextParentIt) { - const auto& [_, id] = *it; - ids.AppendElement(id); - ++it; - } - - LOG_VERBOSE("CamerasParent(%p)::%s", parent, __func__); - // Get a shared memory buffer to copy the frame data into - ShmemBuffer shMemBuffer = - parent->GetBuffer(mCaptureId, properties.bufferSize()); - if (!shMemBuffer.Valid()) { - // Either we ran out of buffers or they're not the right size yet - LOG("Correctly sized Video shmem not available in DeliverFrame"); - // We will do the copy into a(n extra) temporary buffer inside - // the DeliverFrameRunnable constructor. - } else { - // Shared memory buffers of the right size are available, do the copy - // here. - PerformanceRecorder<CopyVideoStage> rec( - "CamerasParent::VideoFrameToShmem"_ns, mTrackingId, - aVideoFrame.width(), aVideoFrame.height()); - VideoFrameUtils::CopyVideoFrameBuffers( - shMemBuffer.GetBytes(), properties.bufferSize(), aVideoFrame); - rec.Record(); - runnable = new DeliverFrameRunnable(parent, mCapEngine, mCaptureId, - std::move(ids), mTrackingId, - std::move(shMemBuffer), properties); - } - if (!runnable) { - runnable = new DeliverFrameRunnable(parent, mCapEngine, mCaptureId, - std::move(ids), mTrackingId, - aVideoFrame, properties); - } - nsIEventTarget* target = parent->GetBackgroundEventTarget(); - target->Dispatch(runnable, NS_DISPATCH_NORMAL); + // Get a shared memory buffer to copy the frame data into + ShmemBuffer shMemBuffer = mParent->GetBuffer(properties.bufferSize()); + if (!shMemBuffer.Valid()) { + // Either we ran out of buffers or they're not the right size yet + LOG("Correctly sized Video shmem not available in DeliverFrame"); + // We will do the copy into a(n extra) temporary buffer inside + // the DeliverFrameRunnable constructor. + } else { + // Shared memory buffers of the right size are available, do the copy here. + PerformanceRecorder<CopyVideoStage> rec( + "CamerasParent::VideoFrameToShmem"_ns, mTrackingId, aVideoFrame.width(), + aVideoFrame.height()); + VideoFrameUtils::CopyVideoFrameBuffers( + shMemBuffer.GetBytes(), properties.bufferSize(), aVideoFrame); + rec.Record(); + runnable = + new DeliverFrameRunnable(mParent, mCapEngine, mStreamId, mTrackingId, + std::move(shMemBuffer), properties); } + if (!runnable) { + runnable = new DeliverFrameRunnable(mParent, mCapEngine, mStreamId, + mTrackingId, aVideoFrame, properties); + } + MOZ_ASSERT(mParent); + nsIEventTarget* target = mParent->GetBackgroundEventTarget(); + MOZ_ASSERT(target != nullptr); + target->Dispatch(runnable, NS_DISPATCH_NORMAL); } -ipc::IPCResult CamerasParent::RecvReleaseFrame(const int& aCaptureId, - ipc::Shmem&& aShmem) { +ipc::IPCResult CamerasParent::RecvReleaseFrame(ipc::Shmem&& aShmem) { MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); - auto guard = mShmemPools.Lock(); - auto it = guard->find(aCaptureId); - if (it == guard->end()) { - MOZ_ASSERT_UNREACHABLE( - "Releasing shmem but pool is already gone. Shmem must have been " - "deallocated."); - return IPC_FAIL(this, "Shmem was already deallocated"); - } - auto& [_, pool] = *it; - pool.Put(ShmemBuffer(aShmem)); + mShmemPool.Put(ShmemBuffer(aShmem)); return IPC_OK(); } @@ -701,16 +466,13 @@ void CamerasParent::CloseEngines() { MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); LOG_FUNCTION(); - // Stop the capturers. - while (!mCapturers->IsEmpty()) { - auto& capturer = mCapturers->LastElement(); - auto removed = capturer->RemoveStreamsFor(this); - if (removed.mNumRemainingStreams == 0) { - auto capturer = mCapturers->PopLastElement(); - auto capEngine = capturer->mCapEngine; - auto captureId = capturer->mCaptureId; - LOG("Forcing shutdown of engine %d, capturer %d", capEngine, captureId); - } + // Stop the callers + while (!mCallbacks.IsEmpty()) { + auto capEngine = mCallbacks[0]->mCapEngine; + auto streamNum = mCallbacks[0]->mStreamId; + LOG("Forcing shutdown of engine %d, capturer %d", capEngine, streamNum); + StopCapture(capEngine, streamNum); + Unused << ReleaseCapture(capEngine, streamNum); } mDeviceChangeEventListener.DisconnectIfExists(); @@ -912,21 +674,30 @@ ipc::IPCResult CamerasParent::RecvGetCaptureCapability( InvokeAsync(mVideoCaptureThread, __func__, [this, self = RefPtr(this), id = nsCString(aUniqueId), aCapEngine, aIndex] { - nsTArray<webrtc::VideoCaptureCapability> const* capabilities = - EnsureCapabilitiesPopulated(aCapEngine, id); webrtc::VideoCaptureCapability webrtcCaps; - if (!capabilities) { - return Promise::CreateAndReject( - -1, "CamerasParent::RecvGetCaptureCapability"); + int error = -1; + if (auto devInfo = GetDeviceInfo(aCapEngine)) { + error = devInfo->GetCapability(id.get(), aIndex, webrtcCaps); + } + + if (!error && aCapEngine == CameraEngine) { + auto iter = mAllCandidateCapabilities.find(id); + if (iter == mAllCandidateCapabilities.end()) { + std::map<uint32_t, webrtc::VideoCaptureCapability> + candidateCapabilities; + candidateCapabilities.emplace(aIndex, webrtcCaps); + mAllCandidateCapabilities.emplace(id, + candidateCapabilities); + } else { + (iter->second).emplace(aIndex, webrtcCaps); + } } - if (aIndex < 0 || - static_cast<size_t>(aIndex) >= capabilities->Length()) { + if (error) { return Promise::CreateAndReject( - -2, "CamerasParent::RecvGetCaptureCapability"); + error, "CamerasParent::RecvGetCaptureCapability"); } return Promise::CreateAndResolve( - capabilities->ElementAt(aIndex), - "CamerasParent::RecvGetCaptureCapability"); + webrtcCaps, "CamerasParent::RecvGetCaptureCapability"); }) ->Then( mPBackgroundEventTarget, __func__, @@ -1082,66 +853,46 @@ ipc::IPCResult CamerasParent::RecvAllocateCapture( using Promise1 = MozPromise<bool, bool, true>; using Data = std::tuple<int, int>; using Promise2 = MozPromise<Data, bool, true>; - InvokeAsync( - GetMainThreadSerialEventTarget(), __func__, - [aWindowID] { - // Verify whether the claimed origin has received permission - // to use the camera, either persistently or this session (one - // shot). - bool allowed = HasCameraPermission(aWindowID); - if (!allowed && Preferences::GetBool( - "media.navigator.permission.disabled", false)) { - // Developer preference for turning off permission check. - allowed = true; - LOG("No permission but checks are disabled"); - } - if (!allowed) { - LOG("No camera permission for this origin"); - } - return Promise1::CreateAndResolve(allowed, - "CamerasParent::RecvAllocateCapture"); - }) - ->Then( - mVideoCaptureThread, __func__, - [this, self = RefPtr(this), aCapEngine, aWindowID, - unique_id = nsCString(aUniqueIdUTF8)]( - Promise1::ResolveOrRejectValue&& aValue) { - VideoEngine* engine = EnsureInitialized(aCapEngine); - if (!engine) { - return Promise2::CreateAndResolve( - std::make_tuple(-1, -1), - "CamerasParent::RecvAllocateCapture no engine"); - } - bool allowed = aValue.ResolveValue(); - if (!allowed && IsWindowCapturing(aWindowID, unique_id)) { - allowed = true; - LOG("No permission but window is already capturing this device"); - } - if (!allowed) { - return Promise2::CreateAndResolve( - std::make_tuple(-1, -1), - "CamerasParent::RecvAllocateCapture"); - } - - nsTArray<webrtc::VideoCaptureCapability> capabilities; - if (const auto* caps = - EnsureCapabilitiesPopulated(aCapEngine, unique_id)) { - capabilities.AppendElements(*caps); - } - - auto created = GetOrCreateCapturer(aCapEngine, aWindowID, unique_id, - std::move(capabilities)); - int error = -1; - engine->WithEntry(created.mCapturer->mCaptureId, - [&](VideoEngine::CaptureEntry& cap) { - if (cap.VideoCapture()) { - error = 0; - } - }); - return Promise2::CreateAndResolve( - std::make_tuple(created.mStreamId, error), - "CamerasParent::RecvAllocateCapture"); - }) + InvokeAsync(GetMainThreadSerialEventTarget(), __func__, + [aWindowID] { + // Verify whether the claimed origin has received permission + // to use the camera, either persistently or this session (one + // shot). + bool allowed = HasCameraPermission(aWindowID); + if (!allowed) { + // Developer preference for turning off permission check. + if (Preferences::GetBool( + "media.navigator.permission.disabled", false)) { + allowed = true; + LOG("No permission but checks are disabled"); + } else { + LOG("No camera permission for this origin"); + } + } + return Promise1::CreateAndResolve( + allowed, "CamerasParent::RecvAllocateCapture"); + }) + ->Then(mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine, + unique_id = nsCString(aUniqueIdUTF8)]( + Promise1::ResolveOrRejectValue&& aValue) { + bool allowed = aValue.ResolveValue(); + int captureId = -1; + int error = -1; + if (allowed && EnsureInitialized(aCapEngine)) { + VideoEngine* engine = mEngines->ElementAt(aCapEngine); + captureId = engine->CreateVideoCapture(unique_id.get()); + engine->WithEntry(captureId, + [&error](VideoEngine::CaptureEntry& cap) { + if (cap.VideoCapture()) { + error = 0; + } + }); + } + return Promise2::CreateAndResolve( + std::make_tuple(captureId, error), + "CamerasParent::RecvAllocateCapture"); + }) ->Then( mPBackgroundEventTarget, __func__, [this, self = RefPtr(this)](Promise2::ResolveOrRejectValue&& aValue) { @@ -1163,24 +914,34 @@ ipc::IPCResult CamerasParent::RecvAllocateCapture( return IPC_OK(); } +int CamerasParent::ReleaseCapture(const CaptureEngine& aCapEngine, + int aCaptureId) { + MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); + int error = -1; + if (auto* engine = EnsureInitialized(aCapEngine)) { + error = engine->ReleaseVideoCapture(aCaptureId); + } + return error; +} + ipc::IPCResult CamerasParent::RecvReleaseCapture( - const CaptureEngine& aCapEngine, const int& aStreamId) { + const CaptureEngine& aCapEngine, const int& aCaptureId) { MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); MOZ_ASSERT(!mDestroyed); LOG_FUNCTION(); - LOG("RecvReleaseCapture stream nr %d", aStreamId); + LOG("RecvReleaseCamera device nr %d", aCaptureId); using Promise = MozPromise<int, bool, true>; InvokeAsync(mVideoCaptureThread, __func__, - [this, self = RefPtr(this), aCapEngine, aStreamId] { + [this, self = RefPtr(this), aCapEngine, aCaptureId] { return Promise::CreateAndResolve( - ReleaseStream(aCapEngine, aStreamId), + ReleaseCapture(aCapEngine, aCaptureId), "CamerasParent::RecvReleaseCapture"); }) ->Then(mPBackgroundEventTarget, __func__, [this, self = RefPtr(this), - aStreamId](Promise::ResolveOrRejectValue&& aValue) { + aCaptureId](Promise::ResolveOrRejectValue&& aValue) { int error = aValue.ResolveValue(); if (mDestroyed) { @@ -1190,19 +951,19 @@ ipc::IPCResult CamerasParent::RecvReleaseCapture( if (error != 0) { Unused << SendReplyFailure(); - LOG("RecvReleaseCapture: Failed to free stream nr %d", - aStreamId); + LOG("RecvReleaseCapture: Failed to free device nr %d", + aCaptureId); return; } Unused << SendReplySuccess(); - LOG("Freed stream nr %d", aStreamId); + LOG("Freed device nr %d", aCaptureId); }); return IPC_OK(); } ipc::IPCResult CamerasParent::RecvStartCapture( - const CaptureEngine& aCapEngine, const int& aStreamId, + const CaptureEngine& aCapEngine, const int& aCaptureId, const VideoCaptureCapability& aIpcCaps, const NormalizedConstraints& aConstraints, const dom::VideoResizeModeEnum& aResizeMode) { @@ -1214,24 +975,18 @@ ipc::IPCResult CamerasParent::RecvStartCapture( using Promise = MozPromise<int, bool, true>; InvokeAsync( mVideoCaptureThread, __func__, - [this, self = RefPtr(this), aCapEngine, aStreamId, aIpcCaps, aConstraints, - aResizeMode] { + [this, self = RefPtr(this), aCapEngine, aCaptureId, aIpcCaps, + aConstraints, aResizeMode] { LOG_FUNCTION(); + int error = -1; if (!EnsureInitialized(aCapEngine)) { - return Promise::CreateAndResolve(-1, - "CamerasParent::RecvStartCapture"); - } - - AggregateCapturer* cbh = GetCapturer(aCapEngine, aStreamId); - if (!cbh) { - return Promise::CreateAndResolve(-1, + return Promise::CreateAndResolve(error, "CamerasParent::RecvStartCapture"); } - int error = -1; mEngines->ElementAt(aCapEngine) - ->WithEntry(cbh->mCaptureId, [&](VideoEngine::CaptureEntry& cap) { + ->WithEntry(aCaptureId, [&](VideoEngine::CaptureEntry& cap) { webrtc::VideoCaptureCapability capability; capability.width = aIpcCaps.width(); capability.height = aIpcCaps.height(); @@ -1240,16 +995,124 @@ ipc::IPCResult CamerasParent::RecvStartCapture( static_cast<webrtc::VideoType>(aIpcCaps.videoType()); capability.interlaced = aIpcCaps.interlaced(); - if (cbh) { - cbh->SetConfigurationFor(aStreamId, capability, aConstraints, - aResizeMode, /*aStarted=*/true); - error = - cap.VideoCapture()->StartCapture(cbh->CombinedCapability()); - if (error) { - cbh->SetConfigurationFor(aStreamId, capability, aConstraints, - aResizeMode, /*aStarted=*/false); + if (sDeviceUniqueIDs.find(aCaptureId) == sDeviceUniqueIDs.end()) { + sDeviceUniqueIDs.emplace( + aCaptureId, cap.VideoCapture()->CurrentDeviceName()); + sAllRequestedCapabilities.emplace(aCaptureId, capability); + } else { + // Starting capture for an id that already exists. Update its + // requested capability. + MOZ_DIAGNOSTIC_ASSERT( + strcmp(sDeviceUniqueIDs[aCaptureId], + cap.VideoCapture()->CurrentDeviceName()) == 0); + MOZ_DIAGNOSTIC_ASSERT( + sAllRequestedCapabilities.find(aCaptureId) != + sAllRequestedCapabilities.end()); + sAllRequestedCapabilities[aCaptureId] = capability; + } + + if (aCapEngine == CameraEngine) { + for (const auto& it : sDeviceUniqueIDs) { + if (strcmp(it.second, + cap.VideoCapture()->CurrentDeviceName()) == 0) { + capability.width = + std::max(capability.width, + sAllRequestedCapabilities[it.first].width); + capability.height = + std::max(capability.height, + sAllRequestedCapabilities[it.first].height); + capability.maxFPS = + std::max(capability.maxFPS, + sAllRequestedCapabilities[it.first].maxFPS); + } + } + + auto candidateCapabilities = mAllCandidateCapabilities.find( + nsCString(cap.VideoCapture()->CurrentDeviceName())); + if ((candidateCapabilities != + mAllCandidateCapabilities.end()) && + (!candidateCapabilities->second.empty())) { + int32_t minIdx = -1; + uint64_t minDistance = UINT64_MAX; + + for (auto& candidateCapability : + candidateCapabilities->second) { + if (candidateCapability.second.videoType != + capability.videoType) { + continue; + } + // The first priority is finding a suitable resolution. + // So here we raise the weight of width and height + uint64_t distance = uint64_t(ResolutionFeasibilityDistance( + candidateCapability.second.width, + capability.width)) + + uint64_t(ResolutionFeasibilityDistance( + candidateCapability.second.height, + capability.height)) + + uint64_t(FeasibilityDistance( + candidateCapability.second.maxFPS, + capability.maxFPS)); + if (distance < minDistance) { + minIdx = static_cast<int32_t>(candidateCapability.first); + minDistance = distance; + } + } + MOZ_ASSERT(minIdx != -1); + capability = candidateCapabilities->second[minIdx]; + } + } else if (aCapEngine == ScreenEngine || + aCapEngine == BrowserEngine || + aCapEngine == WinEngine) { + for (const auto& it : sDeviceUniqueIDs) { + if (strcmp(it.second, + cap.VideoCapture()->CurrentDeviceName()) == 0) { + capability.maxFPS = + std::max(capability.maxFPS, + sAllRequestedCapabilities[it.first].maxFPS); + } } } + + CallbackHelper* cbh = nullptr; + for (auto& cb : mCallbacks) { + if (cb->mCapEngine == aCapEngine && + cb->mStreamId == (uint32_t)aCaptureId) { + cbh = cb.get(); + break; + } + } + bool cbhCreated = !cbh; + if (!cbh) { + cbh = mCallbacks + .AppendElement(MakeUnique<CallbackHelper>( + static_cast<CaptureEngine>(aCapEngine), + aCaptureId, this)) + ->get(); + cap.VideoCapture()->SetTrackingId( + cbh->mTrackingId.mUniqueInProcId); + } + + cbh->SetConfiguration(capability, aConstraints, aResizeMode); + error = cap.VideoCapture()->StartCapture(capability); + + if (!error) { + if (cbhCreated) { + cap.VideoCapture()->RegisterCaptureDataCallback( + static_cast< + webrtc::VideoSinkInterface<webrtc::VideoFrame>*>( + cbh)); + if (auto* event = cap.CaptureEndedEvent(); + event && !cbh->mConnectedToCaptureEnded) { + cbh->mCaptureEndedListener = + event->Connect(mVideoCaptureThread, cbh, + &CallbackHelper::OnCaptureEnded); + cbh->mConnectedToCaptureEnded = true; + } + } + } else { + sDeviceUniqueIDs.erase(aCaptureId); + sAllRequestedCapabilities.erase(aCaptureId); + } }); return Promise::CreateAndResolve(error, @@ -1277,7 +1140,7 @@ ipc::IPCResult CamerasParent::RecvStartCapture( } ipc::IPCResult CamerasParent::RecvFocusOnSelectedSource( - const CaptureEngine& aCapEngine, const int& aStreamId) { + const CaptureEngine& aCapEngine, const int& aCaptureId) { MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); MOZ_ASSERT(!mDestroyed); @@ -1285,17 +1148,11 @@ ipc::IPCResult CamerasParent::RecvFocusOnSelectedSource( using Promise = MozPromise<bool, bool, true>; InvokeAsync(mVideoCaptureThread, __func__, - [this, self = RefPtr(this), aCapEngine, aStreamId] { + [this, self = RefPtr(this), aCapEngine, aCaptureId] { bool result = false; - auto* capturer = GetCapturer(aCapEngine, aStreamId); - if (!capturer) { - return Promise::CreateAndResolve( - result, "CamerasParent::RecvFocusOnSelectedSource"); - } if (auto* engine = EnsureInitialized(aCapEngine)) { engine->WithEntry( - capturer->mCaptureId, - [&](VideoEngine::CaptureEntry& cap) { + aCaptureId, [&](VideoEngine::CaptureEntry& cap) { if (cap.VideoCapture()) { result = cap.VideoCapture()->FocusOnSelectedSource(); } @@ -1324,109 +1181,45 @@ ipc::IPCResult CamerasParent::RecvFocusOnSelectedSource( return IPC_OK(); } -auto CamerasParent::GetOrCreateCapturer( - CaptureEngine aEngine, uint64_t aWindowId, const nsCString& aUniqueId, - nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities) - -> GetOrCreateCapturerResult { - MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); - VideoEngine* engine = EnsureInitialized(aEngine); - const auto ensureShmemPool = [&](int aCaptureId) { - auto guard = mShmemPools.Lock(); - constexpr size_t kMaxShmemBuffers = 1; - guard->try_emplace(aCaptureId, kMaxShmemBuffers); - }; - for (auto& capturer : *mCapturers) { - if (capturer->mCapEngine != aEngine) { - continue; - } - if (capturer->mUniqueId.Equals(aUniqueId)) { - int streamId = engine->GenerateId(); - ensureShmemPool(capturer->mCaptureId); - capturer->AddStream(this, streamId, aWindowId); - return {.mCapturer = capturer.get(), .mStreamId = streamId}; - } - } - NotNull capturer = mCapturers->AppendElement( - AggregateCapturer::Create(mVideoCaptureThread, aEngine, engine, aUniqueId, - aWindowId, std::move(aCapabilities), this)); - ensureShmemPool(capturer->get()->mCaptureId); - return {.mCapturer = capturer->get(), - .mStreamId = capturer->get()->mCaptureId}; -} - -AggregateCapturer* CamerasParent::GetCapturer(CaptureEngine aEngine, - int aStreamId) { - MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); - for (auto& capturer : *mCapturers) { - if (capturer->mCapEngine != aEngine) { - continue; - } - Maybe captureId = capturer->CaptureIdFor(aStreamId); - if (captureId) { - return capturer.get(); - } - } - return nullptr; -} - -int CamerasParent::ReleaseStream(CaptureEngine aEngine, int aStreamId) { +void CamerasParent::StopCapture(const CaptureEngine& aCapEngine, + int aCaptureId) { MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); - auto* capturer = GetCapturer(aEngine, aStreamId); - if (!capturer) { - return -1; - } - auto removed = capturer->RemoveStream(aStreamId); - if (removed.mNumRemainingStreams == 0) { - mCapturers->RemoveElement(capturer); - } - return 0; -} - -nsTArray<webrtc::VideoCaptureCapability> const* -CamerasParent::EnsureCapabilitiesPopulated(CaptureEngine aEngine, - const nsCString& aUniqueId) { - MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); - if (auto iter = mAllCandidateCapabilities.find(aUniqueId); - iter != mAllCandidateCapabilities.end()) { - return &iter->second; - } - auto devInfo = GetDeviceInfo(aEngine); - if (!devInfo) { - return nullptr; - } - const int num = devInfo->NumberOfCapabilities(aUniqueId.get()); - if (num <= 0) { - return nullptr; - } - nsTArray<webrtc::VideoCaptureCapability> capabilities(num); - for (int i = 0; i < num; ++i) { - webrtc::VideoCaptureCapability capability; - if (devInfo->GetCapability(aUniqueId.get(), i, capability)) { - return nullptr; + if (auto* engine = EnsureInitialized(aCapEngine)) { + // we're removing elements, iterate backwards + for (size_t i = mCallbacks.Length(); i > 0; i--) { + if (mCallbacks[i - 1]->mCapEngine == aCapEngine && + mCallbacks[i - 1]->mStreamId == (uint32_t)aCaptureId) { + CallbackHelper* cbh = mCallbacks[i - 1].get(); + engine->WithEntry(aCaptureId, [cbh, &aCaptureId]( + VideoEngine::CaptureEntry& cap) { + if (cap.VideoCapture()) { + cap.VideoCapture()->DeRegisterCaptureDataCallback( + static_cast<webrtc::VideoSinkInterface<webrtc::VideoFrame>*>( + cbh)); + cap.VideoCapture()->StopCaptureIfAllClientsClose(); + + sDeviceUniqueIDs.erase(aCaptureId); + sAllRequestedCapabilities.erase(aCaptureId); + } + }); + cbh->mCaptureEndedListener.DisconnectIfExists(); + mCallbacks.RemoveElementAt(i - 1); + break; + } } - capabilities.AppendElement(capability); } - const auto& [iter, _] = - mAllCandidateCapabilities.emplace(aUniqueId, std::move(capabilities)); - return &iter->second; } ipc::IPCResult CamerasParent::RecvStopCapture(const CaptureEngine& aCapEngine, - const int& aStreamId) { + const int& aCaptureId) { MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); MOZ_ASSERT(!mDestroyed); LOG_FUNCTION(); nsresult rv = mVideoCaptureThread->Dispatch(NS_NewRunnableFunction( - __func__, [this, self = RefPtr(this), aCapEngine, aStreamId] { - auto* capturer = GetCapturer(aCapEngine, aStreamId); - if (capturer) { - capturer->SetConfigurationFor( - aStreamId, webrtc::VideoCaptureCapability{}, - NormalizedConstraints{}, dom::VideoResizeModeEnum::None, - /*aStarted=*/false); - } + __func__, [this, self = RefPtr(this), aCapEngine, aCaptureId] { + StopCapture(aCapEngine, aCaptureId); })); if (mDestroyed) { @@ -1452,12 +1245,7 @@ void CamerasParent::ActorDestroy(ActorDestroyReason aWhy) { LOG_FUNCTION(); // Release shared memory now, it's our last chance - { - auto guard = mShmemPools.Lock(); - for (auto& [captureId, pool] : *guard) { - pool.Cleanup(this); - } - } + mShmemPool.Cleanup(this); // We don't want to receive callbacks or anything if we can't // forward them anymore anyway. mDestroyed = true; @@ -1488,9 +1276,8 @@ CamerasParent::CamerasParent() ? MakeAndAddRefVideoCaptureThreadAndSingletons() : nullptr), mEngines(sEngines), - mCapturers(sCapturers), mVideoCaptureFactory(EnsureVideoCaptureFactory()), - mShmemPools("CamerasParent::mShmemPools"), + mShmemPool(CaptureEngine::MaxEngine), mPBackgroundEventTarget(GetCurrentSerialEventTarget()), mDestroyed(false) { MOZ_ASSERT(mPBackgroundEventTarget != nullptr, diff --git a/dom/media/systemservices/CamerasParent.h b/dom/media/systemservices/CamerasParent.h @@ -28,56 +28,26 @@ namespace mozilla::camera { class CamerasParent; class VideoEngine; -// Class that manages sharing of VideoCaptureImpl instances on top of -// VideoEngine. Sharing is needed as access to most sources is exclusive -// system-wide. -// -// There is at most one AggregateCapturer instance per unique source, as defined -// by its unique capture ID. -// -// There can be multiple requests for a stream from a source, as defined by -// unique stream IDs. -// -// Stream IDs and capture IDs use the same ID space. With capture happening in -// the parent process, application-wide uniqueness is guaranteed. -// -// When multiple stream requests have been made for a source, even across -// multiple CamerasParent instances, this class distributes a single frame to -// each CamerasParent instance that has requested a stream. Distribution to the -// various stream requests happens in CamerasChild::RecvDeliverFrame. -// -// This class similarly handles capture-ended events, and distributes them to -// the correct CamerasParent instances, with distribution to streams happening -// in CamerasChild::RecvCaptureEnded. -class AggregateCapturer final - : public webrtc::VideoSinkInterface<webrtc::VideoFrame> { +class CallbackHelper : public webrtc::VideoSinkInterface<webrtc::VideoFrame> { public: - static std::unique_ptr<AggregateCapturer> Create( - nsISerialEventTarget* aVideoCaptureThread, CaptureEngine aCapEng, - VideoEngine* aEngine, const nsCString& aUniqueId, uint64_t aWindowId, - nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities, - CamerasParent* aParent); - - ~AggregateCapturer(); - - void AddStream(CamerasParent* aParent, int aStreamId, uint64_t aWindowId); - struct RemoveStreamResult { - size_t mNumRemainingStreams; - size_t mNumRemainingStreamsForParent; - }; - RemoveStreamResult RemoveStream(int aStreamId); - RemoveStreamResult RemoveStreamsFor(CamerasParent* aParent); - Maybe<int> CaptureIdFor(int aStreamId); - void SetConfigurationFor(int aStreamId, - const webrtc::VideoCaptureCapability& aCapability, - const NormalizedConstraints& aConstraints, - const dom::VideoResizeModeEnum& aResizeMode, - bool aStarted); - webrtc::VideoCaptureCapability CombinedCapability(); + CallbackHelper(CaptureEngine aCapEng, uint32_t aStreamId, + CamerasParent* aParent) + : mCapEngine(aCapEng), + mStreamId(aStreamId), + mTrackingId(CaptureEngineToTrackingSourceStr(aCapEng), aStreamId), + mParent(aParent), + mConfiguration("CallbackHelper::mConfiguration") {}; + + void SetConfiguration(const webrtc::VideoCaptureCapability& aCapability, + const NormalizedConstraints& aConstraints, + const dom::VideoResizeModeEnum& aResizeMode); void OnCaptureEnded(); void OnFrame(const webrtc::VideoFrame& aVideoFrame) override; + friend CamerasParent; + + private: struct Configuration { webrtc::VideoCaptureCapability mCapability; NormalizedConstraints mConstraints; @@ -85,55 +55,15 @@ class AggregateCapturer final // defaults factored in. dom::VideoResizeModeEnum mResizeMode{}; }; - // Representation of a stream request for the source of this AggregateCapturer - // instance. - struct Stream { - // The CamerasParent instance that requested this stream. mParent is - // responsible for the lifetime of this stream. - CamerasParent* const mParent; - // The id that identifies this stream. This is unique within the application - // session, in the same set of IDs as AggregateCapturer::mCaptureId. - const int mId{-1}; - // The id of the window where the request for this stream originated. - const uint64_t mWindowId{}; - // The configuration applied to this stream. - Configuration mConfiguration; - // Whether the stream has been started and not stopped. As opposed to - // allocated and not deallocated, which controls the presence of this stream - // altogether. - bool mStarted{false}; - // The timestamp of the last frame sent to mParent for this stream. - media::TimeUnit mLastFrameTime{media::TimeUnit::FromNegativeInfinity()}; - }; - // The video capture thread is where all access to this class must happen. - const nsCOMPtr<nsISerialEventTarget> mVideoCaptureThread; - // The identifier for which VideoEngine instance we are using, i.e. which type - // of source we're associated with. const CaptureEngine mCapEngine; - // The (singleton from sEngines) VideoEngine instance that mCaptureId is valid - // in. - const RefPtr<VideoEngine> mEngine; - // The unique ID string of the associated device. - const nsCString mUniqueId; - // The id that identifies the capturer instance of the associated source - // device in VideoEngine. - const int mCaptureId; - // Tracking ID of the capturer for profiler markers. + const uint32_t mStreamId; const TrackingId mTrackingId; - // The (immutable) list of capabilities offered by the associated source - // device. - const nsTArray<webrtc::VideoCaptureCapability> mCapabilities; - // The list of streams that have been requested from all CamerasParent - // instances for the associated source device. - DataMutex<nsTArray<std::unique_ptr<Stream>>> mStreams; - - private: - AggregateCapturer(nsISerialEventTarget* aVideoCaptureThread, - CaptureEngine aCapEng, VideoEngine* aEngine, - const nsCString& aUniqueId, int aCaptureId, - nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities); - + CamerasParent* const mParent; MediaEventListener mCaptureEndedListener; + bool mConnectedToCaptureEnded = false; + DataMutex<Configuration> mConfiguration; + // Capture thread only. + media::TimeUnit mLastFrameTime = media::TimeUnit::FromNegativeInfinity(); }; class DeliverFrameRunnable; @@ -173,7 +103,7 @@ class CamerasParent final : public PCamerasParent { const CaptureEngine& aCapEngine, const nsACString& aUniqueIdUTF8, const uint64_t& aWindowID) override; mozilla::ipc::IPCResult RecvReleaseCapture(const CaptureEngine& aCapEngine, - const int& aStreamId) override; + const int& aCaptureId) override; mozilla::ipc::IPCResult RecvNumberOfCaptureDevices( const CaptureEngine& aCapEngine) override; mozilla::ipc::IPCResult RecvNumberOfCapabilities( @@ -184,21 +114,20 @@ class CamerasParent final : public PCamerasParent { mozilla::ipc::IPCResult RecvGetCaptureDevice( const CaptureEngine& aCapEngine, const int& aDeviceIndex) override; mozilla::ipc::IPCResult RecvStartCapture( - const CaptureEngine& aCapEngine, const int& aStreamId, + const CaptureEngine& aCapEngine, const int& aCaptureId, const VideoCaptureCapability& aIpcCaps, const NormalizedConstraints& aConstraints, const dom::VideoResizeModeEnum& aResizeMode) override; mozilla::ipc::IPCResult RecvFocusOnSelectedSource( - const CaptureEngine& aCapEngine, const int& aStreamId) override; + const CaptureEngine& aCapEngine, const int& aCaptureId) override; mozilla::ipc::IPCResult RecvStopCapture(const CaptureEngine& aCapEngine, - const int& aStreamId) override; + const int& aCaptureId) override; mozilla::ipc::IPCResult RecvReleaseFrame( - const int& aCaptureId, mozilla::ipc::Shmem&& aShmem) override; + mozilla::ipc::Shmem&& aShmem) override; void ActorDestroy(ActorDestroyReason aWhy) override; mozilla::ipc::IPCResult RecvEnsureInitialized( const CaptureEngine& aCapEngine) override; - bool IsWindowCapturing(uint64_t aWindowId, const nsACString& aUniqueId) const; nsIEventTarget* GetBackgroundEventTarget() { return mPBackgroundEventTarget; }; @@ -207,13 +136,12 @@ class CamerasParent final : public PCamerasParent { MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); return mDestroyed; }; - ShmemBuffer GetBuffer(int aCaptureId, size_t aSize); + ShmemBuffer GetBuffer(size_t aSize); // helper to forward to the PBackground thread - int DeliverFrameOverIPC(CaptureEngine aCapEngine, int aCaptureId, - const Span<const int>& aStreamId, - const TrackingId& aTrackingId, - Variant<ShmemBuffer, webrtc::VideoFrame>&& aBuffer, + int DeliverFrameOverIPC(CaptureEngine aCapEngine, uint32_t aStreamId, + const TrackingId& aTrackingId, ShmemBuffer aBuffer, + unsigned char* aAltBuffer, const VideoFrameProperties& aProps); CamerasParent(); @@ -221,18 +149,9 @@ class CamerasParent final : public PCamerasParent { private: virtual ~CamerasParent(); - struct GetOrCreateCapturerResult { - AggregateCapturer* mCapturer{}; - int mStreamId{}; - }; - GetOrCreateCapturerResult GetOrCreateCapturer( - CaptureEngine aEngine, uint64_t aWindowId, const nsCString& aUniqueId, - nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities); - AggregateCapturer* GetCapturer(CaptureEngine aEngine, int aStreamId); - int ReleaseStream(CaptureEngine aEngine, int aStreamId); - - nsTArray<webrtc::VideoCaptureCapability> const* EnsureCapabilitiesPopulated( - CaptureEngine aEngine, const nsCString& aUniqueId); + // We use these helpers for shutdown and for the respective IPC commands. + void StopCapture(const CaptureEngine& aCapEngine, int aCaptureId); + int ReleaseCapture(const CaptureEngine& aCapEngine, int aCaptureId); void OnDeviceChange(); @@ -249,6 +168,7 @@ class CamerasParent final : public PCamerasParent { void OnShutdown(); + nsTArray<UniquePtr<CallbackHelper>> mCallbacks; // If existent, blocks xpcom shutdown while alive. // Note that this makes a reference cycle that gets broken in ActorDestroy(). const UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker; @@ -261,24 +181,12 @@ class CamerasParent final : public PCamerasParent { // Reference to same VideoEngineArray as sEngines. Video capture thread only. const RefPtr<VideoEngineArray> mEngines; - // Reference to same array of AggregateCapturers as sCapturers. There is one - // AggregateCapturer per allocated video source. It tracks the mapping from - // source to streamIds and CamerasParent instances. Video capture thread only. - const RefPtr< - media::Refcountable<nsTArray<std::unique_ptr<AggregateCapturer>>>> - mCapturers; - // Reference to same VideoCaptureFactory as sVideoCaptureFactory. Video // capture thread only. const RefPtr<VideoCaptureFactory> mVideoCaptureFactory; - // Image buffers. One pool per CamerasParent instance and capture id (i.e. - // unique source). Multiple CamerasParent instances capturing the same source - // need distinct ShmemPools as ShmemBuffers are tied to the IPC channel. - // Access is on the PBackground thread for mutations and - // allocating shmem buffers, and on the callback thread (varies by capture - // backend) for querying an existing pool for an available buffer. - DataMutex<std::map<int, ShmemPool>> mShmemPools; + // image buffers + ShmemPool mShmemPool; // PBackgroundParent thread const nsCOMPtr<nsISerialEventTarget> mPBackgroundEventTarget; @@ -286,7 +194,7 @@ class CamerasParent final : public PCamerasParent { // Set to true in ActorDestroy. PBackground only. bool mDestroyed; - std::map<nsCString, nsTArray<webrtc::VideoCaptureCapability>> + std::map<nsCString, std::map<uint32_t, webrtc::VideoCaptureCapability>> mAllCandidateCapabilities; // Listener for the camera VideoEngine::DeviceChangeEvent(). Video capture diff --git a/dom/media/systemservices/PCameras.ipdl b/dom/media/systemservices/PCameras.ipdl @@ -61,13 +61,13 @@ async protocol PCameras manager PBackground; child: - async CaptureEnded(int[] streamIds); + async CaptureEnded(int streamId); // transfers ownership of |buffer| from parent to child - async DeliverFrame(int captureId, int[] streamIds, Shmem buffer, VideoFrameProperties props); + async DeliverFrame(int streamId, Shmem buffer, VideoFrameProperties props); async DeviceChange(); async ReplyNumberOfCaptureDevices(int deviceCount); async ReplyNumberOfCapabilities(int capabilityCount); - async ReplyAllocateCapture(int streamId); + async ReplyAllocateCapture(int captureId); async ReplyGetCaptureCapability(VideoCaptureCapability cap); async ReplyGetCaptureDevice(nsCString device_name, nsCString device_id, bool scary); async ReplyFailure(); @@ -84,15 +84,15 @@ parent: async AllocateCapture(CaptureEngine engine, nsCString unique_idUTF8, uint64_t windowID); - async ReleaseCapture(CaptureEngine engine, int streamId); - async StartCapture(CaptureEngine engine, int streamId, + async ReleaseCapture(CaptureEngine engine, int captureId); + async StartCapture(CaptureEngine engine, int captureId, VideoCaptureCapability capability, NormalizedConstraints constraints, VideoResizeModeEnum resizeMode); - async FocusOnSelectedSource(CaptureEngine engine, int streamId); - async StopCapture(CaptureEngine engine, int streamId); + async FocusOnSelectedSource(CaptureEngine engine, int captureId); + async StopCapture(CaptureEngine engine, int captureId); // transfers frame back - async ReleaseFrame(int captureId, Shmem s); + async ReleaseFrame(Shmem s); // setup camera engine async EnsureInitialized(CaptureEngine engine); diff --git a/dom/media/systemservices/VideoEngine.cpp b/dom/media/systemservices/VideoEngine.cpp @@ -46,21 +46,19 @@ int VideoEngine::SetAndroidObjects() { } #endif -int32_t VideoEngine::CreateVideoCapture(const char* aDeviceUniqueIdUTF8, - uint64_t aWindowID) { +int32_t VideoEngine::CreateVideoCapture(const char* aDeviceUniqueIdUTF8) { LOG(("%s", __PRETTY_FUNCTION__)); MOZ_ASSERT(aDeviceUniqueIdUTF8); int32_t id = GenerateId(); LOG(("CaptureDeviceType=%s id=%d", EnumValueToString(mCaptureDevType), id)); - for (auto& it : mSharedCapturers) { + for (auto& it : mCaps) { if (it.second.VideoCapture() && it.second.VideoCapture()->CurrentDeviceName() && strcmp(it.second.VideoCapture()->CurrentDeviceName(), aDeviceUniqueIdUTF8) == 0) { - mIdToCapturerMap.emplace(id, CaptureHandle{.mCaptureEntryNum = it.first, - .mWindowID = aWindowID}); + mIdMap.emplace(id, it.first); return id; } } @@ -73,9 +71,8 @@ int32_t VideoEngine::CreateVideoCapture(const char* aDeviceUniqueIdUTF8, entry = CaptureEntry(id, std::move(capturer.mCapturer), capturer.mDesktopImpl); - mSharedCapturers.emplace(id, std::move(entry)); - mIdToCapturerMap.emplace( - id, CaptureHandle{.mCaptureEntryNum = id, .mWindowID = aWindowID}); + mCaps.emplace(id, std::move(entry)); + mIdMap.emplace(id, id); return id; } @@ -84,15 +81,14 @@ int VideoEngine::ReleaseVideoCapture(const int32_t aId) { #ifdef DEBUG { - auto it = mIdToCapturerMap.find(aId); - MOZ_ASSERT(it != mIdToCapturerMap.end()); + auto it = mIdMap.find(aId); + MOZ_ASSERT(it != mIdMap.end()); Unused << it; } #endif - for (auto& it : mIdToCapturerMap) { - if (it.first != aId && - it.second.mCaptureEntryNum == mIdToCapturerMap[aId].mCaptureEntryNum) { + for (auto& it : mIdMap) { + if (it.first != aId && it.second == mIdMap[aId]) { // There are other tracks still using this hardware. found = true; } @@ -105,13 +101,13 @@ int VideoEngine::ReleaseVideoCapture(const int32_t aId) { }); MOZ_ASSERT(found); if (found) { - auto it = mSharedCapturers.find(mIdToCapturerMap[aId].mCaptureEntryNum); - MOZ_ASSERT(it != mSharedCapturers.end()); - mSharedCapturers.erase(it); + auto it = mCaps.find(mIdMap[aId]); + MOZ_ASSERT(it != mCaps.end()); + mCaps.erase(it); } } - mIdToCapturerMap.erase(aId); + mIdMap.erase(aId); return found ? 0 : (-1); } @@ -220,44 +216,21 @@ bool VideoEngine::WithEntry( const std::function<void(CaptureEntry& entry)>&& fn) { #ifdef DEBUG { - auto it = mIdToCapturerMap.find(entryCapnum); - MOZ_ASSERT(it != mIdToCapturerMap.end()); + auto it = mIdMap.find(entryCapnum); + MOZ_ASSERT(it != mIdMap.end()); Unused << it; } #endif - auto it = - mSharedCapturers.find(mIdToCapturerMap[entryCapnum].mCaptureEntryNum); - MOZ_ASSERT(it != mSharedCapturers.end()); - if (it == mSharedCapturers.end()) { + auto it = mCaps.find(mIdMap[entryCapnum]); + MOZ_ASSERT(it != mCaps.end()); + if (it == mCaps.end()) { return false; } fn(it->second); return true; } -bool VideoEngine::IsWindowCapturing(uint64_t aWindowID, - const nsCString& aUniqueIdUTF8) { - Maybe<int32_t> sharedId; - for (auto& [id, entry] : mSharedCapturers) { - if (entry.VideoCapture() && entry.VideoCapture()->CurrentDeviceName() && - strcmp(entry.VideoCapture()->CurrentDeviceName(), - aUniqueIdUTF8.get()) == 0) { - sharedId = Some(id); - break; - } - } - if (!sharedId) { - return false; - } - for (auto& [id, handle] : mIdToCapturerMap) { - if (handle.mCaptureEntryNum == *sharedId && handle.mWindowID == aWindowID) { - return true; - } - } - return false; -} - int32_t VideoEngine::GenerateId() { // XXX Something better than this (a map perhaps, or a simple boolean TArray, // given the number in-use is O(1) normally!) @@ -280,8 +253,8 @@ VideoEngine::VideoEngine(const CaptureDeviceType& aCaptureDeviceType, } VideoEngine::~VideoEngine() { - MOZ_ASSERT(mSharedCapturers.empty()); - MOZ_ASSERT(mIdToCapturerMap.empty()); + MOZ_ASSERT(mCaps.empty()); + MOZ_ASSERT(mIdMap.empty()); } } // namespace mozilla::camera diff --git a/dom/media/systemservices/VideoEngine.h b/dom/media/systemservices/VideoEngine.h @@ -44,11 +44,9 @@ class VideoEngine : public webrtc::VideoInputFeedBack { #if defined(ANDROID) static int SetAndroidObjects(); #endif - int32_t GenerateId(); /** Returns a non-negative capture identifier or -1 on failure. */ - int32_t CreateVideoCapture(const char* aDeviceUniqueIdUTF8, - uint64_t aWindowID); + int32_t CreateVideoCapture(const char* aDeviceUniqueIdUTF8); int ReleaseVideoCapture(const int32_t aId); @@ -89,17 +87,10 @@ class VideoEngine : public webrtc::VideoInputFeedBack { friend class VideoEngine; }; - struct CaptureHandle { - int32_t mCaptureEntryNum{}; - uint64_t mWindowID{}; - }; - // Returns true iff an entry for capnum exists bool WithEntry(const int32_t entryCapnum, const std::function<void(CaptureEntry& entry)>&& fn); - bool IsWindowCapturing(uint64_t aWindowID, const nsCString& aUniqueIdUTF8); - void OnDeviceChange() override; MediaEventSource<void>& DeviceChangeEvent() { return mDeviceChangeEvent; } @@ -111,11 +102,12 @@ class VideoEngine : public webrtc::VideoInputFeedBack { const CaptureDeviceType mCaptureDevType; const RefPtr<VideoCaptureFactory> mVideoCaptureFactory; std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> mDeviceInfo; - std::map<int32_t, CaptureEntry> mSharedCapturers; - std::map<int32_t, CaptureHandle> mIdToCapturerMap; + std::map<int32_t, CaptureEntry> mCaps; + std::map<int32_t, int32_t> mIdMap; MediaEventProducer<void> mDeviceChangeEvent; // The validity period for non-camera capture device infos` webrtc::Timestamp mExpiryTime = webrtc::Timestamp::Micros(0); + int32_t GenerateId(); }; } // namespace mozilla::camera #endif diff --git a/dom/media/webrtc/MediaEngine.h b/dom/media/webrtc/MediaEngine.h @@ -50,13 +50,6 @@ class MediaEngine { virtual RefPtr<MediaEngineSource> CreateSource( const MediaDevice* aDevice) = 0; - /** - * Like CreateSource but in addition copies over capabilities and settings - * from another source. - */ - virtual RefPtr<MediaEngineSource> CreateSourceFrom( - const MediaEngineSource* aSource, const MediaDevice* aDevice) = 0; - virtual MediaEventSource<void>& DeviceListChangeEvent() = 0; /** * Return true if devices returned from EnumerateDevices are emulated media diff --git a/dom/media/webrtc/MediaEngineFake.cpp b/dom/media/webrtc/MediaEngineFake.cpp @@ -86,9 +86,6 @@ class MediaEngineFakeVideoSource : public MediaEngineSource { public: MediaEngineFakeVideoSource(); - static already_AddRefed<MediaEngineFakeVideoSource> CreateFrom( - const MediaEngineFakeVideoSource* aSource); - static nsString GetGroupId(); nsresult Allocate(const dom::MediaTrackConstraints& aConstraints, @@ -146,15 +143,6 @@ MediaEngineFakeVideoSource::MediaEngineFakeVideoSource() dom::GetEnumString(dom::VideoResizeModeEnum::None))); } -/*static*/ already_AddRefed<MediaEngineFakeVideoSource> -MediaEngineFakeVideoSource::CreateFrom( - const MediaEngineFakeVideoSource* aSource) { - auto src = MakeRefPtr<MediaEngineFakeVideoSource>(); - *static_cast<MediaTrackSettings*>(src->mSettings) = *aSource->mSettings; - src->mOpts = aSource->mOpts; - return src.forget(); -} - nsString MediaEngineFakeVideoSource::GetGroupId() { return u"Fake Video Group"_ns; } @@ -628,20 +616,4 @@ RefPtr<MediaEngineSource> MediaEngineFake::CreateSource( } } -RefPtr<MediaEngineSource> MediaEngineFake::CreateSourceFrom( - const MediaEngineSource* aSource, const MediaDevice* aMediaDevice) { - MOZ_ASSERT(aMediaDevice->mEngine == this); - switch (aMediaDevice->mMediaSource) { - case MediaSourceEnum::Camera: - return MediaEngineFakeVideoSource::CreateFrom( - static_cast<const MediaEngineFakeVideoSource*>(aSource)); - case MediaSourceEnum::Microphone: - // No main thread members that need to be deep cloned. - return new MediaEngineFakeAudioSource(); - default: - MOZ_ASSERT_UNREACHABLE("Unsupported source type"); - return nullptr; - } -} - } // namespace mozilla diff --git a/dom/media/webrtc/MediaEngineFake.h b/dom/media/webrtc/MediaEngineFake.h @@ -22,8 +22,6 @@ class MediaEngineFake : public MediaEngine { nsTArray<RefPtr<MediaDevice>>*) override; void Shutdown() override {} RefPtr<MediaEngineSource> CreateSource(const MediaDevice* aDevice) override; - RefPtr<MediaEngineSource> CreateSourceFrom( - const MediaEngineSource* aSource, const MediaDevice* aDevice) override; MediaEventSource<void>& DeviceListChangeEvent() override { return mDeviceListChangeEvent; diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp @@ -203,22 +203,6 @@ MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource( } } -/*static*/ -already_AddRefed<MediaEngineRemoteVideoSource> -MediaEngineRemoteVideoSource::CreateFrom( - const MediaEngineRemoteVideoSource* aSource, - const MediaDevice* aMediaDevice) { - auto src = MakeRefPtr<MediaEngineRemoteVideoSource>(aMediaDevice); - *static_cast<MediaTrackSettings*>(src->mSettings) = *aSource->mSettings; - *static_cast<MediaTrackCapabilities*>(src->mTrackCapabilities) = - *aSource->mTrackCapabilities; - { - MutexAutoLock lock(aSource->mMutex); - src->mIncomingImageSize = aSource->mIncomingImageSize; - } - return src.forget(); -} - MediaEngineRemoteVideoSource::~MediaEngineRemoteVideoSource() { mFirstFramePromiseHolder.RejectIfExists(NS_ERROR_ABORT, __func__); } @@ -299,8 +283,8 @@ nsresult MediaEngineRemoteVideoSource::Allocate( .mCapabilityWidth = cw ? Some(cw) : Nothing(), .mCapabilityHeight = ch ? Some(ch) : Nothing(), .mCapEngine = mCapEngine, - .mInputWidth = cw ? cw : mIncomingImageSize.width, - .mInputHeight = ch ? ch : mIncomingImageSize.height, + .mInputWidth = cw, + .mInputHeight = ch, .mRotation = 0, }; framerate = input.mCanCropAndScale.valueOr(false) diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.h b/dom/media/webrtc/MediaEngineRemoteVideoSource.h @@ -88,10 +88,6 @@ class MediaEngineRemoteVideoSource : public MediaEngineSource, public: explicit MediaEngineRemoteVideoSource(const MediaDevice* aMediaDevice); - static already_AddRefed<MediaEngineRemoteVideoSource> CreateFrom( - const MediaEngineRemoteVideoSource* aSource, - const MediaDevice* aMediaDevice); - // ExternalRenderer /** * Signals that the capture stream has ended @@ -165,7 +161,7 @@ class MediaEngineRemoteVideoSource : public MediaEngineSource, // mMutex protects certain members on 3 threads: // MediaManager, Cameras IPC and MediaTrackGraph. - mutable Mutex mMutex MOZ_UNANNOTATED; + Mutex mMutex MOZ_UNANNOTATED; // Current state of this source. // Set under mMutex on the owning thread. Accessed under one of the two. diff --git a/dom/media/webrtc/MediaEngineWebRTC.cpp b/dom/media/webrtc/MediaEngineWebRTC.cpp @@ -299,25 +299,6 @@ RefPtr<MediaEngineSource> MediaEngineWebRTC::CreateSource( } } -RefPtr<MediaEngineSource> MediaEngineWebRTC::CreateSourceFrom( - const MediaEngineSource* aSource, const MediaDevice* aMediaDevice) { - MOZ_ASSERT(aMediaDevice->mEngine == this); - if (MediaEngineSource::IsVideo(aMediaDevice->mMediaSource)) { - return MediaEngineRemoteVideoSource::CreateFrom( - static_cast<const MediaEngineRemoteVideoSource*>(aSource), - aMediaDevice); - } - switch (aMediaDevice->mMediaSource) { - case MediaSourceEnum::Microphone: - return MediaEngineWebRTCMicrophoneSource::CreateFrom( - static_cast<const MediaEngineWebRTCMicrophoneSource*>(aSource), - aMediaDevice); - default: - MOZ_CRASH("Unsupported source type"); - return nullptr; - } -} - void MediaEngineWebRTC::Shutdown() { AssertIsOnOwningThread(); mCameraListChangeListener.DisconnectIfExists(); diff --git a/dom/media/webrtc/MediaEngineWebRTC.h b/dom/media/webrtc/MediaEngineWebRTC.h @@ -27,8 +27,6 @@ class MediaEngineWebRTC : public MediaEngine { void EnumerateDevices(dom::MediaSourceEnum, MediaSinkEnum, nsTArray<RefPtr<MediaDevice>>*) override; RefPtr<MediaEngineSource> CreateSource(const MediaDevice* aDevice) override; - RefPtr<MediaEngineSource> CreateSourceFrom(const MediaEngineSource* aSource, - const MediaDevice*) override; MediaEventSource<void>& DeviceListChangeEvent() override { return mDeviceListChangeEvent; diff --git a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -113,17 +113,6 @@ MediaEngineWebRTCMicrophoneSource::MediaEngineWebRTCMicrophoneSource( })); } -/*static*/ already_AddRefed<MediaEngineWebRTCMicrophoneSource> -MediaEngineWebRTCMicrophoneSource::CreateFrom( - const MediaEngineWebRTCMicrophoneSource* aSource, - const MediaDevice* aMediaDevice) { - auto src = MakeRefPtr<MediaEngineWebRTCMicrophoneSource>(aMediaDevice); - *static_cast<dom::MediaTrackSettings*>(src->mSettings) = *aSource->mSettings; - *static_cast<dom::MediaTrackCapabilities*>(src->mCapabilities) = - *aSource->mCapabilities; - return src.forget(); -} - nsresult MediaEngineWebRTCMicrophoneSource::EvaluateSettings( const NormalizedConstraints& aConstraintsUpdate, const MediaEnginePrefs& aInPrefs, MediaEnginePrefs* aOutPrefs, diff --git a/dom/media/webrtc/MediaEngineWebRTCAudio.h b/dom/media/webrtc/MediaEngineWebRTCAudio.h @@ -34,10 +34,6 @@ class MediaEngineWebRTCMicrophoneSource : public MediaEngineSource { public: explicit MediaEngineWebRTCMicrophoneSource(const MediaDevice* aMediaDevice); - static already_AddRefed<MediaEngineWebRTCMicrophoneSource> CreateFrom( - const MediaEngineWebRTCMicrophoneSource* aSource, - const MediaDevice* aMediaDevice); - nsresult Allocate(const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, uint64_t aWindowID, const char** aOutBadConstraint) override; @@ -318,13 +314,9 @@ class AudioProcessingTrack : public DeviceInputConsumerTrack { void DestroyImpl() override; void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override; uint32_t NumberOfChannels() const override { - if (!mInputProcessing) { - // There's an async gap between adding the track to the graph - // (AudioProcessingTrack::Create) and setting mInputProcessing - // (SetInputProcessing on the media manager thread). - // Return 0 to indicate the default within this gap. - return 0; - } + MOZ_DIAGNOSTIC_ASSERT( + mInputProcessing, + "Must set mInputProcessing before exposing to content"); return mInputProcessing->GetRequestedInputChannelCount(); } // Pass the graph's mixed audio output to mInputProcessing for processing as diff --git a/dom/media/webrtc/tests/mochitests/test_getUserMedia_mediaElementCapture_tracks.html b/dom/media/webrtc/tests/mochitests/test_getUserMedia_mediaElementCapture_tracks.html @@ -22,10 +22,6 @@ let streamUntilEnded; const tracks = []; runTest(async () => { try { - await pushPrefs( - ["media.getusermedia.camera.fake.force", true], - ["media.video_loopback_dev", ""], - ); let stream = await getUserMedia({audio: true, video: true}); // We need to test with multiple tracks. We add an extra of each kind. for (const track of stream.getTracks()) { diff --git a/dom/media/webrtc/tests/mochitests/test_getUserMedia_mediaStreamTrackClone.html b/dom/media/webrtc/tests/mochitests/test_getUserMedia_mediaStreamTrackClone.html @@ -35,10 +35,6 @@ }); runTest(() => Promise.resolve() - .then(() => pushPrefs( - ["media.getusermedia.camera.fake.force", true], - ["media.video_loopback_dev", ""], - )) .then(() => testSingleTrackClonePlayback({audio: true})) .then(() => testSingleTrackClonePlayback({video: true})) .then(() => getUserMedia({video: true})).then(stream => { diff --git a/testing/web-platform/mozilla/meta/mediacapture-streams/MediaStreamTrack-independent-clones.https.html.ini b/testing/web-platform/mozilla/meta/mediacapture-streams/MediaStreamTrack-independent-clones.https.html.ini @@ -1,32 +0,0 @@ -[MediaStreamTrack-independent-clones.https.html] - [gDM gets expected modes by {"resizeMode":"none","width":100} + (cloned) {"resizeMode":"none","height":500}] - expected: - if os == "android": FAIL - [gDM gets expected modes by {"resizeMode":"none","frameRate":50} + (cloned) {"resizeMode":"none","frameRate":1}] - expected: - if os == "android": FAIL - [gDM gets expected modes by {"resizeMode":"none"} + (cloned) {"resizeMode":"crop-and-scale"}] - expected: - if os == "android": FAIL - [gDM gets expected modes by {"resizeMode":"crop-and-scale"} + (cloned) {"resizeMode":"crop-and-scale","height":100}] - expected: - if os == "android": FAIL - [gDM gets expected modes by {"resizeMode":"crop-and-scale","height":100} + (cloned) {"resizeMode":"crop-and-scale"}] - expected: - if os == "android": FAIL - [gDM gets expected modes by {"resizeMode":"crop-and-scale","frameRate":5} + (cloned) {"resizeMode":"crop-and-scale","frameRate":50}] - expected: - if os == "android": FAIL - [gDM gets expected modes by {"resizeMode":"crop-and-scale","frameRate":50} + (cloned) {"resizeMode":"crop-and-scale","frameRate":5}] - expected: - if os == "android": FAIL - [gDM gets expected modes by {"resizeMode":"none"} + (cloned) {"resizeMode":"crop-and-scale","frameRate":5000}] - expected: - if os == "android": FAIL - [applyConstraints on gDM clone doesn't crop with only ideal dimensions] - expected: - if os == "android": FAIL - [applyConstraints on gDM clone doesn't crop with ideal and max dimensions] - expected: - if os == "android": FAIL - prefs: [media.getusermedia.camera.fake.force:true, media.navigator.streams.fake:false, media.cubeb.force_mock_context:true] diff --git a/testing/web-platform/mozilla/meta/screen-capture/MediaStreamTrack-independent-clones.https.html.ini b/testing/web-platform/mozilla/meta/screen-capture/MediaStreamTrack-independent-clones.https.html.ini @@ -1,3 +0,0 @@ -[MediaStreamTrack-independent-clones.https.html] - expected: - if os == "android": ERROR diff --git a/testing/web-platform/mozilla/tests/mediacapture-streams/MediaStreamTrack-independent-clones.https.html b/testing/web-platform/mozilla/tests/mediacapture-streams/MediaStreamTrack-independent-clones.https.html @@ -1,209 +0,0 @@ -<!doctype html> -<title>MediaStreamTrack clones have independent settings. Assumes Mozilla's fake camera source with 480p and 720p capabilities.</title> -<meta name="timeout" content="long"> -<p class="instructions">When prompted, accept to share your video stream.</p> -<script src=/resources/testharness.js></script> -<script src=/resources/testharnessreport.js></script> -<script src=/resources/testdriver.js></script> -<script src=/resources/testdriver-vendor.js></script> -<script src=video-test-helper.js></script> -<script> - "use strict" - - // Native capabilities supported by the fake camera. - const nativeLow = {width: 640, height: 480, frameRate: 30, resizeMode: "none"}; - const nativeHigh = {width: 1280, height: 720, frameRate: 10, resizeMode: "none"}; - - [ - [ - [{resizeMode: "none", width: 1000}, {resizeMode: "none", width: 500}], - [nativeHigh, nativeLow], - ], - [ - [{resizeMode: "none", height: 500}, {resizeMode: "none", width: 1000}], - [nativeLow, nativeHigh], - ], - [ - [{resizeMode: "none", width: 500, height: 500}, {resizeMode: "crop-and-scale", width: 1000, height: 750}], - [nativeLow, {resizeMode: "crop-and-scale", width: 1000, height: 720, frameRate: 10}], - ], - [ - [{resizeMode: "crop-and-scale", width: 800, height: 600}, {resizeMode: "none"}], - [{resizeMode: "crop-and-scale", width: 800, height: 600, frameRate: 10}, nativeLow], - ], - [ - [{resizeMode: "crop-and-scale", height: 500}, {resizeMode: "crop-and-scale", height: 300}], - [ - {resizeMode: "crop-and-scale", width: 889, height: 500, frameRate: 10}, - {resizeMode: "crop-and-scale", width: 400, height: 300, frameRate: 30} - ], - ], - [ - [ - {resizeMode: "crop-and-scale", frameRate: {exact: 5}}, - {resizeMode: "crop-and-scale", frameRate: {exact: 1}}, - ], - [ - {resizeMode: "crop-and-scale", width: 640, height: 480, frameRate: 5}, - {resizeMode: "crop-and-scale", width: 640, height: 480, frameRate: 1}, - ], - [[2, 7], [0.5, 3]], - ], - ].forEach( - ([ - [video, cloneVideo], - [expected, cloneExpected], - [testFramerate, cloneTestFramerate] = [null, null] - ]) => promise_test(async t => { - const stream = await navigator.mediaDevices.getUserMedia({video}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - let settings = track.getSettings(); - let cloneSettings = clone.getSettings(); - for (const key of Object.keys(expected)) { - assert_equals(settings[key], expected[key], `original: ${key}`); - assert_equals(cloneSettings[key], expected[key], `clone: ${key}`); - } - await clone.applyConstraints(cloneVideo); - settings = track.getSettings(); - cloneSettings = clone.getSettings(); - for (const key of Object.keys(expected)) { - assert_equals(settings[key], expected[key], `original-post: ${key}`); - } - for (const key of Object.keys(cloneExpected)) { - assert_equals(cloneSettings[key], cloneExpected[key], `clone-post: ${key}`); - } - await test_resolution_equals(t, track, settings.width, settings.height); - await test_resolution_equals(t, clone, cloneSettings.width, cloneSettings.height); - if (testFramerate) { - const [low, high] = testFramerate; - await test_framerate_between_exclusive(t, track, low, high); - } - if (cloneTestFramerate) { - const [low, high] = cloneTestFramerate; - await test_framerate_between_exclusive(t, clone, low, high); - } - }, `gUM gets ${JSON.stringify(expected)} + clone ` + - `${JSON.stringify(cloneExpected)} by ${JSON.stringify(video)} ` + - `and ${JSON.stringify(cloneVideo)}`)); - - [ - [ - [{echoCancellation: true}, {echoCancellation: false, noiseSuppression: true, autoGainControl: true}], - [ - {echoCancellation: true, noiseSuppression: true, autoGainControl: true}, - {echoCancellation: false, noiseSuppression: true, autoGainControl: true}, - ], - ], - [ - [{echoCancellation: false, noiseSuppression: false, autoGainControl: false}, {}], - [ - {echoCancellation: false, noiseSuppression: false, autoGainControl: false}, - {echoCancellation: true, noiseSuppression: true, autoGainControl: true}, - ], - ], - ].forEach( - ([ - [audio, cloneAudio], - [expected, cloneExpected], - ]) => promise_test(async t => { - const stream = await navigator.mediaDevices.getUserMedia({audio}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - let settings = track.getSettings(); - let cloneSettings = clone.getSettings(); - for (const key of Object.keys(expected)) { - assert_equals(settings[key], expected[key], `original: ${key}`); - assert_equals(cloneSettings[key], expected[key], `clone: ${key}`); - } - await clone.applyConstraints(cloneAudio); - settings = track.getSettings(); - cloneSettings = clone.getSettings(); - for (const key of Object.keys(expected)) { - assert_equals(settings[key], expected[key], `original-post: ${key}`); - } - for (const key of Object.keys(cloneExpected)) { - assert_equals(cloneSettings[key], cloneExpected[key], `clone-post: ${key}`); - } - }, `gUM gets ${JSON.stringify(expected)} + clone ` + - `${JSON.stringify(cloneExpected)} by ${JSON.stringify(audio)} ` + - `and ${JSON.stringify(cloneAudio)}`)); - - - - promise_test(async t => { - const stream = await navigator.mediaDevices.getUserMedia({video: true}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - try { - await clone.applyConstraints({resizeMode: "none", width: {min: 2000}}) - } catch(e) { - assert_equals(e.name, "OverconstrainedError", `got error ${e.message}`); - return; - } - assert_unreached("applyConstraints is rejected with impossible width"); - }, "applyConstraints on gUM clone is rejected by resizeMode none and impossible min-width"); - - promise_test(async t => { - const stream = await navigator.mediaDevices.getUserMedia({video: true}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - try { - await clone.applyConstraints({resizeMode: "none", width: {max: 200}}) - } catch(e) { - assert_equals(e.name, "OverconstrainedError", `got error ${e.message}`); - return; - } - assert_unreached("applyConstraints is rejected with impossible width"); - }, "applyConstraints on gUM clone is rejected by resizeMode none and impossible max-width"); - - promise_test(async t => { - const stream = await navigator.mediaDevices.getUserMedia({video: true}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - try { - await clone.applyConstraints({resizeMode: "crop-and-scale", width: {min: 2000}}) - } catch(e) { - assert_equals(e.name, "OverconstrainedError", `got error ${e.message}`); - return; - } - assert_unreached("applyConstraints is rejected with impossible width"); - }, "applyConstraints on gUM clone is rejected by resizeMode crop-and-scale and impossible width"); - - promise_test(async t => { - const stream = await navigator.mediaDevices.getUserMedia({video: true}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - try { - await clone.applyConstraints({resizeMode: "crop-and-scale", frameRate: {min: 50}}); - } catch(e) { - assert_equals(e.name, "OverconstrainedError", `got error ${e.message}`); - return; - } - assert_unreached("applyConstraints is rejected with impossible fps"); - }, "applyConstraints on gUM clone is rejected by resizeMode crop-and-scale impossible fps"); -</script> diff --git a/testing/web-platform/mozilla/tests/mediacapture-streams/MediaStreamTrack-resizeMode.https.html b/testing/web-platform/mozilla/tests/mediacapture-streams/MediaStreamTrack-resizeMode.https.html @@ -6,11 +6,77 @@ <script src=/resources/testharnessreport.js></script> <script src=/resources/testdriver.js></script> <script src=/resources/testdriver-vendor.js></script> -<script src=settings-helper.js></script> -<script src=video-test-helper.js></script> <script> "use strict" + async function test_framerate_between_exclusive(t, track, lower, upper) { + const video = document.createElement("video"); + document.body.appendChild(video); + t.add_cleanup(async () => document.body.removeChild(video)); + + video.srcObject = new MediaStream([track]); + await video.play(); + + const numSeconds = 2; + await new Promise(r => setTimeout(r, numSeconds * 1000)); + const totalVideoFrames = video.mozPaintedFrames; + assert_between_exclusive(totalVideoFrames / numSeconds, lower, upper, "totalVideoFrames"); + } + + function createSettingsDicts(width, height, step = 1) { + const settingsDicts = [], aspect = width / height; + do { + settingsDicts.push({ width, height }); + if (width > height) { + height = Math.round((width - step) / aspect); + width -= step; + } else { + width = Math.round((height - step) * aspect); + height -= step; + } + } while (width > 2 && height > 2); + return settingsDicts; + } + + function integerFitness(actual, ideal) { + if (actual == ideal) { + return 0; + } + return Math.abs(actual - ideal) / Math.max(Math.abs(actual), Math.abs(ideal)); + } + + function findFittestResolutionSetting(width, height, constraints) { + const widthIsNumber = typeof constraints.width == "number"; + const heightIsNumber = typeof constraints.height == "number"; + const c = { + width: { + ideal: widthIsNumber ? constraints.width : constraints?.width?.ideal, + max: constraints?.width?.max ?? 1000000, + }, + height: { + ideal: heightIsNumber ? constraints.height : constraints?.height?.ideal, + max: constraints?.height?.max ?? 1000000, + }, + }; + const dicts = createSettingsDicts(width, height) + .filter(s => s.width <= c.width.max && s.height <= c.height.max); + for (const dict of dicts) { + dict.distance = + integerFitness(dict.width, c.width.ideal) + + integerFitness(dict.height, c.height.ideal); + } + + const filteredDicts = dicts.filter(s => { + return (!c.width.ideal || s.width <= c.width.ideal) && + (!c.height.ideal || s.height <= c.height.ideal); + }); + + return filteredDicts.reduce( + (a, b) => (a.distance < b.distance ? a : b), + filteredDicts[0], + ); + } + // Native capabilities supported by the fake camera. const nativeLow = {width: 640, height: 480, frameRate: 30, resizeMode: "none"}; const nativeHigh = {width: 1280, height: 720, frameRate: 10, resizeMode: "none"}; diff --git a/testing/web-platform/mozilla/tests/mediacapture-streams/settings-helper.js b/testing/web-platform/mozilla/tests/mediacapture-streams/settings-helper.js @@ -1,54 +0,0 @@ - -function createSettingsDicts(width, height, step = 1) { - const settingsDicts = [], aspect = width / height; - do { - settingsDicts.push({ width, height }); - if (width > height) { - height = Math.round((width - step) / aspect); - width -= step; - } else { - width = Math.round((height - step) * aspect); - height -= step; - } - } while (width > 2 && height > 2); - return settingsDicts; -} - -function integerFitness(actual, ideal) { - if (actual == ideal) { - return 0; - } - return Math.abs(actual - ideal) / Math.max(Math.abs(actual), Math.abs(ideal)); -} - -function findFittestResolutionSetting(width, height, constraints) { - const widthIsNumber = typeof constraints.width == "number"; - const heightIsNumber = typeof constraints.height == "number"; - const c = { - width: { - ideal: widthIsNumber ? constraints.width : constraints?.width?.ideal, - max: constraints?.width?.max ?? 1000000, - }, - height: { - ideal: heightIsNumber ? constraints.height : constraints?.height?.ideal, - max: constraints?.height?.max ?? 1000000, - }, - }; - const dicts = createSettingsDicts(width, height) - .filter(s => s.width <= c.width.max && s.height <= c.height.max); - for (const dict of dicts) { - dict.distance = - integerFitness(dict.width, c.width.ideal) + - integerFitness(dict.height, c.height.ideal); - } - - const filteredDicts = dicts.filter(s => { - return (!c.width.ideal || s.width <= c.width.ideal) && - (!c.height.ideal || s.height <= c.height.ideal); - }); - - return filteredDicts.reduce( - (a, b) => (a.distance < b.distance ? a : b), - filteredDicts[0], - ); -} diff --git a/testing/web-platform/mozilla/tests/mediacapture-streams/video-test-helper.js b/testing/web-platform/mozilla/tests/mediacapture-streams/video-test-helper.js @@ -1,38 +0,0 @@ -// Helper functions checking video flow using HTMLVideoElement. - -async function test_resolution_equals(t, track, width, height) { - const video = document.createElement("video"); - const timeout = new Promise(r => t.step_timeout(r, 1000)); - const waitForResize = () => Promise.race([ - new Promise(r => video.onresize = r), - timeout.then(() => { throw new Error("Timeout waiting for resize"); }), - ]); - video.srcObject = new MediaStream([track]); - video.play(); - // Wait for the first frame. - await waitForResize(); - // There's a potential race with applyConstraints, where a frame of the old - // resolution is in flight and renders after applyConstraints has resolved. - // In that case, wait for another frame. - if (video.videoWidth != width || video.videoHeight != height) { - await waitForResize(); - } - - assert_equals(video.videoWidth, width, "videoWidth"); - assert_equals(video.videoHeight, height, "videoHeight"); -} - -async function test_framerate_between_exclusive(t, track, lower, upper) { - const video = document.createElement("video"); - document.body.appendChild(video); - video.style = "width:320px;height:240px;" - t.add_cleanup(async () => document.body.removeChild(video)); - - video.srcObject = new MediaStream([track]); - await video.play(); - - const numSeconds = 2; - await new Promise(r => setTimeout(r, numSeconds * 1000)); - const totalVideoFrames = video.mozPaintedFrames; - assert_between_exclusive(totalVideoFrames / numSeconds, lower, upper, "totalVideoFrames"); -} diff --git a/testing/web-platform/mozilla/tests/screen-capture/MediaStreamTrack-independent-clones.https.html b/testing/web-platform/mozilla/tests/screen-capture/MediaStreamTrack-independent-clones.https.html @@ -1,232 +0,0 @@ -<!doctype html> -<title>MediaStreamTrack clones have independent settings. Assumes Mozilla's fake camera source with 480p and 720p capabilities.</title> -<meta name="timeout" content="long"> -<p class="instructions">When prompted, accept to share your video stream.</p> -<script src=/resources/testharness.js></script> -<script src=/resources/testharnessreport.js></script> -<script src=/resources/testdriver.js></script> -<script src=/resources/testdriver-vendor.js></script> -<script src=../mediacapture-streams/settings-helper.js></script> -<script src=../mediacapture-streams/video-test-helper.js></script> -<script> - "use strict" - - setup(() => assert_implements("getDisplayMedia" in navigator.mediaDevices, "getDisplayMedia() not supported")); - - // Note these gDM tests will fail if our own window is on a screen different - // than the system's first screen. They're functions in case the browser - // window needs to be moved to the first screen during the test in order to - // pass. - function screenPixelRatio() { return SpecialPowers.wrap(window).desktopToDeviceScale; } - function screenWidth() { return window.screen.width * window.devicePixelRatio; } - function screenHeight() { return window.screen.height * window.devicePixelRatio; } - function desktopWidth() { - // TODO: Bug 1965499 - scale down by screenPixelRatio by default in resizeMode: crop-and-scale. - // return screenWidth() / screenPixelRatio(); - return screenWidth(); - } - function desktopHeight() { - // TODO: Bug 1965499 - scale down by screenPixelRatio by default in resizeMode: crop-and-scale. - // return screenHeight() / screenPixelRatio(); - return screenHeight(); - } - - // TODO: By default we shouldn't be multiplying with window.devicePixelRatio (bug 1703991). - function defaultScreen() { - return { - resizeMode: "crop-and-scale", - width: screenWidth(), - height: screenHeight(), - frameRate: 30, - }; - } - // TODO: Should get the source's real refresh rate for frameRate (bug 1984363). - function nativeScreen() { - return { - resizeMode: "none", - width: screenWidth(), - height: screenHeight(), - frameRate: 60 - }; - } - - [ - [ - [{resizeMode: "none", width: 100}, {resizeMode: "none", height: 500}], - [nativeScreen, nativeScreen], - ], - [ - [{resizeMode: "none", frameRate: 50}, {resizeMode: "none", frameRate: 1}], - [nativeScreen, nativeScreen], - ], - [ - [{resizeMode: "none"}, {resizeMode: "crop-and-scale"}], - [nativeScreen, defaultScreen], - ], - [ - [{resizeMode: "crop-and-scale"}, {resizeMode: "crop-and-scale", height: 100}], - [ - defaultScreen, - () => ({ - resizeMode: "crop-and-scale", - width: Math.round(screenWidth() / screenHeight() * 100), - height: 100, - frameRate: 30 - }), - ], - ], - [ - [{resizeMode: "crop-and-scale", height: 100}, {resizeMode: "crop-and-scale"}], - [ - () => ({ - resizeMode: "crop-and-scale", - width: Math.round(screenWidth() / screenHeight() * 100), - height: 100, - frameRate: 30 - }), - defaultScreen, - ], - ], - [ - [{resizeMode: "crop-and-scale", frameRate: 5}, {resizeMode: "crop-and-scale", frameRate: 50}], - [ - () => { - const { width, height } = defaultScreen(); - return { width, height, frameRate: 5}; - }, - () => { - const { width, height } = defaultScreen(); - return { width, height, frameRate: 50}; - }, - ], - [[2, 7], [2, 50]], - ], - [ - [{resizeMode: "crop-and-scale", frameRate: 50}, {resizeMode: "crop-and-scale", frameRate: 5}], - [ - () => { - const { width, height } = defaultScreen(); - return { width, height, frameRate: 50}; - }, - () => { - const { width, height } = defaultScreen(); - return { width, height, frameRate: 5}; - }, - [[2, 50], [2, 7]] - ], - ], - [ - [{resizeMode: "none"}, {resizeMode: "crop-and-scale", frameRate: 5000}, {}], - [ - nativeScreen, - () => { - const { width, height } = defaultScreen(); - return { width, height, frameRate: 120}; - }, - ] - ], - ].forEach( - ([ - [video, cloneVideo], - [expectedFunc, cloneExpectedFunc], - [testFramerate, cloneTestFramerate] = [null, null] - ]) => { - let expected, cloneExpected; - promise_test(async t => { - expected = expectedFunc(); - cloneExpected = cloneExpectedFunc(); - await test_driver.bless('getDisplayMedia()'); - const stream = await navigator.mediaDevices.getDisplayMedia({video}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - let settings = track.getSettings(); - let cloneSettings = clone.getSettings(); - for (const key of Object.keys(expected)) { - assert_equals(settings[key], expected[key], `original: ${key}`); - assert_equals(cloneSettings[key], expected[key], `clone: ${key}`); - } - await clone.applyConstraints(cloneVideo); - settings = track.getSettings(); - cloneSettings = clone.getSettings(); - for (const key of Object.keys(expected)) { - assert_equals(settings[key], expected[key], `original-post: ${key}`); - } - for (const key of Object.keys(cloneExpected)) { - assert_equals(cloneSettings[key], cloneExpected[key], `clone-post: ${key}`); - } - await test_resolution_equals(t, track, settings.width, settings.height); - await test_resolution_equals(t, clone, cloneSettings.width, cloneSettings.height); - if (testFramerate) { - const [low, high] = testFramerate; - await test_framerate_between_exclusive(t, track, low, high); - } - if (cloneTestFramerate) { - const [low, high] = cloneTestFramerate; - await test_framerate_between_exclusive(t, clone, low, high); - } - }, `gDM gets expected modes by ${JSON.stringify(video)} + (cloned) ${JSON.stringify(cloneVideo)}`); - }); - - promise_test(async t => { - await test_driver.bless('getDisplayMedia()'); - const stream = await navigator.mediaDevices.getDisplayMedia({video: {resizeMode: "none"}}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - await clone.applyConstraints( - { - resizeMode: "crop-and-scale", - width: 400, - height: 400 - } - ); - const expected = findFittestResolutionSetting( - screenWidth(), - screenHeight(), - clone.getConstraints() - ); - assert_equals(clone.getSettings().width, expected.width, "width"); - assert_equals(clone.getSettings().height, expected.height, "height"); - await test_resolution_equals(t, clone, clone.getSettings().width, clone.getSettings().height); - assert_approx_equals( - clone.getSettings().width / clone.getSettings().height, - desktopWidth() / desktopHeight(), - 0.01, - "aspect ratio" - ); - assert_equals(clone.getSettings().resizeMode, "crop-and-scale", "resizeMode"); - }, "applyConstraints on gDM clone doesn't crop with only ideal dimensions"); - - promise_test(async t => { - await test_driver.bless('getDisplayMedia()'); - const stream = await navigator.mediaDevices.getDisplayMedia({video: {resizeMode: "none"}}); - const [track] = stream.getTracks(); - const clone = track.clone(); - t.add_cleanup(() => { - track.stop(); - clone.stop(); - }); - await clone.applyConstraints( - { - resizeMode: "crop-and-scale", - width: {max: 400}, - height: {ideal: 400} - } - ); - assert_equals(clone.getSettings().width, 400, "width"); - assert_equals( - clone.getSettings().height, - Math.round(screenHeight() / screenWidth() * 400), - "height" - ); - await test_resolution_equals(t, clone, clone.getSettings().width, clone.getSettings().height); - assert_equals(clone.getSettings().resizeMode, "crop-and-scale", "resizeMode"); - }, "applyConstraints on gDM clone doesn't crop with ideal and max dimensions"); -</script> diff --git a/xpcom/ds/nsTArray.h b/xpcom/ds/nsTArray.h @@ -768,17 +768,17 @@ namespace detail { // called as a function with two instances of our element type, returns an int, // we treat it as a tri-state comparator. // -// T is the type of the comparator object we want to check. L and R are the -// types that we'll be comparing. +// T is the type of the comparator object we want to check. U is the array +// element type that we'll be comparing. // // V is never passed, and is only used to allow us to specialize on the return // value of the comparator function. -template <typename T, typename L, typename R, typename V = int> +template <typename T, typename U, typename V = int> struct IsCompareMethod : std::false_type {}; -template <typename T, typename L, typename R> +template <typename T, typename U> struct IsCompareMethod< - T, L, R, decltype(std::declval<T>()(std::declval<L>(), std::declval<R>()))> + T, U, decltype(std::declval<T>()(std::declval<U>(), std::declval<U>()))> : std::true_type {}; // These two wrappers allow us to use either a tri-state comparator, or an @@ -793,8 +793,7 @@ struct IsCompareMethod< // purpose. // Comparator wrapper for a tri-state comparator function -template <typename T, typename L, typename R, - bool IsCompare = IsCompareMethod<T, L, R>::value> +template <typename T, typename U, bool IsCompare = IsCompareMethod<T, U>::value> struct CompareWrapper { #ifdef _MSC_VER # pragma warning(push) @@ -826,8 +825,8 @@ struct CompareWrapper { }; // Comparator wrapper for a class with Equals() and LessThan() methods. -template <typename T, typename L, typename R> -struct CompareWrapper<T, L, R, false> { +template <typename T, typename U> +struct CompareWrapper<T, U, false> { MOZ_IMPLICIT CompareWrapper(const T& aComparator) : mComparator(aComparator) {} @@ -1223,7 +1222,7 @@ class nsTArray_Impl template <class Item, class Comparator> [[nodiscard]] index_type IndexOf(const Item& aItem, index_type aStart, const Comparator& aComp) const { - ::detail::CompareWrapper<Comparator, value_type, Item> comp(aComp); + ::detail::CompareWrapper<Comparator, Item> comp(aComp); const value_type* iter = Elements() + aStart; const value_type* iend = Elements() + Length(); @@ -1257,7 +1256,7 @@ class nsTArray_Impl template <class Item, class Comparator> [[nodiscard]] index_type LastIndexOf(const Item& aItem, index_type aStart, const Comparator& aComp) const { - ::detail::CompareWrapper<Comparator, value_type, Item> comp(aComp); + ::detail::CompareWrapper<Comparator, Item> comp(aComp); size_type endOffset = aStart >= Length() ? Length() : aStart + 1; const value_type* iend = Elements() - 1; @@ -1294,7 +1293,7 @@ class nsTArray_Impl [[nodiscard]] index_type BinaryIndexOf(const Item& aItem, const Comparator& aComp) const { using mozilla::BinarySearchIf; - ::detail::CompareWrapper<Comparator, value_type, Item> comp(aComp); + ::detail::CompareWrapper<Comparator, Item> comp(aComp); size_t index; bool found = BinarySearchIf( @@ -1539,7 +1538,7 @@ class nsTArray_Impl [[nodiscard]] index_type IndexOfFirstElementGt( const Item& aItem, const Comparator& aComp) const { using mozilla::BinarySearchIf; - ::detail::CompareWrapper<Comparator, value_type, Item> comp(aComp); + ::detail::CompareWrapper<Comparator, Item> comp(aComp); size_t index; BinarySearchIf( @@ -2016,7 +2015,7 @@ class nsTArray_Impl typename mozilla::FunctionTypeTraits<FunctionElse>::ReturnType>, "ApplyIf's `Function` and `FunctionElse` must return the same type."); - ::detail::CompareWrapper<Comparator, value_type, Item> comp(aComp); + ::detail::CompareWrapper<Comparator, Item> comp(aComp); const value_type* const elements = Elements(); const value_type* const iend = elements + Length(); @@ -2037,7 +2036,7 @@ class nsTArray_Impl typename mozilla::FunctionTypeTraits<FunctionElse>::ReturnType>, "ApplyIf's `Function` and `FunctionElse` must return the same type."); - ::detail::CompareWrapper<Comparator, value_type, Item> comp(aComp); + ::detail::CompareWrapper<Comparator, Item> comp(aComp); value_type* const elements = Elements(); value_type* const iend = elements + Length(); @@ -2247,7 +2246,7 @@ class nsTArray_Impl static_assert(std::is_move_assignable_v<value_type>); static_assert(std::is_move_constructible_v<value_type>); - ::detail::CompareWrapper<Comparator, value_type, value_type> comp(aComp); + ::detail::CompareWrapper<Comparator, value_type> comp(aComp); auto compFn = [&comp](const auto& left, const auto& right) { return comp.LessThan(left, right); }; @@ -2273,8 +2272,7 @@ class nsTArray_Impl static_assert(std::is_move_assignable_v<value_type>); static_assert(std::is_move_constructible_v<value_type>); - const ::detail::CompareWrapper<Comparator, value_type, value_type> comp( - aComp); + const ::detail::CompareWrapper<Comparator, value_type> comp(aComp); auto compFn = [&comp](const auto& lhs, const auto& rhs) { return comp.LessThan(lhs, rhs); }; diff --git a/xpcom/tests/gtest/TestTArray2.cpp b/xpcom/tests/gtest/TestTArray2.cpp @@ -1450,9 +1450,8 @@ TEST(TArray, test_SetLengthAndRetainStorage_no_ctor) #undef RPAREN } -template <typename F, typename Comparator, typename ItemComparator> -bool TestCompareMethodsImpl(F aItemCreator, const Comparator& aComp, - const ItemComparator& aItemComp) { +template <typename Comparator> +bool TestCompareMethods(const Comparator& aComp) { nsTArray<int> ary({57, 4, 16, 17, 3, 5, 96, 12}); ary.Sort(aComp); @@ -1464,43 +1463,31 @@ bool TestCompareMethodsImpl(F aItemCreator, const Comparator& aComp, } } - if (!ary.ContainsSorted(aItemCreator(5), aItemComp)) { + if (!ary.ContainsSorted(5, aComp)) { return false; } - if (ary.ContainsSorted(aItemCreator(42), aItemComp)) { + if (ary.ContainsSorted(42, aComp)) { return false; } - if (ary.BinaryIndexOf(aItemCreator(16), aItemComp) != 4) { + if (ary.BinaryIndexOf(16, aComp) != 4) { return false; } return true; } -template <typename Comparator> -bool TestCompareMethods(const Comparator& aComp) { - return TestCompareMethodsImpl([](int aI) { return aI; }, aComp, aComp); -} - struct IntComparator { bool Equals(int aLeft, int aRight) const { return aLeft == aRight; } bool LessThan(int aLeft, int aRight) const { return aLeft < aRight; } }; -struct IntWrapper { - int mI; -}; - TEST(TArray, test_comparator_objects) { ASSERT_TRUE(TestCompareMethods(IntComparator())); ASSERT_TRUE( TestCompareMethods([](int aLeft, int aRight) { return aLeft - aRight; })); - ASSERT_TRUE(TestCompareMethodsImpl( - [](int aI) { return IntWrapper{.mI = aI}; }, IntComparator(), - [](int aElem, const IntWrapper& aItem) { return aElem - aItem.mI; })); } struct Big {