commit 992dd3e291884d283d7543e97f8319282f32fd95
parent 5d32ec5181a6d7aa1896cdf7bf3dab023739d00f
Author: Chun-Min Chang <chun.m.chang@gmail.com>
Date: Sat, 20 Dec 2025 18:19:43 +0000
Bug 1674892 - Enable connecting AudioStreamTrack and MediaStreamAudioSourceNode in different graphs r=karlt
This patch removes the same-rate restriction between `AudioStreamTrack` and
`MediaStreamAudioSourceNote`. A track running at sample rate X can now feed a
node in a graph running at rate Y.
When the track and node live on different `MediaTrackGraph`s (e.g., due to
different rates), the implementation creates a cross-graph port, which has a
transmitter and receiver pair, to bridge them: The transmitter forwards the
track's audio to its paired receiver, and the receiver supplies data to the
`MediaStreamAudioSourceNode`.
The relay is shared among all consumers targeting the same destination graph and
is managed via a map keyed by graph. The port is created on first use and
destroyed when the last consuming node disconnects.
Differential Revision: https://phabricator.services.mozilla.com/D269914
Diffstat:
9 files changed, 154 insertions(+), 33 deletions(-)
diff --git a/dom/media/AudioStreamTrack.cpp b/dom/media/AudioStreamTrack.cpp
@@ -8,6 +8,9 @@
#include "MediaTrackGraph.h"
#include "nsContentUtils.h"
+extern mozilla::LazyLogModule gMediaStreamTrackLog;
+#define LOG(type, msg) MOZ_LOG(gMediaStreamTrackLog, type, msg)
+
namespace mozilla::dom {
RefPtr<GenericPromise> AudioStreamTrack::AddAudioOutput(
@@ -36,6 +39,85 @@ void AudioStreamTrack::SetAudioOutputVolume(void* aKey, float aVolume) {
mTrack->SetAudioOutputVolume(aKey, aVolume);
}
+already_AddRefed<MediaInputPort> AudioStreamTrack::AddConsumerPort(
+ ProcessedMediaTrack* aTrack) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!mTrack == Ended());
+
+ if (!mTrack || !aTrack || aTrack->IsDestroyed()) {
+ LOG(LogLevel::Warning,
+ ("AudioStreamTrack %p cannot forward contents: track ended or "
+ "data/destination track ended/destroyed",
+ this));
+ return nullptr;
+ }
+
+ MOZ_ASSERT(!mTrack->IsDestroyed());
+ if (mTrack->Graph() == aTrack->Graph()) {
+ return ForwardTrackContentsTo(aTrack);
+ }
+
+ LOG(LogLevel::Verbose,
+ ("AudioStreamTrack %p forwarding cross-graph contents from track %p "
+ "(graph %p) to track %p (graph %p)",
+ this, mTrack.get(), mTrack->Graph(), aTrack, aTrack->Graph()));
+
+ // Route audio from mTrack through a cross-graph transmitter and receiver to
+ // aTrack.
+ MediaTrackGraph* rcvrGraph = aTrack->Graph();
+
+ // Find existing connection for this graph
+ for (auto& conn : mCrossGraphs) {
+ if (conn.mPort->mReceiver->Graph() == rcvrGraph) {
+ conn.mRefCount++;
+ LOG(LogLevel::Verbose,
+ ("AudioStreamTrack %p reusing cross-graph port "
+ "to graph %p (rate %u), refcount now %zu",
+ this, rcvrGraph, rcvrGraph->GraphRate(), conn.mRefCount));
+ return aTrack->AllocateInputPort(conn.mPort->mReceiver);
+ }
+ }
+
+ // Create new connection if none exists
+ LOG(LogLevel::Verbose,
+ ("AudioStreamTrack %p creating cross-graph port to graph %p (rate %u)",
+ this, rcvrGraph, rcvrGraph->GraphRate()));
+ CrossGraphConnection* conn = mCrossGraphs.AppendElement(
+ CrossGraphConnection(CrossGraphPort::Connect(RefPtr{this}, rcvrGraph)));
+ return aTrack->AllocateInputPort(conn->mPort->mReceiver);
+}
+
+void AudioStreamTrack::RemoveConsumerPort(MediaInputPort* aPort) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (!aPort) {
+ return;
+ }
+
+ MediaTrackGraph* receiverGraph = aPort->Graph();
+
+ // Decrement refcount for this graph's connection and remove if it reaches 0
+ for (size_t i = 0; i < mCrossGraphs.Length(); ++i) {
+ auto& conn = mCrossGraphs[i];
+ if (conn.mPort->mReceiver->Graph() == receiverGraph) {
+ MOZ_ASSERT(conn.mRefCount > 0);
+ --conn.mRefCount;
+ LOG(LogLevel::Verbose,
+ ("AudioStreamTrack %p decrementing cross-graph port refcount to "
+ "graph %p (rate %u), refcount now %zu",
+ this, receiverGraph, receiverGraph->GraphRate(), conn.mRefCount));
+ if (conn.mRefCount == 0) {
+ LOG(LogLevel::Verbose,
+ ("AudioStreamTrack %p removing cross-graph forwarding to graph %p "
+ "(rate %u)",
+ this, receiverGraph, receiverGraph->GraphRate()));
+ mCrossGraphs.UnorderedRemoveElementAt(i);
+ }
+ return;
+ }
+ }
+}
+
void AudioStreamTrack::GetLabel(nsAString& aLabel, CallerType aCallerType) {
MediaStreamTrack::GetLabel(aLabel, aCallerType);
}
@@ -44,4 +126,31 @@ already_AddRefed<MediaStreamTrack> AudioStreamTrack::Clone() {
return MediaStreamTrack::CloneInternal<AudioStreamTrack>();
}
+void AudioStreamTrack::SetReadyState(MediaStreamTrackState aState) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ // When transitioning from Live to Ended, mTrack will be destroyed. Since
+ // mTrack is the source for cross-graph data forwarding, keeping cross-graph
+ // ports is unnecessary. Clearing them here ensures all related connections
+ // are properly disconnected and prevents an assertion failure in
+ // CrossGraphTransmitters::ProcessInput due to a missing source.
+ //
+ // This state transition may occur in various situations, such as when the
+ // track is stopped by a user action, or when mTrack is ended during its
+ // ProcessInput (because its source has ended), which is then detected by
+ // MediaTrackGraph and ultimately notifies the ended-signal via MTGListener,
+ // reaching this point.
+ if (!mCrossGraphs.IsEmpty() && aState == MediaStreamTrackState::Ended) {
+ MOZ_ASSERT(!Ended());
+ LOG(LogLevel::Verbose,
+ ("AudioStreamTrack %p ending, destroying %zu cross-graph ports", this,
+ mCrossGraphs.Length()));
+ mCrossGraphs.Clear();
+ }
+
+ MediaStreamTrack::SetReadyState(aState);
+}
+
} // namespace mozilla::dom
+
+#undef LOG
diff --git a/dom/media/AudioStreamTrack.h b/dom/media/AudioStreamTrack.h
@@ -36,10 +36,30 @@ class AudioStreamTrack : public MediaStreamTrack {
void RemoveAudioOutput(void* aKey);
void SetAudioOutputVolume(void* aKey, float aVolume);
+ // Use AddConsumerPort instead of ForwardTrackContentsTo when possible, since
+ // it handles CrossGraphPort creation automatically. Must be balanced with a
+ // corresponding RemoveConsumerPort call.
+ already_AddRefed<MediaInputPort> AddConsumerPort(ProcessedMediaTrack* aTrack);
+ void RemoveConsumerPort(MediaInputPort* aPort);
+
// WebIDL
void GetKind(nsAString& aKind) override { aKind.AssignLiteral("audio"); }
void GetLabel(nsAString& aLabel, CallerType aCallerType) override;
+
+ protected:
+ void SetReadyState(MediaStreamTrackState aState) override;
+
+ private:
+ // Main thread only
+ struct CrossGraphConnection {
+ UniquePtr<CrossGraphPort> mPort;
+ size_t mRefCount;
+
+ explicit CrossGraphConnection(UniquePtr<CrossGraphPort> aPort)
+ : mPort(std::move(aPort)), mRefCount(1) {}
+ };
+ nsTArray<CrossGraphConnection> mCrossGraphs;
};
} // namespace mozilla::dom
diff --git a/dom/media/CrossGraphPort.cpp b/dom/media/CrossGraphPort.cpp
@@ -36,11 +36,23 @@ UniquePtr<CrossGraphPort> CrossGraphPort::Connect(
RefPtr<MediaInputPort> port =
aStreamTrack->ForwardTrackContentsTo(transmitter);
+ LOG(LogLevel::Verbose,
+ ("Created CrossGraphPort transmitter %p (rate %u, from AudioStreamTrack "
+ "%p) and receiver %p (rate %u) between graphs %p and %p",
+ transmitter.get(), transmitter->mSampleRate, aStreamTrack.get(),
+ receiver.get(), receiver->mSampleRate, aStreamTrack->Graph(),
+ aPartnerGraph));
+
return WrapUnique(new CrossGraphPort(std::move(port), std::move(transmitter),
std::move(receiver)));
}
CrossGraphPort::~CrossGraphPort() {
+ LOG(LogLevel::Verbose,
+ ("Destroying CrossGraphPort transmitter %p (rate %u) and receiver %p "
+ "(rate %u) between graphs %p and %p",
+ mTransmitter.get(), mTransmitter->mSampleRate, mReceiver.get(),
+ mReceiver->mSampleRate, mTransmitter->Graph(), mReceiver->Graph()));
mTransmitter->Destroy();
mReceiver->Destroy();
mTransmitterPort->Destroy();
diff --git a/dom/media/CrossGraphPort.h b/dom/media/CrossGraphPort.h
@@ -23,10 +23,6 @@ class AudioStreamTrack;
namespace mozilla {
/**
- * CrossGraphTransmitter and CrossGraphPort are currently unused, but intended
- * for connecting MediaTracks of different MediaTrackGraphs with different
- * sample rates or clock sources for bug 1674892.
- *
* Create with MediaTrackGraph::CreateCrossGraphTransmitter()
*/
class CrossGraphTransmitter : public ProcessedMediaTrack {
diff --git a/dom/media/MediaStreamTrack.cpp b/dom/media/MediaStreamTrack.cpp
@@ -18,7 +18,7 @@
#include "nsServiceManagerUtils.h"
#include "systemservices/MediaUtils.h"
-static mozilla::LazyLogModule gMediaStreamTrackLog("MediaStreamTrack");
+mozilla::LazyLogModule gMediaStreamTrackLog("MediaStreamTrack");
#define LOG(type, msg) MOZ_LOG(gMediaStreamTrackLog, type, msg)
using namespace mozilla::media;
diff --git a/dom/media/MediaStreamTrack.h b/dom/media/MediaStreamTrack.h
@@ -592,7 +592,7 @@ class MediaStreamTrack : public DOMEventTargetHelper, public SupportsWeakPtr {
* Forces the ready state to a particular value, for instance when we're
* cloning an already ended track.
*/
- void SetReadyState(MediaStreamTrackState aState);
+ virtual void SetReadyState(MediaStreamTrackState aState);
/**
* Notified by the MediaTrackGraph, through our owning MediaStream on the
diff --git a/dom/media/MediaTrackGraph.cpp b/dom/media/MediaTrackGraph.cpp
@@ -3267,7 +3267,6 @@ MediaTrackGraphImpl* MediaInputPort::GraphImpl() const {
}
MediaTrackGraph* MediaInputPort::Graph() const {
- mGraph->AssertOnGraphThreadOrNotRunning();
return mGraph;
}
diff --git a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp
@@ -93,34 +93,21 @@ void MediaStreamAudioSourceNode::Destroy() {
MediaStreamAudioSourceNode::~MediaStreamAudioSourceNode() { Destroy(); }
-void MediaStreamAudioSourceNode::AttachToTrack(
- const RefPtr<MediaStreamTrack>& aTrack, ErrorResult& aRv) {
+void MediaStreamAudioSourceNode::AttachToTrack(AudioStreamTrack* aTrack) {
+ MOZ_ASSERT(aTrack);
MOZ_ASSERT(!mInputTrack);
- MOZ_ASSERT(aTrack->AsAudioStreamTrack());
MOZ_DIAGNOSTIC_ASSERT(!aTrack->Ended());
if (!mTrack) {
return;
}
- if (NS_WARN_IF(Context()->Graph() != aTrack->Graph())) {
- nsGlobalWindowInner* pWindow = Context()->GetOwnerWindow();
- Document* document = pWindow ? pWindow->GetExtantDoc() : nullptr;
- nsContentUtils::ReportToConsole(nsIScriptError::warningFlag, "Web Audio"_ns,
- document, nsContentUtils::eDOM_PROPERTIES,
- "MediaStreamAudioSourceNodeDifferentRate");
- // This is not a spec-required exception, just a limitation of our
- // implementation.
- aRv.ThrowNotSupportedError(
- "Connecting AudioNodes from AudioContexts with different sample-rate "
- "is currently not supported.");
- return;
- }
-
mInputTrack = aTrack;
ProcessedMediaTrack* outputTrack =
static_cast<ProcessedMediaTrack*>(mTrack.get());
- mInputPort = mInputTrack->ForwardTrackContentsTo(outputTrack);
+ mInputPort = aTrack->AddConsumerPort(outputTrack);
+ MOZ_DIAGNOSTIC_ASSERT(mInputPort);
+
PrincipalChanged(mInputTrack); // trigger enabling/disabling of the connector
mInputTrack->AddPrincipalChangeObserver(this);
MarkActive();
@@ -129,6 +116,7 @@ void MediaStreamAudioSourceNode::AttachToTrack(
void MediaStreamAudioSourceNode::DetachFromTrack() {
if (mInputTrack) {
mInputTrack->RemovePrincipalChangeObserver(this);
+ mInputTrack->RemoveConsumerPort(mInputPort);
mInputTrack = nullptr;
}
if (mInputPort) {
@@ -168,7 +156,7 @@ void MediaStreamAudioSourceNode::AttachToRightTrack(
}
if (!track->Ended()) {
- AttachToTrack(track, aRv);
+ AttachToTrack(track);
}
return;
}
@@ -190,7 +178,7 @@ void MediaStreamAudioSourceNode::NotifyTrackAdded(
return;
}
- AttachToTrack(aTrack, IgnoreErrors());
+ AttachToTrack(aTrack->AsAudioStreamTrack());
}
void MediaStreamAudioSourceNode::NotifyTrackRemoved(
@@ -269,10 +257,7 @@ size_t MediaStreamAudioSourceNode::SizeOfIncludingThis(
}
void MediaStreamAudioSourceNode::DestroyMediaTrack() {
- if (mInputPort) {
- mInputPort->Destroy();
- mInputPort = nullptr;
- }
+ DetachFromTrack();
AudioNode::DestroyMediaTrack();
}
diff --git a/dom/media/webaudio/MediaStreamAudioSourceNode.h b/dom/media/webaudio/MediaStreamAudioSourceNode.h
@@ -75,7 +75,7 @@ class MediaStreamAudioSourceNode
size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override;
// Attaches to aTrack so that its audio content will be used as input.
- void AttachToTrack(const RefPtr<MediaStreamTrack>& aTrack, ErrorResult& aRv);
+ void AttachToTrack(AudioStreamTrack* aTrack);
// Detaches from the currently attached track if there is one.
void DetachFromTrack();
@@ -137,7 +137,7 @@ class MediaStreamAudioSourceNode
RefPtr<DOMMediaStream> mInputStream;
// On construction we set this to the first audio track of mInputStream.
- RefPtr<MediaStreamTrack> mInputTrack;
+ RefPtr<AudioStreamTrack> mInputTrack;
RefPtr<TrackListener> mListener;
};
