commit 4e31e678ec71b60061ae44ab1615b53cd85e6ca1
parent 4fda8f0471f889c32cfb459d5126314c00f2e99c
Author: Jan-Ivar Bruaroey <jib@mozilla.com>
Date: Wed, 8 Oct 2025 17:05:54 +0000
Bug 1868223 - Implement custom RTCEncodedVideoFrame::Read/WriteStructuredClone for same process. r=bwc,webidl,smaug
Differential Revision: https://phabricator.services.mozilla.com/D267543
Diffstat:
12 files changed, 316 insertions(+), 64 deletions(-)
diff --git a/dom/base/StructuredCloneHolder.cpp b/dom/base/StructuredCloneHolder.cpp
@@ -22,6 +22,9 @@
#include "mozilla/RefPtr.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/StaticPrefs_dom.h"
+#ifdef MOZ_WEBRTC
+# include "mozilla/StaticPrefs_media.h"
+#endif
#include "mozilla/dom/AudioData.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/BindingUtils.h"
@@ -52,6 +55,10 @@
#include "mozilla/dom/MessagePortBinding.h"
#include "mozilla/dom/OffscreenCanvas.h"
#include "mozilla/dom/OffscreenCanvasBinding.h"
+#ifdef MOZ_WEBRTC
+# include "mozilla/dom/RTCEncodedVideoFrame.h"
+# include "mozilla/dom/RTCEncodedVideoFrameBinding.h"
+#endif
#include "mozilla/dom/ReadableStream.h"
#include "mozilla/dom/ReadableStreamBinding.h"
#include "mozilla/dom/ScriptSettings.h"
@@ -1155,6 +1162,18 @@ JSObject* StructuredCloneHolder::CustomReadHandler(
}
}
+#ifdef MOZ_WEBRTC
+ if (StaticPrefs::media_peerconnection_enabled() &&
+ aTag == SCTAG_DOM_RTCENCODEDVIDEOFRAME &&
+ CloneScope() == StructuredCloneScope::SameProcess &&
+ aCloneDataPolicy.areIntraClusterClonableSharedObjectsAllowed()) {
+ JS::Rooted<JSObject*> global(aCx, mGlobal->GetGlobalJSObject());
+ if (RTCEncodedVideoFrame_Binding::ConstructorEnabled(aCx, global)) {
+ return RTCEncodedVideoFrame::ReadStructuredClone(
+ aCx, mGlobal, aReader, RtcEncodedVideoFrames()[aIndex]);
+ }
+ }
+#endif
return ReadFullySerializableObjects(aCx, aReader, aTag, false);
}
@@ -1297,6 +1316,18 @@ bool StructuredCloneHolder::CustomWriteHandler(
}
}
+#ifdef MOZ_WEBRTC
+ // See if this is an RTCEncodedVideoFrame object.
+ if (StaticPrefs::media_peerconnection_enabled()) {
+ RTCEncodedVideoFrame* rtcFrame = nullptr;
+ if (NS_SUCCEEDED(UNWRAP_OBJECT(RTCEncodedVideoFrame, &obj, rtcFrame))) {
+ SameProcessScopeRequired(aSameProcessScopeRequired);
+ return CloneScope() == StructuredCloneScope::SameProcess
+ ? rtcFrame->WriteStructuredClone(aWriter, this)
+ : false;
+ }
+ }
+#endif
{
// We only care about streams, so ReflectorToISupportsStatic is fine.
nsCOMPtr<nsISupports> base = xpc::ReflectorToISupportsStatic(aObj);
diff --git a/dom/base/StructuredCloneHolder.h b/dom/base/StructuredCloneHolder.h
@@ -172,6 +172,9 @@ class MessagePort;
class MessagePortIdentifier;
struct VideoFrameSerializedData;
struct AudioDataSerializedData;
+#ifdef MOZ_WEBRTC
+struct RTCEncodedVideoFrameData;
+#endif
class StructuredCloneHolder : public StructuredCloneHolderBase {
public:
@@ -211,18 +214,25 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
JS::MutableHandle<JS::Value> aValue,
const JS::CloneDataPolicy& aCloneDataPolicy, ErrorResult& aRv);
- // Create a statement for each of the side DOM-ish data members.
- // mTransferredPorts is not included because it is part of the
- // deserialized state.
-#define CLONED_DATA_MEMBERS \
- STMT(mBlobImplArray); \
- STMT(mWasmModuleArray); \
- STMT(mInputStreamArray); \
- STMT(mClonedSurfaces); \
- STMT(mVideoFrames); \
- STMT(mAudioData); \
- STMT(mEncodedVideoChunks); \
- STMT(mEncodedAudioChunks); \
+#ifdef MOZ_WEBRTC
+# define IF_WEBRTC(x) x
+#else
+# define IF_WEBRTC(x)
+#endif
+
+// Create a statement for each of the side DOM-ish data members.
+// mTransferredPorts is not included because it is part of the
+// deserialized state.
+#define CLONED_DATA_MEMBERS \
+ STMT(mBlobImplArray); \
+ STMT(mWasmModuleArray); \
+ STMT(mInputStreamArray); \
+ STMT(mClonedSurfaces); \
+ STMT(mVideoFrames); \
+ STMT(mAudioData); \
+ STMT(mEncodedVideoChunks); \
+ STMT(mEncodedAudioChunks); \
+ IF_WEBRTC(STMT(mRtcEncodedVideoFrames);) \
STMT(mPortIdentifiers);
// Call this method to know if this object is keeping some DOM object alive.
@@ -302,6 +312,12 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
return mEncodedAudioChunks;
}
+#ifdef MOZ_WEBRTC
+ nsTArray<RTCEncodedVideoFrameData>& RtcEncodedVideoFrames() {
+ return mRtcEncodedVideoFrames;
+ }
+#endif
+
// Implementations of the virtual methods to allow cloning of objects which
// JS engine itself doesn't clone.
@@ -416,6 +432,11 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
// Used for cloning EncodedAudioChunk in the structured cloning algorithm.
nsTArray<EncodedAudioChunkData> mEncodedAudioChunks;
+#ifdef MOZ_WEBRTC
+ // Used for cloning RTCEncodedVideoFrame in the structured cloning algorithm.
+ nsTArray<RTCEncodedVideoFrameData> mRtcEncodedVideoFrames;
+#endif
+
// This raw pointer is only set within ::Read() and is unset by the end.
nsIGlobalObject* MOZ_NON_OWNING_REF mGlobal;
diff --git a/dom/base/StructuredCloneTags.h b/dom/base/StructuredCloneTags.h
@@ -163,6 +163,10 @@ enum StructuredCloneTags : uint32_t {
SCTAG_DOM_RTCDATACHANNEL,
+ SCTAG_DOM_RTCENCODEDVIDEOFRAME,
+
+ SCTAG_DOM_RTCENCODEDAUDIOFRAME,
+
// IMPORTANT: If you plan to add an new IDB tag, it _must_ be add before the
// "less stable" tags!
};
diff --git a/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp b/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp
@@ -42,12 +42,13 @@ RTCEncodedAudioFrame::RTCEncodedAudioFrame(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter, RTCRtpScriptTransformer* aOwner)
- : RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter),
+ : RTCEncodedAudioFrameData{{std::move(aFrame), aCounter, /*timestamp*/ 0}},
+ RTCEncodedFrameBase(aGlobal, static_cast<RTCEncodedFrameState&>(*this)),
mOwner(aOwner) {
mMetadata.mSynchronizationSource.Construct(mFrame->GetSsrc());
mMetadata.mPayloadType.Construct(mFrame->GetPayloadType());
const auto& audioFrame(
- static_cast<webrtc::TransformableAudioFrameInterface&>(*mFrame));
+ static_cast<webrtc::TransformableAudioFrameInterface&>(*mState.mFrame));
mMetadata.mContributingSources.Construct();
for (const auto csrc : audioFrame.GetContributingSources()) {
Unused << mMetadata.mContributingSources.Value().AppendElement(csrc,
@@ -85,4 +86,20 @@ void RTCEncodedAudioFrame::GetMetadata(
bool RTCEncodedAudioFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const {
return aOwner == mOwner;
}
+
+// https://www.w3.org/TR/webrtc-encoded-transform/#RTCEncodedAudioFrame-serialization
+/* static */
+already_AddRefed<RTCEncodedAudioFrame>
+RTCEncodedAudioFrame::ReadStructuredClone(JSContext* aCx,
+ nsIGlobalObject* aGlobal,
+ JSStructuredCloneReader* aReader) {
+ // TBD implementation to follow in followup patch
+ return nullptr;
+}
+
+bool RTCEncodedAudioFrame::WriteStructuredClone(
+ JSContext* aCx, JSStructuredCloneWriter* aWriter) const {
+ return false;
+}
+
} // namespace mozilla::dom
diff --git a/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h b/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h
@@ -14,11 +14,16 @@
namespace mozilla::dom {
+struct RTCEncodedAudioFrameData : RTCEncodedFrameState {
+ RTCEncodedAudioFrameMetadata mMetadata;
+};
+
// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and
// providing read-only access to various metadata. After the libwebrtc frame is
// extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is
// detached, but the metadata remains accessible.
-class RTCEncodedAudioFrame final : public RTCEncodedFrameBase {
+class RTCEncodedAudioFrame final : public RTCEncodedAudioFrameData,
+ public RTCEncodedFrameBase {
public:
explicit RTCEncodedAudioFrame(
nsIGlobalObject* aGlobal,
@@ -42,10 +47,23 @@ class RTCEncodedAudioFrame final : public RTCEncodedFrameBase {
bool IsVideo() const override { return false; }
+ // [Serializable] implementations: {Read, Write}StructuredClone
+ static already_AddRefed<RTCEncodedAudioFrame> ReadStructuredClone(
+ JSContext* aCx, nsIGlobalObject* aGlobal,
+ JSStructuredCloneReader* aReader);
+ bool WriteStructuredClone(JSContext* aCx,
+ JSStructuredCloneWriter* aWriter) const;
+
private:
virtual ~RTCEncodedAudioFrame();
+
+ // forbid copy/move to keep mState member in base valid
+ RTCEncodedAudioFrame(const RTCEncodedAudioFrame&) = delete;
+ RTCEncodedAudioFrame& operator=(const RTCEncodedAudioFrame&) = delete;
+ RTCEncodedAudioFrame(RTCEncodedAudioFrame&&) = delete;
+ RTCEncodedAudioFrame& operator=(RTCEncodedAudioFrame&&) = delete;
+
RefPtr<RTCRtpScriptTransformer> mOwner;
- RTCEncodedAudioFrameMetadata mMetadata;
};
} // namespace mozilla::dom
diff --git a/dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp b/dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp
@@ -6,6 +6,7 @@
#include "jsapi/RTCEncodedFrameBase.h"
+#include "api/frame_transformer_interface.h"
#include "js/ArrayBuffer.h"
#include "js/GCAPI.h"
#include "mozilla/dom/ScriptSettings.h"
@@ -21,14 +22,10 @@ NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedFrameBase)
NS_INTERFACE_MAP_ENTRY(nsISupports)
NS_INTERFACE_MAP_END
-RTCEncodedFrameBase::RTCEncodedFrameBase(
- nsIGlobalObject* aGlobal,
- std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
- uint64_t aCounter)
- : mGlobal(aGlobal),
- mFrame(std::move(aFrame)),
- mCounter(aCounter),
- mTimestamp(mFrame->GetTimestamp()) {
+RTCEncodedFrameBase::RTCEncodedFrameBase(nsIGlobalObject* aGlobal,
+ RTCEncodedFrameState& aState)
+ : mGlobal(aGlobal), mState(aState), mData(nullptr) {
+ mState.mTimestamp = mState.mFrame->GetTimestamp();
AutoJSAPI jsapi;
if (NS_WARN_IF(!jsapi.Init(mGlobal))) {
return;
@@ -36,38 +33,52 @@ RTCEncodedFrameBase::RTCEncodedFrameBase(
// Avoid a copy
mData = JS::NewArrayBufferWithUserOwnedContents(
- jsapi.cx(), mFrame->GetData().size(), (void*)(mFrame->GetData().data()));
+ jsapi.cx(), mState.mFrame->GetData().size(),
+ (void*)(mState.mFrame->GetData().data()));
}
RTCEncodedFrameBase::~RTCEncodedFrameBase() = default;
-unsigned long RTCEncodedFrameBase::Timestamp() const { return mTimestamp; }
+unsigned long RTCEncodedFrameBase::Timestamp() const {
+ return mState.mTimestamp;
+}
void RTCEncodedFrameBase::SetData(const ArrayBuffer& aData) {
mData.set(aData.Obj());
- if (mFrame) {
+ if (mState.mFrame) {
aData.ProcessData([&](const Span<uint8_t>& aData, JS::AutoCheckCannotGC&&) {
- mFrame->SetData(
+ mState.mFrame->SetData(
webrtc::ArrayView<const uint8_t>(aData.Elements(), aData.Length()));
});
}
}
-void RTCEncodedFrameBase::GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj) {
+void RTCEncodedFrameBase::GetData(JSContext* aCx,
+ JS::Rooted<JSObject*>* aObj) const {
aObj->set(mData);
}
-uint64_t RTCEncodedFrameBase::GetCounter() const { return mCounter; }
+uint64_t RTCEncodedFrameBase::GetCounter() const { return mState.mCounter; }
std::unique_ptr<webrtc::TransformableFrameInterface>
RTCEncodedFrameBase::TakeFrame() {
- AutoJSAPI jsapi;
- if (!jsapi.Init(mGlobal)) {
- MOZ_CRASH("Could not init JSAPI!");
+ if (mState.mFrame) {
+ AutoJSAPI jsapi;
+ if (!jsapi.Init(mGlobal)) {
+ MOZ_CRASH("Could not init JSAPI!");
+ }
+ // If the JS buffer was transferred (or otherwise detached), neuter native.
+ JS::Rooted<JSObject*> rootedData(jsapi.cx(), mData);
+ if (rootedData && JS::IsDetachedArrayBufferObject(rootedData)) {
+ mState.mFrame.reset();
+ return nullptr;
+ }
+ // Still attached: detach now since we're consuming the frame.
+ JS::DetachArrayBuffer(jsapi.cx(), rootedData);
}
- JS::Rooted<JSObject*> rootedData(jsapi.cx(), mData);
- JS::DetachArrayBuffer(jsapi.cx(), rootedData);
- return std::move(mFrame);
+ return std::move(mState.mFrame);
}
+RTCEncodedFrameState::~RTCEncodedFrameState() = default;
+
} // namespace mozilla::dom
diff --git a/dom/media/webrtc/jsapi/RTCEncodedFrameBase.h b/dom/media/webrtc/jsapi/RTCEncodedFrameBase.h
@@ -9,23 +9,40 @@
#include <memory>
-#include "api/frame_transformer_interface.h"
#include "js/TypeDecls.h"
#include "mozilla/Assertions.h"
#include "mozilla/dom/TypedArray.h" // ArrayBuffer
class nsIGlobalObject;
+namespace webrtc {
+class TransformableFrameInterface;
+}
+
namespace mozilla::dom {
+struct RTCEncodedFrameState {
+ std::unique_ptr<webrtc::TransformableFrameInterface> mFrame;
+ uint64_t mCounter = 0;
+ unsigned long mTimestamp = 0;
+
+ // work around only having forward-declared TransformableFrameInterface
+ ~RTCEncodedFrameState();
+
+ // avoid "move got disabled by a user-declared destructor” trap
+ RTCEncodedFrameState() = default;
+ RTCEncodedFrameState(RTCEncodedFrameState&&) noexcept = default;
+ RTCEncodedFrameState& operator=(RTCEncodedFrameState&&) noexcept = default;
+ RTCEncodedFrameState(const RTCEncodedFrameState&) = delete;
+ RTCEncodedFrameState& operator=(const RTCEncodedFrameState&) = delete;
+};
+
class RTCRtpScriptTransformer;
class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
public:
- explicit RTCEncodedFrameBase(
- nsIGlobalObject* aGlobal,
- std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
- uint64_t aCounter);
+ explicit RTCEncodedFrameBase(nsIGlobalObject* aGlobal,
+ RTCEncodedFrameState& aState);
// nsISupports
NS_DECL_CYCLE_COLLECTING_ISUPPORTS
@@ -36,7 +53,7 @@ class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
void SetData(const ArrayBuffer& aData);
- void GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj);
+ void GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj) const;
uint64_t GetCounter() const;
@@ -48,10 +65,20 @@ class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
protected:
virtual ~RTCEncodedFrameBase();
+
+ // forbid copy/move to protect mState
+ RTCEncodedFrameBase(const RTCEncodedFrameBase&) = delete;
+ RTCEncodedFrameBase& operator=(const RTCEncodedFrameBase&) = delete;
+ RTCEncodedFrameBase(RTCEncodedFrameBase&&) = delete;
+ RTCEncodedFrameBase& operator=(RTCEncodedFrameBase&&) = delete;
+
RefPtr<nsIGlobalObject> mGlobal;
- std::unique_ptr<webrtc::TransformableFrameInterface> mFrame;
- const uint64_t mCounter = 0;
- const unsigned long mTimestamp = 0;
+
+ // Keep serializable state separate in this base and its subclasses
+ // in a manner that avoids diamond inheritance. Subclasses must pass
+ // in *this, to ensure it's constructed before and destroyed after
+ // this base; copy and move are deleted.
+ RTCEncodedFrameState& mState;
JS::Heap<JSObject*> mData;
};
diff --git a/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.cpp b/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.cpp
@@ -12,6 +12,7 @@
#include <string>
#include <utility>
+#include "api/frame_transformer_factory.h"
#include "api/frame_transformer_interface.h"
#include "js/RootingAPI.h"
#include "jsapi/RTCEncodedFrameBase.h"
@@ -20,6 +21,8 @@
#include "mozilla/Unused.h"
#include "mozilla/dom/RTCEncodedVideoFrameBinding.h"
#include "mozilla/dom/RTCRtpScriptTransformer.h"
+#include "mozilla/dom/StructuredCloneHolder.h"
+#include "mozilla/dom/StructuredCloneTags.h"
#include "mozilla/fallible.h"
#include "nsContentUtils.h"
#include "nsCycleCollectionParticipant.h"
@@ -29,11 +32,22 @@
namespace mozilla::dom {
-NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase,
- mOwner)
+NS_IMPL_CYCLE_COLLECTION_CLASS(RTCEncodedVideoFrame)
+NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(RTCEncodedVideoFrame,
+ RTCEncodedFrameBase)
+ NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner)
+ NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER
+NS_IMPL_CYCLE_COLLECTION_UNLINK_END
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(RTCEncodedVideoFrame,
+ RTCEncodedFrameBase)
+ NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner)
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
+NS_IMPL_CYCLE_COLLECTION_TRACE_BEGIN_INHERITED(RTCEncodedVideoFrame,
+ RTCEncodedFrameBase)
+ NS_IMPL_CYCLE_COLLECTION_TRACE_PRESERVED_WRAPPER
+NS_IMPL_CYCLE_COLLECTION_TRACE_END
NS_IMPL_ADDREF_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase)
NS_IMPL_RELEASE_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase)
-
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedVideoFrame)
NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
NS_INTERFACE_MAP_END_INHERITING(RTCEncodedFrameBase)
@@ -42,8 +56,30 @@ RTCEncodedVideoFrame::RTCEncodedVideoFrame(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter, RTCRtpScriptTransformer* aOwner)
- : RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter),
+ : RTCEncodedVideoFrameData{{std::move(aFrame), aCounter, /*timestamp*/ 0}},
+ RTCEncodedFrameBase(aGlobal, static_cast<RTCEncodedFrameState&>(*this)),
mOwner(aOwner) {
+ InitMetadata();
+ // Base class needs this, but can't do it itself because of an assertion in
+ // the cycle-collector.
+ mozilla::HoldJSObjects(this);
+}
+
+RTCEncodedVideoFrame::RTCEncodedVideoFrame(nsIGlobalObject* aGlobal,
+ RTCEncodedVideoFrameData&& aData)
+ : RTCEncodedVideoFrameData{{std::move(aData.mFrame), aData.mCounter,
+ aData.mTimestamp},
+ aData.mType,
+ std::move(aData.mMetadata),
+ aData.mRid},
+ RTCEncodedFrameBase(aGlobal, static_cast<RTCEncodedFrameState&>(*this)),
+ mOwner(nullptr) {
+ // Base class needs this, but can't do it itself because of an assertion in
+ // the cycle-collector.
+ mozilla::HoldJSObjects(this);
+}
+
+void RTCEncodedVideoFrame::InitMetadata() {
const auto& videoFrame(
static_cast<webrtc::TransformableVideoFrameInterface&>(*mFrame));
mType = videoFrame.IsKeyFrame() ? RTCEncodedVideoFrameType::Key
@@ -77,15 +113,14 @@ RTCEncodedVideoFrame::RTCEncodedVideoFrame(
// The metadata timestamp is different, and not presently present in the
// libwebrtc types
if (!videoFrame.GetRid().empty()) {
- mRid = Some(videoFrame.GetRid());
+ mRid = Some(videoFrame.GetRid().c_str());
}
-
- // Base class needs this, but can't do it itself because of an assertion in
- // the cycle-collector.
- mozilla::HoldJSObjects(this);
}
RTCEncodedVideoFrame::~RTCEncodedVideoFrame() {
+ // Clear JS::Heap<> members before unregistering as a script holder,
+ // so their destructors don't barrier against a finalized JS object.
+ mData = nullptr; // from RTCEncodedFrameBase (protected)
// Base class needs this, but can't do it itself because of an assertion in
// the cycle-collector.
mozilla::DropJSObjects(this);
@@ -96,6 +131,17 @@ JSObject* RTCEncodedVideoFrame::WrapObject(JSContext* aCx,
return RTCEncodedVideoFrame_Binding::Wrap(aCx, this, aGivenProto);
}
+RTCEncodedVideoFrameData RTCEncodedVideoFrameData::Clone() const {
+ return RTCEncodedVideoFrameData{
+ {webrtc::CloneVideoFrame(
+ static_cast<webrtc::TransformableVideoFrameInterface*>(
+ mFrame.get())),
+ mCounter, mTimestamp},
+ mType,
+ RTCEncodedVideoFrameMetadata(mMetadata),
+ mRid};
+}
+
nsIGlobalObject* RTCEncodedVideoFrame::GetParentObject() const {
return mGlobal;
}
@@ -111,6 +157,44 @@ bool RTCEncodedVideoFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const {
return aOwner == mOwner;
}
-Maybe<std::string> RTCEncodedVideoFrame::Rid() const { return mRid; }
+Maybe<nsCString> RTCEncodedVideoFrame::Rid() const { return mRid; }
+
+// https://www.w3.org/TR/webrtc-encoded-transform/#RTCEncodedVideoFrame-serialization
+/* static */
+JSObject* RTCEncodedVideoFrame::ReadStructuredClone(
+ JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader,
+ RTCEncodedVideoFrameData& aData) {
+ JS::Rooted<JS::Value> value(aCx, JS::NullValue());
+ // To avoid a rooting hazard error from returning a raw JSObject* before
+ // running the RefPtr destructor, RefPtr needs to be destructed before
+ // returning the raw JSObject*, which is why the RefPtr<RTCEncodedVideoFrame>
+ // is created in the scope below. Otherwise, the static analysis infers the
+ // RefPtr cannot be safely destructed while the unrooted return JSObject* is
+ // on the stack.
+ {
+ auto frame = MakeRefPtr<RTCEncodedVideoFrame>(aGlobal, std::move(aData));
+ if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) {
+ return nullptr;
+ }
+ }
+ return value.toObjectOrNull();
+}
+
+bool RTCEncodedVideoFrame::WriteStructuredClone(
+ JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const {
+ AssertIsOnOwningThread();
+
+ // Indexing the chunk and send the index to the receiver.
+ const uint32_t index =
+ static_cast<uint32_t>(aHolder->RtcEncodedVideoFrames().Length());
+ // The serialization is limited to the same process scope so it's ok to
+ // hand over a (copy of a) webrtc internal object here.
+ //
+ // TODO: optimize later once encoded source API materializes
+ // .AppendElement(aHolder->IsTransferred(mData) ? Take() : Clone())
+ aHolder->RtcEncodedVideoFrames().AppendElement(Clone());
+ return !NS_WARN_IF(
+ !JS_WriteUint32Pair(aWriter, SCTAG_DOM_RTCENCODEDVIDEOFRAME, index));
+}
} // namespace mozilla::dom
diff --git a/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h b/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h
@@ -7,29 +7,43 @@
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_
-#include "jsapi/RTCEncodedFrameBase.h"
#include "mozilla/RefPtr.h"
+#include "mozilla/dom/RTCEncodedFrameBase.h"
#include "mozilla/dom/RTCEncodedVideoFrameBinding.h"
#include "nsIGlobalObject.h"
namespace mozilla::dom {
+
class RTCRtpScriptTransformer;
+class StructuredCloneHolder;
+
+struct RTCEncodedVideoFrameData : RTCEncodedFrameState {
+ RTCEncodedVideoFrameType mType;
+ RTCEncodedVideoFrameMetadata mMetadata;
+ Maybe<nsCString> mRid;
+
+ [[nodiscard]] RTCEncodedVideoFrameData Clone() const;
+};
// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and
// providing read-only access to various metadata. After the libwebrtc frame is
// extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is
// detached, but the metadata remains accessible.
-class RTCEncodedVideoFrame final : public RTCEncodedFrameBase {
+class RTCEncodedVideoFrame final : public RTCEncodedVideoFrameData,
+ public RTCEncodedFrameBase {
public:
explicit RTCEncodedVideoFrame(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter, RTCRtpScriptTransformer* aOwner);
+ explicit RTCEncodedVideoFrame(nsIGlobalObject* aGlobal,
+ RTCEncodedVideoFrameData&& aData);
+
// nsISupports
NS_DECL_ISUPPORTS_INHERITED
- NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCEncodedVideoFrame,
- RTCEncodedFrameBase)
+ NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS_INHERITED(RTCEncodedVideoFrame,
+ RTCEncodedFrameBase)
// webidl (timestamp and data accessors live in base class)
JSObject* WrapObject(JSContext* aCx,
@@ -39,6 +53,8 @@ class RTCEncodedVideoFrame final : public RTCEncodedFrameBase {
RTCEncodedVideoFrameType Type() const;
+ void InitMetadata();
+
void GetMetadata(RTCEncodedVideoFrameMetadata& aMetadata);
bool CheckOwner(RTCRtpScriptTransformer* aOwner) const override;
@@ -47,14 +63,29 @@ class RTCEncodedVideoFrame final : public RTCEncodedFrameBase {
// Not in webidl right now. Might change.
// https://github.com/w3c/webrtc-encoded-transform/issues/147
- Maybe<std::string> Rid() const;
+ Maybe<nsCString> Rid() const;
+
+ static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
+ JSStructuredCloneReader* aReader,
+ RTCEncodedVideoFrameData& aData);
+ bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
+ StructuredCloneHolder* aHolder) const;
private:
virtual ~RTCEncodedVideoFrame();
+
+ // forbid copy/move to keep mState member in base valid
+ RTCEncodedVideoFrame(const RTCEncodedVideoFrame&) = delete;
+ RTCEncodedVideoFrame& operator=(const RTCEncodedVideoFrame&) = delete;
+ RTCEncodedVideoFrame(RTCEncodedVideoFrame&&) = delete;
+ RTCEncodedVideoFrame& operator=(RTCEncodedVideoFrame&&) = delete;
+
+ // RTCEncodedVideoFrame can run on either main thread or worker thread.
+ void AssertIsOnOwningThread() const {
+ NS_ASSERT_OWNINGTHREAD(RTCEncodedVideoFrame);
+ }
+
RefPtr<RTCRtpScriptTransformer> mOwner;
- RTCEncodedVideoFrameType mType;
- RTCEncodedVideoFrameMetadata mMetadata;
- Maybe<std::string> mRid;
};
} // namespace mozilla::dom
diff --git a/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.cpp b/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.cpp
@@ -428,7 +428,10 @@ already_AddRefed<Promise> RTCRtpScriptTransformer::OnTransformedFrame(
if (aFrame->GetCounter() > mLastReceivedFrameCounter &&
aFrame->CheckOwner(this) && mProxy) {
mLastReceivedFrameCounter = aFrame->GetCounter();
- mProxy->OnTransformedFrame(aFrame->TakeFrame());
+ // also skip if frame has been detached (transferred away)
+ if (auto frame = aFrame->TakeFrame()) {
+ mProxy->OnTransformedFrame(std::move(frame));
+ }
}
return Promise::CreateResolvedWithUndefined(GetParentObject(), aError);
diff --git a/dom/webidl/RTCEncodedAudioFrame.webidl b/dom/webidl/RTCEncodedAudioFrame.webidl
@@ -14,6 +14,8 @@ dictionary RTCEncodedAudioFrameMetadata {
short sequenceNumber;
};
+// [Serializable] is implemented without adding attribute here,
+// because we don't implement "full serialization" to disk.
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled",
Exposed=(Window,DedicatedWorker)]
diff --git a/dom/webidl/RTCEncodedVideoFrame.webidl b/dom/webidl/RTCEncodedVideoFrame.webidl
@@ -30,6 +30,9 @@ dictionary RTCEncodedVideoFrameMetadata {
// New interfaces to define encoded video and audio frames. Will eventually
// re-use or extend the equivalent defined in WebCodecs.
+//
+// [Serializable] is implemented without adding attribute here,
+// because we don't implement "full serialization" to disk.
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled",
Exposed=(Window,DedicatedWorker)]
