commit 9a220092ae144225bdffb3d0202ed9075949c674
parent 4efc2953eaac8b992c32818c905501fa56543215
Author: Andreas Pehrson <apehrson@mozilla.com>
Date: Fri, 5 Dec 2025 16:34:21 +0000
Bug 1999970 - Retrofit a Skipped callback on top of GMP encoder API using input frame destroy calls. r=aosmond,media-playback-reviewers,webrtc-reviewers,bwc
This allows the GMP client (in this case the WebRTC GMP encoder) to know whether
the encoder plugin decided to drop a frame. This lets us avoid flushing the
encoder with more encode jobs than it can handle.
Differential Revision: https://phabricator.services.mozilla.com/D272613
Diffstat:
14 files changed, 130 insertions(+), 25 deletions(-)
diff --git a/dom/media/gmp/GMPSharedMemManager.h b/dom/media/gmp/GMPSharedMemManager.h
@@ -11,6 +11,8 @@
namespace mozilla::gmp {
+class GMPVideoi420FrameImpl;
+
enum class GMPSharedMemClass { Decoded, Encoded };
class GMPSharedMemManager {
@@ -27,6 +29,8 @@ class GMPSharedMemManager {
virtual bool MgrAllocShmem(size_t aSize, ipc::Shmem* aMem) { return false; }
virtual void MgrDeallocShmem(ipc::Shmem& aMem) = 0;
+ virtual void MgrDecodedFrameDestroyed(GMPVideoi420FrameImpl* aFrame) {}
+
protected:
virtual bool MgrIsOnOwningThread() const = 0;
diff --git a/dom/media/gmp/GMPVideoEncoderChild.cpp b/dom/media/gmp/GMPVideoEncoderChild.cpp
@@ -10,7 +10,6 @@
#include "GMPContentChild.h"
#include "GMPPlatform.h"
#include "GMPVideoEncodedFrameImpl.h"
-#include "GMPVideoi420FrameImpl.h"
#include "mozilla/StaticPrefs_media.h"
#include "runnable_utils.h"
@@ -68,9 +67,26 @@ void GMPVideoEncoderChild::Encoded(GMPVideoEncodedFrame* aEncodedFrame,
MOZ_CRASH("Encoded without any frame data!");
}
+ mLatestEncodedTimestamp = frameData.mTimestamp();
+
aEncodedFrame->Destroy();
}
+void GMPVideoEncoderChild::MgrDecodedFrameDestroyed(
+ GMPVideoi420FrameImpl* aFrame) {
+ if (NS_WARN_IF(!mPlugin)) {
+ return;
+ }
+
+ // The OpenH264 encoder destroys the input frame if it has skipped encoding
+ // it. When it has encoded it, it calls the Encoded() callback before
+ // destroying the frame.
+ MOZ_ASSERT(mPlugin->GMPMessageLoop() == MessageLoop::current());
+ if (aFrame->Timestamp() > mLatestEncodedTimestamp) {
+ (void)SendDroppedFrame(aFrame->Timestamp());
+ }
+}
+
void GMPVideoEncoderChild::Error(GMPErr aError) {
if (NS_WARN_IF(!mPlugin)) {
return;
@@ -117,8 +133,10 @@ mozilla::ipc::IPCResult GMPVideoEncoderChild::RecvEncode(
return IPC_FAIL(this, "!mVideoDecoder");
}
+ // The `this` destroyed callback outlives the frame, because `mVideoEncoder`
+ // is responsible for destroying the frame, and we outlive `mVideoEncoder`.
auto* f = new GMPVideoi420FrameImpl(aInputFrame, std::move(aInputShmem),
- &mVideoHost);
+ &mVideoHost, HostReportPolicy::Destroyed);
// Ignore any return code. It is OK for this to fail without killing the
// process.
diff --git a/dom/media/gmp/GMPVideoEncoderChild.h b/dom/media/gmp/GMPVideoEncoderChild.h
@@ -8,6 +8,7 @@
#include "GMPSharedMemManager.h"
#include "GMPVideoHost.h"
+#include "GMPVideoi420FrameImpl.h"
#include "gmp-video-encode.h"
#include "mozilla/gmp/PGMPVideoEncoderChild.h"
#include "nsString.h"
@@ -39,6 +40,7 @@ class GMPVideoEncoderChild final : public PGMPVideoEncoderChild,
// GMPSharedMemManager
void MgrDeallocShmem(Shmem& aMem) override { DeallocShmem(aMem); }
+ void MgrDecodedFrameDestroyed(GMPVideoi420FrameImpl* aFrame) override;
protected:
bool MgrIsOnOwningThread() const override;
@@ -66,6 +68,7 @@ class GMPVideoEncoderChild final : public PGMPVideoEncoderChild,
GMPContentChild* mPlugin;
GMPVideoEncoder* mVideoEncoder;
GMPVideoHostImpl mVideoHost;
+ uint64_t mLatestEncodedTimestamp = 0;
};
} // namespace mozilla::gmp
diff --git a/dom/media/gmp/GMPVideoEncoderParent.cpp b/dom/media/gmp/GMPVideoEncoderParent.cpp
@@ -270,6 +270,14 @@ mozilla::ipc::IPCResult GMPVideoEncoderParent::RecvEncodedData(
return IPC_OK();
}
+mozilla::ipc::IPCResult GMPVideoEncoderParent::RecvDroppedFrame(
+ const uint64_t& aTimestamp) {
+ if (mCallback) {
+ mCallback->Dropped(aTimestamp);
+ }
+ return IPC_OK();
+}
+
mozilla::ipc::IPCResult GMPVideoEncoderParent::RecvError(const GMPErr& aError) {
if (mCallback) {
mCallback->Error(aError);
diff --git a/dom/media/gmp/GMPVideoEncoderParent.h b/dom/media/gmp/GMPVideoEncoderParent.h
@@ -73,6 +73,7 @@ class GMPVideoEncoderParent final : public GMPVideoEncoderProxy,
const GMPVideoEncodedFrameData& aEncodedFrame,
nsTArray<uint8_t>&& aEncodedData,
nsTArray<uint8_t>&& aCodecSpecificInfo) override;
+ mozilla::ipc::IPCResult RecvDroppedFrame(const uint64_t& aTimestamp) override;
mozilla::ipc::IPCResult RecvError(const GMPErr& aError) override;
mozilla::ipc::IPCResult RecvShutdown() override;
diff --git a/dom/media/gmp/GMPVideoEncoderProxy.h b/dom/media/gmp/GMPVideoEncoderProxy.h
@@ -18,6 +18,7 @@ class GMPVideoEncoderCallbackProxy : public GMPCallbackBase {
virtual ~GMPVideoEncoderCallbackProxy() = default;
virtual void Encoded(GMPVideoEncodedFrame* aEncodedFrame,
const nsTArray<uint8_t>& aCodecSpecificInfo) = 0;
+ virtual void Dropped(uint64_t aTimestamp) = 0;
virtual void Error(GMPErr aError) = 0;
};
diff --git a/dom/media/gmp/GMPVideoHost.cpp b/dom/media/gmp/GMPVideoHost.cpp
@@ -94,6 +94,9 @@ void GMPVideoHostImpl::DecodedFrameCreated(
void GMPVideoHostImpl::DecodedFrameDestroyed(GMPVideoi420FrameImpl* aFrame) {
MOZ_ALWAYS_TRUE(mDecodedFrames.RemoveElement(aFrame));
+ if (mSharedMemMgr && aFrame->mReportPolicy == HostReportPolicy::Destroyed) {
+ mSharedMemMgr->MgrDecodedFrameDestroyed(aFrame);
+ }
}
} // namespace mozilla::gmp
diff --git a/dom/media/gmp/GMPVideoi420FrameImpl.cpp b/dom/media/gmp/GMPVideoi420FrameImpl.cpp
@@ -40,16 +40,25 @@ void GMPVideoi420FrameImpl::GMPFramePlane::Copy(uint8_t* aDst,
}
}
-GMPVideoi420FrameImpl::GMPVideoi420FrameImpl(GMPVideoHostImpl* aHost)
- : mHost(aHost), mWidth(0), mHeight(0), mTimestamp(0ll), mDuration(0ll) {
+GMPVideoi420FrameImpl::GMPVideoi420FrameImpl(
+ GMPVideoHostImpl* aHost,
+ HostReportPolicy aReportPolicy /*= HostReportPolicy::None*/)
+ : mReportPolicy(aReportPolicy),
+ mHost(aHost),
+ mWidth(0),
+ mHeight(0),
+ mTimestamp(0ll),
+ mDuration(0ll) {
MOZ_ASSERT(aHost);
aHost->DecodedFrameCreated(this);
}
GMPVideoi420FrameImpl::GMPVideoi420FrameImpl(
const GMPVideoi420FrameData& aFrameData, ipc::Shmem&& aShmemBuffer,
- GMPVideoHostImpl* aHost)
- : mHost(aHost),
+ GMPVideoHostImpl* aHost,
+ HostReportPolicy aReportPolicy /*= HostReportPolicy::None*/)
+ : mReportPolicy(aReportPolicy),
+ mHost(aHost),
mShmemBuffer(std::move(aShmemBuffer)),
mYPlane(aFrameData.mYPlane()),
mUPlane(aFrameData.mUPlane()),
@@ -65,8 +74,10 @@ GMPVideoi420FrameImpl::GMPVideoi420FrameImpl(
GMPVideoi420FrameImpl::GMPVideoi420FrameImpl(
const GMPVideoi420FrameData& aFrameData, nsTArray<uint8_t>&& aArrayBuffer,
- GMPVideoHostImpl* aHost)
- : mHost(aHost),
+ GMPVideoHostImpl* aHost,
+ HostReportPolicy aReportPolicy /*= HostReportPolicy::None*/)
+ : mReportPolicy(aReportPolicy),
+ mHost(aHost),
mArrayBuffer(std::move(aArrayBuffer)),
mYPlane(aFrameData.mYPlane()),
mUPlane(aFrameData.mUPlane()),
diff --git a/dom/media/gmp/GMPVideoi420FrameImpl.h b/dom/media/gmp/GMPVideoi420FrameImpl.h
@@ -17,14 +17,24 @@ class GMPPlaneData;
class GMPVideoi420FrameData;
class GMPVideoHostImpl;
-class GMPVideoi420FrameImpl final : public GMPVideoi420Frame {
+enum class HostReportPolicy : uint8_t {
+ None,
+ Destroyed,
+};
+
+class GMPVideoi420FrameImpl : public GMPVideoi420Frame {
public:
- explicit GMPVideoi420FrameImpl(GMPVideoHostImpl* aHost);
- GMPVideoi420FrameImpl(const GMPVideoi420FrameData& aFrameData,
- ipc::Shmem&& aShmemBuffer, GMPVideoHostImpl* aHost);
- GMPVideoi420FrameImpl(const GMPVideoi420FrameData& aFrameData,
- nsTArray<uint8_t>&& aArrayBuffer,
- GMPVideoHostImpl* aHost);
+ explicit GMPVideoi420FrameImpl(
+ GMPVideoHostImpl* aHost,
+ HostReportPolicy aReportPolicy = HostReportPolicy::None);
+ GMPVideoi420FrameImpl(
+ const GMPVideoi420FrameData& aFrameData, ipc::Shmem&& aShmemBuffer,
+ GMPVideoHostImpl* aHost,
+ HostReportPolicy aReportPolicy = HostReportPolicy::None);
+ GMPVideoi420FrameImpl(
+ const GMPVideoi420FrameData& aFrameData, nsTArray<uint8_t>&& aArrayBuffer,
+ GMPVideoHostImpl* aHost,
+ HostReportPolicy aReportPolicy = HostReportPolicy::None);
virtual ~GMPVideoi420FrameImpl();
// This is called during a normal destroy sequence, which is
@@ -75,7 +85,7 @@ class GMPVideoi420FrameImpl final : public GMPVideoi420Frame {
const uint8_t* Buffer() const;
int32_t AllocatedSize() const;
- private:
+ protected:
struct GMPFramePlane {
explicit GMPFramePlane(const GMPPlaneData& aPlaneData);
GMPFramePlane() = default;
@@ -98,6 +108,10 @@ class GMPVideoi420FrameImpl final : public GMPVideoi420Frame {
GMPErr MaybeResize(int32_t aNewSize);
void DestroyBuffer();
+ public:
+ const HostReportPolicy mReportPolicy;
+
+ protected:
GMPVideoHostImpl* mHost;
nsTArray<uint8_t> mArrayBuffer;
ipc::Shmem mShmemBuffer;
diff --git a/dom/media/gmp/PGMPVideoEncoder.ipdl b/dom/media/gmp/PGMPVideoEncoder.ipdl
@@ -43,6 +43,7 @@ parent:
async EncodedData(GMPVideoEncodedFrameData aEncodedFrame,
uint8_t[] aEncodedData,
uint8_t[] aCodecSpecificInfo);
+ async DroppedFrame(uint64_t aTimestamp);
async Error(GMPErr aErr);
async Shutdown();
};
diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoEncoder.cpp b/dom/media/platforms/agnostic/gmp/GMPVideoEncoder.cpp
@@ -446,6 +446,11 @@ void GMPVideoEncoder::Encoded(GMPVideoEncodedFrame* aEncodedFrame,
}
}
+void GMPVideoEncoder::Dropped(uint64_t aTimestamp) {
+ MOZ_ASSERT(IsOnGMPThread());
+ // TODO: implement
+}
+
void GMPVideoEncoder::Teardown(const MediaResult& aResult,
StaticString aCallSite) {
GMP_LOG_DEBUG("[%p] GMPVideoEncoder::Teardown", this);
diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoEncoder.h b/dom/media/platforms/agnostic/gmp/GMPVideoEncoder.h
@@ -39,6 +39,7 @@ class GMPVideoEncoder final : public MediaDataEncoder,
void Encoded(GMPVideoEncodedFrame* aEncodedFrame,
const nsTArray<uint8_t>& aCodecSpecificInfo) override;
+ void Dropped(uint64_t aTimestamp) override;
void Error(GMPErr aError) override;
void Terminated() override;
diff --git a/dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.cpp b/dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.cpp
@@ -387,7 +387,9 @@ void WebrtcGmpVideoEncoder::Encode_g(
GMP_LOG_DEBUG("GMP Encode: failed to create frame");
return;
}
- frame->SetTimestamp(AssertedCast<uint64_t>(aInputImage.ntp_time_ms() * 1000));
+ const auto gmpTimestamp =
+ AssertedCast<uint64_t>(aInputImage.ntp_time_ms() * 1000);
+ frame->SetTimestamp(gmpTimestamp);
GMPCodecSpecificInfo info{};
info.mCodecType = kGMPVideoCodecH264;
@@ -422,7 +424,7 @@ void WebrtcGmpVideoEncoder::Encode_g(
mInputImageMap.LastElement().ntp_timestamp_ms <
aInputImage.ntp_time_ms());
mInputImageMap.AppendElement(
- InputImageData{.gmp_timestamp_us = frame->Timestamp(),
+ InputImageData{.gmp_timestamp_us = gmpTimestamp,
.ntp_timestamp_ms = aInputImage.ntp_time_ms(),
.timestamp_us = aInputImage.timestamp_us(),
.rtp_timestamp = aInputImage.rtp_timestamp(),
@@ -526,20 +528,23 @@ void WebrtcGmpVideoEncoder::Terminated() {
// Could now notify that it's dead
}
+static int32_t GmpTimestampComparator(const InputImageData& aA,
+ const InputImageData& aB) {
+ const auto& a = aA.gmp_timestamp_us;
+ const auto& b = aB.gmp_timestamp_us;
+ return a < b ? -1 : a != b;
+}
+
void WebrtcGmpVideoEncoder::Encoded(
GMPVideoEncodedFrame* aEncodedFrame,
const nsTArray<uint8_t>& aCodecSpecificInfo) {
MOZ_ASSERT(mGMPThread->IsOnCurrentThread());
Maybe<InputImageData> data;
- auto gmp_timestamp_comparator = [](const InputImageData& aA,
- const InputImageData& aB) -> int32_t {
- const auto& a = aA.gmp_timestamp_us;
- const auto& b = aB.gmp_timestamp_us;
- return a < b ? -1 : a != b;
- };
+ MOZ_ASSERT(!mInputImageMap.IsEmpty());
+ MOZ_ASSERT(mInputImageMap.Length() <= kMaxImagesInFlight);
size_t nextIdx = mInputImageMap.IndexOfFirstElementGt(
InputImageData{.gmp_timestamp_us = aEncodedFrame->TimeStamp()},
- gmp_timestamp_comparator);
+ GmpTimestampComparator);
const size_t numToRemove = nextIdx;
size_t numFramesDropped = numToRemove;
MOZ_ASSERT(nextIdx != 0);
@@ -676,6 +681,34 @@ void WebrtcGmpVideoEncoder::Encoded(
mCallback->OnEncodedImage(unit, &info);
}
+void WebrtcGmpVideoEncoder::Dropped(uint64_t aTimestamp) {
+ MOZ_ASSERT(mGMPThread->IsOnCurrentThread());
+ MOZ_ASSERT(!mInputImageMap.IsEmpty());
+ MOZ_ASSERT(mInputImageMap.Length() <= kMaxImagesInFlight);
+
+ size_t nextIdx = mInputImageMap.IndexOfFirstElementGt(
+ InputImageData{.gmp_timestamp_us = aTimestamp}, GmpTimestampComparator);
+ const size_t numDropped = nextIdx;
+ MOZ_ASSERT(nextIdx != 0);
+ MOZ_ASSERT(mInputImageMap.ElementAt(nextIdx - 1).gmp_timestamp_us ==
+ aTimestamp);
+ mInputImageMap.RemoveElementsAt(0, numDropped);
+
+ GMP_LOG_DEBUG("GMP Dropped: %" PRIu64
+ " dropped by encoder. Reporting %u frames dropped.",
+ aTimestamp, static_cast<uint32_t>(numDropped));
+
+ MutexAutoLock lock(mCallbackMutex);
+ if (!mCallback) {
+ return;
+ }
+
+ for (size_t i = 0; i < numDropped; ++i) {
+ mCallback->OnDroppedFrame(
+ webrtc::EncodedImageCallback::DropReason::kDroppedByEncoder);
+ }
+}
+
// Decoder.
WebrtcGmpVideoDecoder::WebrtcGmpVideoDecoder(std::string aPCHandle,
TrackingId aTrackingId)
diff --git a/dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h b/dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h
@@ -201,6 +201,8 @@ class WebrtcGmpVideoEncoder final : public GMPVideoEncoderCallbackProxy,
void Encoded(GMPVideoEncodedFrame* aEncodedFrame,
const nsTArray<uint8_t>& aCodecSpecificInfo) override;
+ void Dropped(uint64_t aTimestamp) override;
+
void Error(GMPErr aError) override {}
private:
