commit 4df889591754afec179b7cb419fb803d67bb6a06
parent 5268af90d48886646439fb7f46fe2374e4ac2561
Author: Cristina Horotan <chorotan@mozilla.com>
Date: Mon, 10 Nov 2025 14:03:33 +0200
Revert "Bug 1944033: apply code formatting via Lando" for causing build bustages at MediaSession.h
This reverts commit 3394e884c5a49f2fb34e52e4503b0fefbca050c4.
Revert "Bug 1944033 - Remove FetchImageHelper. r=media-playback-reviewers,aosmond"
This reverts commit 59332a010eb378b6656819bc798eb7477d70f2a6.
Revert "Bug 1944033 - Update macOS/cocoa to use the image from the child. r=media-playback-reviewers,aosmond"
This reverts commit 5d7b2b78ccbdd03698a47be1f9cff4eb335f00a2.
Revert "Bug 1944033 - Update Windows to use the image from the child. r=media-playback-reviewers,win-reviewers,gstoll,aosmond"
This reverts commit 44b88dad2e5fa531f43c11169c2c170e9073dfda.
Revert "Bug 1944033 - Update Linux/MPRIS to use the image from the child. r=media-playback-reviewers,aosmond"
This reverts commit 570166de598ed3f4b5f10bfdaefac07a6fb98218.
Revert "Bug 1944033 - Load MediaSession artwork in the child. r=media-playback-reviewers,aosmond"
This reverts commit 6dc346b92490dcac3b401a34ad7a346febc75193.
Revert "Bug 1944033 - Add MediaImageData. r=media-playback-reviewers,aosmond"
This reverts commit 01d56417ef9c72252ffe694010c2d7cf96adb2b9.
Diffstat:
15 files changed, 561 insertions(+), 250 deletions(-)
diff --git a/dom/media/mediacontrol/FetchImageHelper.cpp b/dom/media/mediacontrol/FetchImageHelper.cpp
@@ -0,0 +1,164 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "FetchImageHelper.h"
+
+#include "mozilla/Logging.h"
+#include "mozilla/NullPrincipal.h"
+#include "mozilla/gfx/2D.h"
+#include "nsIChannel.h"
+#include "nsNetUtil.h"
+
+mozilla::LazyLogModule gFetchImageLog("FetchImageHelper");
+
+#undef LOG
+#define LOG(msg, ...) \
+ MOZ_LOG(gFetchImageLog, LogLevel::Debug, \
+ ("FetchImageHelper=%p, " msg, this, ##__VA_ARGS__))
+
+using namespace mozilla::gfx;
+
+namespace mozilla::dom {
+
+FetchImageHelper::FetchImageHelper(const MediaImage& aImage)
+ : mSrc(aImage.mSrc) {}
+
+FetchImageHelper::~FetchImageHelper() { AbortFetchingImage(); }
+
+RefPtr<ImagePromise> FetchImageHelper::FetchImage() {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (IsFetchingImage()) {
+ return mPromise.Ensure(__func__);
+ }
+
+ LOG("Start fetching image from %s", NS_ConvertUTF16toUTF8(mSrc).get());
+ nsCOMPtr<nsIURI> uri;
+ if (NS_FAILED(NS_NewURI(getter_AddRefs(uri), mSrc))) {
+ LOG("Failed to create URI");
+ return ImagePromise::CreateAndReject(false, __func__);
+ }
+
+ MOZ_ASSERT(!mListener);
+ mListener = new ImageFetchListener();
+ if (NS_FAILED(mListener->FetchDecodedImageFromURI(uri, this))) {
+ LOG("Failed to decode image from async channel");
+ return ImagePromise::CreateAndReject(false, __func__);
+ }
+ return mPromise.Ensure(__func__);
+}
+
+void FetchImageHelper::AbortFetchingImage() {
+ MOZ_ASSERT(NS_IsMainThread());
+ LOG("AbortFetchingImage");
+ mPromise.RejectIfExists(false, __func__);
+ ClearListenerIfNeeded();
+}
+
+void FetchImageHelper::ClearListenerIfNeeded() {
+ if (mListener) {
+ mListener->Clear();
+ mListener = nullptr;
+ }
+}
+
+bool FetchImageHelper::IsFetchingImage() const {
+ MOZ_ASSERT(NS_IsMainThread());
+ return !mPromise.IsEmpty() && mListener;
+}
+
+void FetchImageHelper::HandleFetchSuccess(imgIContainer* aImage) {
+ MOZ_ASSERT(aImage);
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(IsFetchingImage());
+ LOG("Finished fetching image");
+ mPromise.Resolve(aImage, __func__);
+ ClearListenerIfNeeded();
+}
+
+void FetchImageHelper::HandleFetchFail() {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(IsFetchingImage());
+ LOG("Reject the promise because of fetching failed");
+ mPromise.RejectIfExists(false, __func__);
+ ClearListenerIfNeeded();
+}
+
+/**
+ * Implementation for FetchImageHelper::ImageFetchListener
+ */
+NS_IMPL_ISUPPORTS(FetchImageHelper::ImageFetchListener, imgIContainerCallback)
+
+FetchImageHelper::ImageFetchListener::~ImageFetchListener() {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!mHelper, "Cancel() should be called before desturction!");
+}
+
+nsresult FetchImageHelper::ImageFetchListener::FetchDecodedImageFromURI(
+ nsIURI* aURI, FetchImageHelper* aHelper) {
+ MOZ_ASSERT(!mHelper && !mChannel,
+ "Should call Clear() berfore running another fetching process!");
+ RefPtr<nsIPrincipal> nullPrincipal =
+ NullPrincipal::CreateWithoutOriginAttributes();
+ nsCOMPtr<nsIChannel> channel;
+ nsresult rv =
+ NS_NewChannel(getter_AddRefs(channel), aURI, nullPrincipal,
+ nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL,
+ nsIContentPolicy::TYPE_INTERNAL_IMAGE, nullptr, nullptr,
+ nullptr, nullptr, nsIRequest::LOAD_ANONYMOUS);
+ if (NS_FAILED(rv)) {
+ return rv;
+ }
+
+ nsCOMPtr<imgITools> imgTools = do_GetService("@mozilla.org/image/tools;1");
+ if (!imgTools) {
+ return NS_ERROR_FAILURE;
+ }
+
+ rv = imgTools->DecodeImageFromChannelAsync(aURI, channel, this, nullptr);
+ if (NS_FAILED(rv)) {
+ return NS_ERROR_FAILURE;
+ }
+ MOZ_ASSERT(aHelper);
+ mHelper = aHelper;
+ mChannel = channel;
+ return NS_OK;
+}
+
+void FetchImageHelper::ImageFetchListener::Clear() {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (mChannel) {
+ mChannel->CancelWithReason(
+ NS_BINDING_ABORTED, "FetchImageHelper::ImageFetchListener::Clear"_ns);
+ mChannel = nullptr;
+ }
+ mHelper = nullptr;
+}
+
+bool FetchImageHelper::ImageFetchListener::IsFetchingImage() const {
+ MOZ_ASSERT(NS_IsMainThread());
+ return mHelper ? mHelper->IsFetchingImage() : false;
+}
+
+NS_IMETHODIMP FetchImageHelper::ImageFetchListener::OnImageReady(
+ imgIContainer* aImage, nsresult aStatus) {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (!IsFetchingImage()) {
+ return NS_OK;
+ }
+ // We have received image, so we don't need the channel anymore.
+ mChannel = nullptr;
+
+ MOZ_ASSERT(mHelper);
+ if (NS_FAILED(aStatus) || !aImage) {
+ mHelper->HandleFetchFail();
+ Clear();
+ return aStatus;
+ }
+
+ mHelper->HandleFetchSuccess(aImage);
+
+ return NS_OK;
+}
+
+} // namespace mozilla::dom
diff --git a/dom/media/mediacontrol/FetchImageHelper.h b/dom/media/mediacontrol/FetchImageHelper.h
@@ -0,0 +1,82 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_MEDIACONTROL_FETCHIMAGEHELPER_H_
+#define DOM_MEDIA_MEDIACONTROL_FETCHIMAGEHELPER_H_
+
+#include "imgIContainer.h"
+#include "imgITools.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/dom/MediaSessionBinding.h"
+
+namespace mozilla::dom {
+/**
+ * FetchImageHelper is used to fetch image data from MediaImage, and the fetched
+ * image data would be used to show on the virtual control inferface. The URL of
+ * MediaImage is defined by websites by using MediaSession API [1].
+ *
+ * By using `FetchImage()`, it would return a promise that would resolve with a
+ * `imgIContainer`, then we can get the image data from the container.
+ *
+ * [1] https://w3c.github.io/mediasession/#dictdef-mediaimage
+ */
+using ImagePromise = MozPromise<nsCOMPtr<imgIContainer>, bool,
+ /* IsExclusive = */ true>;
+class FetchImageHelper final {
+ public:
+ explicit FetchImageHelper(const MediaImage& aImage);
+ ~FetchImageHelper();
+
+ // Return a promise which would be resolved with the decoded image surface
+ // when we finish fetching and decoding image data, and it would be rejected
+ // when we fail to fecth the image.
+ RefPtr<ImagePromise> FetchImage();
+
+ // Stop fetching and decoding image and reject the image promise. If we have
+ // not started yet fetching image, then nothing would happen.
+ void AbortFetchingImage();
+
+ // Return true if we're fecthing image.
+ bool IsFetchingImage() const;
+
+ private:
+ /**
+ * ImageFetchListener is used to listen the notification of finishing fetching
+ * image data (via `OnImageReady()`) and finishing decoding image data (via
+ * `Notify()`).
+ */
+ class ImageFetchListener final : public imgIContainerCallback {
+ public:
+ NS_DECL_ISUPPORTS
+ ImageFetchListener() = default;
+
+ // Start an async channel to load the image, and return error if the channel
+ // opens failed. It would use `aHelper::HandleFetchSuccess/Fail()` to notify
+ // the result asynchronously.
+ nsresult FetchDecodedImageFromURI(nsIURI* aURI, FetchImageHelper* aHelper);
+ void Clear();
+ bool IsFetchingImage() const;
+
+ // Method of imgIContainerCallback
+ NS_IMETHOD OnImageReady(imgIContainer* aImage, nsresult aStatus) override;
+
+ private:
+ ~ImageFetchListener();
+
+ FetchImageHelper* MOZ_NON_OWNING_REF mHelper = nullptr;
+ nsCOMPtr<nsIChannel> mChannel;
+ };
+
+ void ClearListenerIfNeeded();
+ void HandleFetchSuccess(imgIContainer* aImage);
+ void HandleFetchFail();
+
+ nsString mSrc;
+ MozPromiseHolder<ImagePromise> mPromise;
+ RefPtr<ImageFetchListener> mListener;
+};
+
+} // namespace mozilla::dom
+
+#endif // DOM_MEDIA_MEDIACONTROL_FETCHIMAGEHELPER_H_
diff --git a/dom/media/mediacontrol/MediaControlUtils.h b/dom/media/mediacontrol/MediaControlUtils.h
@@ -7,9 +7,7 @@
#ifndef DOM_MEDIA_MEDIACONTROL_MEDIACONTROLUTILS_H_
#define DOM_MEDIA_MEDIACONTROL_MEDIACONTROLUTILS_H_
-#include "ImageOps.h"
#include "MediaController.h"
-#include "gfxDrawable.h"
#include "imgIEncoder.h"
#include "imgITools.h"
#include "mozilla/Logging.h"
@@ -96,23 +94,19 @@ inline bool IsImageIn(const nsTArray<MediaImage>& aArtwork,
// The image buffer would be allocated in aStream whose size is aSize and the
// buffer head is aBuffer
-inline nsresult GetEncodedImageBuffer(gfx::DataSourceSurface* aSurface,
+inline nsresult GetEncodedImageBuffer(imgIContainer* aImage,
const nsACString& aMimeType,
nsIInputStream** aStream, uint32_t* aSize,
char** aBuffer) {
- MOZ_ASSERT(aSurface);
+ MOZ_ASSERT(aImage);
nsCOMPtr<imgITools> imgTools = do_GetService("@mozilla.org/image/tools;1");
if (!imgTools) {
return NS_ERROR_FAILURE;
}
- RefPtr<gfxDrawable> drawable =
- new gfxSurfaceDrawable(aSurface, aSurface->GetSize());
- nsCOMPtr<imgIContainer> image = image::ImageOps::CreateFromDrawable(drawable);
-
nsCOMPtr<nsIInputStream> inputStream;
- nsresult rv = imgTools->EncodeImage(image, aMimeType, u""_ns,
+ nsresult rv = imgTools->EncodeImage(aImage, aMimeType, u""_ns,
getter_AddRefs(inputStream));
if (NS_FAILED(rv)) {
return rv;
diff --git a/dom/media/mediacontrol/moz.build b/dom/media/mediacontrol/moz.build
@@ -7,6 +7,7 @@ EXPORTS.mozilla.dom += [
"AudioFocusManager.h",
"ContentMediaController.h",
"ContentPlaybackController.h",
+ "FetchImageHelper.h",
"MediaControlKeyManager.h",
"MediaControlKeySource.h",
"MediaController.h",
@@ -24,6 +25,7 @@ UNIFIED_SOURCES += [
"AudioFocusManager.cpp",
"ContentMediaController.cpp",
"ContentPlaybackController.cpp",
+ "FetchImageHelper.cpp",
"MediaControlKeyManager.cpp",
"MediaControlKeySource.cpp",
"MediaController.cpp",
diff --git a/dom/media/mediasession/MediaMetadata.cpp b/dom/media/mediasession/MediaMetadata.cpp
@@ -10,19 +10,10 @@
#include "mozilla/dom/MediaSessionBinding.h"
#include "mozilla/dom/ScriptSettings.h"
#include "mozilla/dom/ToJSValue.h"
-#include "mozilla/image/FetchDecodedImage.h"
#include "nsNetUtil.h"
namespace mozilla::dom {
-MediaImage MediaImageData::ToMediaImage() const {
- MediaImage image;
- image.mSizes = mSizes;
- image.mSrc = mSrc;
- image.mType = mType;
- return image;
-}
-
// Only needed for refcounted objects.
NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(MediaMetadata, mParent)
NS_IMPL_CYCLE_COLLECTING_ADDREF(MediaMetadata)
@@ -82,7 +73,7 @@ void MediaMetadata::GetArtwork(JSContext* aCx, nsTArray<JSObject*>& aRetVal,
for (size_t i = 0; i < mArtwork.Length(); ++i) {
JS::Rooted<JS::Value> value(aCx);
- if (!ToJSValue(aCx, mArtwork[i].ToMediaImage(), &value)) {
+ if (!ToJSValue(aCx, mArtwork[i], &value)) {
aRv.NoteJSContextException(aCx);
return;
}
@@ -121,54 +112,6 @@ void MediaMetadata::SetArtwork(JSContext* aCx,
SetArtworkInternal(artwork, aRv);
};
-RefPtr<MediaMetadataBasePromise> MediaMetadata::FetchArtwork(
- const MediaMetadataBase& aMetadata, nsIPrincipal* aPrincipal,
- const size_t aIndex) {
- if (!aPrincipal || aIndex >= aMetadata.mArtwork.Length()) {
- // No image loaded successfully or no principal, but still resolve without
- // any image data.
- return MediaMetadataBasePromise::CreateAndResolve(aMetadata, __func__);
- }
-
- nsCOMPtr<nsIURI> uri;
- if (NS_WARN_IF(NS_FAILED(
- NS_NewURI(getter_AddRefs(uri), aMetadata.mArtwork[aIndex].mSrc)))) {
- return FetchArtwork(aMetadata, aPrincipal, aIndex + 1);
- }
-
- return image::FetchDecodedImage(uri, gfx::IntSize{}, aPrincipal)
- ->Then(
- GetCurrentSerialEventTarget(), __func__,
- [metadata = aMetadata, principal = RefPtr{aPrincipal},
- aIndex](already_AddRefed<imgIContainer> aImage) {
- nsCOMPtr<imgIContainer> image(std::move(aImage));
- // The promise should only be resolved for decoded images, so using
- // FLAG_SYNC_DECODE is just a precaution.
- if (RefPtr<mozilla::gfx::SourceSurface> surface =
- image->GetFrame(imgIContainer::FRAME_FIRST,
- imgIContainer::FLAG_SYNC_DECODE |
- imgIContainer::FLAG_ASYNC_NOTIFY)) {
- if (RefPtr<mozilla::gfx::DataSourceSurface> dataSurface =
- surface->GetDataSurface()) {
- MediaMetadataBase data(metadata);
- data.mArtwork[aIndex].mDataSurface = dataSurface;
- return MediaMetadataBasePromise::CreateAndResolve(data,
- __func__);
- }
- }
- return FetchArtwork(metadata, principal, aIndex + 1);
- },
- [metadata = aMetadata, principal = RefPtr{aPrincipal},
- aIndex](nsresult aStatus) {
- return FetchArtwork(metadata, principal, aIndex + 1);
- });
-}
-
-RefPtr<MediaMetadataBasePromise> MediaMetadata::LoadMetadataArtwork() {
- // TODO: Track Promise?
- return FetchArtwork(*this, mParent->PrincipalOrNull(), 0);
-}
-
static nsIURI* GetEntryBaseURL() {
nsCOMPtr<Document> doc = GetEntryDocument();
return doc ? doc->GetDocBaseURI() : nullptr;
@@ -195,19 +138,17 @@ static nsresult ResolveURL(nsString& aURL, nsIURI* aBaseURI) {
void MediaMetadata::SetArtworkInternal(const Sequence<MediaImage>& aArtwork,
ErrorResult& aRv) {
- nsCOMPtr<nsIURI> baseURI = GetEntryBaseURL();
+ nsTArray<MediaImage> artwork;
+ artwork.Assign(aArtwork);
- nsTArray<MediaImageData> artwork;
- for (const MediaImage& image : aArtwork) {
- MediaImageData imageData(image);
- nsresult rv = ResolveURL(imageData.mSrc, baseURI);
+ nsCOMPtr<nsIURI> baseURI = GetEntryBaseURL();
+ for (MediaImage& image : artwork) {
+ nsresult rv = ResolveURL(image.mSrc, baseURI);
if (NS_WARN_IF(NS_FAILED(rv))) {
aRv.ThrowTypeError<MSG_INVALID_URL>(NS_ConvertUTF16toUTF8(image.mSrc));
return;
}
- artwork.AppendElement(std::move(imageData));
}
-
mArtwork = std::move(artwork);
}
diff --git a/dom/media/mediasession/MediaMetadata.h b/dom/media/mediasession/MediaMetadata.h
@@ -10,7 +10,6 @@
#include "js/TypeDecls.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/MediaSessionBinding.h"
-#include "mozilla/gfx/2D.h"
#include "nsCycleCollectionParticipant.h"
#include "nsWrapperCache.h"
@@ -21,22 +20,6 @@ class ErrorResult;
namespace dom {
-class MediaImageData {
- public:
- MediaImageData() = default;
- explicit MediaImageData(const MediaImage& aImage)
- : mSizes(aImage.mSizes), mSrc(aImage.mSrc), mType(aImage.mType) {}
-
- MediaImage ToMediaImage() const;
-
- nsString mSizes;
- nsString mSrc;
- nsString mType;
- // Maybe null, only the first valid artwork is fetched by
- // MediaMetadata::FetchArtwork.
- RefPtr<mozilla::gfx::DataSourceSurface> mDataSurface;
-};
-
class MediaMetadataBase {
public:
MediaMetadataBase() = default;
@@ -50,12 +33,9 @@ class MediaMetadataBase {
nsString mArtist;
nsString mAlbum;
nsCString mUrl;
- CopyableTArray<MediaImageData> mArtwork;
+ CopyableTArray<MediaImage> mArtwork;
};
-using MediaMetadataBasePromise =
- mozilla::MozPromise<MediaMetadataBase, nsresult, true>;
-
class MediaMetadata final : public nsISupports,
public nsWrapperCache,
private MediaMetadataBase {
@@ -92,10 +72,10 @@ class MediaMetadata final : public nsISupports,
void SetArtwork(JSContext* aCx, const Sequence<JSObject*>& aArtwork,
ErrorResult& aRv);
- // This function will always resolve successfully, even when no artwork was
- // loaded.
- // At most, it returns one decoded image of the artwork.
- RefPtr<MediaMetadataBasePromise> LoadMetadataArtwork();
+ // This would expose MediaMetadataBase's members as public, so use this method
+ // carefully. Now we only use this when we want to update the metadata to the
+ // media session controller in the chrome process.
+ MediaMetadataBase* AsMetadataBase() { return this; }
private:
MediaMetadata(nsIGlobalObject* aParent, const nsString& aTitle,
@@ -108,10 +88,6 @@ class MediaMetadata final : public nsISupports,
void SetArtworkInternal(const Sequence<MediaImage>& aArtwork,
ErrorResult& aRv);
- static RefPtr<MediaMetadataBasePromise> FetchArtwork(
- const MediaMetadataBase& aMetadata, nsIPrincipal* aPrincipal,
- const size_t aIndex);
-
nsCOMPtr<nsIGlobalObject> mParent;
};
diff --git a/dom/media/mediasession/MediaSession.cpp b/dom/media/mediasession/MediaSession.cpp
@@ -87,7 +87,6 @@ void MediaSession::Shutdown() {
if (mParent) {
SetMediaSessionDocStatus(SessionDocStatus::eInactive);
}
- mMetadataRequest.DisconnectIfExists();
}
void MediaSession::NotifyOwnerDocumentActivityChanged() {
@@ -316,30 +315,13 @@ void MediaSession::NotifyMetadataUpdated() {
RefPtr<BrowsingContext> currentBC = GetParentObject()->GetBrowsingContext();
MOZ_ASSERT(currentBC, "Update session metadata after context destroyed!");
- if (!mMediaMetadata) {
- if (RefPtr<IMediaInfoUpdater> updater = ContentMediaAgent::Get(currentBC)) {
- updater->UpdateMetadata(currentBC->Id(), Nothing());
- }
- return;
+ Maybe<MediaMetadataBase> metadata;
+ if (GetMetadata()) {
+ metadata.emplace(*(GetMetadata()->AsMetadataBase()));
+ }
+ if (RefPtr<IMediaInfoUpdater> updater = ContentMediaAgent::Get(currentBC)) {
+ updater->UpdateMetadata(currentBC->Id(), metadata);
}
-
- mMetadataRequest.DisconnectIfExists();
-
- mMediaMetadata->LoadMetadataArtwork()
- ->Then(
- GetCurrentSerialEventTarget(), __func__,
- [self = RefPtr{this}, currentBC](MediaMetadataBase&& aMetadata) {
- if (RefPtr<IMediaInfoUpdater> updater =
- ContentMediaAgent::Get(currentBC)) {
- updater->UpdateMetadata(currentBC->Id(), Some(aMetadata));
- }
-
- self->mMetadataRequest.Complete();
- },
- [](nsresult rv) {
- MOZ_ASSERT_UNREACHABLE("LoadMetadataArtwork should always resolve");
- })
- ->Track(mMetadataRequest);
}
void MediaSession::NotifyEnableSupportedAction(MediaSessionAction aAction) {
diff --git a/dom/media/mediasession/MediaSession.h b/dom/media/mediasession/MediaSession.h
@@ -131,9 +131,6 @@ class MediaSession final : public nsIDocumentActivity, public nsWrapperCache {
Maybe<PositionState> mPositionState;
RefPtr<Document> mDoc;
SessionDocStatus mSessionDocState = SessionDocStatus::eInactive;
-
- MozPromiseRequestHolder<mozilla::dom::MediaMetadataBasePromise>
- mMetadataRequest;
};
} // namespace dom
diff --git a/dom/media/mediasession/MediaSessionIPCUtils.h b/dom/media/mediasession/MediaSessionIPCUtils.h
@@ -11,7 +11,6 @@
#include "mozilla/dom/BindingIPCUtils.h"
#include "mozilla/dom/MediaSession.h"
#include "mozilla/dom/MediaSessionBinding.h"
-#include "nsContentUtils.h"
namespace mozilla {
namespace dom {
@@ -24,19 +23,13 @@ typedef Maybe<MediaMetadataBase> MaybeMediaMetadataBase;
namespace IPC {
template <>
-struct ParamTraits<mozilla::dom::MediaImageData> {
- typedef mozilla::dom::MediaImageData paramType;
+struct ParamTraits<mozilla::dom::MediaImage> {
+ typedef mozilla::dom::MediaImage paramType;
static void Write(MessageWriter* aWriter, const paramType& aParam) {
WriteParam(aWriter, aParam.mSizes);
WriteParam(aWriter, aParam.mSrc);
WriteParam(aWriter, aParam.mType);
-
- mozilla::Maybe<mozilla::dom::IPCImage> image;
- if (aParam.mDataSurface) {
- image = nsContentUtils::SurfaceToIPCImage(*aParam.mDataSurface);
- }
- WriteParam(aWriter, std::move(image));
}
static bool Read(MessageReader* aReader, paramType* aResult) {
@@ -45,14 +38,6 @@ struct ParamTraits<mozilla::dom::MediaImageData> {
!ReadParam(aReader, &(aResult->mType))) {
return false;
}
-
- mozilla::Maybe<mozilla::dom::IPCImage> image;
- if (!ReadParam(aReader, &image)) {
- return false;
- }
- if (image) {
- aResult->mDataSurface = nsContentUtils::IPCImageToSurface(*image);
- }
return true;
}
};
diff --git a/widget/cocoa/MediaHardwareKeysEventSourceMacMediaCenter.h b/widget/cocoa/MediaHardwareKeysEventSourceMacMediaCenter.h
@@ -5,6 +5,7 @@
#ifndef WIDGET_COCOA_MEDIAHARDWAREKEYSEVENTSOURCEMACMEDIACENTER_H_
#define WIDGET_COCOA_MEDIAHARDWAREKEYSEVENTSOURCEMACMEDIACENTER_H_
+#include "mozilla/dom/FetchImageHelper.h"
#include "mozilla/dom/MediaControlKeySource.h"
#ifdef __OBJC__
@@ -51,7 +52,15 @@ class MediaHardwareKeysEventSourceMacMediaCenter final
bool mOpened = false;
dom::MediaMetadataBase mMediaMetadata;
+ // Should only be used on main thread
+ UniquePtr<dom::FetchImageHelper> mImageFetcher;
+ MozPromiseRequestHolder<dom::ImagePromise> mImageFetchRequest;
+
+ nsString mFetchingUrl;
nsString mCurrentImageUrl;
+ size_t mNextImageIndex = 0;
+
+ void LoadImageAtIndex(const size_t aIndex);
MediaCenterEventHandler mPlayPauseHandler;
MediaCenterEventHandler mNextTrackHandler;
diff --git a/widget/cocoa/MediaHardwareKeysEventSourceMacMediaCenter.mm b/widget/cocoa/MediaHardwareKeysEventSourceMacMediaCenter.mm
@@ -156,7 +156,10 @@ bool MediaHardwareKeysEventSourceMacMediaCenter::Open() {
void MediaHardwareKeysEventSourceMacMediaCenter::Close() {
LOG("Close MediaHardwareKeysEventSourceMacMediaCenter");
SetPlaybackState(MediaSessionPlaybackState::None);
+ mImageFetchRequest.DisconnectIfExists();
mCurrentImageUrl.Truncate();
+ mFetchingUrl.Truncate();
+ mNextImageIndex = 0;
EndListeningForEvents();
mOpened = false;
MediaControlKeySource::Close();
@@ -202,50 +205,15 @@ void MediaHardwareKeysEventSourceMacMediaCenter::SetMediaMetadata(
forKey:MPMediaItemPropertyArtist];
[nowPlayingInfo setObject:nsCocoaUtils::ToNSString(aMetadata.mAlbum)
forKey:MPMediaItemPropertyAlbumTitle];
+ if (mCurrentImageUrl.IsEmpty() ||
+ !IsImageIn(aMetadata.mArtwork, mCurrentImageUrl)) {
+ [nowPlayingInfo removeObjectForKey:MPMediaItemPropertyArtwork];
- bool remove = true;
- for (const dom::MediaImageData& imageData : aMetadata.mArtwork) {
- if (!imageData.mDataSurface) {
- continue;
- }
-
- if (mCurrentImageUrl == imageData.mSrc) {
- LOG("Artwork image url did not change.");
- remove = false;
- break;
- }
-
- RefPtr<gfxDrawable> drawable = new gfxSurfaceDrawable(
- imageData.mDataSurface, imageData.mDataSurface->GetSize());
- nsCOMPtr<imgIContainer> imageContainer =
- image::ImageOps::CreateFromDrawable(drawable);
-
- NSImage* image;
- nsresult rv =
- nsCocoaUtils::CreateDualRepresentationNSImageFromImageContainer(
- imageContainer, imgIContainer::FRAME_CURRENT, nullptr,
- NSMakeSize(0, 0), &image);
- if (NS_FAILED(rv) || !image) {
- LOG("Failed to create cocoa image. Try next image");
- continue;
+ if (mFetchingUrl.IsEmpty() ||
+ !IsImageIn(aMetadata.mArtwork, mFetchingUrl)) {
+ mNextImageIndex = 0;
+ LoadImageAtIndex(mNextImageIndex++);
}
-
- MPMediaItemArtwork* artwork = [[MPMediaItemArtwork alloc]
- initWithBoundsSize:image.size
- requestHandler:^NSImage* _Nonnull(CGSize aSize) {
- return image;
- }];
- [nowPlayingInfo setObject:artwork forKey:MPMediaItemPropertyArtwork];
- [artwork release];
- [image release];
-
- mCurrentImageUrl = imageData.mSrc;
- remove = false;
- break;
- }
-
- if (remove) {
- [nowPlayingInfo removeObjectForKey:MPMediaItemPropertyArtwork];
}
// The procedure of updating `nowPlayingInfo` is actually an async operation
@@ -296,5 +264,77 @@ void MediaHardwareKeysEventSourceMacMediaCenter::SetPositionState(
}
}
+void MediaHardwareKeysEventSourceMacMediaCenter::LoadImageAtIndex(
+ const size_t aIndex) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (aIndex >= mMediaMetadata.mArtwork.Length()) {
+ LOG("Stop loading image. No available image");
+ mImageFetchRequest.DisconnectIfExists();
+ mFetchingUrl.Truncate();
+ return;
+ }
+
+ const MediaImage& image = mMediaMetadata.mArtwork[aIndex];
+
+ if (!IsValidImageUrl(image.mSrc)) {
+ LOG("Skip the image with invalid URL. Try next image");
+ LoadImageAtIndex(mNextImageIndex++);
+ return;
+ }
+
+ mImageFetchRequest.DisconnectIfExists();
+ mFetchingUrl = image.mSrc;
+
+ mImageFetcher = MakeUnique<dom::FetchImageHelper>(image);
+ RefPtr<MediaHardwareKeysEventSourceMacMediaCenter> self = this;
+ mImageFetcher->FetchImage()
+ ->Then(
+ AbstractThread::MainThread(), __func__,
+ [this, self](const nsCOMPtr<imgIContainer>& aImage) {
+ LOG("The image is fetched successfully");
+ mImageFetchRequest.Complete();
+
+ NSImage* image;
+ nsresult rv =
+ nsCocoaUtils::CreateDualRepresentationNSImageFromImageContainer(
+ aImage, imgIContainer::FRAME_CURRENT, nullptr,
+ NSMakeSize(0, 0), &image);
+ if (NS_FAILED(rv) || !image) {
+ LOG("Failed to create cocoa image. Try next image");
+ LoadImageAtIndex(mNextImageIndex++);
+ return;
+ }
+ mCurrentImageUrl = mFetchingUrl;
+
+ MPNowPlayingInfoCenter* center =
+ [MPNowPlayingInfoCenter defaultCenter];
+ NSMutableDictionary* nowPlayingInfo =
+ [[center.nowPlayingInfo mutableCopy] autorelease]
+ ?: [NSMutableDictionary dictionary];
+
+ MPMediaItemArtwork* artwork = [[MPMediaItemArtwork alloc]
+ initWithBoundsSize:image.size
+ requestHandler:^NSImage* _Nonnull(CGSize aSize) {
+ return image;
+ }];
+ [nowPlayingInfo setObject:artwork
+ forKey:MPMediaItemPropertyArtwork];
+ [artwork release];
+ [image release];
+
+ center.nowPlayingInfo = nowPlayingInfo;
+
+ mFetchingUrl.Truncate();
+ },
+ [this, self](bool) {
+ LOG("Failed to fetch image. Try next image");
+ mImageFetchRequest.Complete();
+ mFetchingUrl.Truncate();
+ LoadImageAtIndex(mNextImageIndex++);
+ })
+ ->Track(mImageFetchRequest);
+}
+
} // namespace widget
} // namespace mozilla
diff --git a/widget/gtk/MPRISServiceHandler.cpp b/widget/gtk/MPRISServiceHandler.cpp
@@ -531,38 +531,38 @@ GVariant* MPRISServiceHandler::GetPlaybackStatus() const {
void MPRISServiceHandler::SetMediaMetadata(
const dom::MediaMetadataBase& aMetadata) {
- SetMediaMetadataInternal(aMetadata);
-
- for (const dom::MediaImageData& image : aMetadata.mArtwork) {
- if (!image.mDataSurface) {
- continue;
- }
-
- if (mCurrentImageUrl == image.mSrc) {
- LOGMPRIS("Artwork image URL did not change");
- break;
- }
-
- uint32_t size = 0;
- char* data = nullptr;
- // Only used to hold the image data
- nsCOMPtr<nsIInputStream> inputStream;
-
- nsresult rv =
- dom::GetEncodedImageBuffer(image.mDataSurface, mMimeType,
- getter_AddRefs(inputStream), &size, &data);
- if (NS_FAILED(rv) || !inputStream || size == 0 || !data) {
- LOGMPRIS("Failed to get the image buffer info. Try next image");
- continue;
+ // Reset the index of the next available image to be fetched in the artwork,
+ // before checking the fetching process should be started or not. The image
+ // fetching process could be skipped if the image being fetching currently is
+ // in the artwork. If the current image fetching fails, the next availabe
+ // candidate should be the first image in the latest artwork
+ mNextImageIndex = 0;
+
+ // No need to fetch a MPRIS image if
+ // 1) MPRIS image is being fetched, and the one in fetching is in the artwork
+ // 2) MPRIS image is not being fetched, and the one in use is in the artwork
+ if (!mFetchingUrl.IsEmpty()) {
+ if (dom::IsImageIn(aMetadata.mArtwork, mFetchingUrl)) {
+ LOGMPRIS(
+ "No need to load MPRIS image. The one being processed is in the "
+ "artwork");
+ // Set MPRIS without the image first. The image will be loaded to MPRIS
+ // asynchronously once it's fetched and saved into a local file
+ SetMediaMetadataInternal(aMetadata);
+ return;
}
-
- if (SetImageToDisplay(data, size)) {
- mCurrentImageUrl = image.mSrc;
- LOGMPRIS("The MPRIS image is updated to the image from: %s",
- NS_ConvertUTF16toUTF8(mCurrentImageUrl).get());
- break;
+ } else if (!mCurrentImageUrl.IsEmpty()) {
+ if (dom::IsImageIn(aMetadata.mArtwork, mCurrentImageUrl)) {
+ LOGMPRIS("No need to load MPRIS image. The one in use is in the artwork");
+ SetMediaMetadataInternal(aMetadata, false);
+ return;
}
}
+
+ // Set MPRIS without the image first then load the image to MPRIS
+ // asynchronously
+ SetMediaMetadataInternal(aMetadata);
+ LoadImageAtIndex(mNextImageIndex++);
}
bool MPRISServiceHandler::EmitMetadataChanged() const {
@@ -588,12 +588,76 @@ void MPRISServiceHandler::SetMediaMetadataInternal(
void MPRISServiceHandler::ClearMetadata() {
mMPRISMetadata.Clear();
+ mImageFetchRequest.DisconnectIfExists();
RemoveAllLocalImages();
mCurrentImageUrl.Truncate();
+ mFetchingUrl.Truncate();
+ mNextImageIndex = 0;
mSupportedKeys = 0;
EmitMetadataChanged();
}
+void MPRISServiceHandler::LoadImageAtIndex(const size_t aIndex) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (aIndex >= mMPRISMetadata.mArtwork.Length()) {
+ LOGMPRIS("Stop loading image to MPRIS. No available image");
+ mImageFetchRequest.DisconnectIfExists();
+ return;
+ }
+
+ const dom::MediaImage& image = mMPRISMetadata.mArtwork[aIndex];
+
+ if (!dom::IsValidImageUrl(image.mSrc)) {
+ LOGMPRIS("Skip the image with invalid URL. Try next image");
+ LoadImageAtIndex(mNextImageIndex++);
+ return;
+ }
+
+ mImageFetchRequest.DisconnectIfExists();
+ mFetchingUrl = image.mSrc;
+
+ mImageFetcher = MakeUnique<dom::FetchImageHelper>(image);
+ RefPtr<MPRISServiceHandler> self = this;
+ mImageFetcher->FetchImage()
+ ->Then(
+ AbstractThread::MainThread(), __func__,
+ [this, self](const nsCOMPtr<imgIContainer>& aImage) {
+ LOGMPRIS("The image is fetched successfully");
+ mImageFetchRequest.Complete();
+
+ uint32_t size = 0;
+ char* data = nullptr;
+ // Only used to hold the image data
+ nsCOMPtr<nsIInputStream> inputStream;
+ nsresult rv = dom::GetEncodedImageBuffer(
+ aImage, mMimeType, getter_AddRefs(inputStream), &size, &data);
+ if (NS_FAILED(rv) || !inputStream || size == 0 || !data) {
+ LOGMPRIS("Failed to get the image buffer info. Try next image");
+ LoadImageAtIndex(mNextImageIndex++);
+ return;
+ }
+
+ if (SetImageToDisplay(data, size)) {
+ mCurrentImageUrl = mFetchingUrl;
+ LOGMPRIS("The MPRIS image is updated to the image from: %s",
+ NS_ConvertUTF16toUTF8(mCurrentImageUrl).get());
+ } else {
+ LOGMPRIS("Failed to set image to MPRIS");
+ mCurrentImageUrl.Truncate();
+ }
+
+ mFetchingUrl.Truncate();
+ },
+ [this, self](bool) {
+ LOGMPRIS("Failed to fetch image. Try next image");
+ mImageFetchRequest.Complete();
+ mFetchingUrl.Truncate();
+ LoadImageAtIndex(mNextImageIndex++);
+ })
+ ->Track(mImageFetchRequest);
+}
+
bool MPRISServiceHandler::SetImageToDisplay(const char* aImageData,
uint32_t aDataSize) {
if (!RenewLocalImageFile(aImageData, aDataSize)) {
diff --git a/widget/gtk/MPRISServiceHandler.h b/widget/gtk/MPRISServiceHandler.h
@@ -8,6 +8,7 @@
#define WIDGET_GTK_MPRIS_SERVICE_HANDLER_H_
#include <gio/gio.h>
+#include "mozilla/dom/FetchImageHelper.h"
#include "mozilla/dom/MediaControlKeySource.h"
#include "mozilla/UniquePtr.h"
#include "nsIFile.h"
@@ -139,8 +140,17 @@ class MPRISServiceHandler final : public dom::MediaControlKeySource {
nsCOMPtr<nsIFile> mLocalImageFile;
nsCOMPtr<nsIFile> mLocalImageFolder;
+ UniquePtr<dom::FetchImageHelper> mImageFetcher;
+ MozPromiseRequestHolder<dom::ImagePromise> mImageFetchRequest;
+
+ nsString mFetchingUrl;
nsString mCurrentImageUrl;
+ size_t mNextImageIndex = 0;
+
+ // Load the image at index aIndex of the metadta's artwork to MPRIS
+ // asynchronously
+ void LoadImageAtIndex(const size_t aIndex);
bool SetImageToDisplay(const char* aImageData, uint32_t aDataSize);
bool RenewLocalImageFile(const char* aImageData, uint32_t aDataSize);
diff --git a/widget/windows/WindowsSMTCProvider.cpp b/widget/windows/WindowsSMTCProvider.cpp
@@ -224,6 +224,7 @@ void WindowsSMTCProvider::ClearMetadata() {
if (FAILED(mDisplay->ClearAll())) {
LOG("Failed to clear SMTC display");
}
+ mImageFetchRequest.DisconnectIfExists();
CancelPendingStoreAsyncOperation();
mThumbnailUrl.Truncate();
mProcessingUrl.Truncate();
@@ -595,46 +596,102 @@ void WindowsSMTCProvider::SetPositionState(
}
void WindowsSMTCProvider::LoadThumbnail(
- const nsTArray<mozilla::dom::MediaImageData>& aArtwork) {
+ const nsTArray<mozilla::dom::MediaImage>& aArtwork) {
MOZ_ASSERT(NS_IsMainThread());
- for (const dom::MediaImageData& image : aArtwork) {
- if (!image.mDataSurface) {
- continue;
- }
+ // TODO: Sort the images by the preferred size or format.
+ mArtwork = aArtwork;
+ mNextImageIndex = 0;
- if (mThumbnailUrl == image.mSrc) {
- LOG("Artwork image URL did not change");
+ // Abort the loading if
+ // 1) thumbnail is being updated, and one in processing is in the artwork
+ // 2) thumbnail is not being updated, and one in use is in the artwork
+ if (!mProcessingUrl.IsEmpty()) {
+ LOG("Load thumbnail while image: %s is being processed",
+ NS_ConvertUTF16toUTF8(mProcessingUrl).get());
+ if (mozilla::dom::IsImageIn(mArtwork, mProcessingUrl)) {
+ LOG("No need to load thumbnail. The one being processed is in the "
+ "artwork");
return;
}
-
- // Although IMAGE_JPEG or IMAGE_BMP are valid types as well, but a
- // png image with transparent background will be converted into a
- // jpeg/bmp file with a colored background. IMAGE_PNG format seems
- // to be the best choice for now.
- uint32_t size = 0;
- char* src = nullptr;
- // Only used to hold the image data
- nsCOMPtr<nsIInputStream> inputStream;
- nsresult rv = mozilla::dom::GetEncodedImageBuffer(
- image.mDataSurface, nsLiteralCString(IMAGE_PNG),
- getter_AddRefs(inputStream), &size, &src);
- if (NS_FAILED(rv) || !inputStream || size == 0 || !src) {
- LOG("Failed to get the image buffer info. Try next image");
- continue;
+ } else if (!mThumbnailUrl.IsEmpty()) {
+ if (mozilla::dom::IsImageIn(mArtwork, mThumbnailUrl)) {
+ LOG("No need to load thumbnail. The one in use is in the artwork");
+ return;
}
+ }
+
+ // If there is a pending image store operation, that image must be different
+ // from the new image will be loaded below, so the pending one should be
+ // cancelled.
+ CancelPendingStoreAsyncOperation();
+ // Remove the current thumbnail on the interface
+ ClearThumbnail();
+ // Then load the new thumbnail asynchronously
+ LoadImageAtIndex(mNextImageIndex++);
+}
+
+void WindowsSMTCProvider::LoadImageAtIndex(const size_t aIndex) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (aIndex >= mArtwork.Length()) {
+ LOG("Stop loading thumbnail. No more available images");
+ mImageFetchRequest.DisconnectIfExists();
+ mProcessingUrl.Truncate();
+ return;
+ }
+
+ const mozilla::dom::MediaImage& image = mArtwork[aIndex];
- // If there is a pending image store operation, that image must be different
- // from the new image will be loaded below, so the pending one should be
- // cancelled.
- CancelPendingStoreAsyncOperation();
- // Remove the current thumbnail on the interface
- ClearThumbnail();
+ // TODO: No need to fetch the default image and do image processing since the
+ // the default image is local file and it's trustworthy. For the default
+ // image, we can use `CreateFromFile` to create the IRandomAccessStream. We
+ // should probably cache it since it could be used very often (Bug 1643102)
- mProcessingUrl = image.mSrc;
- LoadImage(src, size);
- break;
+ if (!mozilla::dom::IsValidImageUrl(image.mSrc)) {
+ LOG("Skip the image with invalid URL. Try next image");
+ mImageFetchRequest.DisconnectIfExists();
+ LoadImageAtIndex(mNextImageIndex++);
+ return;
}
+
+ mImageFetchRequest.DisconnectIfExists();
+ mProcessingUrl = image.mSrc;
+
+ mImageFetcher = mozilla::MakeUnique<mozilla::dom::FetchImageHelper>(image);
+ RefPtr<WindowsSMTCProvider> self = this;
+ mImageFetcher->FetchImage()
+ ->Then(
+ AbstractThread::MainThread(), __func__,
+ [this, self](const nsCOMPtr<imgIContainer>& aImage) {
+ LOG("The image is fetched successfully");
+ mImageFetchRequest.Complete();
+
+ // Although IMAGE_JPEG or IMAGE_BMP are valid types as well, but a
+ // png image with transparent background will be converted into a
+ // jpeg/bmp file with a colored background. IMAGE_PNG format seems
+ // to be the best choice for now.
+ uint32_t size = 0;
+ char* src = nullptr;
+ // Only used to hold the image data
+ nsCOMPtr<nsIInputStream> inputStream;
+ nsresult rv = mozilla::dom::GetEncodedImageBuffer(
+ aImage, nsLiteralCString(IMAGE_PNG),
+ getter_AddRefs(inputStream), &size, &src);
+ if (NS_FAILED(rv) || !inputStream || size == 0 || !src) {
+ LOG("Failed to get the image buffer info. Try next image");
+ LoadImageAtIndex(mNextImageIndex++);
+ return;
+ }
+
+ LoadImage(src, size);
+ },
+ [this, self](bool) {
+ LOG("Failed to fetch image. Try next image");
+ mImageFetchRequest.Complete();
+ LoadImageAtIndex(mNextImageIndex++);
+ })
+ ->Track(mImageFetchRequest);
}
void WindowsSMTCProvider::LoadImage(const char* aImageData,
diff --git a/widget/windows/WindowsSMTCProvider.h b/widget/windows/WindowsSMTCProvider.h
@@ -12,6 +12,7 @@
# include <Windows.Media.h>
# include <wrl.h>
+# include "mozilla/dom/FetchImageHelper.h"
# include "mozilla/dom/MediaController.h"
# include "mozilla/dom/MediaControlKeySource.h"
# include "mozilla/UniquePtr.h"
@@ -71,7 +72,10 @@ class WindowsSMTCProvider final : public mozilla::dom::MediaControlKeySource {
bool SetMusicMetadata(const nsString& aArtist, const nsString& aTitle);
// Sets one of the artwork to the SMTC interface asynchronously
- void LoadThumbnail(const nsTArray<mozilla::dom::MediaImageData>& aArtwork);
+ void LoadThumbnail(const nsTArray<mozilla::dom::MediaImage>& aArtwork);
+ // Stores the image at index aIndex of the mArtwork to the Thumbnail
+ // asynchronously
+ void LoadImageAtIndex(const size_t aIndex);
// Stores the raw binary data of an image to mImageStream and set it to the
// Thumbnail asynchronously
void LoadImage(const char* aImageData, uint32_t aDataSize);
@@ -114,6 +118,10 @@ class WindowsSMTCProvider final : public mozilla::dom::MediaControlKeySource {
CopyableTArray<mozilla::dom::MediaImage> mArtwork;
size_t mNextImageIndex;
+ mozilla::UniquePtr<mozilla::dom::FetchImageHelper> mImageFetcher;
+ mozilla::MozPromiseRequestHolder<mozilla::dom::ImagePromise>
+ mImageFetchRequest;
+
HWND mWindow; // handle to the invisible window
// EventRegistrationTokens are used to have a handle on a callback (to remove
