RemoteMediaDataEncoderParent.cpp (10895B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "RemoteMediaDataEncoderParent.h" 8 9 #include "ImageContainer.h" 10 #include "PEMFactory.h" 11 #include "RemoteMediaManagerParent.h" 12 #include "mozilla/dom/WebCodecsUtils.h" 13 14 namespace mozilla { 15 16 extern LazyLogModule sPEMLog; 17 18 #define LOG_INTERNAL(level, fmt, ...) \ 19 MOZ_LOG_FMT(sPEMLog, LogLevel::level, \ 20 "[RemoteMediaDataEncoderParent] {}: " fmt, __func__, \ 21 __VA_ARGS__) 22 #define LOGE(fmt, ...) LOG_INTERNAL(Error, fmt, __VA_ARGS__) 23 #define LOGW(fmt, ...) LOG_INTERNAL(Warning, fmt, __VA_ARGS__) 24 #define LOGD(fmt, ...) LOG_INTERNAL(Debug, fmt, __VA_ARGS__) 25 #define LOGV(fmt, ...) LOG_INTERNAL(Verbose, fmt, __VA_ARGS__) 26 27 #define LOGE_IF(condition, fmt, ...) \ 28 do { \ 29 if (condition) { \ 30 LOGE(fmt, __VA_ARGS__); \ 31 } \ 32 } while (0) 33 34 #define AUTO_MARKER(var, postfix) \ 35 AutoWebCodecsMarker var("RemoteMediaDataEncoderParent", postfix); 36 37 RemoteMediaDataEncoderParent::RemoteMediaDataEncoderParent( 38 const EncoderConfig& aConfig) 39 : ShmemRecycleAllocator(this), 40 mBufferRecycleBin(new layers::BufferRecycleBin()), 41 mConfig(aConfig) {} 42 43 RemoteMediaDataEncoderParent::~RemoteMediaDataEncoderParent() = default; 44 45 IPCResult RemoteMediaDataEncoderParent::RecvConstruct( 46 ConstructResolver&& aResolver) { 47 if (mEncoder) { 48 aResolver(MediaResult(NS_ERROR_ALREADY_INITIALIZED, __func__)); 49 return IPC_OK(); 50 } 51 52 RefPtr<TaskQueue> taskQueue = 53 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_ENCODER), 54 "RemoteMediaDataEncoderParent"); 55 56 auto factory = MakeRefPtr<PEMFactory>(); 57 factory->CreateEncoderAsync(mConfig, taskQueue) 58 ->Then(GetCurrentSerialEventTarget(), __func__, 59 [self = RefPtr{this}, resolver = std::move(aResolver)]( 60 PlatformEncoderModule::CreateEncoderPromise:: 61 ResolveOrRejectValue&& aValue) { 62 if (aValue.IsReject()) { 63 resolver(aValue.RejectValue()); 64 return; 65 } 66 67 if (self->mEncoder) { 68 resolver(MediaResult(NS_ERROR_ALREADY_INITIALIZED, __func__)); 69 return; 70 } 71 72 self->mEncoder = std::move(aValue.ResolveValue()); 73 resolver(MediaResult(NS_OK)); 74 }); 75 return IPC_OK(); 76 } 77 78 IPCResult RemoteMediaDataEncoderParent::RecvInit(InitResolver&& aResolver) { 79 if (!mEncoder) { 80 aResolver(MediaResult(NS_ERROR_ABORT, __func__)); 81 return IPC_OK(); 82 } 83 84 mEncoder->Init()->Then( 85 GetCurrentSerialEventTarget(), __func__, 86 [encoder = RefPtr{mEncoder}, resolver = std::move(aResolver)]( 87 MediaDataEncoder::InitPromise::ResolveOrRejectValue&& aValue) { 88 if (aValue.IsReject()) { 89 resolver(aValue.RejectValue()); 90 return; 91 } 92 93 nsCString hardwareReason; 94 bool hardware = encoder->IsHardwareAccelerated(hardwareReason); 95 resolver(EncodeInitCompletionIPDL{encoder->GetDescriptionName(), 96 hardware, std::move(hardwareReason)}); 97 }); 98 return IPC_OK(); 99 } 100 101 IPCResult RemoteMediaDataEncoderParent::RecvEncode( 102 const EncodedInputIPDL& aData, EncodeResolver&& aResolver) { 103 if (!mEncoder) { 104 aResolver(MediaResult(NS_ERROR_ABORT, __func__)); 105 return IPC_OK(); 106 } 107 108 nsTArray<RefPtr<MediaData>> frames; 109 110 if (mConfig.IsAudio() && 111 aData.type() == EncodedInputIPDL::TArrayOfRemoteAudioData) { 112 auto remoteAudioArray = aData.get_ArrayOfRemoteAudioData(); 113 if (remoteAudioArray->IsEmpty()) { 114 LOGE("[{}] no audio frames received", fmt::ptr(this)); 115 aResolver(MediaResult(NS_ERROR_INVALID_ARG, __func__)); 116 return IPC_OK(); 117 } 118 119 LOGV("[{}] recv {} audio frames", fmt::ptr(this), 120 remoteAudioArray->Count()); 121 for (size_t i = 0; i < remoteAudioArray->Count(); i++) { 122 frames.AppendElement( 123 remoteAudioArray->ElementAt(i).downcast<MediaData>()); 124 } 125 } else if (mConfig.IsVideo() && 126 aData.type() == EncodedInputIPDL::TArrayOfRemoteVideoData) { 127 auto remoteVideoArray = aData.get_ArrayOfRemoteVideoData(); 128 if (remoteVideoArray->Array().IsEmpty()) { 129 LOGE("[{}] no video frames received", fmt::ptr(this)); 130 aResolver(MediaResult(NS_ERROR_INVALID_ARG, __func__)); 131 return IPC_OK(); 132 } 133 134 LOGV("[{}] recv {} video frames", fmt::ptr(this), 135 remoteVideoArray->Array().Length()); 136 for (size_t i = 0; i < remoteVideoArray->Array().Length(); i++) { 137 RefPtr<MediaData> frame; 138 auto data = std::move(remoteVideoArray->Array().ElementAt(i)); 139 if (!data.image().IsEmpty()) { 140 AUTO_MARKER(marker, ".RecvEncode.TransferToImage"); 141 RefPtr<layers::Image> image = 142 data.image().TransferToImage(mBufferRecycleBin); 143 marker.End(); 144 LOGE_IF(!image, "[{}] failed to get image from video frame at index {}", 145 fmt::ptr(this), i); 146 if (image) { 147 frame = VideoData::CreateFromImage( 148 data.display(), data.base().offset(), data.base().time(), 149 data.base().duration(), image, data.base().keyframe(), 150 data.base().timecode()) 151 .downcast<MediaData>(); 152 } 153 } else { 154 LOGW("[{}] empty image in video frame at index {}", fmt::ptr(this), i); 155 frame = MakeRefPtr<NullData>(data.base().offset(), data.base().time(), 156 data.base().duration()); 157 } 158 159 if (NS_WARN_IF(!frame)) { 160 LOGE("[{}] failed to create video frame", fmt::ptr(this)); 161 aResolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__)); 162 return IPC_OK(); 163 } 164 165 frames.AppendElement(std::move(frame)); 166 } 167 } else { 168 LOGE("[{}] invalid input data type", fmt::ptr(this)); 169 aResolver(MediaResult(NS_ERROR_INVALID_ARG, __func__)); 170 return IPC_OK(); 171 } 172 173 LOGV("[{}] encoding {} frames", fmt::ptr(this), frames.Length()); 174 mEncoder->Encode(std::move(frames)) 175 ->Then( 176 GetCurrentSerialEventTarget(), __func__, 177 [self = RefPtr{this}, resolver = std::move(aResolver)]( 178 MediaDataEncoder::EncodePromise::ResolveOrRejectValue&& aValue) { 179 if (aValue.IsReject()) { 180 resolver(aValue.RejectValue()); 181 return; 182 } 183 184 auto ticket = MakeRefPtr<ShmemRecycleTicket>(); 185 auto samples = MakeRefPtr<ArrayOfRemoteMediaRawData>(); 186 if (!samples->Fill(aValue.ResolveValue(), [&](size_t aSize) { 187 return self->AllocateBuffer(aSize, ticket); 188 })) { 189 self->ReleaseTicket(ticket); 190 resolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__)); 191 return; 192 } 193 194 uint32_t ticketId = ++self->mTicketCounter; 195 self->mTickets[ticketId] = std::move(ticket); 196 resolver(EncodeCompletionIPDL(samples, ticketId)); 197 }); 198 return IPC_OK(); 199 } 200 201 IPCResult RemoteMediaDataEncoderParent::RecvReconfigure( 202 EncoderConfigurationChangeList* aConfigurationChanges, 203 ReconfigureResolver&& aResolver) { 204 if (!mEncoder) { 205 aResolver(MediaResult(NS_ERROR_ABORT, __func__)); 206 return IPC_OK(); 207 } 208 209 mEncoder->Reconfigure(aConfigurationChanges) 210 ->Then( 211 GetCurrentSerialEventTarget(), __func__, 212 [encoder = RefPtr{mEncoder}, resolver = std::move(aResolver)]( 213 MediaDataEncoder::ReconfigurationPromise::ResolveOrRejectValue&& 214 aValue) { 215 if (aValue.IsReject()) { 216 resolver(aValue.RejectValue()); 217 return; 218 } 219 220 resolver(MediaResult(NS_OK)); 221 }); 222 return IPC_OK(); 223 } 224 225 IPCResult RemoteMediaDataEncoderParent::RecvDrain(DrainResolver&& aResolver) { 226 if (!mEncoder) { 227 aResolver(MediaResult(NS_ERROR_ABORT, __func__)); 228 return IPC_OK(); 229 } 230 231 mEncoder->Drain()->Then( 232 GetCurrentSerialEventTarget(), __func__, 233 [self = RefPtr{this}, resolver = std::move(aResolver)]( 234 MediaDataEncoder::EncodePromise::ResolveOrRejectValue&& aValue) { 235 if (aValue.IsReject()) { 236 resolver(aValue.RejectValue()); 237 return; 238 } 239 240 auto ticket = MakeRefPtr<ShmemRecycleTicket>(); 241 auto samples = MakeRefPtr<ArrayOfRemoteMediaRawData>(); 242 if (!samples->Fill(aValue.ResolveValue(), [&](size_t aSize) { 243 return self->AllocateBuffer(aSize, ticket); 244 })) { 245 self->ReleaseTicket(ticket); 246 resolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__)); 247 return; 248 } 249 250 uint32_t ticketId = ++self->mTicketCounter; 251 self->mTickets[ticketId] = std::move(ticket); 252 resolver(EncodeCompletionIPDL(samples, ticketId)); 253 }); 254 return IPC_OK(); 255 } 256 257 IPCResult RemoteMediaDataEncoderParent::RecvReleaseTicket( 258 const uint32_t& aTicketId) { 259 auto i = mTickets.find(aTicketId); 260 if (i != mTickets.end()) { 261 ReleaseTicket(i->second.get()); 262 mTickets.erase(i); 263 } 264 return IPC_OK(); 265 } 266 267 IPCResult RemoteMediaDataEncoderParent::RecvShutdown( 268 ShutdownResolver&& aResolver) { 269 if (!mEncoder) { 270 aResolver(false); 271 return IPC_OK(); 272 } 273 274 mEncoder->Shutdown()->Then( 275 GetCurrentSerialEventTarget(), __func__, 276 [resolver = std::move(aResolver)]( 277 const ShutdownPromise::ResolveOrRejectValue& aValue) { 278 resolver(aValue.IsResolve()); 279 }); 280 mEncoder = nullptr; 281 return IPC_OK(); 282 } 283 284 IPCResult RemoteMediaDataEncoderParent::RecvSetBitrate( 285 const uint32_t& aBitrate, SetBitrateResolver&& aResolver) { 286 if (!mEncoder) { 287 aResolver(NS_ERROR_ABORT); 288 return IPC_OK(); 289 } 290 291 mEncoder->SetBitrate(aBitrate)->Then( 292 GetCurrentSerialEventTarget(), __func__, 293 [encoder = RefPtr{mEncoder}, resolver = std::move(aResolver)]( 294 GenericPromise::ResolveOrRejectValue&& aValue) { 295 resolver(aValue.IsResolve() ? NS_OK : aValue.RejectValue()); 296 }); 297 return IPC_OK(); 298 } 299 300 void RemoteMediaDataEncoderParent::ActorDestroy(ActorDestroyReason aWhy) { 301 if (mEncoder) { 302 mEncoder->Shutdown(); 303 mEncoder = nullptr; 304 } 305 306 for (auto& i : mTickets) { 307 ReleaseTicket(i.second.get()); 308 } 309 mTickets.clear(); 310 311 CleanupShmemRecycleAllocator(); 312 } 313 314 #undef LOGE 315 #undef LOGW 316 #undef LOGD 317 #undef LOGV 318 #undef LOGE_IF 319 #undef LOG_INTERNAL 320 #undef AUTO_MARKER 321 322 } // namespace mozilla