WMFMediaDataEncoder.cpp (18469B)
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "WMFMediaDataEncoder.h" 8 9 #include <comdef.h> 10 11 #include "ImageContainer.h" 12 #include "ImageConversion.h" 13 #include "MFTEncoder.h" 14 #include "PlatformEncoderModule.h" 15 #include "TimeUnits.h" 16 #include "WMFDataEncoderUtils.h" 17 #include "WMFUtils.h" 18 #include "mozilla/WindowsProcessMitigations.h" 19 #include "mozilla/dom/WebCodecsUtils.h" 20 21 namespace mozilla { 22 23 #define AUTO_MARKER(desc) AUTO_WEBCODECS_MARKER("WMFMediaDataEncoder", desc) 24 25 using InitPromise = MediaDataEncoder::InitPromise; 26 using EncodePromise = MediaDataEncoder::EncodePromise; 27 using ReconfigurationPromise = MediaDataEncoder::ReconfigurationPromise; 28 29 WMFMediaDataEncoder::WMFMediaDataEncoder(const EncoderConfig& aConfig, 30 const RefPtr<TaskQueue>& aTaskQueue) 31 : mConfig(aConfig), 32 mTaskQueue(aTaskQueue), 33 mHardwareNotAllowed(aConfig.mHardwarePreference == 34 HardwarePreference::RequireSoftware) { 35 WMF_ENC_LOGE("WMFMediaDataEncoder ctor: %s, (hw not allowed: %s)", 36 aConfig.ToString().get(), mHardwareNotAllowed ? "yes" : "no"); 37 MOZ_ASSERT(mTaskQueue); 38 } 39 40 RefPtr<InitPromise> WMFMediaDataEncoder::Init() { 41 return InvokeAsync(mTaskQueue, this, __func__, 42 &WMFMediaDataEncoder::ProcessInit); 43 } 44 RefPtr<EncodePromise> WMFMediaDataEncoder::Encode(const MediaData* aSample) { 45 WMF_ENC_LOGD("Encode ts=%s", aSample->mTime.ToString().get()); 46 MOZ_ASSERT(aSample); 47 48 RefPtr<const VideoData> sample(aSample->As<const VideoData>()); 49 50 return InvokeAsync<RefPtr<const VideoData>>( 51 mTaskQueue, this, __func__, &WMFMediaDataEncoder::ProcessEncode, 52 std::move(sample)); 53 } 54 RefPtr<EncodePromise> WMFMediaDataEncoder::Encode( 55 nsTArray<RefPtr<MediaData>>&& aSamples) { 56 WMF_ENC_LOGD("Encode: num of samples=%zu", aSamples.Length()); 57 MOZ_ASSERT(!aSamples.IsEmpty()); 58 59 nsTArray<RefPtr<const VideoData>> videoSamples; 60 for (auto& sample : aSamples) { 61 videoSamples.AppendElement(sample->As<const VideoData>()); 62 } 63 64 return InvokeAsync(mTaskQueue, this, __func__, 65 &WMFMediaDataEncoder::ProcessEncodeBatch, 66 std::move(videoSamples)); 67 } 68 RefPtr<EncodePromise> WMFMediaDataEncoder::Drain() { 69 WMF_ENC_LOGD("Drain"); 70 return InvokeAsync(mTaskQueue, this, __func__, 71 &WMFMediaDataEncoder::ProcessDrain); 72 } 73 RefPtr<ShutdownPromise> WMFMediaDataEncoder::Shutdown() { 74 WMF_ENC_LOGD("Shutdown"); 75 return InvokeAsync( 76 mTaskQueue, __func__, [self = RefPtr<WMFMediaDataEncoder>(this)]() { 77 auto r = MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, 78 "Canceled by WMFMediaDataEncoder::Shutdown"); 79 80 // Cancel encode in flight if any. 81 self->mEncodeRequest.DisconnectIfExists(); 82 self->mEncodePromise.RejectIfExists(r, __func__); 83 84 // Cancel drain in flight if any. 85 self->mDrainRequest.DisconnectIfExists(); 86 self->mDrainPromise.RejectIfExists(r, __func__); 87 88 if (self->mEncoder) { 89 self->mEncoder->Destroy(); 90 self->mEncoder = nullptr; 91 } 92 return ShutdownPromise::CreateAndResolve(true, __func__); 93 }); 94 } 95 RefPtr<GenericPromise> WMFMediaDataEncoder::SetBitrate(uint32_t aBitsPerSec) { 96 return InvokeAsync( 97 mTaskQueue, __func__, 98 [self = RefPtr<WMFMediaDataEncoder>(this), aBitsPerSec]() { 99 MOZ_ASSERT(self->mEncoder); 100 return SUCCEEDED(self->mEncoder->SetBitrate(aBitsPerSec)) 101 ? GenericPromise::CreateAndResolve(true, __func__) 102 : GenericPromise::CreateAndReject( 103 NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, __func__); 104 }); 105 } 106 107 RefPtr<ReconfigurationPromise> WMFMediaDataEncoder::Reconfigure( 108 const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) { 109 // General reconfiguration interface not implemented right now 110 return MediaDataEncoder::ReconfigurationPromise::CreateAndReject( 111 NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); 112 }; 113 114 nsCString WMFMediaDataEncoder::GetDescriptionName() const { 115 return MFTEncoder::GetFriendlyName(CodecToSubtype(mConfig.mCodec)); 116 } 117 118 RefPtr<InitPromise> WMFMediaDataEncoder::ProcessInit() { 119 AssertOnTaskQueue(); 120 121 WMF_ENC_LOGD("ProcessInit"); 122 123 MOZ_ASSERT(!mEncoder, 124 "Should not initialize encoder again without shutting down"); 125 126 auto cleanup = MakeScopeExit([&] { mIsHardwareAccelerated = false; }); 127 128 if (!wmf::MediaFoundationInitializer::HasInitialized()) { 129 return InitPromise::CreateAndReject( 130 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 131 RESULT_DETAIL("Can't create the MFT encoder.")), 132 __func__); 133 } 134 135 RefPtr<MFTEncoder> encoder = new MFTEncoder( 136 mHardwareNotAllowed ? MFTEncoder::HWPreference::SoftwareOnly 137 : MFTEncoder::HWPreference::PreferHardware); 138 HRESULT hr; 139 mscom::EnsureMTA([&]() { hr = InitMFTEncoder(encoder); }); 140 141 if (FAILED(hr)) { 142 _com_error error(hr); 143 WMF_ENC_LOGE("init MFTEncoder: error = 0x%lX, %ls", hr, 144 error.ErrorMessage()); 145 return InitPromise::CreateAndReject( 146 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 147 RESULT_DETAIL("Can't create the MFT encoder.")), 148 __func__); 149 } 150 151 mEncoder = std::move(encoder); 152 InitializeConfigData(); 153 mIsHardwareAccelerated = mEncoder->IsHardwareAccelerated(); 154 WMF_ENC_LOGD("HW accelerated: %s", mIsHardwareAccelerated ? "yes" : "no"); 155 cleanup.release(); 156 return InitPromise::CreateAndResolve(true, __func__); 157 } 158 159 HRESULT WMFMediaDataEncoder::InitMFTEncoder(RefPtr<MFTEncoder>& aEncoder) { 160 HRESULT hr = aEncoder->Create(CodecToSubtype(mConfig.mCodec), mConfig.mSize, 161 mConfig.mCodecSpecific); 162 if (FAILED(hr)) { 163 _com_error error(hr); 164 WMF_ENC_LOGE("MFTEncoder::Create: error = 0x%lX, %ls", hr, 165 error.ErrorMessage()); 166 return hr; 167 } 168 169 hr = aEncoder->SetModes(mConfig); 170 if (FAILED(hr)) { 171 _com_error error(hr); 172 WMF_ENC_LOGE("MFTEncoder::SetMode: error = 0x%lX, %ls", hr, 173 error.ErrorMessage()); 174 return hr; 175 } 176 177 hr = SetMediaTypes(aEncoder, mConfig); 178 if (FAILED(hr)) { 179 _com_error error(hr); 180 WMF_ENC_LOGE("MFTEncoder::SetMediaType: error = 0x%lX, %ls", hr, 181 error.ErrorMessage()); 182 return hr; 183 } 184 185 return S_OK; 186 } 187 188 void WMFMediaDataEncoder::InitializeConfigData() { 189 AssertOnTaskQueue(); 190 MOZ_ASSERT(mEncoder); 191 192 if (mConfig.mCodec != CodecType::H264) { 193 return; 194 } 195 196 auto r = mEncoder->GetMPEGSequenceHeader(); 197 if (r.isErr()) { 198 WMF_ENC_LOGE("GetMPEGSequenceHeader failed"); 199 return; 200 } 201 202 nsTArray<UINT8> header = r.unwrap(); 203 SetConfigData(header); 204 } 205 206 void WMFMediaDataEncoder::SetConfigData(const nsTArray<UINT8>& aHeader) { 207 AssertOnTaskQueue(); 208 MOZ_ASSERT(mEncoder); 209 210 if (mConfig.mCodec != CodecType::H264) { 211 return; 212 } 213 214 mConfigData = 215 aHeader.Length() > 0 ? ParseH264Parameters(aHeader, IsAnnexB()) : nullptr; 216 WMF_ENC_LOGD("ConfigData has been updated to %zu bytes", 217 mConfigData ? mConfigData->Length() : 0); 218 } 219 220 RefPtr<EncodePromise> WMFMediaDataEncoder::ProcessEncode( 221 RefPtr<const VideoData>&& aSample) { 222 AssertOnTaskQueue(); 223 MOZ_ASSERT(mEncoder); 224 MOZ_ASSERT(aSample); 225 MOZ_ASSERT(mEncodePromise.IsEmpty()); 226 MOZ_ASSERT(!mEncodeRequest.Exists()); 227 228 WMF_ENC_LOGD("ProcessEncode ts=%s duration=%s", 229 aSample->mTime.ToString().get(), 230 aSample->mDuration.ToString().get()); 231 232 RefPtr<IMFSample> nv12 = ConvertToNV12InputSample(std::move(aSample)); 233 if (!nv12) { 234 WMF_ENC_LOGE("failed to convert sample into NV12 format"); 235 return EncodePromise::CreateAndReject( 236 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 237 RESULT_DETAIL("Failed to convert sample into NV12 format")), 238 __func__); 239 } 240 241 RefPtr<EncodePromise> p = mEncodePromise.Ensure(__func__); 242 243 nsTArray<MFTEncoder::InputSample> inputs; 244 inputs.AppendElement(MFTEncoder::InputSample{ 245 .mSample = nv12.forget(), .mKeyFrameRequested = aSample->mKeyframe}); 246 mEncoder->Encode(std::move(inputs)) 247 ->Then( 248 GetCurrentSerialEventTarget(), __func__, 249 [self = RefPtr<WMFMediaDataEncoder>(this)]( 250 MFTEncoder::EncodedData&& aOutput) { 251 self->mEncodeRequest.Complete(); 252 self->mEncodePromise.Resolve( 253 self->ProcessOutputSamples(std::move(aOutput)), __func__); 254 }, 255 [self = 256 RefPtr<WMFMediaDataEncoder>(this)](const MediaResult& aError) { 257 WMF_ENC_SLOGE("Encode failed: %s", aError.Description().get()); 258 self->mEncodeRequest.Complete(); 259 self->mEncodePromise.Reject(aError, __func__); 260 }) 261 ->Track(mEncodeRequest); 262 263 return p; 264 } 265 266 RefPtr<EncodePromise> WMFMediaDataEncoder::ProcessEncodeBatch( 267 nsTArray<RefPtr<const VideoData>>&& aSamples) { 268 AssertOnTaskQueue(); 269 MOZ_ASSERT(mEncoder); 270 MOZ_ASSERT(!aSamples.IsEmpty()); 271 MOZ_ASSERT(mEncodePromise.IsEmpty()); 272 MOZ_ASSERT(!mEncodeRequest.Exists()); 273 274 WMF_ENC_LOGD("ProcessEncodeBatch: num of samples=%zu", aSamples.Length()); 275 276 nsTArray<MFTEncoder::InputSample> inputs; 277 for (auto& sample : aSamples) { 278 RefPtr<IMFSample> nv12 = ConvertToNV12InputSample(std::move(sample)); 279 if (!nv12) { 280 WMF_ENC_LOGE( 281 "failed to convert samples(ts=%s duration=%s) into NV12 format", 282 sample->mTime.ToString().get(), sample->mDuration.ToString().get()); 283 return EncodePromise::CreateAndReject( 284 MediaResult( 285 NS_ERROR_DOM_MEDIA_FATAL_ERR, 286 RESULT_DETAIL("Failed to convert sample into NV12 format")), 287 __func__); 288 } 289 inputs.AppendElement(MFTEncoder::InputSample{ 290 .mSample = std::move(nv12), .mKeyFrameRequested = sample->mKeyframe}); 291 } 292 293 RefPtr<EncodePromise> p = mEncodePromise.Ensure(__func__); 294 295 mEncoder->Encode(std::move(inputs)) 296 ->Then( 297 GetCurrentSerialEventTarget(), __func__, 298 [self = RefPtr<WMFMediaDataEncoder>(this)]( 299 MFTEncoder::EncodedData&& aOutput) { 300 self->mEncodeRequest.Complete(); 301 self->mEncodePromise.Resolve( 302 self->ProcessOutputSamples(std::move(aOutput)), __func__); 303 }, 304 [self = 305 RefPtr<WMFMediaDataEncoder>(this)](const MediaResult& aError) { 306 WMF_ENC_SLOGE("Encode failed: %s", aError.Description().get()); 307 self->mEncodeRequest.Complete(); 308 self->mEncodePromise.Reject(aError, __func__); 309 }) 310 ->Track(mEncodeRequest); 311 312 return p; 313 } 314 315 RefPtr<EncodePromise> WMFMediaDataEncoder::ProcessDrain() { 316 AssertOnTaskQueue(); 317 MOZ_ASSERT(mEncoder); 318 MOZ_ASSERT(mDrainPromise.IsEmpty()); 319 MOZ_ASSERT(!mDrainRequest.Exists()); 320 321 WMF_ENC_LOGD("ProcessDrain"); 322 323 RefPtr<EncodePromise> p = mDrainPromise.Ensure(__func__); 324 325 mEncoder->Drain() 326 ->Then( 327 GetCurrentSerialEventTarget(), __func__, 328 [self = RefPtr<WMFMediaDataEncoder>(this)]( 329 MFTEncoder::EncodedData&& aOutput) { 330 self->mDrainRequest.Complete(); 331 self->mDrainPromise.Resolve( 332 self->ProcessOutputSamples(std::move(aOutput)), __func__); 333 }, 334 [self = 335 RefPtr<WMFMediaDataEncoder>(this)](const MediaResult& aError) { 336 WMF_ENC_SLOGE("Drain failed: %s", aError.Description().get()); 337 self->mDrainRequest.Complete(); 338 self->mDrainPromise.Reject(aError, __func__); 339 }) 340 ->Track(mDrainRequest); 341 342 return p; 343 } 344 345 already_AddRefed<IMFSample> WMFMediaDataEncoder::ConvertToNV12InputSample( 346 RefPtr<const VideoData>&& aData) { 347 AssertOnTaskQueue(); 348 MOZ_ASSERT(mEncoder); 349 350 AUTO_MARKER("::ConvertToNV12InputSample"); 351 352 size_t mBufferLength = 0; 353 354 const int32_t ySrtride = mConfig.mSize.width; 355 const int32_t uvStride = ySrtride; 356 357 const int32_t yHeight = mConfig.mSize.height; 358 const int32_t uvHeight = yHeight / 2 + (yHeight % 2); 359 360 CheckedInt<size_t> yLength(ySrtride); 361 yLength *= yHeight; 362 if (!yLength.isValid()) { 363 WMF_ENC_LOGE("dest yLength overflows"); 364 return nullptr; 365 } 366 367 CheckedInt<size_t> uvLength(uvStride); 368 uvLength *= uvHeight; 369 if (!uvLength.isValid()) { 370 WMF_ENC_LOGE("dest uvLength overflows"); 371 return nullptr; 372 } 373 374 CheckedInt<size_t> length(yLength); 375 length += uvLength; 376 if (!length.isValid()) { 377 WMF_ENC_LOGE("dest length overflows"); 378 return nullptr; 379 } 380 mBufferLength = length.value(); 381 382 RefPtr<IMFSample> input; 383 HRESULT hr = mEncoder->CreateInputSample(&input, mBufferLength); 384 if (FAILED(hr)) { 385 _com_error error(hr); 386 WMF_ENC_LOGE("CreateInputSample: error = 0x%lX, %ls", hr, 387 error.ErrorMessage()); 388 return nullptr; 389 } 390 391 RefPtr<IMFMediaBuffer> buffer; 392 hr = input->GetBufferByIndex(0, getter_AddRefs(buffer)); 393 if (FAILED(hr)) { 394 _com_error error(hr); 395 WMF_ENC_LOGE("GetBufferByIndex: error = 0x%lX, %ls", hr, 396 error.ErrorMessage()); 397 return nullptr; 398 } 399 400 hr = buffer->SetCurrentLength(mBufferLength); 401 if (FAILED(hr)) { 402 _com_error error(hr); 403 WMF_ENC_LOGE("SetCurrentLength: error = 0x%lX, %ls", hr, 404 error.ErrorMessage()); 405 return nullptr; 406 } 407 408 LockBuffer lockBuffer(buffer); 409 hr = lockBuffer.Result(); 410 if (FAILED(hr)) { 411 _com_error error(hr); 412 WMF_ENC_LOGE("LockBuffer: error = 0x%lX, %ls", hr, error.ErrorMessage()); 413 return nullptr; 414 } 415 416 nsresult rv = ConvertToNV12(aData->mImage, lockBuffer.Data(), ySrtride, 417 lockBuffer.Data() + yLength.value(), uvStride, 418 mConfig.mSize); 419 if (NS_FAILED(rv)) { 420 WMF_ENC_LOGE("Failed to convert to NV12"); 421 return nullptr; 422 } 423 424 hr = input->SetSampleTime(UsecsToHNs(aData->mTime.ToMicroseconds())); 425 if (FAILED(hr)) { 426 _com_error error(hr); 427 WMF_ENC_LOGE("SetSampleTime: error = 0x%lX, %ls", hr, error.ErrorMessage()); 428 return nullptr; 429 } 430 431 hr = input->SetSampleDuration(UsecsToHNs(aData->mDuration.ToMicroseconds())); 432 if (FAILED(hr)) { 433 _com_error error(hr); 434 WMF_ENC_LOGE("SetSampleDuration: error = 0x%lX, %ls", hr, 435 error.ErrorMessage()); 436 return nullptr; 437 } 438 439 return input.forget(); 440 } 441 442 MediaDataEncoder::EncodedData WMFMediaDataEncoder::ProcessOutputSamples( 443 nsTArray<MFTEncoder::OutputSample>&& aSamples) { 444 EncodedData frames; 445 446 WMF_ENC_LOGD("ProcessOutputSamples: %zu frames", aSamples.Length()); 447 448 for (MFTEncoder::OutputSample& sample : aSamples) { 449 RefPtr<MediaRawData> frame = OutputSampleToMediaData(sample); 450 if (frame) { 451 frames.AppendElement(std::move(frame)); 452 } else { 453 WMF_ENC_LOGE("failed to convert output frame"); 454 } 455 } 456 return frames; 457 } 458 459 already_AddRefed<MediaRawData> WMFMediaDataEncoder::OutputSampleToMediaData( 460 MFTEncoder::OutputSample& aSample) { 461 AssertOnTaskQueue(); 462 MOZ_ASSERT(aSample.mSample); 463 464 RefPtr<IMFMediaBuffer> buffer; 465 HRESULT hr = aSample.mSample->GetBufferByIndex(0, getter_AddRefs(buffer)); 466 NS_ENSURE_TRUE(SUCCEEDED(hr), nullptr); 467 468 LockBuffer lockBuffer(buffer); 469 NS_ENSURE_TRUE(SUCCEEDED(lockBuffer.Result()), nullptr); 470 471 LONGLONG time = 0; 472 hr = aSample.mSample->GetSampleTime(&time); 473 NS_ENSURE_TRUE(SUCCEEDED(hr), nullptr); 474 475 LONGLONG duration = 0; 476 hr = aSample.mSample->GetSampleDuration(&duration); 477 NS_ENSURE_TRUE(SUCCEEDED(hr), nullptr); 478 479 bool isKeyframe = MFGetAttributeUINT32(aSample.mSample, 480 MFSampleExtension_CleanPoint, false); 481 482 auto frame = MakeRefPtr<MediaRawData>(); 483 484 if (mConfig.mCodec == CodecType::H264 && 485 mConfig.mScalabilityMode != ScalabilityMode::None) { 486 auto maybeId = 487 H264::ExtractSVCTemporalId(lockBuffer.Data(), lockBuffer.Length()); 488 frame->mTemporalLayerId = Some(maybeId.unwrapOr(0)); 489 } 490 491 if (!aSample.mHeader.IsEmpty()) { 492 SetConfigData(aSample.mHeader); 493 } 494 495 if (!WriteFrameData(frame, lockBuffer, isKeyframe)) { 496 return nullptr; 497 } 498 499 frame->mTime = media::TimeUnit::FromMicroseconds(HNsToUsecs(time)); 500 frame->mDuration = media::TimeUnit::FromMicroseconds(HNsToUsecs(duration)); 501 frame->mKeyframe = isKeyframe; 502 503 WMF_ENC_LOGD("converted MediaData: ts=%s", frame->mTime.ToString().get()); 504 505 return frame.forget(); 506 } 507 508 bool WMFMediaDataEncoder::IsAnnexB() const { 509 MOZ_ASSERT(mConfig.mCodec == CodecType::H264); 510 return mConfig.mCodecSpecific.as<H264Specific>().mFormat == 511 H264BitStreamFormat::ANNEXB; 512 } 513 514 bool WMFMediaDataEncoder::WriteFrameData(RefPtr<MediaRawData>& aDest, 515 LockBuffer& aSrc, bool aIsKeyframe) { 516 // From raw encoded data, write in avCC or AnnexB format depending on the 517 // config. 518 519 if (mConfig.mCodec == CodecType::H264) { 520 size_t prependLength = 0; 521 RefPtr<MediaByteBuffer> avccHeader; 522 if (aIsKeyframe && mConfigData) { 523 if (IsAnnexB()) { 524 const nsTArray<NAL_TYPES> aTypes = {H264_NAL_SPS, H264_NAL_PPS}; 525 if (!AnnexB::FindAllNalTypes( 526 Span<const uint8_t>(aSrc.Data(), aSrc.Length()), aTypes)) { 527 prependLength = mConfigData->Length(); 528 } 529 } else { 530 avccHeader = mConfigData; 531 } 532 } 533 534 UniquePtr<MediaRawDataWriter> writer(aDest->CreateWriter()); 535 if (!writer->SetSize(prependLength + aSrc.Length())) { 536 WMF_ENC_LOGE("fail to allocate output buffer"); 537 return false; 538 } 539 540 if (prependLength > 0) { 541 PodCopy(writer->Data(), mConfigData->Elements(), prependLength); 542 } 543 PodCopy(writer->Data() + prependLength, aSrc.Data(), aSrc.Length()); 544 545 if (!IsAnnexB() && !AnnexB::ConvertSampleToAVCC(aDest, avccHeader)) { 546 WMF_ENC_LOGE("fail to convert annex-b sample to AVCC"); 547 return false; 548 } 549 550 return true; 551 } 552 UniquePtr<MediaRawDataWriter> writer(aDest->CreateWriter()); 553 if (!writer->SetSize(aSrc.Length())) { 554 WMF_ENC_LOGE("fail to allocate output buffer"); 555 return false; 556 } 557 558 PodCopy(writer->Data(), aSrc.Data(), aSrc.Length()); 559 return true; 560 } 561 562 #undef AUTO_MARKER 563 564 } // namespace mozilla