FFmpegDataDecoder.cpp (17282B)
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include <string.h> 8 9 #include "libavcodec/avcodec.h" 10 #include "libavutil/dict.h" 11 #ifdef __GNUC__ 12 # include <unistd.h> 13 #endif 14 15 #include "FFmpegDataDecoder.h" 16 #include "FFmpegLibs.h" 17 #include "FFmpegLog.h" 18 #include "FFmpegUtils.h" 19 #include "VideoUtils.h" 20 #include "mozilla/StaticPrefs_media.h" 21 #include "mozilla/TaskQueue.h" 22 #include "prsystem.h" 23 24 #if defined(MOZ_WIDGET_ANDROID) && defined(FFVPX_VERSION) 25 # include "mozilla/MediaDrmRemoteCDMParent.h" 26 #endif 27 28 namespace mozilla { 29 30 StaticMutex FFmpegDataDecoder<LIBAV_VER>::sMutex; 31 32 FFmpegDataDecoder<LIBAV_VER>::FFmpegDataDecoder(const FFmpegLibWrapper* aLib, 33 AVCodecID aCodecID, 34 PRemoteCDMActor* aCDM) 35 : mLib(aLib), 36 mCodecContext(nullptr), 37 mCodecParser(nullptr), 38 mFrame(nullptr), 39 mExtraData(nullptr), 40 mCodecID(aCodecID), 41 mVideoCodec(IsVideoCodec(aCodecID)), 42 mTaskQueue(TaskQueue::Create( 43 GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), 44 "FFmpegDataDecoder")), 45 mLastInputDts(media::TimeUnit::FromNegativeInfinity()) { 46 MOZ_ASSERT(aLib); 47 MOZ_COUNT_CTOR(FFmpegDataDecoder); 48 49 #if defined(MOZ_WIDGET_ANDROID) && defined(FFVPX_VERSION) 50 if (aCDM) { 51 if (PRemoteCDMParent* parentCDM = aCDM->AsPRemoteCDMParent()) { 52 mCDM = static_cast<MediaDrmRemoteCDMParent*>(parentCDM); 53 } 54 } 55 #elif defined(DEBUG) 56 MOZ_ASSERT(!aCDM); 57 #else 58 (void)aCDM; 59 #endif 60 } 61 62 FFmpegDataDecoder<LIBAV_VER>::~FFmpegDataDecoder() { 63 MOZ_COUNT_DTOR(FFmpegDataDecoder); 64 if (mCodecParser) { 65 mLib->av_parser_close(mCodecParser); 66 mCodecParser = nullptr; 67 } 68 } 69 70 MediaResult FFmpegDataDecoder<LIBAV_VER>::AllocateExtraData() { 71 if (mExtraData) { 72 mCodecContext->extradata_size = mExtraData->Length(); 73 // FFmpeg may use SIMD instructions to access the data which reads the 74 // data in 32 bytes block. Must ensure we have enough data to read. 75 uint32_t padding_size = 76 #if LIBAVCODEC_VERSION_MAJOR >= 58 77 AV_INPUT_BUFFER_PADDING_SIZE; 78 #else 79 FF_INPUT_BUFFER_PADDING_SIZE; 80 #endif 81 mCodecContext->extradata = static_cast<uint8_t*>( 82 mLib->av_malloc(mExtraData->Length() + padding_size)); 83 if (!mCodecContext->extradata) { 84 return MediaResult(NS_ERROR_OUT_OF_MEMORY, 85 RESULT_DETAIL("Couldn't init ffmpeg extradata")); 86 } 87 memcpy(mCodecContext->extradata, mExtraData->Elements(), 88 mExtraData->Length()); 89 } else { 90 mCodecContext->extradata_size = 0; 91 } 92 93 return NS_OK; 94 } 95 96 // Note: This doesn't run on the ffmpeg TaskQueue, it runs on some other media 97 // taskqueue 98 MediaResult FFmpegDataDecoder<LIBAV_VER>::InitSWDecoder( 99 AVDictionary** aOptions) { 100 FFMPEG_LOG("Initialising FFmpeg decoder"); 101 102 AVCodec* codec = FindSoftwareAVCodec(mLib, mCodecID); 103 if (!codec) { 104 FFMPEG_LOG(" couldn't find ffmpeg decoder for codec id %d", mCodecID); 105 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 106 RESULT_DETAIL("unable to find codec")); 107 } 108 109 return InitDecoder(codec, aOptions); 110 } 111 112 #if defined(MOZ_WIDGET_ANDROID) && defined(USING_MOZFFVPX) 113 /* static */ 114 void FFmpegDataDecoder<LIBAV_VER>::CryptoInfoAddRef(void* aCryptoInfo) { 115 reinterpret_cast<MediaDrmCryptoInfo*>(aCryptoInfo)->AddRef(); 116 } 117 118 /* static */ 119 void FFmpegDataDecoder<LIBAV_VER>::CryptoInfoRelease(void* aCryptoInfo) { 120 reinterpret_cast<MediaDrmCryptoInfo*>(aCryptoInfo)->Release(); 121 } 122 123 MediaResult FFmpegDataDecoder<LIBAV_VER>::MaybeAttachCryptoInfo( 124 MediaRawData* aSample, AVPacket* aPacket) { 125 if (!aSample->mCrypto.IsEncrypted()) { 126 return NS_OK; 127 } 128 129 if (NS_WARN_IF(!mCDM)) { 130 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 131 RESULT_DETAIL("missing CDM for encrypted sample")); 132 } 133 134 RefPtr<MediaDrmCryptoInfo> cryptoInfo = mCDM->CreateCryptoInfo(aSample); 135 if (NS_WARN_IF(!cryptoInfo)) { 136 return MediaResult( 137 NS_ERROR_DOM_MEDIA_FATAL_ERR, 138 RESULT_DETAIL("missing MediaDrmCryptoInfo for encrypted sample")); 139 } 140 141 aPacket->moz_ndk_crypto_info = cryptoInfo->GetNdkCryptoInfo(); 142 if (NS_WARN_IF(!aPacket->moz_ndk_crypto_info)) { 143 return MediaResult( 144 NS_ERROR_DOM_MEDIA_FATAL_ERR, 145 RESULT_DETAIL("missing AMediaCodecCryptoInfo for encrypted sample")); 146 } 147 148 aPacket->moz_crypto_info = cryptoInfo.forget().take(); 149 aPacket->moz_crypto_info_addref = CryptoInfoAddRef; 150 aPacket->moz_crypto_info_release = CryptoInfoRelease; 151 152 FFMPEG_LOG(" encrypted packet, ndk_crypto_info=%p", 153 aPacket->moz_ndk_crypto_info); 154 return NS_OK; 155 } 156 157 MediaResult FFmpegDataDecoder<LIBAV_VER>::MaybeAttachCDM() { 158 MOZ_ASSERT(mCodecContext); 159 160 if (!mCDM) { 161 return NS_OK; 162 } 163 164 if (!(mCodecContext->codec->capabilities & AV_CODEC_CAP_HARDWARE)) { 165 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 166 RESULT_DETAIL("CDM requires MediaCodec decoder")); 167 } 168 169 mCrypto = mCDM->GetCrypto(); 170 if (NS_WARN_IF(!mCrypto)) { 171 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 172 RESULT_DETAIL("missing MediaDrmCrypto from CDM")); 173 } 174 175 mCodecContext->moz_ndk_crypto = mCrypto->GetNdkCrypto(); 176 if (NS_WARN_IF(!mCodecContext->moz_ndk_crypto)) { 177 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 178 RESULT_DETAIL("missing AMediaCrypto from CDM")); 179 } 180 181 FFMPEG_LOG(" attached CDM, ndk_crypto=%p", mCodecContext->moz_ndk_crypto); 182 return NS_OK; 183 } 184 185 void FFmpegDataDecoder<LIBAV_VER>::MaybeDetachCDM() { 186 if (mCodecContext) { 187 mCodecContext->moz_ndk_crypto = nullptr; 188 } 189 190 if (mCDM) { 191 mCDM = nullptr; 192 } 193 } 194 #endif 195 196 MediaResult FFmpegDataDecoder<LIBAV_VER>::InitDecoder(AVCodec* aCodec, 197 AVDictionary** aOptions) { 198 FFMPEG_LOG(" codec %s : %s", aCodec->name, aCodec->long_name); 199 200 StaticMutexAutoLock mon(sMutex); 201 202 if (!(mCodecContext = mLib->avcodec_alloc_context3(aCodec))) { 203 FFMPEG_LOG(" couldn't allocate ffmpeg context for codec %s", aCodec->name); 204 return MediaResult(NS_ERROR_OUT_OF_MEMORY, 205 RESULT_DETAIL("Couldn't init ffmpeg context")); 206 } 207 208 if (NeedParser()) { 209 MOZ_ASSERT(mCodecParser == nullptr); 210 mCodecParser = mLib->av_parser_init(mCodecID); 211 if (mCodecParser) { 212 mCodecParser->flags |= ParserFlags(); 213 } 214 } 215 mCodecContext->opaque = this; 216 217 InitCodecContext(); 218 MediaResult ret = AllocateExtraData(); 219 if (NS_FAILED(ret)) { 220 FFMPEG_LOG(" couldn't allocate ffmpeg extra data for codec %s", 221 aCodec->name); 222 ReleaseCodecContext(); 223 return ret; 224 } 225 226 #if LIBAVCODEC_VERSION_MAJOR < 57 227 if (aCodec->capabilities & CODEC_CAP_DR1) { 228 mCodecContext->flags |= CODEC_FLAG_EMU_EDGE; 229 } 230 #endif 231 232 #if defined(MOZ_WIDGET_ANDROID) && defined(USING_MOZFFVPX) 233 ret = MaybeAttachCDM(); 234 if (NS_FAILED(ret)) { 235 ReleaseCodecContext(); 236 return ret; 237 } 238 #endif 239 240 if (mLib->avcodec_open2(mCodecContext, aCodec, aOptions) < 0) { 241 ReleaseCodecContext(); 242 FFMPEG_LOG(" Couldn't open avcodec for %s", aCodec->name); 243 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, 244 RESULT_DETAIL("Couldn't open avcodec")); 245 } 246 247 FFMPEG_LOG(" FFmpeg decoder init successful."); 248 return NS_OK; 249 } 250 251 void FFmpegDataDecoder<LIBAV_VER>::ReleaseCodecContext() { 252 if (!mCodecContext) { 253 return; 254 } 255 #if LIBAVCODEC_VERSION_MAJOR < 57 256 mLib->avcodec_close(mCodecContext); 257 // avcodec_close only frees the extradata for encoders. 258 if (mCodecContext->extradata) { 259 mLib->av_freep(&mCodecContext->extradata); 260 } 261 mLib->av_freep(&mCodecContext); 262 #else 263 mLib->avcodec_free_context(&mCodecContext); 264 #endif 265 } 266 267 RefPtr<ShutdownPromise> FFmpegDataDecoder<LIBAV_VER>::Shutdown() { 268 RefPtr<FFmpegDataDecoder<LIBAV_VER>> self = this; 269 return InvokeAsync(mTaskQueue, __func__, [self]() { 270 self->ProcessShutdown(); 271 return self->mTaskQueue->BeginShutdown(); 272 }); 273 } 274 275 RefPtr<MediaDataDecoder::DecodePromise> FFmpegDataDecoder<LIBAV_VER>::Decode( 276 MediaRawData* aSample) { 277 return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__, 278 &FFmpegDataDecoder::ProcessDecode, aSample); 279 } 280 281 RefPtr<MediaDataDecoder::DecodePromise> 282 FFmpegDataDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample) { 283 AUTO_PROFILER_LABEL("FFmpegDataDecoder::ProcessDecode", MEDIA_PLAYBACK); 284 MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); 285 PROCESS_DECODE_LOG(aSample); 286 bool gotFrame = false; 287 DecodedData results; 288 MediaResult rv = DoDecode(aSample, &gotFrame, results); 289 if (NS_FAILED(rv)) { 290 return DecodePromise::CreateAndReject(rv, __func__); 291 } 292 return DecodePromise::CreateAndResolve(std::move(results), __func__); 293 } 294 295 MediaResult FFmpegDataDecoder<LIBAV_VER>::DoDecode( 296 MediaRawData* aSample, bool* aGotFrame, 297 MediaDataDecoder::DecodedData& aResults) { 298 MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); 299 300 uint8_t* inputData = const_cast<uint8_t*>(aSample->Data()); 301 size_t inputSize = aSample->Size(); 302 303 mLastInputDts = aSample->mTimecode; 304 305 if (inputData && mCodecParser) { // inputData is null when draining. 306 if (aGotFrame) { 307 *aGotFrame = false; 308 } 309 while (inputSize) { 310 uint8_t* data = inputData; 311 int size = inputSize; 312 int len = mLib->av_parser_parse2( 313 mCodecParser, mCodecContext, &data, &size, inputData, inputSize, 314 aSample->mTime.ToMicroseconds(), aSample->mTimecode.ToMicroseconds(), 315 aSample->mOffset); 316 if (size_t(len) > inputSize) { 317 return NS_ERROR_DOM_MEDIA_DECODE_ERR; 318 } 319 if (size) { 320 bool gotFrame = false; 321 MediaResult rv = DoDecode(aSample, data, size, &gotFrame, aResults); 322 if (NS_FAILED(rv)) { 323 return rv; 324 } 325 if (gotFrame && aGotFrame) { 326 *aGotFrame = true; 327 } 328 } 329 inputData += len; 330 inputSize -= len; 331 } 332 return NS_OK; 333 } 334 return DoDecode(aSample, inputData, inputSize, aGotFrame, aResults); 335 } 336 337 RefPtr<MediaDataDecoder::FlushPromise> FFmpegDataDecoder<LIBAV_VER>::Flush() { 338 return InvokeAsync(mTaskQueue, this, __func__, 339 &FFmpegDataDecoder<LIBAV_VER>::ProcessFlush); 340 } 341 342 RefPtr<MediaDataDecoder::DecodePromise> FFmpegDataDecoder<LIBAV_VER>::Drain() { 343 return InvokeAsync(mTaskQueue, this, __func__, 344 &FFmpegDataDecoder<LIBAV_VER>::ProcessDrain); 345 } 346 347 RefPtr<MediaDataDecoder::DecodePromise> 348 FFmpegDataDecoder<LIBAV_VER>::ProcessDrain() { 349 AUTO_PROFILER_LABEL("FFmpegDataDecoder::ProcessDrain", MEDIA_PLAYBACK); 350 MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); 351 FFMPEG_LOG("FFmpegDataDecoder: draining buffers"); 352 RefPtr<MediaRawData> empty(new MediaRawData()); 353 empty->mTimecode = mLastInputDts; 354 bool gotFrame = false; 355 DecodedData results; 356 // When draining the underlying FFmpeg decoder without encountering any 357 // problems, DoDecode will either return a single frame at a time until 358 // gotFrame is set to false, or it will return a block of frames with 359 // NS_ERROR_DOM_MEDIA_END_OF_STREAM (EOS). However, if any issue arises, such 360 // as pending data in the pipeline being corrupt or invalid, non-EOS errors 361 // like NS_ERROR_DOM_MEDIA_DECODE_ERR will be returned and must be handled 362 // accordingly. 363 RefPtr<MediaDataDecoder::DecodePromise> p = mDrainPromise.Ensure(__func__); 364 do { 365 MediaResult r = DoDecode(empty, &gotFrame, results); 366 if (NS_FAILED(r)) { 367 if (r.Code() == NS_ERROR_DOM_MEDIA_END_OF_STREAM) { 368 break; 369 } 370 if (r.Code() == NS_ERROR_NOT_AVAILABLE) { 371 if (results.IsEmpty()) { 372 return p; 373 } 374 break; 375 } 376 mDrainPromise.Reject(r, __func__); 377 return p; 378 } 379 } while (gotFrame); 380 mDrainPromise.Resolve(std::move(results), __func__); 381 return p; 382 } 383 384 RefPtr<MediaDataDecoder::FlushPromise> 385 FFmpegDataDecoder<LIBAV_VER>::ProcessFlush() { 386 AUTO_PROFILER_LABEL("FFmpegDataDecoder::ProcessFlush", MEDIA_PLAYBACK); 387 MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); 388 if (mCodecContext) { 389 FFMPEG_LOG("FFmpegDataDecoder: flushing buffers"); 390 mLib->avcodec_flush_buffers(mCodecContext); 391 } 392 if (mCodecParser) { 393 FFMPEG_LOG("FFmpegDataDecoder: reinitializing parser"); 394 mLib->av_parser_close(mCodecParser); 395 mCodecParser = mLib->av_parser_init(mCodecID); 396 } 397 return FlushPromise::CreateAndResolve(true, __func__); 398 } 399 400 void FFmpegDataDecoder<LIBAV_VER>::ProcessShutdown() { 401 AUTO_PROFILER_LABEL("FFmpegDataDecoder::ProcessShutdown", MEDIA_PLAYBACK); 402 MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); 403 StaticMutexAutoLock mon(sMutex); 404 405 if (mCodecContext) { 406 FFMPEG_LOG("FFmpegDataDecoder: shutdown"); 407 ReleaseCodecContext(); 408 #if LIBAVCODEC_VERSION_MAJOR >= 55 409 mLib->av_frame_free(&mFrame); 410 #elif LIBAVCODEC_VERSION_MAJOR == 54 411 mLib->avcodec_free_frame(&mFrame); 412 #else 413 mLib->av_freep(&mFrame); 414 #endif 415 } 416 } 417 418 AVFrame* FFmpegDataDecoder<LIBAV_VER>::PrepareFrame() { 419 MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); 420 #if LIBAVCODEC_VERSION_MAJOR >= 55 421 if (mFrame) { 422 mLib->av_frame_unref(mFrame); 423 } else { 424 mFrame = mLib->av_frame_alloc(); 425 } 426 #elif LIBAVCODEC_VERSION_MAJOR == 54 427 if (mFrame) { 428 mLib->avcodec_get_frame_defaults(mFrame); 429 } else { 430 mFrame = mLib->avcodec_alloc_frame(); 431 } 432 #else 433 mLib->av_freep(&mFrame); 434 mFrame = mLib->avcodec_alloc_frame(); 435 #endif 436 return mFrame; 437 } 438 439 /* static */ AVCodec* FFmpegDataDecoder<LIBAV_VER>::FindSoftwareAVCodec( 440 const FFmpegLibWrapper* aLib, AVCodecID aCodec) { 441 MOZ_ASSERT(aLib); 442 443 // We use this instead of MOZ_USE_HWDECODE because it is possible to disable 444 // support for hardware decoding in Firefox, while the system ffmpeg library 445 // still exposes the hardware codecs. 446 #if LIBAVCODEC_VERSION_MAJOR >= 58 447 AVCodec* fallbackCodec = nullptr; 448 void* opaque = nullptr; 449 while (AVCodec* codec = aLib->av_codec_iterate(&opaque)) { 450 if (codec->id != aCodec || !aLib->av_codec_is_decoder(codec)) { 451 continue; 452 } 453 454 if (codec->capabilities & AV_CODEC_CAP_HARDWARE) { 455 continue; 456 } 457 458 // We prefer to use our own OpenH264 decoder through the plugin over ffmpeg 459 // by default due to broken decoding with some versions. openh264 has broken 460 // decoding of some h264 videos so don't use it unless explicitly allowed 461 // for now. 462 if (strcmp(codec->name, "libopenh264") == 0) { 463 if (!StaticPrefs::media_ffmpeg_allow_openh264()) { 464 FFMPEGV_LOG("libopenh264 available but disabled by pref"); 465 } else if (!fallbackCodec) { 466 fallbackCodec = codec; 467 } 468 continue; 469 } 470 471 if (codec->capabilities & AV_CODEC_CAP_EXPERIMENTAL) { 472 if (!fallbackCodec) { 473 fallbackCodec = codec; 474 } 475 continue; 476 } 477 478 FFMPEGV_LOG("Using preferred software codec %s", codec->name); 479 return codec; 480 } 481 482 if (fallbackCodec) { 483 FFMPEGV_LOG("Using fallback software codec %s", fallbackCodec->name); 484 } 485 return fallbackCodec; 486 #else 487 AVCodec* codec = aLib->avcodec_find_decoder(aCodec); 488 if (codec) { 489 // We prefer to use our own OpenH264 decoder through the plugin over ffmpeg 490 // by default due to broken decoding with some versions. openh264 has broken 491 // decoding of some h264 videos so don't use it unless explicitly allowed 492 // for now. 493 if (strcmp(codec->name, "libopenh264") == 0 && 494 !StaticPrefs::media_ffmpeg_allow_openh264()) { 495 FFMPEGV_LOG("libopenh264 selected but disabled by pref"); 496 return nullptr; 497 } 498 499 FFMPEGV_LOG("Using preferred software codec %s", codec->name); 500 } 501 return codec; 502 #endif 503 } 504 505 #ifdef MOZ_USE_HWDECODE 506 /* static */ AVCodec* FFmpegDataDecoder<LIBAV_VER>::FindHardwareAVCodec( 507 const FFmpegLibWrapper* aLib, AVCodecID aCodec, 508 AVHWDeviceType aDeviceType) { 509 AVCodec* fallbackCodec = nullptr; 510 void* opaque = nullptr; 511 const bool ignoreDeviceType = aDeviceType == AV_HWDEVICE_TYPE_NONE; 512 while (AVCodec* codec = aLib->av_codec_iterate(&opaque)) { 513 if (codec->id != aCodec || !aLib->av_codec_is_decoder(codec)) { 514 continue; 515 } 516 517 bool hasHwConfig = 518 codec->capabilities & AV_CODEC_CAP_HARDWARE && ignoreDeviceType; 519 if (!hasHwConfig) { 520 for (int i = 0; const AVCodecHWConfig* config = 521 aLib->avcodec_get_hw_config(codec, i); 522 ++i) { 523 if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && 524 (ignoreDeviceType || config->device_type == aDeviceType)) { 525 hasHwConfig = true; 526 break; 527 } 528 } 529 } 530 531 if (!hasHwConfig) { 532 continue; 533 } 534 535 if (codec->capabilities & AV_CODEC_CAP_EXPERIMENTAL) { 536 if (!fallbackCodec) { 537 fallbackCodec = codec; 538 } 539 continue; 540 } 541 542 FFMPEGV_LOG("Using preferred hardware codec %s", codec->name); 543 return codec; 544 } 545 546 if (fallbackCodec) { 547 FFMPEGV_LOG("Using fallback hardware codec %s", fallbackCodec->name); 548 } 549 return nullptr; 550 } 551 #endif 552 553 } // namespace mozilla