WebrtcMediaDataDecoderCodec.cpp (7671B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #include "WebrtcMediaDataDecoderCodec.h" 6 7 #include "ImageContainer.h" 8 #include "MediaDataDecoderProxy.h" 9 #include "PDMFactory.h" 10 #include "VideoUtils.h" 11 #include "mozilla/StaticPrefs_media.h" 12 #include "mozilla/layers/ImageBridgeChild.h" 13 #include "mozilla/media/MediaUtils.h" 14 // #include "modules/video_coding/include/video_codec_interface.h" 15 #include "modules/video_coding/include/video_error_codes.h" 16 17 namespace mozilla { 18 19 WebrtcMediaDataDecoder::WebrtcMediaDataDecoder(nsACString& aCodecMimeType, 20 TrackingId aTrackingId) 21 : mThreadPool(GetMediaThreadPool(MediaThreadType::SUPERVISOR)), 22 mTaskQueue(TaskQueue::Create(do_AddRef(mThreadPool), 23 "WebrtcMediaDataDecoder::mTaskQueue")), 24 mImageContainer(MakeAndAddRef<layers::ImageContainer>( 25 layers::ImageUsageType::Webrtc, 26 layers::ImageContainer::ASYNCHRONOUS)), 27 mFactory(new PDMFactory()), 28 mTrackType(TrackInfo::kUndefinedTrack), 29 mCodecType(aCodecMimeType), 30 mTrackingId(std::move(aTrackingId)) {} 31 32 WebrtcMediaDataDecoder::~WebrtcMediaDataDecoder() {} 33 34 bool WebrtcMediaDataDecoder::Configure( 35 const webrtc::VideoDecoder::Settings& settings) { 36 nsCString codec; 37 mTrackType = TrackInfo::kVideoTrack; 38 mInfo = VideoInfo(settings.max_render_resolution().Width(), 39 settings.max_render_resolution().Height()); 40 mInfo.mMimeType = mCodecType; 41 42 #ifdef MOZ_WIDGET_GTK 43 if (mInfo.mMimeType.EqualsLiteral("video/vp8") && 44 !StaticPrefs::media_navigator_mediadatadecoder_vp8_hardware_enabled()) { 45 mDisabledHardwareAcceleration = true; 46 } 47 #endif 48 49 return WEBRTC_VIDEO_CODEC_OK == CreateDecoder(); 50 } 51 52 int32_t WebrtcMediaDataDecoder::Decode(const webrtc::EncodedImage& aInputImage, 53 bool aMissingFrames, 54 int64_t aRenderTimeMs) { 55 if (!mCallback || !mDecoder) { 56 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 57 } 58 59 if (!aInputImage.data() || !aInputImage.size()) { 60 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 61 } 62 63 // Always start with a complete key frame. 64 if (mNeedKeyframe) { 65 if (aInputImage._frameType != webrtc::VideoFrameType::kVideoFrameKey) 66 return WEBRTC_VIDEO_CODEC_ERROR; 67 // We have a key frame - is it complete? 68 mNeedKeyframe = false; 69 } 70 71 auto disabledHardwareAcceleration = 72 MakeScopeExit([&] { mDisabledHardwareAcceleration = true; }); 73 74 RefPtr<MediaRawData> compressedFrame = 75 new MediaRawData(aInputImage.data(), aInputImage.size()); 76 if (!compressedFrame->Data()) { 77 return WEBRTC_VIDEO_CODEC_MEMORY; 78 } 79 80 compressedFrame->mTime = 81 media::TimeUnit::FromMicroseconds(aInputImage.RtpTimestamp()); 82 compressedFrame->mTimecode = 83 media::TimeUnit::FromMicroseconds(aRenderTimeMs * 1000); 84 compressedFrame->mKeyframe = 85 aInputImage._frameType == webrtc::VideoFrameType::kVideoFrameKey; 86 { 87 media::Await( 88 do_AddRef(mThreadPool), mDecoder->Decode(compressedFrame), 89 [&](const MediaDataDecoder::DecodedData& aResults) { 90 mResults = aResults.Clone(); 91 mError = NS_OK; 92 }, 93 [&](const MediaResult& aError) { mError = aError; }); 94 95 for (auto& frame : mResults) { 96 MOZ_ASSERT(frame->mType == MediaData::Type::VIDEO_DATA); 97 RefPtr<VideoData> video = frame->As<VideoData>(); 98 MOZ_ASSERT(video); 99 if (!video->mImage) { 100 // Nothing to display. 101 continue; 102 } 103 webrtc::scoped_refptr<ImageBuffer> image( 104 new webrtc::RefCountedObject<ImageBuffer>(std::move(video->mImage))); 105 106 auto videoFrame = webrtc::VideoFrame::Builder() 107 .set_video_frame_buffer(image) 108 .set_timestamp_rtp(aInputImage.RtpTimestamp()) 109 .set_rotation(aInputImage.rotation_) 110 .build(); 111 mCallback->Decoded(videoFrame); 112 } 113 mResults.Clear(); 114 } 115 116 if (NS_FAILED(mError) && mError != NS_ERROR_DOM_MEDIA_CANCELED) { 117 CreateDecoder(); 118 return WEBRTC_VIDEO_CODEC_ERROR; 119 } 120 121 if (NS_FAILED(mError)) { 122 return WEBRTC_VIDEO_CODEC_ERROR; 123 } 124 125 disabledHardwareAcceleration.release(); 126 return WEBRTC_VIDEO_CODEC_OK; 127 } 128 129 int32_t WebrtcMediaDataDecoder::RegisterDecodeCompleteCallback( 130 webrtc::DecodedImageCallback* aCallback) { 131 mCallback = aCallback; 132 return WEBRTC_VIDEO_CODEC_OK; 133 } 134 135 int32_t WebrtcMediaDataDecoder::Release() { 136 if (mDecoder) { 137 RefPtr<MediaDataDecoder> decoder = std::move(mDecoder); 138 decoder->Flush()->Then(mTaskQueue, __func__, 139 [decoder]() { decoder->Shutdown(); }); 140 } 141 142 mNeedKeyframe = true; 143 mError = NS_OK; 144 145 return WEBRTC_VIDEO_CODEC_OK; 146 } 147 148 bool WebrtcMediaDataDecoder::OnTaskQueue() const { 149 return mTaskQueue->IsOnCurrentThread(); 150 } 151 152 int32_t WebrtcMediaDataDecoder::CreateDecoder() { 153 RefPtr<layers::KnowsCompositor> knowsCompositor = 154 layers::ImageBridgeChild::GetSingleton(); 155 156 if (mDecoder) { 157 Release(); 158 } 159 160 RefPtr<TaskQueue> tq = 161 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), 162 "webrtc decode TaskQueue"); 163 RefPtr<MediaDataDecoder> decoder; 164 165 media::Await(do_AddRef(mThreadPool), InvokeAsync(tq, __func__, [&] { 166 RefPtr<GenericPromise> p = 167 mFactory 168 ->CreateDecoder( 169 {mInfo, 170 CreateDecoderParams::OptionSet( 171 CreateDecoderParams::Option::LowLatency, 172 CreateDecoderParams::Option::FullH264Parsing, 173 CreateDecoderParams::Option:: 174 ErrorIfNoInitializationData, 175 mDisabledHardwareAcceleration 176 ? CreateDecoderParams::Option:: 177 HardwareDecoderNotAllowed 178 : CreateDecoderParams::Option::Default), 179 mTrackType, mImageContainer, knowsCompositor, 180 Some(mTrackingId)}) 181 ->Then( 182 tq, __func__, 183 [&](RefPtr<MediaDataDecoder>&& aDecoder) { 184 decoder = std::move(aDecoder); 185 return GenericPromise::CreateAndResolve( 186 true, __func__); 187 }, 188 [](const MediaResult& aResult) { 189 return GenericPromise::CreateAndReject( 190 NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); 191 }); 192 return p; 193 })); 194 195 if (!decoder) { 196 return WEBRTC_VIDEO_CODEC_ERROR; 197 } 198 199 // We need to wrap our decoder in a MediaDataDecoderProxy so that it always 200 // run on an nsISerialEventTarget (which the webrtc code doesn't do) 201 mDecoder = new MediaDataDecoderProxy(decoder.forget(), tq.forget()); 202 203 media::Await( 204 do_AddRef(mThreadPool), mDecoder->Init(), 205 [&](TrackInfo::TrackType) { mError = NS_OK; }, 206 [&](const MediaResult& aError) { mError = aError; }); 207 208 return NS_SUCCEEDED(mError) ? WEBRTC_VIDEO_CODEC_OK 209 : WEBRTC_VIDEO_CODEC_ERROR; 210 } 211 212 } // namespace mozilla