[ tor-browser ].git.dasho

MFMediaEngineVideoStream.cpp (18488B)
      1 /* This Source Code Form is subject to the terms of the Mozilla Public
      2 * License, v. 2.0. If a copy of the MPL was not distributed with this
      3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      4 
      5 #include "MFMediaEngineVideoStream.h"
      6 
      7 #include "MFMediaEngineUtils.h"
      8 #include "mozilla/StaticPrefs_media.h"
      9 #include "mozilla/layers/DcompSurfaceImage.h"
     10 
     11 namespace mozilla {
     12 
     13 #define LOG(msg, ...)                           \
     14  MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug,   \
     15          ("MFMediaStream=%p (%s), " msg, this, \
     16           this->GetDescriptionName().get(), ##__VA_ARGS__))
     17 
     18 #define LOGV(msg, ...)                          \
     19  MOZ_LOG(gMFMediaEngineLog, LogLevel::Verbose, \
     20          ("MFMediaStream=%p (%s), " msg, this, \
     21           this->GetDescriptionName().get(), ##__VA_ARGS__))
     22 
     23 using Microsoft::WRL::ComPtr;
     24 using Microsoft::WRL::MakeAndInitialize;
     25 
     26 /* static */
     27 MFMediaEngineVideoStream* MFMediaEngineVideoStream::Create(
     28    uint64_t aStreamId, const TrackInfo& aInfo, bool aIsEncryptedCustomInit,
     29    MFMediaSource* aParentSource) {
     30  MFMediaEngineVideoStream* stream;
     31  MOZ_ASSERT(aInfo.IsVideo());
     32  if (FAILED(MakeAndInitialize<MFMediaEngineVideoStream>(
     33          &stream, aStreamId, aInfo, aIsEncryptedCustomInit, aParentSource))) {
     34    return nullptr;
     35  }
     36  stream->mStreamType =
     37      GetStreamTypeFromMimeType(aInfo.GetAsVideoInfo()->mMimeType);
     38  MOZ_ASSERT(StreamTypeIsVideo(stream->mStreamType));
     39  stream->mHasReceivedInitialCreateDecoderConfig = false;
     40  stream->SetDCompSurfaceHandle(INVALID_HANDLE_VALUE, gfx::IntSize{});
     41  return stream;
     42 }
     43 
     44 void MFMediaEngineVideoStream::SetKnowsCompositor(
     45    layers::KnowsCompositor* aKnowsCompositor) {
     46  ComPtr<MFMediaEngineVideoStream> self = this;
     47  (void)mTaskQueue->Dispatch(NS_NewRunnableFunction(
     48      "MFMediaEngineStream::SetKnowsCompositor",
     49      [self, knowCompositor = RefPtr<layers::KnowsCompositor>{aKnowsCompositor},
     50       this]() {
     51        mKnowsCompositor = knowCompositor;
     52        LOG("Set SetKnowsCompositor=%p", mKnowsCompositor.get());
     53        ResolvePendingPromisesIfNeeded();
     54      }));
     55 }
     56 
     57 void MFMediaEngineVideoStream::SetDCompSurfaceHandle(HANDLE aDCompSurfaceHandle,
     58                                                     gfx::IntSize aDisplay) {
     59  ComPtr<MFMediaEngineVideoStream> self = this;
     60  (void)mTaskQueue->Dispatch(NS_NewRunnableFunction(
     61      "MFMediaEngineStream::SetDCompSurfaceHandle",
     62      [self, aDCompSurfaceHandle, aDisplay, this]() {
     63        if (mDCompSurfaceHandle == aDCompSurfaceHandle) {
     64          return;
     65        }
     66        mDCompSurfaceHandle = aDCompSurfaceHandle;
     67        mNeedRecreateImage = true;
     68        {
     69          MutexAutoLock lock(mMutex);
     70          if (aDCompSurfaceHandle != INVALID_HANDLE_VALUE &&
     71              aDisplay != mDisplay) {
     72            LOG("Update display [%dx%d] -> [%dx%d]", mDisplay.Width(),
     73                mDisplay.Height(), aDisplay.Width(), aDisplay.Height());
     74            mDisplay = aDisplay;
     75          }
     76        }
     77        LOG("Set DCompSurfaceHandle, handle=%p", mDCompSurfaceHandle);
     78        ResolvePendingPromisesIfNeeded();
     79      }));
     80 }
     81 
     82 HRESULT MFMediaEngineVideoStream::CreateMediaType(const TrackInfo& aInfo,
     83                                                  IMFMediaType** aMediaType) {
     84  auto& videoInfo = *aInfo.GetAsVideoInfo();
     85  mIsEncrypted = videoInfo.mCrypto.IsEncrypted();
     86 
     87  GUID subType = VideoMimeTypeToMediaFoundationSubtype(videoInfo.mMimeType);
     88  NS_ENSURE_TRUE(subType != GUID_NULL, MF_E_TOPO_CODEC_NOT_FOUND);
     89 
     90  // https://docs.microsoft.com/en-us/windows/win32/medfound/media-type-attributes
     91  ComPtr<IMFMediaType> mediaType;
     92  RETURN_IF_FAILED(wmf::MFCreateMediaType(&mediaType));
     93  RETURN_IF_FAILED(mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
     94  RETURN_IF_FAILED(mediaType->SetGUID(MF_MT_SUBTYPE, subType));
     95 
     96  const auto& image = videoInfo.mImage;
     97  UINT32 imageWidth = image.Width();
     98  UINT32 imageHeight = image.Height();
     99  RETURN_IF_FAILED(MFSetAttributeSize(mediaType.Get(), MF_MT_FRAME_SIZE,
    100                                      imageWidth, imageHeight));
    101 
    102  UINT32 displayWidth = videoInfo.mDisplay.Width();
    103  UINT32 displayHeight = videoInfo.mDisplay.Height();
    104  {
    105    MutexAutoLock lock(mMutex);
    106    mDisplay = videoInfo.mDisplay;
    107  }
    108  // PAR = DAR / SAR = (DW / DH) / (SW / SH) = (DW * SH) / (DH * SW)
    109  RETURN_IF_FAILED(MFSetAttributeRatio(
    110      mediaType.Get(), MF_MT_PIXEL_ASPECT_RATIO, displayWidth * imageHeight,
    111      displayHeight * imageWidth));
    112 
    113  // https://docs.microsoft.com/en-us/windows/win32/api/mfobjects/ns-mfobjects-mfoffset
    114  // The value of the MFOffset number is value + (fract / 65536.0f).
    115  static const auto ToMFOffset = [](float aValue) {
    116    MFOffset offset;
    117    offset.value = static_cast<short>(aValue);
    118    offset.fract = static_cast<WORD>(65536 * (aValue - offset.value));
    119    return offset;
    120  };
    121  MFVideoArea area;
    122  area.OffsetX = ToMFOffset(videoInfo.ImageRect().x);
    123  area.OffsetY = ToMFOffset(videoInfo.ImageRect().y);
    124  area.Area = {(LONG)imageWidth, (LONG)imageHeight};
    125  RETURN_IF_FAILED(mediaType->SetBlob(MF_MT_GEOMETRIC_APERTURE, (UINT8*)&area,
    126                                      sizeof(area)));
    127 
    128  // https://docs.microsoft.com/en-us/windows/win32/api/mfapi/ne-mfapi-mfvideorotationformat
    129  static const auto ToMFVideoRotationFormat = [](VideoRotation aRotation) {
    130    using Rotation = VideoRotation;
    131    switch (aRotation) {
    132      case Rotation::kDegree_0:
    133        return MFVideoRotationFormat_0;
    134      case Rotation::kDegree_90:
    135        return MFVideoRotationFormat_90;
    136      case Rotation::kDegree_180:
    137        return MFVideoRotationFormat_180;
    138      default:
    139        MOZ_ASSERT(aRotation == Rotation::kDegree_270);
    140        return MFVideoRotationFormat_270;
    141    }
    142  };
    143  const auto rotation = ToMFVideoRotationFormat(videoInfo.mRotation);
    144  RETURN_IF_FAILED(mediaType->SetUINT32(MF_MT_VIDEO_ROTATION, rotation));
    145 
    146  static const auto ToMFVideoTransFunc =
    147      [](const Maybe<gfx::YUVColorSpace>& aColorSpace) {
    148        using YUVColorSpace = gfx::YUVColorSpace;
    149        if (!aColorSpace) {
    150          return MFVideoTransFunc_Unknown;
    151        }
    152        // https://docs.microsoft.com/en-us/windows/win32/api/mfobjects/ne-mfobjects-mfvideotransferfunction
    153        switch (*aColorSpace) {
    154          case YUVColorSpace::BT601:
    155          case YUVColorSpace::BT709:
    156            return MFVideoTransFunc_709;
    157          case YUVColorSpace::BT2020:
    158            return MFVideoTransFunc_2020;
    159          case YUVColorSpace::Identity:
    160            return MFVideoTransFunc_sRGB;
    161          default:
    162            return MFVideoTransFunc_Unknown;
    163        }
    164      };
    165  const auto transFunc = ToMFVideoTransFunc(videoInfo.mColorSpace);
    166  RETURN_IF_FAILED(mediaType->SetUINT32(MF_MT_TRANSFER_FUNCTION, transFunc));
    167 
    168  static const auto ToMFVideoPrimaries =
    169      [](const Maybe<gfx::YUVColorSpace>& aColorSpace) {
    170        using YUVColorSpace = gfx::YUVColorSpace;
    171        if (!aColorSpace) {
    172          return MFVideoPrimaries_Unknown;
    173        }
    174        // https://docs.microsoft.com/en-us/windows/win32/api/mfobjects/ne-mfobjects-mfvideoprimaries
    175        switch (*aColorSpace) {
    176          case YUVColorSpace::BT601:
    177            return MFVideoPrimaries_Unknown;
    178          case YUVColorSpace::BT709:
    179            return MFVideoPrimaries_BT709;
    180          case YUVColorSpace::BT2020:
    181            return MFVideoPrimaries_BT2020;
    182          case YUVColorSpace::Identity:
    183            return MFVideoPrimaries_BT709;
    184          default:
    185            return MFVideoPrimaries_Unknown;
    186        }
    187      };
    188  const auto videoPrimaries = ToMFVideoPrimaries(videoInfo.mColorSpace);
    189  RETURN_IF_FAILED(mediaType->SetUINT32(MF_MT_VIDEO_PRIMARIES, videoPrimaries));
    190 
    191  LOG("Created video type, subtype=%s, image=[%ux%u], display=[%ux%u], "
    192      "rotation=%s, tranFuns=%s, primaries=%s, encrypted=%d",
    193      GUIDToStr(subType), imageWidth, imageHeight, displayWidth, displayHeight,
    194      MFVideoRotationFormatToStr(rotation),
    195      MFVideoTransferFunctionToStr(transFunc),
    196      MFVideoPrimariesToStr(videoPrimaries), mIsEncrypted);
    197  if (IsEncrypted()) {
    198    ComPtr<IMFMediaType> protectedMediaType;
    199    RETURN_IF_FAILED(wmf::MFWrapMediaType(mediaType.Get(),
    200                                          MFMediaType_Protected, subType,
    201                                          protectedMediaType.GetAddressOf()));
    202    LOG("Wrap MFMediaType_Video into MFMediaType_Protected");
    203    *aMediaType = protectedMediaType.Detach();
    204  } else {
    205    *aMediaType = mediaType.Detach();
    206  }
    207  return S_OK;
    208 }
    209 
    210 bool MFMediaEngineVideoStream::HasEnoughRawData() const {
    211  // If more than this much raw video is queued, we'll hold off request more
    212  // video.
    213  return mRawDataQueueForFeedingEngine.PreciseDuration() >=
    214         StaticPrefs::media_wmf_media_engine_raw_data_threshold_video();
    215 }
    216 
    217 bool MFMediaEngineVideoStream::IsDCompImageReady() {
    218  AssertOnTaskQueue();
    219  if (!mDCompSurfaceHandle || mDCompSurfaceHandle == INVALID_HANDLE_VALUE) {
    220    LOGV("Can't create image without a valid dcomp surface handle");
    221    return false;
    222  }
    223 
    224  if (!mKnowsCompositor) {
    225    LOGV("Can't create image without the knows compositor");
    226    return false;
    227  }
    228 
    229  if (!mDcompSurfaceImage || mNeedRecreateImage) {
    230    MutexAutoLock lock(mMutex);
    231    // DirectComposition only supports RGBA. We use DXGI_FORMAT_B8G8R8A8_UNORM
    232    // as a default because we can't know what format the dcomp surface is.
    233    // https://docs.microsoft.com/en-us/windows/win32/api/dcomp/nf-dcomp-idcompositionsurfacefactory-createsurface
    234    mDcompSurfaceImage = new layers::DcompSurfaceImage(
    235        mDCompSurfaceHandle, mDisplay, gfx::SurfaceFormat::B8G8R8A8,
    236        mKnowsCompositor);
    237    mNeedRecreateImage = false;
    238    LOG("Created dcomp surface image, handle=%p, size=[%u,%u]",
    239        mDCompSurfaceHandle, mDisplay.Width(), mDisplay.Height());
    240  }
    241  return true;
    242 }
    243 
    244 RefPtr<MediaDataDecoder::DecodePromise> MFMediaEngineVideoStream::OutputData(
    245    RefPtr<MediaRawData> aSample) {
    246  if (IsShutdown()) {
    247    return MediaDataDecoder::DecodePromise::CreateAndReject(
    248        MediaResult(NS_ERROR_FAILURE,
    249                    RESULT_DETAIL("MFMediaEngineStream is shutdown")),
    250        __func__);
    251  }
    252  AssertOnTaskQueue();
    253  NotifyNewData(aSample);
    254  MediaDataDecoder::DecodedData outputs;
    255  if (RefPtr<MediaData> outputData = OutputDataInternal()) {
    256    outputs.AppendElement(outputData);
    257    LOGV("Output data [%" PRId64 ",%" PRId64 "]",
    258         outputData->mTime.ToMicroseconds(),
    259         outputData->GetEndTime().ToMicroseconds());
    260  }
    261  if (ShouldDelayVideoDecodeBeforeDcompReady()) {
    262    LOG("Dcomp isn't ready and we already have enough video data. We will send "
    263        "them back together at one when Dcomp is ready");
    264    return mVideoDecodeBeforeDcompPromise.Ensure(__func__);
    265  }
    266  return MediaDataDecoder::DecodePromise::CreateAndResolve(std::move(outputs),
    267                                                           __func__);
    268 }
    269 
    270 already_AddRefed<MediaData> MFMediaEngineVideoStream::OutputDataInternal() {
    271  AssertOnTaskQueue();
    272  if (mRawDataQueueForGeneratingOutput.GetSize() == 0 || !IsDCompImageReady()) {
    273    return nullptr;
    274  }
    275  RefPtr<MediaRawData> sample = mRawDataQueueForGeneratingOutput.PopFront();
    276  RefPtr<VideoData> output;
    277  {
    278    MutexAutoLock lock(mMutex);
    279    output = VideoData::CreateFromImage(
    280        mDisplay, sample->mOffset, sample->mTime, sample->mDuration,
    281        mDcompSurfaceImage, sample->mKeyframe, sample->mTimecode);
    282  }
    283  return output.forget();
    284 }
    285 
    286 RefPtr<MediaDataDecoder::DecodePromise> MFMediaEngineVideoStream::Drain() {
    287  AssertOnTaskQueue();
    288  MediaDataDecoder::DecodedData outputs;
    289  if (!IsDCompImageReady()) {
    290    LOGV("Waiting for dcomp image for draining");
    291    // A workaround for a special case where we have sent all input data to the
    292    // media engine, and waiting for an output. Sometime media engine would
    293    // never return the first frame to us, unless we notify it the end event,
    294    // which happens on the case where the video only contains one frame. If we
    295    // don't send end event to the media engine, the drain promise would be
    296    // pending forever.
    297    if (!mSampleRequestTokens.empty() &&
    298        mRawDataQueueForFeedingEngine.GetSize() == 0) {
    299      NotifyEndEvent();
    300    }
    301    return mPendingDrainPromise.Ensure(__func__);
    302  }
    303  return MFMediaEngineStream::Drain();
    304 }
    305 
    306 RefPtr<MediaDataDecoder::FlushPromise> MFMediaEngineVideoStream::Flush() {
    307  AssertOnTaskQueue();
    308  auto promise = MFMediaEngineStream::Flush();
    309  mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    310  mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED,
    311                                                __func__);
    312  return promise;
    313 }
    314 
    315 void MFMediaEngineVideoStream::ResolvePendingPromisesIfNeeded() {
    316  AssertOnTaskQueue();
    317  if (!IsDCompImageReady()) {
    318    return;
    319  }
    320 
    321  // Resolve decoding promise first, then drain promise
    322  if (!mVideoDecodeBeforeDcompPromise.IsEmpty()) {
    323    MediaDataDecoder::DecodedData outputs;
    324    while (RefPtr<MediaData> outputData = OutputDataInternal()) {
    325      outputs.AppendElement(outputData);
    326      LOGV("Output data [%" PRId64 ",%" PRId64 "]",
    327           outputData->mTime.ToMicroseconds(),
    328           outputData->GetEndTime().ToMicroseconds());
    329    }
    330    mVideoDecodeBeforeDcompPromise.Resolve(std::move(outputs), __func__);
    331    LOG("Resolved video decode before Dcomp promise");
    332  }
    333 
    334  // This drain promise could return no data, if all data has been processed in
    335  // the decoding promise.
    336  if (!mPendingDrainPromise.IsEmpty()) {
    337    MediaDataDecoder::DecodedData outputs;
    338    while (RefPtr<MediaData> outputData = OutputDataInternal()) {
    339      outputs.AppendElement(outputData);
    340      LOGV("Output data [%" PRId64 ",%" PRId64 "]",
    341           outputData->mTime.ToMicroseconds(),
    342           outputData->GetEndTime().ToMicroseconds());
    343    }
    344    mPendingDrainPromise.Resolve(std::move(outputs), __func__);
    345    LOG("Resolved pending drain promise");
    346  }
    347 }
    348 
    349 MediaDataDecoder::ConversionRequired MFMediaEngineVideoStream::NeedsConversion()
    350    const {
    351  return mStreamType == WMFStreamType::H264 ||
    352                 mStreamType == WMFStreamType::HEVC
    353             ? MediaDataDecoder::ConversionRequired::kNeedAnnexB
    354             : MediaDataDecoder::ConversionRequired::kNeedNone;
    355 }
    356 
    357 void MFMediaEngineVideoStream::SetConfig(const TrackInfo& aConfig) {
    358  MOZ_ASSERT(aConfig.IsVideo());
    359  ComPtr<MFMediaEngineStream> self = this;
    360  (void)mTaskQueue->Dispatch(
    361      NS_NewRunnableFunction("MFMediaEngineStream::SetConfig",
    362                             [self, info = *aConfig.GetAsVideoInfo(), this]() {
    363                               if (mHasReceivedInitialCreateDecoderConfig) {
    364                                 // Here indicating a new config for video,
    365                                 // which is triggered by the media change
    366                                 // monitor, so we need to update the config.
    367                                 UpdateConfig(info);
    368                               }
    369                               mHasReceivedInitialCreateDecoderConfig = true;
    370                             }));
    371 }
    372 
    373 void MFMediaEngineVideoStream::UpdateConfig(const VideoInfo& aInfo) {
    374  AssertOnTaskQueue();
    375  // Disable explicit format change event for H264/HEVC to allow switching to
    376  // the new stream without a full re-create, which will be much faster. This is
    377  // also due to the fact that the MFT decoder can handle some format changes
    378  // without a format change event. For format changes that the MFT decoder
    379  // cannot support (e.g. codec change), the playback will fail later with
    380  // MF_E_INVALIDMEDIATYPE (0xC00D36B4).
    381  if (mStreamType == WMFStreamType::H264 ||
    382      mStreamType == WMFStreamType::HEVC) {
    383    return;
    384  }
    385 
    386  LOG("Video config changed, will update stream descriptor");
    387  PROFILER_MARKER_TEXT("VideoConfigChange", MEDIA_PLAYBACK, {},
    388                       nsPrintfCString("stream=%s, id=%" PRIu64,
    389                                       GetDescriptionName().get(), mStreamId));
    390  ComPtr<IMFMediaType> mediaType;
    391  RETURN_VOID_IF_FAILED(CreateMediaType(aInfo, mediaType.GetAddressOf()));
    392  RETURN_VOID_IF_FAILED(GenerateStreamDescriptor(mediaType));
    393  RETURN_VOID_IF_FAILED(mMediaEventQueue->QueueEventParamUnk(
    394      MEStreamFormatChanged, GUID_NULL, S_OK, mediaType.Get()));
    395 }
    396 
    397 void MFMediaEngineVideoStream::ShutdownCleanUpOnTaskQueue() {
    398  AssertOnTaskQueue();
    399  mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    400  mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED,
    401                                                __func__);
    402 }
    403 
    404 void MFMediaEngineVideoStream::SendRequestSampleEvent(bool aIsEnough) {
    405  AssertOnTaskQueue();
    406  MFMediaEngineStream::SendRequestSampleEvent(aIsEnough);
    407  // We need more data to be sent in, we should resolve the promise to allow
    408  // more input data to be sent.
    409  if (!aIsEnough && !mVideoDecodeBeforeDcompPromise.IsEmpty()) {
    410    LOG("Resolved pending input promise to allow more input be sent in");
    411    mVideoDecodeBeforeDcompPromise.Resolve(MediaDataDecoder::DecodedData{},
    412                                           __func__);
    413  }
    414 }
    415 
    416 bool MFMediaEngineVideoStream::IsEnded() const {
    417  AssertOnTaskQueue();
    418  // If a video only contains one frame, the media engine won't return a decoded
    419  // frame before we tell it the track is already ended. However, due to the
    420  // constraint of our media pipeline, the format reader won't notify EOS until
    421  // the draining finishes, which causes a deadlock. Therefore, we would
    422  // consider having pending drain promise as a sign of EOS as well, in order to
    423  // get the decoded frame and revolve the drain promise.
    424  return (mReceivedEOS || !mPendingDrainPromise.IsEmpty()) &&
    425         mRawDataQueueForFeedingEngine.GetSize() == 0;
    426 }
    427 
    428 bool MFMediaEngineVideoStream::IsEncrypted() const {
    429  return mIsEncrypted || mIsEncryptedCustomInit;
    430 }
    431 
    432 bool MFMediaEngineVideoStream::ShouldDelayVideoDecodeBeforeDcompReady() {
    433  return HasEnoughRawData() && !IsDCompImageReady();
    434 }
    435 
    436 nsCString MFMediaEngineVideoStream::GetCodecName() const {
    437  switch (mStreamType) {
    438    case WMFStreamType::H264:
    439      return "h264"_ns;
    440    case WMFStreamType::VP8:
    441      return "vp8"_ns;
    442    case WMFStreamType::VP9:
    443      return "vp9"_ns;
    444    case WMFStreamType::AV1:
    445      return "av1"_ns;
    446    case WMFStreamType::HEVC:
    447      return "hevc"_ns;
    448    default:
    449      return "unknown"_ns;
    450  };
    451 }
    452 
    453 #undef LOG
    454 #undef LOGV
    455 
    456 }  // namespace mozilla
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE