[ tor-browser ].git.dasho

AppleVTDecoder.cpp (29942B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "AppleVTDecoder.h"
      8 
      9 #include <CoreVideo/CVPixelBufferIOSurface.h>
     10 #include <IOSurface/IOSurfaceRef.h>
     11 
     12 #include <limits>
     13 
     14 #include "AOMDecoder.h"
     15 #include "AppleDecoderModule.h"
     16 #include "CallbackThreadRegistry.h"
     17 #include "H264.h"
     18 #include "H265.h"
     19 #include "MP4Decoder.h"
     20 #include "MacIOSurfaceImage.h"
     21 #include "MediaData.h"
     22 #include "VPXDecoder.h"
     23 #include "VideoUtils.h"
     24 #include "gfxMacUtils.h"
     25 #include "mozilla/Logging.h"
     26 #include "mozilla/TaskQueue.h"
     27 #include "mozilla/gfx/gfxVars.h"
     28 
     29 #define LOG(...) DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, __VA_ARGS__)
     30 #define LOGEX(_this, ...) \
     31  DDMOZ_LOGEX(_this, sPDMLog, mozilla::LogLevel::Debug, __VA_ARGS__)
     32 
     33 namespace mozilla {
     34 
     35 using namespace layers;
     36 
     37 AppleVTDecoder::AppleVTDecoder(const VideoInfo& aConfig,
     38                               layers::ImageContainer* aImageContainer,
     39                               const CreateDecoderParams::OptionSet& aOptions,
     40                               layers::KnowsCompositor* aKnowsCompositor,
     41                               Maybe<TrackingId> aTrackingId)
     42    : mExtraData(aConfig.mExtraData),
     43      mPictureWidth(aConfig.mImage.width),
     44      mPictureHeight(aConfig.mImage.height),
     45      mDisplayWidth(aConfig.mDisplay.width),
     46      mDisplayHeight(aConfig.mDisplay.height),
     47      mColorSpace(aConfig.mColorSpace
     48                      ? *aConfig.mColorSpace
     49                      : DefaultColorSpace({mPictureWidth, mPictureHeight})),
     50      mColorPrimaries(aConfig.mColorPrimaries ? *aConfig.mColorPrimaries
     51                                              : gfx::ColorSpace2::BT709),
     52      mTransferFunction(aConfig.mTransferFunction
     53                            ? *aConfig.mTransferFunction
     54                            : gfx::TransferFunction::BT709),
     55      mColorRange(aConfig.mColorRange),
     56      mColorDepth(aConfig.mColorDepth),
     57      mStreamType(AppleVTDecoder::GetStreamType(aConfig.mMimeType)),
     58      mTaskQueue(TaskQueue::Create(
     59          GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
     60          "AppleVTDecoder")),
     61      mMaxRefFrames(GetMaxRefFrames(
     62          aOptions.contains(CreateDecoderParams::Option::LowLatency))),
     63      mImageContainer(aImageContainer),
     64      mKnowsCompositor(aKnowsCompositor)
     65 #ifdef MOZ_WIDGET_UIKIT
     66      ,
     67      mUseSoftwareImages(true)
     68 #else
     69      ,
     70      mUseSoftwareImages(aKnowsCompositor &&
     71                         aKnowsCompositor->GetWebRenderCompositorType() ==
     72                             layers::WebRenderCompositor::SOFTWARE)
     73 #endif
     74      ,
     75      mTrackingId(aTrackingId),
     76      mIsFlushing(false),
     77      mCallbackThreadId(),
     78      mMonitor("AppleVTDecoder"),
     79      mPromise(&mMonitor),  // To ensure our PromiseHolder is only ever accessed
     80                            // with the monitor held.
     81      mFormat(nullptr),
     82      mSession(nullptr),
     83      mIsHardwareAccelerated(false) {
     84  MOZ_COUNT_CTOR(AppleVTDecoder);
     85  MOZ_ASSERT(mStreamType != StreamType::Unknown);
     86  LOG("Creating AppleVTDecoder for %dx%d %s video, mMaxRefFrames=%u",
     87      mDisplayWidth, mDisplayHeight, EnumValueToString(mStreamType),
     88      mMaxRefFrames);
     89 }
     90 
     91 AppleVTDecoder::~AppleVTDecoder() { MOZ_COUNT_DTOR(AppleVTDecoder); }
     92 
     93 RefPtr<MediaDataDecoder::InitPromise> AppleVTDecoder::Init() {
     94  AUTO_PROFILER_LABEL("AppleVTDecoder::Init", MEDIA_PLAYBACK);
     95  MediaResult rv = InitializeSession();
     96 
     97  if (NS_SUCCEEDED(rv)) {
     98    return InitPromise::CreateAndResolve(TrackType::kVideoTrack, __func__);
     99  }
    100 
    101  return InitPromise::CreateAndReject(rv, __func__);
    102 }
    103 
    104 RefPtr<MediaDataDecoder::DecodePromise> AppleVTDecoder::Decode(
    105    MediaRawData* aSample) {
    106  LOG("mp4 input sample %p pts %lld duration %lld us%s %zu bytes", aSample,
    107      aSample->mTime.ToMicroseconds(), aSample->mDuration.ToMicroseconds(),
    108      aSample->mKeyframe ? " keyframe" : "", aSample->Size());
    109 
    110  RefPtr<AppleVTDecoder> self = this;
    111  RefPtr<MediaRawData> sample = aSample;
    112  return InvokeAsync(mTaskQueue, __func__, [self, this, sample] {
    113    RefPtr<DecodePromise> p;
    114    {
    115      MonitorAutoLock mon(mMonitor);
    116      p = mPromise.Ensure(__func__);
    117    }
    118    ProcessDecode(sample);
    119    return p;
    120  });
    121 }
    122 
    123 RefPtr<MediaDataDecoder::FlushPromise> AppleVTDecoder::Flush() {
    124  mIsFlushing = true;
    125  return InvokeAsync(mTaskQueue, this, __func__, &AppleVTDecoder::ProcessFlush);
    126 }
    127 
    128 RefPtr<MediaDataDecoder::DecodePromise> AppleVTDecoder::Drain() {
    129  return InvokeAsync(mTaskQueue, this, __func__, &AppleVTDecoder::ProcessDrain);
    130 }
    131 
    132 RefPtr<ShutdownPromise> AppleVTDecoder::Shutdown() {
    133  RefPtr<AppleVTDecoder> self = this;
    134  return InvokeAsync(mTaskQueue, __func__, [self]() {
    135    self->ProcessShutdown();
    136    return self->mTaskQueue->BeginShutdown();
    137  });
    138 }
    139 
    140 // Helper to fill in a timestamp structure.
    141 static CMSampleTimingInfo TimingInfoFromSample(MediaRawData* aSample) {
    142  CMSampleTimingInfo timestamp;
    143 
    144  timestamp.duration =
    145      CMTimeMake(aSample->mDuration.ToMicroseconds(), USECS_PER_S);
    146  timestamp.presentationTimeStamp =
    147      CMTimeMake(aSample->mTime.ToMicroseconds(), USECS_PER_S);
    148  timestamp.decodeTimeStamp =
    149      CMTimeMake(aSample->mTimecode.ToMicroseconds(), USECS_PER_S);
    150 
    151  return timestamp;
    152 }
    153 
    154 void AppleVTDecoder::ProcessDecode(MediaRawData* aSample) {
    155  AUTO_PROFILER_LABEL("AppleVTDecoder::ProcessDecode", MEDIA_PLAYBACK);
    156  AssertOnTaskQueue();
    157  PROCESS_DECODE_LOG(aSample);
    158 
    159  if (mIsFlushing) {
    160    MonitorAutoLock mon(mMonitor);
    161    mPromise.Reject(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    162    return;
    163  }
    164 
    165  mTrackingId.apply([&](const auto& aId) {
    166    MediaInfoFlag flag = MediaInfoFlag::None;
    167    flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
    168                                : MediaInfoFlag::NonKeyFrame);
    169    flag |= (mIsHardwareAccelerated ? MediaInfoFlag::HardwareDecoding
    170                                    : MediaInfoFlag::SoftwareDecoding);
    171    switch (mStreamType) {
    172      case StreamType::H264:
    173        flag |= MediaInfoFlag::VIDEO_H264;
    174        break;
    175      case StreamType::VP9:
    176        flag |= MediaInfoFlag::VIDEO_VP9;
    177        break;
    178      case StreamType::AV1:
    179        flag |= MediaInfoFlag::VIDEO_AV1;
    180        break;
    181      case StreamType::HEVC:
    182        flag |= MediaInfoFlag::VIDEO_HEVC;
    183        break;
    184      default:
    185        break;
    186    }
    187    mPerformanceRecorder.Start(aSample->mTimecode.ToMicroseconds(),
    188                               "AppleVTDecoder"_ns, aId, flag);
    189  });
    190 
    191  AutoCFTypeRef<CMBlockBufferRef> block;
    192  AutoCFTypeRef<CMSampleBufferRef> sample;
    193  VTDecodeInfoFlags infoFlags;
    194  OSStatus rv;
    195 
    196  // FIXME: This copies the sample data. I think we can provide
    197  // a custom block source which reuses the aSample buffer.
    198  // But note that there may be a problem keeping the samples
    199  // alive over multiple frames.
    200  rv = CMBlockBufferCreateWithMemoryBlock(
    201      kCFAllocatorDefault,  // Struct allocator.
    202      const_cast<uint8_t*>(aSample->Data()), aSample->Size(),
    203      kCFAllocatorNull,  // Block allocator.
    204      NULL,              // Block source.
    205      0,                 // Data offset.
    206      aSample->Size(), false, block.Receive());
    207  if (rv != noErr) {
    208    NS_ERROR("Couldn't create CMBlockBuffer");
    209    MonitorAutoLock mon(mMonitor);
    210    mPromise.Reject(
    211        MediaResult(NS_ERROR_OUT_OF_MEMORY,
    212                    RESULT_DETAIL("CMBlockBufferCreateWithMemoryBlock:%x", rv)),
    213        __func__);
    214    return;
    215  }
    216 
    217  CMSampleTimingInfo timestamp = TimingInfoFromSample(aSample);
    218  rv = CMSampleBufferCreate(kCFAllocatorDefault, block, true, 0, 0, mFormat, 1,
    219                            1, &timestamp, 0, NULL, sample.Receive());
    220  if (rv != noErr) {
    221    NS_ERROR("Couldn't create CMSampleBuffer");
    222    MonitorAutoLock mon(mMonitor);
    223    mPromise.Reject(MediaResult(NS_ERROR_OUT_OF_MEMORY,
    224                                RESULT_DETAIL("CMSampleBufferCreate:%x", rv)),
    225                    __func__);
    226    return;
    227  }
    228 
    229  VTDecodeFrameFlags decodeFlags =
    230      kVTDecodeFrame_EnableAsynchronousDecompression;
    231  rv = VTDecompressionSessionDecodeFrame(
    232      mSession, sample, decodeFlags, CreateAppleFrameRef(aSample), &infoFlags);
    233  if (infoFlags & kVTDecodeInfo_FrameDropped) {
    234    MonitorAutoLock mon(mMonitor);
    235    // Smile and nod
    236    NS_WARNING("Decoder synchronously dropped frame");
    237    MaybeResolveBufferedFrames();
    238    return;
    239  }
    240 
    241  if (rv != noErr) {
    242    LOG("AppleVTDecoder: Error %d VTDecompressionSessionDecodeFrame", rv);
    243    NS_WARNING("Couldn't pass frame to decoder");
    244    // It appears that even when VTDecompressionSessionDecodeFrame returned a
    245    // failure. Decoding sometimes actually get processed.
    246    MonitorAutoLock mon(mMonitor);
    247    mPromise.RejectIfExists(
    248        MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
    249                    RESULT_DETAIL("VTDecompressionSessionDecodeFrame:%x", rv)),
    250        __func__);
    251    return;
    252  }
    253 }
    254 
    255 void AppleVTDecoder::ProcessShutdown() {
    256  AUTO_PROFILER_LABEL("AppleVTDecoder::ProcessShutdown", MEDIA_PLAYBACK);
    257  if (mSession) {
    258    LOG("%s: cleaning up session", __func__);
    259    VTDecompressionSessionInvalidate(mSession);
    260    mSession.Reset();
    261  }
    262  if (mFormat) {
    263    LOG("%s: releasing format", __func__);
    264    mFormat.Reset();
    265  }
    266 }
    267 
    268 RefPtr<MediaDataDecoder::FlushPromise> AppleVTDecoder::ProcessFlush() {
    269  AUTO_PROFILER_LABEL("AppleVTDecoder::ProcessFlush", MEDIA_PLAYBACK);
    270  AssertOnTaskQueue();
    271  nsresult rv = WaitForAsynchronousFrames();
    272  if (NS_FAILED(rv)) {
    273    LOG("AppleVTDecoder::Flush failed waiting for platform decoder");
    274  }
    275  MonitorAutoLock mon(mMonitor);
    276  mPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    277 
    278  while (!mReorderQueue.IsEmpty()) {
    279    mReorderQueue.Pop();
    280  }
    281  mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
    282  mSeekTargetThreshold.reset();
    283  mIsFlushing = false;
    284  return FlushPromise::CreateAndResolve(true, __func__);
    285 }
    286 
    287 RefPtr<MediaDataDecoder::DecodePromise> AppleVTDecoder::ProcessDrain() {
    288  AUTO_PROFILER_LABEL("AppleVTDecoder::ProcessDrain", MEDIA_PLAYBACK);
    289  AssertOnTaskQueue();
    290  nsresult rv = WaitForAsynchronousFrames();
    291  if (NS_FAILED(rv)) {
    292    LOG("AppleVTDecoder::Drain failed waiting for platform decoder");
    293  }
    294  MonitorAutoLock mon(mMonitor);
    295  DecodedData samples;
    296  while (!mReorderQueue.IsEmpty()) {
    297    samples.AppendElement(mReorderQueue.Pop());
    298  }
    299  return DecodePromise::CreateAndResolve(std::move(samples), __func__);
    300 }
    301 
    302 AppleVTDecoder::AppleFrameRef* AppleVTDecoder::CreateAppleFrameRef(
    303    const MediaRawData* aSample) {
    304  MOZ_ASSERT(aSample);
    305  return new AppleFrameRef(*aSample);
    306 }
    307 
    308 void AppleVTDecoder::SetSeekThreshold(const media::TimeUnit& aTime) {
    309  if (aTime.IsValid()) {
    310    mSeekTargetThreshold = Some(aTime);
    311  } else {
    312    mSeekTargetThreshold.reset();
    313  }
    314 }
    315 
    316 //
    317 // Implementation details.
    318 //
    319 
    320 // Callback passed to the VideoToolbox decoder for returning data.
    321 // This needs to be static because the API takes a C-style pair of
    322 // function and userdata pointers. This validates parameters and
    323 // forwards the decoded image back to an object method.
    324 static void PlatformCallback(void* decompressionOutputRefCon,
    325                             void* sourceFrameRefCon, OSStatus status,
    326                             VTDecodeInfoFlags flags, CVImageBufferRef image,
    327                             CMTime presentationTimeStamp,
    328                             CMTime presentationDuration) {
    329  AppleVTDecoder* decoder =
    330      static_cast<AppleVTDecoder*>(decompressionOutputRefCon);
    331  LOGEX(decoder, "AppleVideoDecoder %s status %d flags %d", __func__,
    332        static_cast<int>(status), flags);
    333 
    334  UniquePtr<AppleVTDecoder::AppleFrameRef> frameRef(
    335      static_cast<AppleVTDecoder::AppleFrameRef*>(sourceFrameRefCon));
    336 
    337  // Validate our arguments.
    338  if (status != noErr) {
    339    NS_WARNING("VideoToolbox decoder returned an error");
    340    decoder->OnDecodeError(status);
    341    return;
    342  }
    343  if (!image) {
    344    NS_WARNING("VideoToolbox decoder returned no data");
    345  } else if (flags & kVTDecodeInfo_FrameDropped) {
    346    NS_WARNING("  ...frame tagged as dropped...");
    347  } else {
    348    MOZ_ASSERT(CFGetTypeID(image) == CVPixelBufferGetTypeID(),
    349               "VideoToolbox returned an unexpected image type");
    350  }
    351 
    352  decoder->OutputFrame(image, *frameRef);
    353 }
    354 
    355 void AppleVTDecoder::MaybeResolveBufferedFrames() {
    356  mMonitor.AssertCurrentThreadOwns();
    357 
    358  if (mPromise.IsEmpty()) {
    359    return;
    360  }
    361 
    362  DecodedData results;
    363  while (mReorderQueue.Length() > mMaxRefFrames) {
    364    results.AppendElement(mReorderQueue.Pop());
    365  }
    366  mPromise.Resolve(std::move(results), __func__);
    367 }
    368 
    369 void AppleVTDecoder::MaybeRegisterCallbackThread() {
    370  ProfilerThreadId id = profiler_current_thread_id();
    371  if (MOZ_LIKELY(id == mCallbackThreadId)) {
    372    return;
    373  }
    374  mCallbackThreadId = id;
    375  CallbackThreadRegistry::Get()->Register(mCallbackThreadId,
    376                                          "AppleVTDecoderCallback");
    377 }
    378 
    379 nsCString AppleVTDecoder::GetCodecName() const {
    380  return nsCString(EnumValueToString(mStreamType));
    381 }
    382 
    383 // Copy and return a decoded frame.
    384 void AppleVTDecoder::OutputFrame(CVPixelBufferRef aImage,
    385                                 AppleVTDecoder::AppleFrameRef aFrameRef) {
    386  MaybeRegisterCallbackThread();
    387 
    388  if (mIsFlushing) {
    389    // We are in the process of flushing or shutting down; ignore frame.
    390    return;
    391  }
    392 
    393  LOG("mp4 output frame %lld dts %lld pts %lld duration %lld us%s",
    394      aFrameRef.byte_offset, aFrameRef.decode_timestamp.ToMicroseconds(),
    395      aFrameRef.composition_timestamp.ToMicroseconds(),
    396      aFrameRef.duration.ToMicroseconds(),
    397      aFrameRef.is_sync_point ? " keyframe" : "");
    398 
    399  if (!aImage) {
    400    // Image was dropped by decoder or none return yet.
    401    // We need more input to continue.
    402    MonitorAutoLock mon(mMonitor);
    403    MaybeResolveBufferedFrames();
    404    return;
    405  }
    406 
    407  bool useNullSample = false;
    408  if (mSeekTargetThreshold.isSome()) {
    409    if ((aFrameRef.composition_timestamp + aFrameRef.duration) <
    410        mSeekTargetThreshold.ref()) {
    411      useNullSample = true;
    412    } else {
    413      mSeekTargetThreshold.reset();
    414    }
    415  }
    416 
    417  // Where our resulting image will end up.
    418  RefPtr<MediaData> data;
    419  // Bounds.
    420  VideoInfo info;
    421  info.mDisplay = gfx::IntSize(mDisplayWidth, mDisplayHeight);
    422 
    423  if (useNullSample) {
    424    data = new NullData(aFrameRef.byte_offset, aFrameRef.composition_timestamp,
    425                        aFrameRef.duration);
    426  } else if (mUseSoftwareImages) {
    427    size_t width = CVPixelBufferGetWidth(aImage);
    428    size_t height = CVPixelBufferGetHeight(aImage);
    429    DebugOnly<size_t> planes = CVPixelBufferGetPlaneCount(aImage);
    430    MOZ_ASSERT(planes == 3, "Likely not YUV420 format and it must be.");
    431 
    432    VideoData::YCbCrBuffer buffer;
    433 
    434    // Lock the returned image data.
    435    CVReturn rv =
    436        CVPixelBufferLockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly);
    437    if (rv != kCVReturnSuccess) {
    438      NS_ERROR("error locking pixel data");
    439      MonitorAutoLock mon(mMonitor);
    440      mPromise.Reject(
    441          MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
    442                      RESULT_DETAIL("CVPixelBufferLockBaseAddress:%x", rv)),
    443          __func__);
    444      return;
    445    }
    446    // Y plane.
    447    buffer.mPlanes[0].mData =
    448        static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 0));
    449    buffer.mPlanes[0].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 0);
    450    buffer.mPlanes[0].mWidth = width;
    451    buffer.mPlanes[0].mHeight = height;
    452    buffer.mPlanes[0].mSkip = 0;
    453    // Cb plane.
    454    buffer.mPlanes[1].mData =
    455        static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 1));
    456    buffer.mPlanes[1].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1);
    457    buffer.mPlanes[1].mWidth = (width + 1) / 2;
    458    buffer.mPlanes[1].mHeight = (height + 1) / 2;
    459    buffer.mPlanes[1].mSkip = 0;
    460    // Cr plane.
    461    buffer.mPlanes[2].mData =
    462        static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 2));
    463    buffer.mPlanes[2].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 2);
    464    buffer.mPlanes[2].mWidth = (width + 1) / 2;
    465    buffer.mPlanes[2].mHeight = (height + 1) / 2;
    466    buffer.mPlanes[2].mSkip = 0;
    467 
    468    buffer.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
    469    buffer.mYUVColorSpace = mColorSpace;
    470    buffer.mColorPrimaries = mColorPrimaries;
    471    buffer.mColorRange = mColorRange;
    472 
    473    gfx::IntRect visible = gfx::IntRect(0, 0, mPictureWidth, mPictureHeight);
    474 
    475    // Copy the image data into our own format.
    476    Result<already_AddRefed<VideoData>, MediaResult> result =
    477        VideoData::CreateAndCopyData(
    478            info, mImageContainer, aFrameRef.byte_offset,
    479            aFrameRef.composition_timestamp, aFrameRef.duration, buffer,
    480            aFrameRef.is_sync_point, aFrameRef.decode_timestamp, visible,
    481            mKnowsCompositor);
    482    // TODO: Reject mPromise below with result's error return.
    483    data = result.unwrapOr(nullptr);
    484    // Unlock the returned image data.
    485    CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly);
    486  } else {
    487    // Set pixel buffer properties on aImage before we extract its surface.
    488    // This ensures that we can use defined enums to set values instead
    489    // of later setting magic CFSTR values on the surface itself.
    490    if (mColorSpace == gfx::YUVColorSpace::BT601) {
    491      CVBufferSetAttachment(aImage, kCVImageBufferYCbCrMatrixKey,
    492                            kCVImageBufferYCbCrMatrix_ITU_R_601_4,
    493                            kCVAttachmentMode_ShouldPropagate);
    494    } else if (mColorSpace == gfx::YUVColorSpace::BT709) {
    495      CVBufferSetAttachment(aImage, kCVImageBufferYCbCrMatrixKey,
    496                            kCVImageBufferYCbCrMatrix_ITU_R_709_2,
    497                            kCVAttachmentMode_ShouldPropagate);
    498    } else if (mColorSpace == gfx::YUVColorSpace::BT2020) {
    499      CVBufferSetAttachment(aImage, kCVImageBufferYCbCrMatrixKey,
    500                            kCVImageBufferYCbCrMatrix_ITU_R_2020,
    501                            kCVAttachmentMode_ShouldPropagate);
    502    }
    503 
    504    if (mColorPrimaries == gfx::ColorSpace2::BT709) {
    505      CVBufferSetAttachment(aImage, kCVImageBufferColorPrimariesKey,
    506                            kCVImageBufferColorPrimaries_ITU_R_709_2,
    507                            kCVAttachmentMode_ShouldPropagate);
    508    } else if (mColorPrimaries == gfx::ColorSpace2::BT2020) {
    509      CVBufferSetAttachment(aImage, kCVImageBufferColorPrimariesKey,
    510                            kCVImageBufferColorPrimaries_ITU_R_2020,
    511                            kCVAttachmentMode_ShouldPropagate);
    512    }
    513 
    514    // Transfer function is applied independently from the colorSpace.
    515    CVBufferSetAttachment(
    516        aImage, kCVImageBufferTransferFunctionKey,
    517        gfxMacUtils::CFStringForTransferFunction(mTransferFunction),
    518        kCVAttachmentMode_ShouldPropagate);
    519 
    520    CFTypeRefPtr<IOSurfaceRef> surface =
    521        CFTypeRefPtr<IOSurfaceRef>::WrapUnderGetRule(
    522            CVPixelBufferGetIOSurface(aImage));
    523    MOZ_ASSERT(surface, "Decoder didn't return an IOSurface backed buffer");
    524 
    525    RefPtr<MacIOSurface> macSurface = new MacIOSurface(std::move(surface));
    526    macSurface->SetYUVColorSpace(mColorSpace);
    527    macSurface->mColorPrimaries = mColorPrimaries;
    528 
    529    RefPtr<layers::Image> image = new layers::MacIOSurfaceImage(macSurface);
    530 
    531    data = VideoData::CreateFromImage(
    532        info.mDisplay, aFrameRef.byte_offset, aFrameRef.composition_timestamp,
    533        aFrameRef.duration, image.forget(), aFrameRef.is_sync_point,
    534        aFrameRef.decode_timestamp);
    535  }
    536 
    537  if (!data) {
    538    NS_ERROR("Couldn't create VideoData for frame");
    539    MonitorAutoLock mon(mMonitor);
    540    mPromise.Reject(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
    541    return;
    542  }
    543 
    544  mPerformanceRecorder.Record(
    545      aFrameRef.decode_timestamp.ToMicroseconds(), [&](DecodeStage& aStage) {
    546        aStage.SetResolution(static_cast<int>(CVPixelBufferGetWidth(aImage)),
    547                             static_cast<int>(CVPixelBufferGetHeight(aImage)));
    548        auto format = [&]() -> Maybe<DecodeStage::ImageFormat> {
    549          switch (CVPixelBufferGetPixelFormatType(aImage)) {
    550            case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
    551            case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
    552              return Some(DecodeStage::NV12);
    553            case kCVPixelFormatType_422YpCbCr8_yuvs:
    554            case kCVPixelFormatType_422YpCbCr8FullRange:
    555              return Some(DecodeStage::YUV422P);
    556            case kCVPixelFormatType_32BGRA:
    557              return Some(DecodeStage::RGBA32);
    558            default:
    559              return Nothing();
    560          }
    561        }();
    562        format.apply([&](auto aFormat) { aStage.SetImageFormat(aFormat); });
    563        aStage.SetColorDepth(mColorDepth);
    564        aStage.SetYUVColorSpace(mColorSpace);
    565        aStage.SetColorRange(mColorRange);
    566        aStage.SetStartTimeAndEndTime(data->mTime.ToMicroseconds(),
    567                                      data->GetEndTime().ToMicroseconds());
    568      });
    569 
    570  // Frames come out in DTS order but we need to output them
    571  // in composition order.
    572  MonitorAutoLock mon(mMonitor);
    573  mReorderQueue.Push(std::move(data));
    574  MaybeResolveBufferedFrames();
    575 
    576  LOG("%llu decoded frames queued",
    577      static_cast<unsigned long long>(mReorderQueue.Length()));
    578 }
    579 
    580 void AppleVTDecoder::OnDecodeError(OSStatus aError) {
    581  MonitorAutoLock mon(mMonitor);
    582  mPromise.RejectIfExists(
    583      MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
    584                  RESULT_DETAIL("OnDecodeError:%x", aError)),
    585      __func__);
    586 }
    587 
    588 nsresult AppleVTDecoder::WaitForAsynchronousFrames() {
    589  OSStatus rv = VTDecompressionSessionWaitForAsynchronousFrames(mSession);
    590  if (rv != noErr) {
    591    NS_ERROR("AppleVTDecoder: Error waiting for asynchronous frames");
    592    return NS_ERROR_FAILURE;
    593  }
    594  return NS_OK;
    595 }
    596 
    597 MediaResult AppleVTDecoder::InitializeSession() {
    598  OSStatus rv;
    599 
    600  AutoCFTypeRef<CFDictionaryRef> extensions(CreateDecoderExtensions());
    601  CMVideoCodecType streamType;
    602  if (mStreamType == StreamType::H264) {
    603    streamType = kCMVideoCodecType_H264;
    604  } else if (mStreamType == StreamType::VP9) {
    605    streamType = CMVideoCodecType(AppleDecoderModule::kCMVideoCodecType_VP9);
    606  } else if (mStreamType == StreamType::HEVC) {
    607    streamType = kCMVideoCodecType_HEVC;
    608  } else {
    609    streamType = kCMVideoCodecType_AV1;
    610  }
    611 
    612  rv = CMVideoFormatDescriptionCreate(
    613      kCFAllocatorDefault, streamType, AssertedCast<int32_t>(mPictureWidth),
    614      AssertedCast<int32_t>(mPictureHeight), extensions, mFormat.Receive());
    615  if (rv != noErr) {
    616    return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    617                       RESULT_DETAIL("Couldn't create format description!"));
    618  }
    619 
    620  // Contruct video decoder selection spec.
    621  AutoCFTypeRef<CFDictionaryRef> spec(CreateDecoderSpecification());
    622 
    623  // Contruct output configuration.
    624  AutoCFTypeRef<CFDictionaryRef> outputConfiguration(
    625      CreateOutputConfiguration());
    626 
    627  VTDecompressionOutputCallbackRecord cb = {PlatformCallback, this};
    628  rv =
    629      VTDecompressionSessionCreate(kCFAllocatorDefault, mFormat,
    630                                   spec,  // Video decoder selection.
    631                                   outputConfiguration,  // Output video format.
    632                                   &cb, mSession.Receive());
    633 
    634  if (rv != noErr) {
    635    LOG("AppleVTDecoder: VTDecompressionSessionCreate failed: %d", rv);
    636    return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    637                       RESULT_DETAIL("Couldn't create decompression session!"));
    638  }
    639 
    640  CFBooleanRef isUsingHW = nullptr;
    641  rv = VTSessionCopyProperty(
    642      mSession,
    643      kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder,
    644      kCFAllocatorDefault, &isUsingHW);
    645  if (rv == noErr) {
    646    mIsHardwareAccelerated = isUsingHW == kCFBooleanTrue;
    647    LOG("AppleVTDecoder: %s hardware accelerated decoding",
    648        mIsHardwareAccelerated ? "using" : "not using");
    649  } else {
    650    LOG("AppleVTDecoder: maybe hardware accelerated decoding "
    651        "(VTSessionCopyProperty query failed %d)",
    652        static_cast<int>(rv));
    653  }
    654  if (isUsingHW) {
    655    CFRelease(isUsingHW);
    656  }
    657 
    658  return NS_OK;
    659 }
    660 
    661 CFDictionaryRef AppleVTDecoder::CreateDecoderExtensions() {
    662  AutoCFTypeRef<CFDataRef> data(
    663      CFDataCreate(kCFAllocatorDefault, mExtraData->Elements(),
    664                   AssertedCast<CFIndex>(mExtraData->Length())));
    665 
    666  const void* atomsKey[1];
    667  if (mStreamType == StreamType::H264) {
    668    atomsKey[0] = CFSTR("avcC");
    669  } else if (mStreamType == StreamType::VP9) {
    670    atomsKey[0] = CFSTR("vpcC");
    671  } else if (mStreamType == StreamType::HEVC) {
    672    atomsKey[0] = CFSTR("hvcC");
    673  } else {
    674    atomsKey[0] = CFSTR("av1C");
    675  }
    676 
    677  const void* atomsValue[] = {data};
    678  static_assert(std::size(atomsKey) == std::size(atomsValue),
    679                "Non matching keys/values array size");
    680 
    681  AutoCFTypeRef<CFDictionaryRef> atoms(CFDictionaryCreate(
    682      kCFAllocatorDefault, atomsKey, atomsValue, std::size(atomsKey),
    683      &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
    684 
    685  const void* extensionKeys[] = {
    686      kCVImageBufferChromaLocationBottomFieldKey,
    687      kCVImageBufferChromaLocationTopFieldKey,
    688      kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms};
    689 
    690  const void* extensionValues[] = {kCVImageBufferChromaLocation_Left,
    691                                   kCVImageBufferChromaLocation_Left, atoms};
    692  static_assert(std::size(extensionKeys) == std::size(extensionValues),
    693                "Non matching keys/values array size");
    694 
    695  return CFDictionaryCreate(kCFAllocatorDefault, extensionKeys, extensionValues,
    696                            std::size(extensionKeys),
    697                            &kCFTypeDictionaryKeyCallBacks,
    698                            &kCFTypeDictionaryValueCallBacks);
    699 }
    700 
    701 CFDictionaryRef AppleVTDecoder::CreateDecoderSpecification() {
    702  const void* specKeys[] = {
    703      kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder};
    704  const void* specValues[1];
    705  if (gfx::gfxVars::CanUseHardwareVideoDecoding()) {
    706    specValues[0] = kCFBooleanTrue;
    707  } else {
    708    // This GPU is blacklisted for hardware decoding.
    709    specValues[0] = kCFBooleanFalse;
    710  }
    711  static_assert(std::size(specKeys) == std::size(specValues),
    712                "Non matching keys/values array size");
    713 
    714  return CFDictionaryCreate(kCFAllocatorDefault, specKeys, specValues,
    715                            std::size(specKeys), &kCFTypeDictionaryKeyCallBacks,
    716                            &kCFTypeDictionaryValueCallBacks);
    717 }
    718 
    719 CFDictionaryRef AppleVTDecoder::CreateOutputConfiguration() {
    720  if (mUseSoftwareImages) {
    721    // Output format type:
    722    SInt32 PixelFormatTypeValue = kCVPixelFormatType_420YpCbCr8Planar;
    723    AutoCFTypeRef<CFNumberRef> PixelFormatTypeNumber(CFNumberCreate(
    724        kCFAllocatorDefault, kCFNumberSInt32Type, &PixelFormatTypeValue));
    725    const void* outputKeys[] = {kCVPixelBufferPixelFormatTypeKey};
    726    const void* outputValues[] = {PixelFormatTypeNumber};
    727    static_assert(std::size(outputKeys) == std::size(outputValues),
    728                  "Non matching keys/values array size");
    729 
    730    return CFDictionaryCreate(
    731        kCFAllocatorDefault, outputKeys, outputValues, std::size(outputKeys),
    732        &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
    733  }
    734 
    735  // Output format type:
    736 
    737  bool is10Bit = (gfx::BitDepthForColorDepth(mColorDepth) == 10);
    738  SInt32 PixelFormatTypeValue =
    739      mColorRange == gfx::ColorRange::FULL
    740          ? (is10Bit ? kCVPixelFormatType_420YpCbCr10BiPlanarFullRange
    741                     : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
    742          : (is10Bit ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
    743                     : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
    744  AutoCFTypeRef<CFNumberRef> PixelFormatTypeNumber(CFNumberCreate(
    745      kCFAllocatorDefault, kCFNumberSInt32Type, &PixelFormatTypeValue));
    746  // Construct IOSurface Properties
    747  const void* IOSurfaceKeys[] = {kIOSurfaceIsGlobal};
    748  const void* IOSurfaceValues[] = {kCFBooleanTrue};
    749  static_assert(std::size(IOSurfaceKeys) == std::size(IOSurfaceValues),
    750                "Non matching keys/values array size");
    751 
    752  // Contruct output configuration.
    753  AutoCFTypeRef<CFDictionaryRef> IOSurfaceProperties(CFDictionaryCreate(
    754      kCFAllocatorDefault, IOSurfaceKeys, IOSurfaceValues,
    755      std::size(IOSurfaceKeys), &kCFTypeDictionaryKeyCallBacks,
    756      &kCFTypeDictionaryValueCallBacks));
    757 
    758  const void* outputKeys[] = {kCVPixelBufferIOSurfacePropertiesKey,
    759                              kCVPixelBufferPixelFormatTypeKey,
    760                              kCVPixelBufferOpenGLCompatibilityKey};
    761  const void* outputValues[] = {IOSurfaceProperties, PixelFormatTypeNumber,
    762                                kCFBooleanTrue};
    763  static_assert(std::size(outputKeys) == std::size(outputValues),
    764                "Non matching keys/values array size");
    765 
    766  return CFDictionaryCreate(
    767      kCFAllocatorDefault, outputKeys, outputValues, std::size(outputKeys),
    768      &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
    769 }
    770 
    771 AppleVTDecoder::StreamType AppleVTDecoder::GetStreamType(
    772    const nsCString& aMimeType) const {
    773  if (MP4Decoder::IsH264(aMimeType)) {
    774    return StreamType::H264;
    775  }
    776  if (MP4Decoder::IsHEVC(aMimeType)) {
    777    return StreamType::HEVC;
    778  }
    779  if (VPXDecoder::IsVP9(aMimeType)) {
    780    return StreamType::VP9;
    781  }
    782  if (AOMDecoder::IsAV1(aMimeType)) {
    783    return StreamType::AV1;
    784  }
    785  return StreamType::Unknown;
    786 }
    787 
    788 uint32_t AppleVTDecoder::GetMaxRefFrames(bool aIsLowLatency) const {
    789  if (mStreamType == StreamType::H264 && !aIsLowLatency) {
    790    return H264::ComputeMaxRefFrames(mExtraData);
    791  }
    792  if (mStreamType == StreamType::HEVC && !aIsLowLatency) {
    793    return H265::ComputeMaxRefFrames(mExtraData);
    794  }
    795  return 0;
    796 }
    797 
    798 }  // namespace mozilla
    799 
    800 #undef LOG
    801 #undef LOGEX
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE