[ tor-browser ].git.dasho

AppleVTEncoder.cpp (46853B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "AppleVTEncoder.h"
      8 
      9 #include <CoreFoundation/CFArray.h>
     10 #include <CoreFoundation/CFByteOrder.h>
     11 #include <CoreFoundation/CFDictionary.h>
     12 #include <MacTypes.h>
     13 
     14 #include "AnnexB.h"
     15 #include "H264.h"
     16 #include "ImageContainer.h"
     17 #include "mozilla/dom/BindingUtils.h"
     18 #include "mozilla/dom/ImageUtils.h"
     19 
     20 namespace mozilla {
     21 extern LazyLogModule sPEMLog;
     22 #define LOGE(fmt, ...)                       \
     23  MOZ_LOG(sPEMLog, mozilla::LogLevel::Error, \
     24          ("[AppleVTEncoder] %s: " fmt, __func__, ##__VA_ARGS__))
     25 #define LOGW(fmt, ...)                         \
     26  MOZ_LOG(sPEMLog, mozilla::LogLevel::Warning, \
     27          ("[AppleVTEncoder] %s: " fmt, __func__, ##__VA_ARGS__))
     28 #define LOGD(fmt, ...)                       \
     29  MOZ_LOG(sPEMLog, mozilla::LogLevel::Debug, \
     30          ("[AppleVTEncoder] %s: " fmt, __func__, ##__VA_ARGS__))
     31 #define LOGV(fmt, ...)                         \
     32  MOZ_LOG(sPEMLog, mozilla::LogLevel::Verbose, \
     33          ("[AppleVTEncoder] %s: " fmt, __func__, ##__VA_ARGS__))
     34 
     35 static CFDictionaryRef BuildEncoderSpec(const bool aHardwareNotAllowed,
     36                                        const bool aLowLatencyRateControl) {
     37  if (__builtin_available(macos 11.3, *)) {
     38    if (aLowLatencyRateControl) {
     39      // If doing low-latency rate control, the hardware encoder is required.
     40      const void* keys[] = {
     41          kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
     42          kVTVideoEncoderSpecification_EnableLowLatencyRateControl};
     43      const void* values[] = {kCFBooleanTrue, kCFBooleanTrue};
     44 
     45      static_assert(std::size(keys) == std::size(values),
     46                    "Non matching keys/values array size");
     47      return CFDictionaryCreate(kCFAllocatorDefault, keys, values,
     48                                std::size(keys), &kCFTypeDictionaryKeyCallBacks,
     49                                &kCFTypeDictionaryValueCallBacks);
     50    }
     51  }
     52  const void* keys[] = {
     53      kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder};
     54  const void* values[] = {aHardwareNotAllowed ? kCFBooleanFalse
     55                                              : kCFBooleanTrue};
     56 
     57  static_assert(std::size(keys) == std::size(values),
     58                "Non matching keys/values array size");
     59  return CFDictionaryCreate(kCFAllocatorDefault, keys, values, std::size(keys),
     60                            &kCFTypeDictionaryKeyCallBacks,
     61                            &kCFTypeDictionaryValueCallBacks);
     62 }
     63 
     64 static void FrameCallback(void* aEncoder, void* aFrameRefCon, OSStatus aStatus,
     65                          VTEncodeInfoFlags aInfoFlags,
     66                          CMSampleBufferRef aSampleBuffer) {
     67  (static_cast<AppleVTEncoder*>(aEncoder))
     68      ->OutputFrame(aStatus, aInfoFlags, aSampleBuffer);
     69 }
     70 
     71 bool AppleVTEncoder::SetAverageBitrate(uint32_t aBitsPerSec) {
     72  MOZ_ASSERT(mSession);
     73 
     74  SessionPropertyManager mgr(mSession);
     75  return mgr.Set(kVTCompressionPropertyKey_AverageBitRate,
     76                 int64_t(aBitsPerSec)) == noErr;
     77 }
     78 
     79 bool AppleVTEncoder::SetConstantBitrate(uint32_t aBitsPerSec) {
     80  MOZ_ASSERT(mSession);
     81 
     82  if (__builtin_available(macos 13.0, *)) {
     83    SessionPropertyManager mgr(mSession);
     84    OSStatus rv = mgr.Set(kVTCompressionPropertyKey_ConstantBitRate,
     85                          AssertedCast<int32_t>(aBitsPerSec));
     86    if (rv == kVTPropertyNotSupportedErr) {
     87      LOGE("Constant bitrate not supported.");
     88    }
     89    return rv == noErr;
     90  }
     91  return false;
     92 }
     93 
     94 bool AppleVTEncoder::SetBitrateAndMode(BitrateMode aBitrateMode,
     95                                       uint32_t aBitsPerSec) {
     96  if (aBitrateMode == BitrateMode::Variable) {
     97    return SetAverageBitrate(aBitsPerSec);
     98  }
     99  return SetConstantBitrate(aBitsPerSec);
    100 }
    101 
    102 bool AppleVTEncoder::SetFrameRate(int64_t aFPS) {
    103  MOZ_ASSERT(mSession);
    104 
    105  SessionPropertyManager mgr(mSession);
    106  return mgr.Set(kVTCompressionPropertyKey_ExpectedFrameRate, aFPS) == noErr;
    107 }
    108 
    109 bool AppleVTEncoder::SetRealtime(bool aEnabled) {
    110  MOZ_ASSERT(mSession);
    111 
    112  // B-frames has been disabled in Init(), so no need to set it here.
    113 
    114  SessionPropertyManager mgr(mSession);
    115  OSStatus status = mgr.Set(kVTCompressionPropertyKey_RealTime, aEnabled);
    116  LOGD("%s real time, status: %d", aEnabled ? "Enable" : "Disable", status);
    117  if (status != noErr) {
    118    return false;
    119  }
    120 
    121  if (__builtin_available(macos 11.0, *)) {
    122    status = mgr.Set(
    123        kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, aEnabled);
    124    LOGD("%s PrioritizeEncodingSpeedOverQuality, status: %d",
    125         aEnabled ? "Enable" : "Disable", status);
    126    if (status != noErr && status != kVTPropertyNotSupportedErr) {
    127      return false;
    128    }
    129  }
    130 
    131  int32_t maxFrameDelayCount = aEnabled ? 0 : kVTUnlimitedFrameDelayCount;
    132  status =
    133      mgr.Set(kVTCompressionPropertyKey_MaxFrameDelayCount, maxFrameDelayCount);
    134  LOGD("Set max frame delay count to %d, status: %d", maxFrameDelayCount,
    135       status);
    136  if (status != noErr && status != kVTPropertyNotSupportedErr) {
    137    return false;
    138  }
    139 
    140  return true;
    141 }
    142 
    143 bool AppleVTEncoder::SetProfileLevel(H264_PROFILE aValue) {
    144  MOZ_ASSERT(mSession);
    145 
    146  CFStringRef profileLevel = nullptr;
    147  switch (aValue) {
    148    case H264_PROFILE::H264_PROFILE_BASE:
    149      profileLevel = kVTProfileLevel_H264_Baseline_AutoLevel;
    150      break;
    151    case H264_PROFILE::H264_PROFILE_MAIN:
    152      profileLevel = kVTProfileLevel_H264_Main_AutoLevel;
    153      break;
    154    case H264_PROFILE::H264_PROFILE_HIGH:
    155      profileLevel = kVTProfileLevel_H264_High_AutoLevel;
    156      break;
    157    default:
    158      LOGE("Profile %d not handled", static_cast<int>(aValue));
    159  }
    160 
    161  if (profileLevel == nullptr) {
    162    return false;
    163  }
    164 
    165  SessionPropertyManager mgr(mSession);
    166  return mgr.Set(kVTCompressionPropertyKey_ProfileLevel, profileLevel) == noErr;
    167 }
    168 
    169 static Maybe<CFStringRef> MapColorPrimaries(
    170    const gfx::ColorSpace2& aPrimaries) {
    171  switch (aPrimaries) {
    172    case gfx::ColorSpace2::Display:
    173      return Nothing();
    174    case gfx::ColorSpace2::SRGB:
    175      return Some(kCVImageBufferColorPrimaries_P22);
    176    case gfx::ColorSpace2::DISPLAY_P3:
    177      return Some(kCVImageBufferColorPrimaries_P3_D65);
    178    case gfx::ColorSpace2::BT601_525:
    179      return Some(kCVImageBufferColorPrimaries_SMPTE_C);
    180    case gfx::ColorSpace2::BT709:
    181      return Some(kCVImageBufferColorPrimaries_ITU_R_709_2);
    182    case gfx::ColorSpace2::BT2020:
    183      return Some(kCVImageBufferColorPrimaries_ITU_R_2020);
    184  }
    185 
    186  MOZ_ASSERT_UNREACHABLE("Unsupported color primaries");
    187  return Nothing();
    188 }
    189 
    190 static Maybe<CFStringRef> MapYCbCrMatrix(const gfx::YUVColorSpace& aMatrix) {
    191  switch (aMatrix) {
    192    case gfx::YUVColorSpace::BT601:
    193      return Some(kCVImageBufferYCbCrMatrix_ITU_R_601_4);
    194    case gfx::YUVColorSpace::BT709:
    195      return Some(kCVImageBufferYCbCrMatrix_ITU_R_709_2);
    196    case gfx::YUVColorSpace::BT2020:
    197      return Some(kCVImageBufferYCbCrMatrix_ITU_R_2020);
    198    case gfx::YUVColorSpace::Identity:
    199      return Nothing();
    200  }
    201 
    202  MOZ_ASSERT_UNREACHABLE("Unsupported YCbCr matrix");
    203  return Nothing();
    204 }
    205 
    206 static Maybe<CFStringRef> MapTransferFunction(
    207    const gfx::TransferFunction& aTransferFunction) {
    208  switch (aTransferFunction) {
    209    case gfx::TransferFunction::BT709:
    210      return Some(kCVImageBufferTransferFunction_ITU_R_709_2);
    211    case gfx::TransferFunction::SRGB:
    212      return Some(kCVImageBufferTransferFunction_sRGB);
    213    case gfx::TransferFunction::PQ:
    214      return Some(kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ);
    215    case gfx::TransferFunction::HLG:
    216      return Some(kCVImageBufferTransferFunction_ITU_R_2100_HLG);
    217  }
    218 
    219  MOZ_ASSERT_UNREACHABLE("Unsupported transfer function");
    220  return Nothing();
    221 }
    222 
    223 struct EncoderColorSpace {
    224  CFStringRef mColorPrimaries = nullptr;
    225  CFStringRef mYCbCrMatrix = nullptr;
    226  CFStringRef mTransferFunction = nullptr;
    227 };
    228 
    229 static Result<EncoderColorSpace, MediaResult> MapColorSpace(
    230    const EncoderConfig::VideoColorSpace& aColorSpace) {
    231  EncoderColorSpace colorSpace;
    232  if (aColorSpace.mPrimaries) {
    233    Maybe<CFStringRef> p = MapColorPrimaries(aColorSpace.mPrimaries.ref());
    234    if (p.isNothing()) {
    235      return Err(MediaResult(
    236          NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR,
    237          RESULT_DETAIL("Unsupported color primaries: %u",
    238                        static_cast<uint8_t>(aColorSpace.mPrimaries.ref()))));
    239    }
    240    colorSpace.mColorPrimaries = p.value();
    241  }
    242  if (aColorSpace.mMatrix) {
    243    Maybe<CFStringRef> m = MapYCbCrMatrix(aColorSpace.mMatrix.ref());
    244    if (m.isNothing()) {
    245      return Err(MediaResult(
    246          NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR,
    247          RESULT_DETAIL("Unsupported YCbCr matrix: %u",
    248                        static_cast<uint8_t>(aColorSpace.mMatrix.ref()))));
    249    }
    250    colorSpace.mYCbCrMatrix = m.value();
    251  }
    252  if (aColorSpace.mTransferFunction) {
    253    Maybe<CFStringRef> f =
    254        MapTransferFunction(aColorSpace.mTransferFunction.ref());
    255    if (f.isNothing()) {
    256      return Err(MediaResult(
    257          NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR,
    258          RESULT_DETAIL(
    259              "Unsupported transfer function: %u",
    260              static_cast<uint8_t>(aColorSpace.mTransferFunction.ref()))));
    261    }
    262    colorSpace.mTransferFunction = f.value();
    263  }
    264  return colorSpace;
    265 }
    266 
    267 bool AppleVTEncoder::IsSettingColorSpaceSupported() const {
    268  SessionPropertyManager mgr(mSession);
    269  return mgr.IsSupported(kVTCompressionPropertyKey_ColorPrimaries) &&
    270         mgr.IsSupported(kVTCompressionPropertyKey_YCbCrMatrix) &&
    271         mgr.IsSupported(kVTCompressionPropertyKey_TransferFunction);
    272 }
    273 
    274 MediaResult AppleVTEncoder::SetColorSpace(
    275    const EncoderConfig::SampleFormat& aFormat) {
    276  MOZ_ASSERT(mSession);
    277 
    278  if (!aFormat.IsYUV()) {
    279    return MediaResult(NS_OK, "Skip setting color space for non-YUV formats");
    280  }
    281 
    282  if (!IsSettingColorSpaceSupported()) {
    283    return MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR,
    284                       "Setting color space not supported");
    285  }
    286 
    287  auto r = MapColorSpace(aFormat.mColorSpace);
    288  if (r.isErr()) {
    289    return r.unwrapErr();
    290  }
    291 
    292  EncoderColorSpace colorSpace = r.unwrap();
    293 
    294  SessionPropertyManager mgr(mSession);
    295  AutoTArray<const char*, 3> properties;
    296 
    297  if (colorSpace.mColorPrimaries) {
    298    OSStatus status = mgr.Set(kVTCompressionPropertyKey_ColorPrimaries,
    299                              colorSpace.mColorPrimaries);
    300    if (status != noErr) {
    301      return MediaResult(
    302          NS_ERROR_DOM_MEDIA_FATAL_ERR,
    303          RESULT_DETAIL("Failed to set color primaries. Error: %d", status));
    304    }
    305    properties.AppendElement("ColorPrimaries");
    306  }
    307  if (colorSpace.mYCbCrMatrix) {
    308    OSStatus status =
    309        mgr.Set(kVTCompressionPropertyKey_YCbCrMatrix, colorSpace.mYCbCrMatrix);
    310    if (status != noErr) {
    311      return MediaResult(
    312          NS_ERROR_DOM_MEDIA_FATAL_ERR,
    313          RESULT_DETAIL("Failed to set YCbCr matrix. Error: %d", status));
    314    }
    315    properties.AppendElement("YCbCrMatrix");
    316  }
    317  if (colorSpace.mTransferFunction) {
    318    OSStatus status = mgr.Set(kVTCompressionPropertyKey_TransferFunction,
    319                              colorSpace.mTransferFunction);
    320    if (status != noErr) {
    321      return MediaResult(
    322          NS_ERROR_DOM_MEDIA_FATAL_ERR,
    323          RESULT_DETAIL("Failed to set transfer function. Error: %d", status));
    324    }
    325    properties.AppendElement("TransferFunction");
    326  }
    327 
    328  nsCString msg;
    329  if (properties.IsEmpty()) {
    330    msg = "No color space properties set"_ns;
    331  } else {
    332    msg = StringJoin(","_ns, properties);
    333    msg.Append(" set");
    334  }
    335 
    336  return MediaResult(NS_OK, msg);
    337 }
    338 
    339 static Result<OSType, MediaResult> MapPixelFormat(
    340    dom::ImageBitmapFormat aFormat, gfx::ColorRange aColorRange) {
    341  const bool isFullRange = aColorRange == gfx::ColorRange::FULL;
    342 
    343  Maybe<OSType> fmt;
    344  switch (aFormat) {
    345    case dom::ImageBitmapFormat::YUV444P:
    346      return kCVPixelFormatType_444YpCbCr8;
    347    case dom::ImageBitmapFormat::YUV420P:
    348      return isFullRange ? kCVPixelFormatType_420YpCbCr8PlanarFullRange
    349                         : kCVPixelFormatType_420YpCbCr8Planar;
    350    case dom::ImageBitmapFormat::YUV420SP_NV12:
    351      return isFullRange ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
    352                         : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
    353    case dom::ImageBitmapFormat::RGBA32:
    354      fmt.emplace(kCVPixelFormatType_32RGBA);
    355      break;
    356    case dom::ImageBitmapFormat::BGRA32:
    357      fmt.emplace(kCVPixelFormatType_32BGRA);
    358      break;
    359    case dom::ImageBitmapFormat::RGB24:
    360      fmt.emplace(kCVPixelFormatType_24RGB);
    361      break;
    362    case dom::ImageBitmapFormat::BGR24:
    363      fmt.emplace(kCVPixelFormatType_24BGR);
    364      break;
    365    case dom::ImageBitmapFormat::GRAY8:
    366      fmt.emplace(kCVPixelFormatType_OneComponent8);
    367      break;
    368    default:
    369      MOZ_ASSERT_UNREACHABLE("Unsupported image format");
    370  }
    371 
    372  // Limited RGB formats are not supported on MacOS (Bug 1957758).
    373  if (fmt) {
    374    if (!isFullRange) {
    375      return Err(MediaResult(
    376          NS_ERROR_NOT_IMPLEMENTED,
    377          RESULT_DETAIL("format %s with limited colorspace is not supported",
    378                        dom::GetEnumString(aFormat).get())));
    379    }
    380    return fmt.value();
    381  }
    382 
    383  return Err(MediaResult(NS_ERROR_NOT_IMPLEMENTED,
    384                         RESULT_DETAIL("format %s is not supported",
    385                                       dom::GetEnumString(aFormat).get())));
    386 }
    387 
    388 RefPtr<MediaDataEncoder::InitPromise> AppleVTEncoder::Init() {
    389  MOZ_ASSERT(!mSession,
    390             "Cannot initialize encoder again without shutting down");
    391 
    392  MediaResult r = InitSession();
    393  if (NS_FAILED(r.Code())) {
    394    LOGE("%s", r.Description().get());
    395    return InitPromise::CreateAndReject(r, __func__);
    396  }
    397 
    398  mError = NS_OK;
    399  return InitPromise::CreateAndResolve(true, __func__);
    400 }
    401 
    402 MediaResult AppleVTEncoder::InitSession() {
    403  MOZ_ASSERT(!mSession);
    404 
    405  auto errorExit = MakeScopeExit([&] { InvalidateSessionIfNeeded(); });
    406 
    407  if (mConfig.mSize.width == 0 || mConfig.mSize.height == 0) {
    408    return MediaResult(
    409        NS_ERROR_ILLEGAL_VALUE,
    410        RESULT_DETAIL("Neither width (%d) nor height (%d) can be zero",
    411                      mConfig.mSize.width, mConfig.mSize.height));
    412  }
    413 
    414  if (mConfig.mScalabilityMode != ScalabilityMode::None && !OSSupportsSVC()) {
    415    return MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR,
    416                       "SVC only supported on macOS 11.3 and more recent"_ns);
    417  }
    418 
    419  bool lowLatencyRateControl =
    420      mConfig.mUsage == Usage::Realtime ||
    421      mConfig.mScalabilityMode != ScalabilityMode::None;
    422  LOGD("low latency rate control: %s, Hardware allowed: %s",
    423       lowLatencyRateControl ? "yes" : "no",
    424       mHardwareNotAllowed ? "no" : "yes");
    425  AutoCFTypeRef<CFDictionaryRef> spec(
    426      BuildEncoderSpec(mHardwareNotAllowed, lowLatencyRateControl));
    427 
    428  // Bug 1955153: Set sourceImageBufferAttributes using the pixel format derived
    429  // from mConfig.mFormat.
    430  OSStatus status = VTCompressionSessionCreate(
    431      kCFAllocatorDefault, mConfig.mSize.width, mConfig.mSize.height,
    432      kCMVideoCodecType_H264, spec, nullptr /* sourceImageBufferAttributes */,
    433      kCFAllocatorDefault, &FrameCallback, this /* outputCallbackRefCon */,
    434      mSession.Receive());
    435  if (status != noErr) {
    436    return MediaResult(
    437        NS_ERROR_DOM_MEDIA_FATAL_ERR,
    438        RESULT_DETAIL("fail to create encoder session. Error: %d", status));
    439  }
    440 
    441  SessionPropertyManager mgr(mSession);
    442 
    443  status = mgr.Set(kVTCompressionPropertyKey_AllowFrameReordering, false);
    444  if (status != noErr) {
    445    return MediaResult(
    446        NS_ERROR_DOM_MEDIA_FATAL_ERR,
    447        RESULT_DETAIL("Couldn't disable bframes. Error: %d", status));
    448  }
    449 
    450  if (mConfig.mUsage == Usage::Realtime && !SetRealtime(true)) {
    451    return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    452                       "fail to configure real-time"_ns);
    453  }
    454 
    455  if (mConfig.mBitrate) {
    456    if (mConfig.mCodec == CodecType::H264 &&
    457        mConfig.mBitrateMode == BitrateMode::Constant) {
    458      // Not supported, fall-back to VBR.
    459      LOGD("H264 CBR not supported in VideoToolbox, falling back to VBR");
    460      mConfig.mBitrateMode = BitrateMode::Variable;
    461    }
    462    bool rv = SetBitrateAndMode(mConfig.mBitrateMode, mConfig.mBitrate);
    463    if (!rv) {
    464      return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    465                         "fail to configurate bitrate"_ns);
    466    }
    467  }
    468 
    469  if (mConfig.mScalabilityMode != ScalabilityMode::None) {
    470    if (__builtin_available(macos 11.3, *)) {
    471      float baseLayerFPSRatio = 1.0f;
    472      switch (mConfig.mScalabilityMode) {
    473        case ScalabilityMode::L1T2:
    474          baseLayerFPSRatio = 0.5;
    475          break;
    476        case ScalabilityMode::L1T3:
    477          // Not supported in hw on macOS, but is accepted and errors out when
    478          // encoding. Reject the configuration now.
    479          return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    480                             RESULT_DETAIL("macOS only support L1T2 h264 SVC"));
    481        default:
    482          MOZ_ASSERT_UNREACHABLE("Unhandled value");
    483      }
    484 
    485      status = mgr.Set(kVTCompressionPropertyKey_BaseLayerFrameRateFraction,
    486                       baseLayerFPSRatio);
    487      if (status != noErr) {
    488        return MediaResult(
    489            NS_ERROR_DOM_MEDIA_FATAL_ERR,
    490            RESULT_DETAIL("fail to configure SVC (base ratio: %f). Error: %d",
    491                          baseLayerFPSRatio, status));
    492      }
    493    } else {
    494      return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    495                         "macOS version too old to enable SVC"_ns);
    496    }
    497  }
    498 
    499  int64_t interval =
    500      mConfig.mKeyframeInterval > std::numeric_limits<int64_t>::max()
    501          ? std::numeric_limits<int64_t>::max()
    502          : AssertedCast<int64_t>(mConfig.mKeyframeInterval);
    503 
    504  status = mgr.Set(kVTCompressionPropertyKey_MaxKeyFrameInterval, interval);
    505  if (status != noErr) {
    506    return MediaResult(
    507        NS_ERROR_DOM_MEDIA_FATAL_ERR,
    508        RESULT_DETAIL("fail to configurate keyframe interval: %" PRId64
    509                      ". Error: %d",
    510                      interval, status));
    511  }
    512 
    513  if (mConfig.mCodecSpecific.is<H264Specific>()) {
    514    const H264Specific& specific = mConfig.mCodecSpecific.as<H264Specific>();
    515    if (!SetProfileLevel(specific.mProfile)) {
    516      return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    517                         RESULT_DETAIL("fail to configurate profile level:%d",
    518                                       int(specific.mProfile)));
    519    }
    520  }
    521 
    522  MediaResult colorSpaceResult = SetColorSpace(mConfig.mFormat);
    523  if (NS_SUCCEEDED(colorSpaceResult.Code())) {
    524    LOGD("%s", colorSpaceResult.Description().get());
    525  } else if (colorSpaceResult.Code() == NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR) {
    526    // Color space not supported, ignore.
    527    LOGW("%s", colorSpaceResult.Description().get());
    528  } else {
    529    MOZ_ASSERT(NS_FAILED(colorSpaceResult.Code()));
    530    LOGE("%s", colorSpaceResult.Description().get());
    531    return colorSpaceResult;
    532  }
    533 
    534  bool isUsingHW = false;
    535  status =
    536      mgr.Copy(kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder,
    537               isUsingHW);
    538  mIsHardwareAccelerated = status == noErr && isUsingHW;
    539  LOGD("Using hw acceleration: %s", mIsHardwareAccelerated ? "yes" : "no");
    540 
    541  errorExit.release();
    542  return NS_OK;
    543 }
    544 
    545 void AppleVTEncoder::InvalidateSessionIfNeeded() {
    546  if (mSession) {
    547    VTCompressionSessionInvalidate(mSession);
    548    mSession.Reset();
    549  }
    550 }
    551 
    552 CFDictionaryRef AppleVTEncoder::BuildSourceImageBufferAttributes(
    553    OSType aPixelFormat) {
    554  // Source image buffer attributes
    555  const void* keys[] = {kCVPixelBufferOpenGLCompatibilityKey,  // TODO
    556                        kCVPixelBufferIOSurfacePropertiesKey,  // TODO
    557                        kCVPixelBufferPixelFormatTypeKey};
    558 
    559  AutoCFTypeRef<CFDictionaryRef> ioSurfaceProps(CFDictionaryCreate(
    560      kCFAllocatorDefault, nullptr, nullptr, 0, &kCFTypeDictionaryKeyCallBacks,
    561      &kCFTypeDictionaryValueCallBacks));
    562  AutoCFTypeRef<CFNumberRef> pixelFormat(
    563      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &aPixelFormat));
    564  const void* values[] = {kCFBooleanTrue, ioSurfaceProps, pixelFormat};
    565 
    566  MOZ_ASSERT(std::size(keys) == std::size(values),
    567             "Non matching keys/values array size");
    568 
    569  return CFDictionaryCreate(kCFAllocatorDefault, keys, values, std::size(keys),
    570                            &kCFTypeDictionaryKeyCallBacks,
    571                            &kCFTypeDictionaryValueCallBacks);
    572 }
    573 
    574 static bool IsKeyframe(CMSampleBufferRef aSample) {
    575  CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(aSample, 0);
    576  if (attachments == nullptr || CFArrayGetCount(attachments) == 0) {
    577    return false;
    578  }
    579 
    580  return !CFDictionaryContainsKey(
    581      static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)),
    582      kCMSampleAttachmentKey_NotSync);
    583 }
    584 
    585 static size_t GetNumParamSets(CMFormatDescriptionRef aDescription) {
    586  size_t numParamSets = 0;
    587  OSStatus status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
    588      aDescription, 0, nullptr, nullptr, &numParamSets, nullptr);
    589  if (status != noErr) {
    590    LOGE("Cannot get number of parameter sets from format description");
    591  }
    592 
    593  return numParamSets;
    594 }
    595 
    596 static const uint8_t kNALUStart[4] = {0, 0, 0, 1};
    597 
    598 static size_t GetParamSet(CMFormatDescriptionRef aDescription, size_t aIndex,
    599                          const uint8_t** aDataPtr) {
    600  size_t length = 0;
    601  int headerSize = 0;
    602  if (CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
    603          aDescription, aIndex, aDataPtr, &length, nullptr, &headerSize) !=
    604      noErr) {
    605    LOGE("failed to get parameter set from format description");
    606    return 0;
    607  }
    608  MOZ_ASSERT(headerSize == sizeof(kNALUStart), "Only support 4 byte header");
    609 
    610  return length;
    611 }
    612 
    613 static bool WriteSPSPPS(MediaRawData* aDst,
    614                        CMFormatDescriptionRef aDescription) {
    615  // Get SPS/PPS
    616  const size_t numParamSets = GetNumParamSets(aDescription);
    617  UniquePtr<MediaRawDataWriter> writer(aDst->CreateWriter());
    618  for (size_t i = 0; i < numParamSets; i++) {
    619    const uint8_t* data = nullptr;
    620    size_t length = GetParamSet(aDescription, i, &data);
    621    if (length == 0) {
    622      return false;
    623    }
    624    if (!writer->Append(kNALUStart, sizeof(kNALUStart))) {
    625      LOGE("Cannot write NAL unit start code");
    626      return false;
    627    }
    628    if (!writer->Append(data, length)) {
    629      LOGE("Cannot write parameter set");
    630      return false;
    631    }
    632  }
    633  return true;
    634 }
    635 
    636 static RefPtr<MediaByteBuffer> extractAvcc(
    637    CMFormatDescriptionRef aDescription) {
    638  CFPropertyListRef list = CMFormatDescriptionGetExtension(
    639      aDescription,
    640      kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms);
    641  if (!list) {
    642    LOGE("fail to get atoms");
    643    return nullptr;
    644  }
    645  CFDataRef avcC = static_cast<CFDataRef>(
    646      CFDictionaryGetValue(static_cast<CFDictionaryRef>(list), CFSTR("avcC")));
    647  if (!avcC) {
    648    LOGE("fail to extract avcC");
    649    return nullptr;
    650  }
    651  CFIndex length = CFDataGetLength(avcC);
    652  const UInt8* bytes = CFDataGetBytePtr(avcC);
    653  if (length <= 0 || !bytes) {
    654    LOGE("empty avcC");
    655    return nullptr;
    656  }
    657 
    658  RefPtr<MediaByteBuffer> config = new MediaByteBuffer(length);
    659  config->AppendElements(bytes, length);
    660  return config;
    661 }
    662 
    663 bool AppleVTEncoder::WriteExtraData(MediaRawData* aDst, CMSampleBufferRef aSrc,
    664                                    const bool aAsAnnexB) {
    665  if (!IsKeyframe(aSrc)) {
    666    return true;
    667  }
    668 
    669  LOGV("Writing extra data (%s) for keyframe", aAsAnnexB ? "AnnexB" : "AVCC");
    670 
    671  aDst->mKeyframe = true;
    672  CMFormatDescriptionRef desc = CMSampleBufferGetFormatDescription(aSrc);
    673  if (!desc) {
    674    LOGE("fail to get format description from sample");
    675    return false;
    676  }
    677 
    678  if (aAsAnnexB) {
    679    return WriteSPSPPS(aDst, desc);
    680  }
    681 
    682  RefPtr<MediaByteBuffer> avcc = extractAvcc(desc);
    683  if (!avcc) {
    684    LOGE("failed to extract avcc");
    685    return false;
    686  }
    687 
    688  if (!mAvcc || !H264::CompareExtraData(avcc, mAvcc)) {
    689    LOGV("avcC changed, updating");
    690    mAvcc = avcc;
    691    aDst->mExtraData = mAvcc;
    692  }
    693 
    694  return true;
    695 }
    696 
    697 static bool WriteNALUs(MediaRawData* aDst, CMSampleBufferRef aSrc,
    698                       bool aAsAnnexB = false) {
    699  size_t srcRemaining = CMSampleBufferGetTotalSampleSize(aSrc);
    700  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(aSrc);
    701  if (!block) {
    702    LOGE("Cannot get block buffer frome sample");
    703    return false;
    704  }
    705  UniquePtr<MediaRawDataWriter> writer(aDst->CreateWriter());
    706  size_t writtenLength = aDst->Size();
    707  // Ensure capacity.
    708  if (!writer->SetSize(writtenLength + srcRemaining)) {
    709    LOGE("Cannot allocate buffer");
    710    return false;
    711  }
    712  size_t readLength = 0;
    713  while (srcRemaining > 0) {
    714    // Extract the size of next NAL unit
    715    uint8_t unitSizeBytes[4];
    716    MOZ_ASSERT(srcRemaining > sizeof(unitSizeBytes));
    717    if (CMBlockBufferCopyDataBytes(block, readLength, sizeof(unitSizeBytes),
    718                                   reinterpret_cast<uint32_t*>(
    719                                       unitSizeBytes)) != kCMBlockBufferNoErr) {
    720      LOGE("Cannot copy unit size bytes");
    721      return false;
    722    }
    723    size_t unitSize =
    724        CFSwapInt32BigToHost(*reinterpret_cast<uint32_t*>(unitSizeBytes));
    725 
    726    if (aAsAnnexB) {
    727      // Replace unit size bytes with NALU start code.
    728      PodCopy(writer->Data() + writtenLength, kNALUStart, sizeof(kNALUStart));
    729      readLength += sizeof(unitSizeBytes);
    730      srcRemaining -= sizeof(unitSizeBytes);
    731      writtenLength += sizeof(kNALUStart);
    732    } else {
    733      // Copy unit size bytes + data.
    734      unitSize += sizeof(unitSizeBytes);
    735    }
    736    MOZ_ASSERT(writtenLength + unitSize <= aDst->Size());
    737    // Copy NAL unit data
    738    if (CMBlockBufferCopyDataBytes(block, readLength, unitSize,
    739                                   writer->Data() + writtenLength) !=
    740        kCMBlockBufferNoErr) {
    741      LOGE("Cannot copy unit data");
    742      return false;
    743    }
    744    readLength += unitSize;
    745    srcRemaining -= unitSize;
    746    writtenLength += unitSize;
    747  }
    748  MOZ_ASSERT(writtenLength == aDst->Size());
    749  return true;
    750 }
    751 
    752 void AppleVTEncoder::OutputFrame(OSStatus aStatus, VTEncodeInfoFlags aFlags,
    753                                 CMSampleBufferRef aBuffer) {
    754  LOGV("status: %d, flags: %d, buffer %p", aStatus, aFlags, aBuffer);
    755 
    756  if (aStatus != noErr) {
    757    ProcessOutput(nullptr, EncodeResult::EncodeError);
    758    return;
    759  }
    760 
    761  if (aFlags & kVTEncodeInfo_FrameDropped) {
    762    ProcessOutput(nullptr, EncodeResult::FrameDropped);
    763    return;
    764  }
    765 
    766  if (!aBuffer) {
    767    ProcessOutput(nullptr, EncodeResult::EmptyBuffer);
    768    return;
    769  }
    770 
    771  RefPtr<MediaRawData> output(new MediaRawData());
    772 
    773  if (__builtin_available(macos 11.3, *)) {
    774    if (mConfig.mScalabilityMode != ScalabilityMode::None) {
    775      CFDictionaryRef dict = (CFDictionaryRef)(CFArrayGetValueAtIndex(
    776          CMSampleBufferGetSampleAttachmentsArray(aBuffer, true), 0));
    777      CFBooleanRef isBaseLayerRef = (CFBooleanRef)CFDictionaryGetValue(
    778          dict, (const void*)kCMSampleAttachmentKey_IsDependedOnByOthers);
    779      Boolean isBaseLayer = CFBooleanGetValue(isBaseLayerRef);
    780      output->mTemporalLayerId.emplace(isBaseLayer ? 0 : 1);
    781    }
    782  }
    783 
    784  bool forceAvcc = false;
    785  if (mConfig.mCodecSpecific.is<H264Specific>()) {
    786    forceAvcc = mConfig.mCodecSpecific.as<H264Specific>().mFormat ==
    787                H264BitStreamFormat::AVC;
    788  }
    789  bool asAnnexB = !forceAvcc;
    790  bool succeeded = WriteExtraData(output, aBuffer, asAnnexB) &&
    791                   WriteNALUs(output, aBuffer, asAnnexB);
    792 
    793  output->mTime = media::TimeUnit::FromSeconds(
    794      CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(aBuffer)));
    795  output->mDuration = media::TimeUnit::FromSeconds(
    796      CMTimeGetSeconds(CMSampleBufferGetOutputDuration(aBuffer)));
    797  LOGV("Make a %s output[time: %s, duration: %s]: %s",
    798       asAnnexB ? "AnnexB" : "AVCC", output->mTime.ToString().get(),
    799       output->mDuration.ToString().get(), succeeded ? "succeed" : "failed");
    800  ProcessOutput(succeeded ? std::move(output) : nullptr, EncodeResult::Success);
    801 }
    802 
    803 void AppleVTEncoder::ProcessOutput(RefPtr<MediaRawData>&& aOutput,
    804                                   EncodeResult aResult) {
    805  if (!mTaskQueue->IsCurrentThreadIn()) {
    806    LOGV("Dispatch ProcessOutput to task queue");
    807    nsresult rv = mTaskQueue->Dispatch(
    808        NewRunnableMethod<RefPtr<MediaRawData>, EncodeResult>(
    809            "AppleVTEncoder::ProcessOutput", this,
    810            &AppleVTEncoder::ProcessOutput, std::move(aOutput), aResult));
    811    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
    812    (void)rv;
    813    return;
    814  }
    815 
    816  if (aResult != EncodeResult::Success) {
    817    switch (aResult) {
    818      case EncodeResult::EncodeError:
    819        mError =
    820            MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to encode"_ns);
    821        break;
    822      case EncodeResult::EmptyBuffer:
    823        mError =
    824            MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Buffer is empty"_ns);
    825        break;
    826      case EncodeResult::FrameDropped:
    827        if (mConfig.mUsage == Usage::Realtime) {
    828          // Dropping a frame in real-time usage is okay.
    829          LOGW("Frame is dropped");
    830        } else {
    831          // Some usages like transcoding should not drop a frame.
    832          LOGE("Frame is dropped");
    833          mError =
    834              MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Frame is dropped"_ns);
    835        }
    836        break;
    837      default:
    838        MOZ_ASSERT_UNREACHABLE("Unknown EncodeResult");
    839        break;
    840    }
    841    MaybeResolveOrRejectEncodePromise();
    842    return;
    843  }
    844 
    845  LOGV("Got %zu bytes of output", !aOutput.get() ? 0 : aOutput->Size());
    846 
    847  if (!aOutput) {
    848    mError =
    849        MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "No converted output"_ns);
    850    MaybeResolveOrRejectEncodePromise();
    851    return;
    852  }
    853 
    854  mEncodedData.AppendElement(std::move(aOutput));
    855  MaybeResolveOrRejectEncodePromise();
    856 }
    857 
    858 RefPtr<MediaDataEncoder::EncodePromise> AppleVTEncoder::Encode(
    859    const MediaData* aSample) {
    860  MOZ_ASSERT(aSample != nullptr);
    861 
    862  RefPtr<const VideoData> sample(aSample->As<const VideoData>());
    863 
    864  RefPtr<AppleVTEncoder> self = this;
    865  return InvokeAsync(mTaskQueue, __func__, [self, this, sample] {
    866    MOZ_ASSERT(mEncodePromise.IsEmpty(),
    867               "Encode should not be called again before getting results");
    868    RefPtr<EncodePromise> p = mEncodePromise.Ensure(__func__);
    869    ProcessEncode(sample);
    870    return p;
    871  });
    872 }
    873 
    874 // TODO(Bug 1984936): For realtime mode, resolve the promise after the first
    875 // sample's result is available, then continue processing remaining samples.
    876 // This allows the caller to keep submitting new samples while the encoder
    877 // handles pending ones.
    878 RefPtr<MediaDataEncoder::EncodePromise> AppleVTEncoder::Encode(
    879    nsTArray<RefPtr<MediaData>>&& aSamples) {
    880  MOZ_ASSERT(!aSamples.IsEmpty());
    881 
    882  RefPtr<AppleVTEncoder> self = this;
    883  return InvokeAsync(
    884      mTaskQueue, __func__, [self, samples = std::move(aSamples)]() mutable {
    885        MOZ_ASSERT(self->mEncodeBatchPromise.IsEmpty(),
    886                   "Encode should not be called again before getting results");
    887        RefPtr<EncodePromise> p = self->mEncodeBatchPromise.Ensure(__func__);
    888        self->EncodeNextSample(std::move(samples), EncodedData());
    889        return p;
    890      });
    891 }
    892 
    893 RefPtr<MediaDataEncoder::ReconfigurationPromise> AppleVTEncoder::Reconfigure(
    894    const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) {
    895  return InvokeAsync(mTaskQueue, this, __func__,
    896                     &AppleVTEncoder::ProcessReconfigure,
    897                     aConfigurationChanges);
    898 }
    899 
    900 void AppleVTEncoder::ProcessEncode(const RefPtr<const VideoData>& aSample) {
    901  LOGV("::ProcessEncode");
    902  AssertOnTaskQueue();
    903  MOZ_ASSERT(mSession);
    904 
    905  if (NS_FAILED(mError)) {
    906    LOGE("Pending error: %s", mError.Description().get());
    907    MaybeResolveOrRejectEncodePromise();
    908  }
    909 
    910  AutoCVBufferRef<CVImageBufferRef> buffer(
    911      CreateCVPixelBuffer(aSample->mImage));
    912  if (!buffer) {
    913    LOGE("Failed to allocate buffer");
    914    mError =
    915        MediaResult(NS_ERROR_OUT_OF_MEMORY, "failed to allocate buffer"_ns);
    916    MaybeResolveOrRejectEncodePromise();
    917    return;
    918  }
    919 
    920  CFDictionaryRef frameProps = nullptr;
    921  if (aSample->mKeyframe) {
    922    CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
    923    CFTypeRef values[] = {kCFBooleanTrue};
    924    MOZ_ASSERT(std::size(keys) == std::size(values));
    925    frameProps = CFDictionaryCreate(
    926        kCFAllocatorDefault, keys, values, std::size(keys),
    927        &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
    928  };
    929 
    930  VTEncodeInfoFlags info;
    931  OSStatus status = VTCompressionSessionEncodeFrame(
    932      mSession, buffer,
    933      CMTimeMake(aSample->mTime.ToMicroseconds(), USECS_PER_S),
    934      CMTimeMake(aSample->mDuration.ToMicroseconds(), USECS_PER_S), frameProps,
    935      nullptr /* sourceFrameRefcon */, &info);
    936  if (status != noErr) {
    937    LOGE("VTCompressionSessionEncodeFrame error: %d", status);
    938    mError = MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
    939                         "VTCompressionSessionEncodeFrame error"_ns);
    940    MaybeResolveOrRejectEncodePromise();
    941    return;
    942  }
    943 
    944  if (mConfig.mUsage != Usage::Realtime) {
    945    MaybeResolveOrRejectEncodePromise();
    946    return;
    947  }
    948 
    949  // The latency between encoding a sample and receiving the encoded output is
    950  // critical in real-time usage. To minimize the latency, the output result
    951  // should be returned immediately once they are ready, instead of being
    952  // returned in the next or later Encode() iterations.
    953  LOGV("Encoding in progress");
    954 
    955  // Workaround for real-time encoding in OS versions < 11.
    956  ForceOutputIfNeeded();
    957 }
    958 
    959 RefPtr<MediaDataEncoder::ReconfigurationPromise>
    960 AppleVTEncoder::ProcessReconfigure(
    961    const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) {
    962  AssertOnTaskQueue();
    963  MOZ_ASSERT(mSession);
    964 
    965  bool ok = false;
    966  for (const auto& confChange : aConfigurationChanges->mChanges) {
    967    // A reconfiguration on the fly succeeds if all changes can be applied
    968    // successfuly. In case of failure, the encoder will be drained and
    969    // recreated.
    970    ok &= confChange.match(
    971        // Not supported yet
    972        [&](const DimensionsChange& aChange) -> bool { return false; },
    973        [&](const DisplayDimensionsChange& aChange) -> bool { return false; },
    974        [&](const BitrateModeChange& aChange) -> bool {
    975          mConfig.mBitrateMode = aChange.get();
    976          return SetBitrateAndMode(mConfig.mBitrateMode, mConfig.mBitrate);
    977        },
    978        [&](const BitrateChange& aChange) -> bool {
    979          mConfig.mBitrate = aChange.get().refOr(0);
    980          // 0 is the default in AppleVTEncoder: the encoder chooses the bitrate
    981          // based on the content.
    982          return SetBitrateAndMode(mConfig.mBitrateMode, mConfig.mBitrate);
    983        },
    984        [&](const FramerateChange& aChange) -> bool {
    985          // 0 means default, in VideoToolbox, and is valid, perform some light
    986          // sanitation on other values.
    987          double fps = aChange.get().refOr(0);
    988          if (std::isnan(fps) || fps < 0 ||
    989              int64_t(fps) > std::numeric_limits<int32_t>::max()) {
    990            LOGE("Invalid fps of %lf", fps);
    991            return false;
    992          }
    993          return SetFrameRate(AssertedCast<int64_t>(fps));
    994        },
    995        [&](const UsageChange& aChange) -> bool {
    996          mConfig.mUsage = aChange.get();
    997          return SetRealtime(aChange.get() == Usage::Realtime);
    998        },
    999        [&](const ContentHintChange& aChange) -> bool { return false; },
   1000        [&](const SampleRateChange& aChange) -> bool { return false; },
   1001        [&](const NumberOfChannelsChange& aChange) -> bool { return false; });
   1002  };
   1003  using P = MediaDataEncoder::ReconfigurationPromise;
   1004  if (ok) {
   1005    return P::CreateAndResolve(true, __func__);
   1006  }
   1007  return P::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
   1008 }
   1009 
   1010 static size_t NumberOfPlanes(OSType aPixelFormat) {
   1011  switch (aPixelFormat) {
   1012    case kCVPixelFormatType_32RGBA:
   1013    case kCVPixelFormatType_32BGRA:
   1014    case kCVPixelFormatType_24RGB:
   1015    case kCVPixelFormatType_24BGR:
   1016    case kCVPixelFormatType_OneComponent8:
   1017      return 1;
   1018    case kCVPixelFormatType_444YpCbCr8:
   1019    case kCVPixelFormatType_420YpCbCr8PlanarFullRange:
   1020    case kCVPixelFormatType_420YpCbCr8Planar:
   1021      return 3;
   1022    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
   1023    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
   1024      return 2;
   1025    default:
   1026      LOGE("Unsupported input pixel format");
   1027      return 0;
   1028  }
   1029 }
   1030 
   1031 using namespace layers;
   1032 
   1033 static void ReleaseSurface(void* aReleaseRef, const void* aBaseAddress) {
   1034  RefPtr<gfx::DataSourceSurface> released =
   1035      dont_AddRef(static_cast<gfx::DataSourceSurface*>(aReleaseRef));
   1036 }
   1037 
   1038 static void ReleaseImage(void* aImageGrip, const void* aDataPtr,
   1039                         size_t aDataSize, size_t aNumOfPlanes,
   1040                         const void** aPlanes) {
   1041  (static_cast<PlanarYCbCrImage*>(aImageGrip))->Release();
   1042 }
   1043 
   1044 CVPixelBufferRef AppleVTEncoder::CreateCVPixelBuffer(Image* aSource) {
   1045  AssertOnTaskQueue();
   1046 
   1047  auto sfr = EncoderConfig::SampleFormat::FromImage(aSource);
   1048  if (sfr.isErr()) {
   1049    MediaResult err = sfr.unwrapErr();
   1050    LOGE("%s", err.Description().get());
   1051    return nullptr;
   1052  }
   1053  const EncoderConfig::SampleFormat sf = sfr.unwrap();
   1054 
   1055  gfx::ColorRange defaultColorRange =
   1056      sf.IsYUV() ? gfx::ColorRange::LIMITED : gfx::ColorRange::FULL;
   1057  auto pfr = MapPixelFormat(sf.mPixelFormat, sf.mColorSpace.mRange
   1058                                                 ? sf.mColorSpace.mRange.value()
   1059                                                 : defaultColorRange);
   1060  if (pfr.isErr()) {
   1061    MediaResult err = pfr.unwrapErr();
   1062    LOGE("%s", err.Description().get());
   1063    return nullptr;
   1064  }
   1065 
   1066  OSType pixelFormat = pfr.unwrap();
   1067 
   1068  if (sf != mConfig.mFormat) {
   1069    LOGV(
   1070        "Input image in format %s but encoder configured with format %s. "
   1071        "Fingers crossed",
   1072        sf.ToString().get(), mConfig.mFormat.ToString().get());
   1073    // Bug 1955153: If the encoder encounters a kVTPixelTransferNotSupportedErr
   1074    // error due to an unsupported image format, it must be re-initialized.
   1075    // Additionally, any changes to the color space also require re-initializing
   1076    // the encoder.
   1077  }
   1078 
   1079  if (aSource->GetFormat() == ImageFormat::PLANAR_YCBCR) {
   1080    PlanarYCbCrImage* image = aSource->AsPlanarYCbCrImage();
   1081    if (!image || !image->GetData()) {
   1082      LOGE("Failed to get PlanarYCbCrImage or its data");
   1083      return nullptr;
   1084    }
   1085 
   1086    size_t numPlanes = NumberOfPlanes(pixelFormat);
   1087    const PlanarYCbCrImage::Data* yuv = image->GetData();
   1088 
   1089    auto ySize = yuv->YDataSize();
   1090    auto cbcrSize = yuv->CbCrDataSize();
   1091    void* addresses[3] = {};
   1092    size_t widths[3] = {};
   1093    size_t heights[3] = {};
   1094    size_t strides[3] = {};
   1095    switch (numPlanes) {
   1096      case 3:
   1097        addresses[2] = yuv->mCrChannel;
   1098        widths[2] = cbcrSize.width;
   1099        heights[2] = cbcrSize.height;
   1100        strides[2] = yuv->mCbCrStride;
   1101        [[fallthrough]];
   1102      case 2:
   1103        addresses[1] = yuv->mCbChannel;
   1104        widths[1] = cbcrSize.width;
   1105        heights[1] = cbcrSize.height;
   1106        strides[1] = yuv->mCbCrStride;
   1107        [[fallthrough]];
   1108      case 1:
   1109        addresses[0] = yuv->mYChannel;
   1110        widths[0] = ySize.width;
   1111        heights[0] = ySize.height;
   1112        strides[0] = yuv->mYStride;
   1113        break;
   1114      default:
   1115        LOGE("Unexpected number of planes: %zu", numPlanes);
   1116        MOZ_ASSERT_UNREACHABLE("Unexpected number of planes");
   1117        return nullptr;
   1118    }
   1119 
   1120    CVPixelBufferRef buffer = nullptr;
   1121    image->AddRef();  // Grip input buffers.
   1122    CVReturn rv = CVPixelBufferCreateWithPlanarBytes(
   1123        kCFAllocatorDefault, yuv->mPictureRect.width, yuv->mPictureRect.height,
   1124        pixelFormat, nullptr /* dataPtr */, 0 /* dataSize */, numPlanes,
   1125        addresses, widths, heights, strides, ReleaseImage /* releaseCallback */,
   1126        image /* releaseRefCon */, nullptr /* pixelBufferAttributes */,
   1127        &buffer);
   1128    if (rv == kCVReturnSuccess) {
   1129      return buffer;
   1130      // |image| will be released in |ReleaseImage()|.
   1131    }
   1132    LOGE("CVPIxelBufferCreateWithPlanarBytes error");
   1133    image->Release();
   1134    return nullptr;
   1135  }
   1136 
   1137  RefPtr<gfx::SourceSurface> surface = aSource->GetAsSourceSurface();
   1138  if (!surface) {
   1139    LOGE("Failed to get SourceSurface");
   1140    return nullptr;
   1141  }
   1142 
   1143  RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
   1144  if (!dataSurface) {
   1145    LOGE("Failed to get DataSurface");
   1146    return nullptr;
   1147  }
   1148 
   1149  gfx::DataSourceSurface::ScopedMap map(dataSurface,
   1150                                        gfx::DataSourceSurface::READ);
   1151  if (NS_WARN_IF(!map.IsMapped())) {
   1152    LOGE("Failed to map DataSurface");
   1153    return nullptr;
   1154  }
   1155 
   1156  CVPixelBufferRef buffer = nullptr;
   1157  gfx::DataSourceSurface* dss = dataSurface.forget().take();
   1158  CVReturn rv = CVPixelBufferCreateWithBytes(
   1159      kCFAllocatorDefault, dss->GetSize().Width(), dss->GetSize().Height(),
   1160      pixelFormat, map.GetData(), map.GetStride(), ReleaseSurface, dss, nullptr,
   1161      &buffer);
   1162  if (rv == kCVReturnSuccess) {
   1163    return buffer;
   1164    // |dss| will be released in |ReleaseSurface()|.
   1165  }
   1166  LOGE("CVPIxelBufferCreateWithBytes error: %d", rv);
   1167  RefPtr<gfx::DataSourceSurface> released = dont_AddRef(dss);
   1168  return nullptr;
   1169 }
   1170 
   1171 RefPtr<MediaDataEncoder::EncodePromise> AppleVTEncoder::Drain() {
   1172  return InvokeAsync(mTaskQueue, this, __func__, &AppleVTEncoder::ProcessDrain);
   1173 }
   1174 
   1175 RefPtr<MediaDataEncoder::EncodePromise> AppleVTEncoder::ProcessDrain() {
   1176  LOGV("::ProcessDrain");
   1177  AssertOnTaskQueue();
   1178  MOZ_ASSERT(mSession);
   1179 
   1180  OSStatus status =
   1181      VTCompressionSessionCompleteFrames(mSession, kCMTimeIndefinite);
   1182  if (status != noErr) {
   1183    LOGE("VTCompressionSessionCompleteFrames error");
   1184    return EncodePromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
   1185                                          __func__);
   1186  }
   1187 
   1188  // Resolve the pending encode promise if any.
   1189  MaybeResolveOrRejectEncodePromise();
   1190 
   1191  // VTCompressionSessionCompleteFrames() could have queued multiple tasks with
   1192  // the new drained frames. Dispatch a task after them to resolve the promise
   1193  // with those frames.
   1194  RefPtr<AppleVTEncoder> self = this;
   1195  return InvokeAsync(mTaskQueue, __func__, [self]() {
   1196    EncodedData pendingFrames(std::move(self->mEncodedData));
   1197    LOGV("Resolve drain promise with %zu encoded outputs",
   1198         pendingFrames.Length());
   1199    self->mEncodedData = EncodedData();
   1200    return EncodePromise::CreateAndResolve(std::move(pendingFrames), __func__);
   1201  });
   1202 }
   1203 
   1204 RefPtr<ShutdownPromise> AppleVTEncoder::Shutdown() {
   1205  return InvokeAsync(mTaskQueue, this, __func__,
   1206                     &AppleVTEncoder::ProcessShutdown);
   1207 }
   1208 
   1209 RefPtr<ShutdownPromise> AppleVTEncoder::ProcessShutdown() {
   1210  LOGD("::ProcessShutdown");
   1211  AssertOnTaskQueue();
   1212  InvalidateSessionIfNeeded();
   1213 
   1214  mIsHardwareAccelerated = false;
   1215  mError = MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled in shutdown"_ns);
   1216  MaybeResolveOrRejectEncodePromise();
   1217  mError = NS_OK;
   1218 
   1219  return ShutdownPromise::CreateAndResolve(true, __func__);
   1220 }
   1221 
   1222 RefPtr<GenericPromise> AppleVTEncoder::SetBitrate(uint32_t aBitsPerSec) {
   1223  RefPtr<AppleVTEncoder> self = this;
   1224  return InvokeAsync(mTaskQueue, __func__, [self, aBitsPerSec]() {
   1225    MOZ_ASSERT(self->mSession);
   1226    bool rv = self->SetBitrateAndMode(self->mConfig.mBitrateMode, aBitsPerSec);
   1227    return rv ? GenericPromise::CreateAndResolve(true, __func__)
   1228              : GenericPromise::CreateAndReject(
   1229                    NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, __func__);
   1230  });
   1231 }
   1232 
   1233 void AppleVTEncoder::MaybeResolveOrRejectEncodePromise() {
   1234  AssertOnTaskQueue();
   1235 
   1236  if (mEncodePromise.IsEmpty()) {
   1237    LOGV(
   1238        "No pending promise to resolve(pending outputs: %zu) or reject(err: "
   1239        "%s)",
   1240        mEncodedData.Length(), mError.Description().get());
   1241    return;
   1242  }
   1243 
   1244  if (mTimer) {
   1245    mTimer->Cancel();
   1246    mTimer = nullptr;
   1247  }
   1248 
   1249  if (NS_FAILED(mError.Code())) {
   1250    LOGE("Rejecting encode promise with error: %s", mError.Description().get());
   1251    mEncodePromise.Reject(mError, __func__);
   1252    return;
   1253  }
   1254 
   1255  LOGV("Resolving with %zu encoded outputs", mEncodedData.Length());
   1256  mEncodePromise.Resolve(std::move(mEncodedData), __func__);
   1257 }
   1258 
   1259 void AppleVTEncoder::ForceOutputIfNeeded() {
   1260  if (__builtin_available(macos 11.0, *)) {
   1261    return;
   1262  }
   1263 
   1264  AssertOnTaskQueue();
   1265 
   1266  // Ideally, OutputFrame (called via FrameCallback) should resolve the encode
   1267  // promise. However, sometimes output is produced only after multiple
   1268  // inputs. To ensure continuous encoding, we force the encoder to produce a
   1269  // potentially empty output if no result is received in 50 ms.
   1270  RefPtr<AppleVTEncoder> self = this;
   1271  auto r = NS_NewTimerWithCallback(
   1272      [self](nsITimer* aTimer) {
   1273        if (!self->mSession) {
   1274          LOGV("Do nothing since the encoder has been shut down");
   1275          return;
   1276        }
   1277 
   1278        LOGV("Resolving the pending promise");
   1279        self->MaybeResolveOrRejectEncodePromise();
   1280      },
   1281      TimeDuration::FromMilliseconds(50), nsITimer::TYPE_ONE_SHOT,
   1282      "EncodingProgressChecker"_ns, mTaskQueue);
   1283  if (r.isErr()) {
   1284    LOGE(
   1285        "Failed to set an encoding progress checker. Resolve the pending "
   1286        "promise now");
   1287    MaybeResolveOrRejectEncodePromise();
   1288    return;
   1289  }
   1290  mTimer = r.unwrap();
   1291 }
   1292 
   1293 void AppleVTEncoder::EncodeNextSample(
   1294    nsTArray<RefPtr<MediaData>>&& aInputs,
   1295    MediaDataEncoder::EncodedData&& aOutputs) {
   1296  AssertOnTaskQueue();
   1297  MOZ_ASSERT(!mEncodeBatchPromise.IsEmpty());
   1298  MOZ_ASSERT(!mEncodeBatchRequest.Exists());
   1299 
   1300  if (aInputs.IsEmpty()) {
   1301    LOGV("All samples processed. Resolving the encode promise");
   1302    mEncodeBatchPromise.Resolve(std::move(aOutputs), __func__);
   1303    return;
   1304  }
   1305 
   1306  LOGV("Processing next sample out of %zu remaining", aInputs.Length());
   1307  Encode(aInputs[0])
   1308      ->Then(
   1309          GetCurrentSerialEventTarget(), __func__,
   1310          [self = RefPtr{this}, inputs = std::move(aInputs),
   1311           outputs = std::move(aOutputs)](
   1312              MediaDataEncoder::EncodedData&& aData) mutable {
   1313            self->mEncodeBatchRequest.Complete();
   1314            inputs.RemoveElementAt(0);
   1315            outputs.AppendElements(aData);
   1316            self->EncodeNextSample(std::move(inputs), std::move(outputs));
   1317          },
   1318          [self = RefPtr{this}](const MediaResult& aError) {
   1319            self->mEncodeBatchRequest.Complete();
   1320            LOGE("EncodeNextSample failed: %s", aError.Description().get());
   1321            self->mEncodeBatchPromise.Reject(aError, __func__);
   1322          })
   1323      ->Track(mEncodeBatchRequest);
   1324 }
   1325 
   1326 #undef LOGE
   1327 #undef LOGW
   1328 #undef LOGD
   1329 #undef LOGV
   1330 
   1331 }  // namespace mozilla
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE