[ tor-browser ].git.dasho

VideoDecoder.cpp (34005B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "mozilla/dom/VideoDecoder.h"
      8 
      9 #include "DecoderTraits.h"
     10 #include "DecoderTypes.h"
     11 #include "GPUVideoImage.h"
     12 #include "H264.h"
     13 #include "ImageContainer.h"
     14 #include "MediaContainerType.h"
     15 #include "MediaData.h"
     16 #include "VideoUtils.h"
     17 #include "WebCodecsUtils.h"
     18 #include "mozilla/Assertions.h"
     19 #include "mozilla/Logging.h"
     20 #include "mozilla/Maybe.h"
     21 #include "mozilla/RefPtr.h"
     22 #include "mozilla/Try.h"
     23 #include "mozilla/dom/EncodedVideoChunk.h"
     24 #include "mozilla/dom/EncodedVideoChunkBinding.h"
     25 #include "mozilla/dom/ImageUtils.h"
     26 #include "mozilla/dom/Promise.h"
     27 #include "mozilla/dom/VideoColorSpaceBinding.h"
     28 #include "mozilla/dom/VideoDecoderBinding.h"
     29 #include "mozilla/dom/VideoFrameBinding.h"
     30 #include "mozilla/dom/WebCodecsUtils.h"
     31 #include "nsPrintfCString.h"
     32 
     33 #ifdef XP_MACOSX
     34 #  include "MacIOSurfaceImage.h"
     35 #elif MOZ_WAYLAND
     36 #  include "mozilla/layers/DMABUFSurfaceImage.h"
     37 #  include "mozilla/widget/DMABufSurface.h"
     38 #endif
     39 
     40 extern mozilla::LazyLogModule gWebCodecsLog;
     41 
     42 namespace mozilla::dom {
     43 
     44 #ifdef LOG_INTERNAL
     45 #  undef LOG_INTERNAL
     46 #endif  // LOG_INTERNAL
     47 #define LOG_INTERNAL(level, msg, ...) \
     48  MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
     49 
     50 #ifdef LOG
     51 #  undef LOG
     52 #endif  // LOG
     53 #define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
     54 
     55 #ifdef LOGW
     56 #  undef LOGW
     57 #endif  // LOGW
     58 #define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
     59 
     60 #ifdef LOGE
     61 #  undef LOGE
     62 #endif  // LOGE
     63 #define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
     64 
     65 #ifdef LOGV
     66 #  undef LOGV
     67 #endif  // LOGV
     68 #define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
     69 
     70 NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoDecoder, DOMEventTargetHelper,
     71                                   mErrorCallback, mOutputCallback)
     72 NS_IMPL_ADDREF_INHERITED(VideoDecoder, DOMEventTargetHelper)
     73 NS_IMPL_RELEASE_INHERITED(VideoDecoder, DOMEventTargetHelper)
     74 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoDecoder)
     75 NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
     76 
     77 /*
     78 * Below are helper classes
     79 */
     80 
     81 VideoDecoderConfigInternal::VideoDecoderConfigInternal(
     82    const nsAString& aCodec, Maybe<uint32_t>&& aCodedHeight,
     83    Maybe<uint32_t>&& aCodedWidth, Maybe<VideoColorSpaceInternal>&& aColorSpace,
     84    already_AddRefed<MediaByteBuffer> aDescription,
     85    Maybe<uint32_t>&& aDisplayAspectHeight,
     86    Maybe<uint32_t>&& aDisplayAspectWidth,
     87    const HardwareAcceleration& aHardwareAcceleration,
     88    Maybe<bool>&& aOptimizeForLatency)
     89    : mCodec(aCodec),
     90      mCodedHeight(std::move(aCodedHeight)),
     91      mCodedWidth(std::move(aCodedWidth)),
     92      mColorSpace(std::move(aColorSpace)),
     93      mDescription(aDescription),
     94      mDisplayAspectHeight(std::move(aDisplayAspectHeight)),
     95      mDisplayAspectWidth(std::move(aDisplayAspectWidth)),
     96      mHardwareAcceleration(aHardwareAcceleration),
     97      mOptimizeForLatency(std::move(aOptimizeForLatency)) {};
     98 
     99 /*static*/
    100 RefPtr<VideoDecoderConfigInternal> VideoDecoderConfigInternal::Create(
    101    const VideoDecoderConfig& aConfig) {
    102  nsCString errorMessage;
    103  if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) {
    104    LOGE("Failed to create VideoDecoderConfigInternal: %s", errorMessage.get());
    105    return nullptr;
    106  }
    107 
    108  RefPtr<MediaByteBuffer> description;
    109  if (aConfig.mDescription.WasPassed()) {
    110    auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value());
    111    if (rv.isErr()) {  // Invalid description data.
    112      LOGE(
    113          "Failed to create VideoDecoderConfigInternal due to invalid "
    114          "description data. Error: 0x%08" PRIx32,
    115          static_cast<uint32_t>(rv.unwrapErr()));
    116      return nullptr;
    117    }
    118    description = rv.unwrap();
    119  }
    120 
    121  return MakeRefPtr<VideoDecoderConfigInternal>(
    122      aConfig.mCodec, OptionalToMaybe(aConfig.mCodedHeight),
    123      OptionalToMaybe(aConfig.mCodedWidth),
    124      OptionalToMaybe(aConfig.mColorSpace), description.forget(),
    125      OptionalToMaybe(aConfig.mDisplayAspectHeight),
    126      OptionalToMaybe(aConfig.mDisplayAspectWidth),
    127      aConfig.mHardwareAcceleration,
    128      OptionalToMaybe(aConfig.mOptimizeForLatency));
    129 }
    130 
    131 nsCString VideoDecoderConfigInternal::ToString() const {
    132  nsCString rv;
    133 
    134  rv.Append(NS_ConvertUTF16toUTF8(mCodec));
    135  if (mCodedWidth.isSome()) {
    136    rv.AppendPrintf("coded: %dx%d", mCodedWidth.value(), mCodedHeight.value());
    137  }
    138  if (mDisplayAspectWidth.isSome()) {
    139    rv.AppendPrintf("display %dx%d", mDisplayAspectWidth.value(),
    140                    mDisplayAspectHeight.value());
    141  }
    142  if (mColorSpace.isSome()) {
    143    rv.AppendPrintf("colorspace %s", "todo");
    144  }
    145  if (mDescription) {
    146    rv.AppendPrintf("extradata: %zu bytes", mDescription->Length());
    147  }
    148  rv.AppendPrintf("hw accel: %s", GetEnumString(mHardwareAcceleration).get());
    149  if (mOptimizeForLatency.isSome()) {
    150    rv.AppendPrintf("optimize for latency: %s",
    151                    mOptimizeForLatency.value() ? "true" : "false");
    152  }
    153 
    154  return rv;
    155 }
    156 
    157 /*
    158 * The followings are helpers for VideoDecoder methods
    159 */
    160 
    161 struct MIMECreateParam {
    162  explicit MIMECreateParam(const VideoDecoderConfigInternal& aConfig)
    163      : mCodec(aConfig.mCodec),
    164        mWidth(aConfig.mCodedWidth),
    165        mHeight(aConfig.mCodedHeight) {}
    166  explicit MIMECreateParam(const VideoDecoderConfig& aConfig)
    167      : mCodec(aConfig.mCodec),
    168        mWidth(OptionalToMaybe(aConfig.mCodedWidth)),
    169        mHeight(OptionalToMaybe(aConfig.mCodedHeight)) {}
    170 
    171  const nsString mCodec;
    172  const Maybe<uint32_t> mWidth;
    173  const Maybe<uint32_t> mHeight;
    174 };
    175 
    176 static nsTArray<nsCString> GuessMIMETypes(const MIMECreateParam& aParam) {
    177  const auto codec = NS_ConvertUTF16toUTF8(aParam.mCodec);
    178  nsTArray<nsCString> types;
    179  for (const nsCString& container : GuessContainers(aParam.mCodec)) {
    180    nsPrintfCString mime("video/%s; codecs=%s", container.get(), codec.get());
    181    if (aParam.mWidth) {
    182      mime.AppendPrintf("; width=%d", *aParam.mWidth);
    183    }
    184    if (aParam.mHeight) {
    185      mime.AppendPrintf("; height=%d", *aParam.mHeight);
    186    }
    187    types.AppendElement(mime);
    188  }
    189  return types;
    190 }
    191 
    192 // https://w3c.github.io/webcodecs/#check-configuration-support
    193 template <typename Config>
    194 static bool CanDecode(const Config& aConfig) {
    195  // TODO: Enable WebCodecs on Android (Bug 1840508)
    196  if (IsOnAndroid()) {
    197    return false;
    198  }
    199  if (!IsSupportedVideoCodec(aConfig.mCodec)) {
    200    return false;
    201  }
    202 
    203  // TODO (1880326): code below is wrongly using the logic of HTMLMediaElement
    204  // for determining if a codec can be played, and incorrect codec string for
    205  // h264 are accepted for HTMLMediaElement for compat reasons. Perform stricter
    206  // check here until we fix it for real.
    207  if (IsH264CodecString(aConfig.mCodec)) {
    208    uint8_t profile, constraint;
    209    H264_LEVEL level;
    210    bool supported =
    211        ExtractH264CodecDetails(aConfig.mCodec, profile, constraint, level,
    212                                H264CodecStringStrictness::Strict);
    213    if (!supported) {
    214      return false;
    215    }
    216  }
    217 
    218  // TODO (1880326): Instead of calling CanHandleContainerType with the guessed
    219  // the containers, DecoderTraits should provide an API to tell if a codec is
    220  // decodable or not.
    221  for (const nsCString& mime : GuessMIMETypes(MIMECreateParam(aConfig))) {
    222    if (Maybe<MediaContainerType> containerType =
    223            MakeMediaExtendedMIMEType(mime)) {
    224      if (DecoderTraits::CanHandleContainerType(
    225              *containerType, nullptr /* DecoderDoctorDiagnostics */) !=
    226          CANPLAY_NO) {
    227        return true;
    228      }
    229    }
    230  }
    231  return false;
    232 }
    233 
    234 static nsTArray<UniquePtr<TrackInfo>> GetTracksInfo(
    235    const VideoDecoderConfigInternal& aConfig) {
    236  // TODO: Instead of calling GetTracksInfo with the guessed containers,
    237  // DecoderTraits should provide an API to create the TrackInfo directly.
    238  for (const nsCString& mime : GuessMIMETypes(MIMECreateParam(aConfig))) {
    239    if (Maybe<MediaContainerType> containerType =
    240            MakeMediaExtendedMIMEType(mime)) {
    241      if (nsTArray<UniquePtr<TrackInfo>> tracks =
    242              DecoderTraits::GetTracksInfo(*containerType);
    243          !tracks.IsEmpty()) {
    244        return tracks;
    245      }
    246    }
    247  }
    248  return {};
    249 }
    250 
    251 static Result<Ok, nsresult> CloneConfiguration(
    252    RootedDictionary<VideoDecoderConfig>& aDest, JSContext* aCx,
    253    const VideoDecoderConfig& aConfig, ErrorResult& aRv) {
    254  DebugOnly<nsCString> str;
    255  MOZ_ASSERT(VideoDecoderTraits::Validate(aConfig, str));
    256 
    257  aDest.mCodec = aConfig.mCodec;
    258  if (aConfig.mCodedHeight.WasPassed()) {
    259    aDest.mCodedHeight.Construct(aConfig.mCodedHeight.Value());
    260  }
    261  if (aConfig.mCodedWidth.WasPassed()) {
    262    aDest.mCodedWidth.Construct(aConfig.mCodedWidth.Value());
    263  }
    264  if (aConfig.mColorSpace.WasPassed()) {
    265    aDest.mColorSpace.Construct(aConfig.mColorSpace.Value());
    266  }
    267  if (aConfig.mDescription.WasPassed()) {
    268    aDest.mDescription.Construct();
    269    MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(),
    270                        aConfig.mDescription.Value(), aRv));
    271  }
    272  if (aConfig.mDisplayAspectHeight.WasPassed()) {
    273    aDest.mDisplayAspectHeight.Construct(aConfig.mDisplayAspectHeight.Value());
    274  }
    275  if (aConfig.mDisplayAspectWidth.WasPassed()) {
    276    aDest.mDisplayAspectWidth.Construct(aConfig.mDisplayAspectWidth.Value());
    277  }
    278  aDest.mHardwareAcceleration = aConfig.mHardwareAcceleration;
    279  if (aConfig.mOptimizeForLatency.WasPassed()) {
    280    aDest.mOptimizeForLatency.Construct(aConfig.mOptimizeForLatency.Value());
    281  }
    282 
    283  return Ok();
    284 }
    285 
    286 static Maybe<VideoPixelFormat> GuessPixelFormat(layers::Image* aImage) {
    287  if (aImage) {
    288    // TODO: Implement ImageUtils::Impl for MacIOSurfaceImage and
    289    // DMABUFSurfaceImage?
    290    if (aImage->AsPlanarYCbCrImage() || aImage->AsNVImage()) {
    291      const ImageUtils imageUtils(aImage);
    292      Maybe<dom::ImageBitmapFormat> format = imageUtils.GetFormat();
    293      Maybe<VideoPixelFormat> f =
    294          format.isSome() ? ImageBitmapFormatToVideoPixelFormat(format.value())
    295                          : Nothing();
    296 
    297      // ImageBitmapFormat cannot distinguish YUV420 or YUV420A.
    298      bool hasAlpha = aImage->AsPlanarYCbCrImage() &&
    299                      aImage->AsPlanarYCbCrImage()->GetData() &&
    300                      aImage->AsPlanarYCbCrImage()->GetData()->mAlpha;
    301      if (f && *f == VideoPixelFormat::I420 && hasAlpha) {
    302        return Some(VideoPixelFormat::I420A);
    303      }
    304      return f;
    305    }
    306    if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) {
    307      RefPtr<layers::ImageBridgeChild> imageBridge =
    308          layers::ImageBridgeChild::GetSingleton();
    309      layers::TextureClient* texture = image->GetTextureClient(imageBridge);
    310      if (NS_WARN_IF(!texture)) {
    311        return Nothing();
    312      }
    313      return SurfaceFormatToVideoPixelFormat(texture->GetFormat());
    314    }
    315 #ifdef XP_MACOSX
    316    if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) {
    317      MOZ_ASSERT(image->GetSurface());
    318      return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat());
    319    }
    320 #endif
    321 #ifdef MOZ_WAYLAND
    322    if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) {
    323      MOZ_ASSERT(image->GetSurface());
    324      return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat());
    325    }
    326 #endif
    327  }
    328  LOGW("Failed to get pixel format from layers::Image");
    329  return Nothing();
    330 }
    331 
    332 static VideoColorSpaceInternal GuessColorSpace(
    333    const layers::PlanarYCbCrData* aData) {
    334  if (!aData) {
    335    LOGE("nullptr in GuessColorSpace");
    336    return {};
    337  }
    338 
    339  VideoColorSpaceInternal colorSpace;
    340  colorSpace.mFullRange = Some(ToFullRange(aData->mColorRange));
    341  if (Maybe<VideoMatrixCoefficients> m =
    342          ToMatrixCoefficients(aData->mYUVColorSpace)) {
    343    colorSpace.mMatrix = ToMatrixCoefficients(aData->mYUVColorSpace);
    344    colorSpace.mPrimaries = ToPrimaries(aData->mColorPrimaries);
    345  }
    346  if (!colorSpace.mPrimaries) {
    347    LOG("Missing primaries, guessing from colorspace");
    348    // Make an educated guess based on the coefficients.
    349    colorSpace.mPrimaries = colorSpace.mMatrix.map([](const auto& aMatrix) {
    350      switch (aMatrix) {
    351        case VideoMatrixCoefficients::Bt2020_ncl:
    352          return VideoColorPrimaries::Bt2020;
    353        case VideoMatrixCoefficients::Rgb:
    354        case VideoMatrixCoefficients::Bt470bg:
    355        case VideoMatrixCoefficients::Smpte170m:
    356          LOGW(
    357              "Warning: Falling back to BT709 when attempting to determine the "
    358              "primaries function of a YCbCr buffer");
    359          [[fallthrough]];
    360        case VideoMatrixCoefficients::Bt709:
    361          return VideoColorPrimaries::Bt709;
    362      }
    363      MOZ_ASSERT_UNREACHABLE("Unexpected matrix coefficients");
    364      LOGW(
    365          "Warning: Falling back to BT709 due to unexpected matrix "
    366          "coefficients "
    367          "when attempting to determine the primaries function of a YCbCr "
    368          "buffer");
    369      return VideoColorPrimaries::Bt709;
    370    });
    371  }
    372 
    373  if (Maybe<VideoTransferCharacteristics> c =
    374          ToTransferCharacteristics(aData->mTransferFunction)) {
    375    colorSpace.mTransfer = Some(*c);
    376  }
    377  if (!colorSpace.mTransfer) {
    378    LOG("Missing transfer characteristics, guessing from colorspace");
    379    colorSpace.mTransfer = Some(([&] {
    380      switch (aData->mYUVColorSpace) {
    381        case gfx::YUVColorSpace::Identity:
    382          return VideoTransferCharacteristics::Iec61966_2_1;
    383        case gfx::YUVColorSpace::BT2020:
    384          return VideoTransferCharacteristics::Pq;
    385        case gfx::YUVColorSpace::BT601:
    386          LOGW(
    387              "Warning: Falling back to BT709 when attempting to determine the "
    388              "transfer function of a MacIOSurface");
    389          [[fallthrough]];
    390        case gfx::YUVColorSpace::BT709:
    391          return VideoTransferCharacteristics::Bt709;
    392      }
    393      MOZ_ASSERT_UNREACHABLE("Unexpected color space");
    394      LOGW(
    395          "Warning: Falling back to BT709 due to unexpected color space "
    396          "when attempting to determine the transfer function of a "
    397          "MacIOSurface");
    398      return VideoTransferCharacteristics::Bt709;
    399    })());
    400  }
    401 
    402  return colorSpace;
    403 }
    404 
    405 #ifdef XP_MACOSX
    406 static VideoColorSpaceInternal GuessColorSpace(const MacIOSurface* aSurface) {
    407  if (!aSurface) {
    408    return {};
    409  }
    410  VideoColorSpaceInternal colorSpace;
    411  colorSpace.mFullRange = Some(aSurface->IsFullRange());
    412  if (Maybe<dom::VideoMatrixCoefficients> m =
    413          ToMatrixCoefficients(aSurface->GetYUVColorSpace())) {
    414    colorSpace.mMatrix = Some(*m);
    415  }
    416  if (Maybe<VideoColorPrimaries> p = ToPrimaries(aSurface->mColorPrimaries)) {
    417    colorSpace.mPrimaries = Some(*p);
    418  }
    419  // Make an educated guess based on the coefficients.
    420  if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::Identity) {
    421    colorSpace.mTransfer = Some(VideoTransferCharacteristics::Iec61966_2_1);
    422  } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT709) {
    423    colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709);
    424  } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT2020) {
    425    colorSpace.mTransfer = Some(VideoTransferCharacteristics::Pq);
    426  } else {
    427    LOGW(
    428        "Warning: Falling back to BT709 when attempting to determine the "
    429        "transfer function of a MacIOSurface");
    430    colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709);
    431  }
    432 
    433  return colorSpace;
    434 }
    435 #endif
    436 #ifdef MOZ_WAYLAND
    437 // TODO: Set DMABufSurface::IsFullRange() to const so aSurface can be const.
    438 static VideoColorSpaceInternal GuessColorSpace(DMABufSurface* aSurface) {
    439  if (!aSurface) {
    440    return {};
    441  }
    442  VideoColorSpaceInternal colorSpace;
    443  colorSpace.mFullRange = Some(aSurface->IsFullRange());
    444  if (Maybe<dom::VideoMatrixCoefficients> m =
    445          ToMatrixCoefficients(aSurface->GetYUVColorSpace())) {
    446    colorSpace.mMatrix = Some(*m);
    447  }
    448  // No other color space information.
    449  return colorSpace;
    450 }
    451 #endif
    452 
    453 static VideoColorSpaceInternal GuessColorSpace(layers::Image* aImage) {
    454  if (aImage) {
    455    if (layers::PlanarYCbCrImage* image = aImage->AsPlanarYCbCrImage()) {
    456      return GuessColorSpace(image->GetData());
    457    }
    458    if (layers::NVImage* image = aImage->AsNVImage()) {
    459      return GuessColorSpace(image->GetData());
    460    }
    461    if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) {
    462      VideoColorSpaceInternal colorSpace;
    463      colorSpace.mFullRange =
    464          Some(image->GetColorRange() != gfx::ColorRange::LIMITED);
    465      colorSpace.mMatrix = ToMatrixCoefficients(image->GetYUVColorSpace());
    466      colorSpace.mPrimaries = ToPrimaries(image->GetColorPrimaries());
    467      colorSpace.mTransfer =
    468          ToTransferCharacteristics(image->GetTransferFunction());
    469      // In some circumstances, e.g. on Linux software decoding when using
    470      // VPXDecoder and RDD, the primaries aren't set correctly. Make a good
    471      // guess based on the other params. Fixing this is tracked in
    472      // https://bugzilla.mozilla.org/show_bug.cgi?id=1869825
    473      if (!colorSpace.mPrimaries) {
    474        if (colorSpace.mMatrix.isSome()) {
    475          switch (colorSpace.mMatrix.value()) {
    476            case VideoMatrixCoefficients::Rgb:
    477            case VideoMatrixCoefficients::Bt709:
    478              colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt709);
    479              break;
    480            case VideoMatrixCoefficients::Bt470bg:
    481            case VideoMatrixCoefficients::Smpte170m:
    482              colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt470bg);
    483              break;
    484            case VideoMatrixCoefficients::Bt2020_ncl:
    485              colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt2020);
    486              break;
    487          };
    488        }
    489      }
    490      return colorSpace;
    491    }
    492 #ifdef XP_MACOSX
    493    // TODO: Make sure VideoFrame can interpret its internal data in different
    494    // formats.
    495    if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) {
    496      return GuessColorSpace(image->GetSurface());
    497    }
    498 #endif
    499 #ifdef MOZ_WAYLAND
    500    // TODO: Make sure VideoFrame can interpret its internal data in different
    501    // formats.
    502    if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) {
    503      return GuessColorSpace(image->GetSurface());
    504    }
    505 #endif
    506  }
    507  LOGW("Failed to get color space from layers::Image");
    508  return {};
    509 }
    510 
    511 static Result<gfx::IntSize, nsresult> AdjustDisplaySize(
    512    const uint32_t aDisplayAspectWidth, const uint32_t aDisplayAspectHeight,
    513    const gfx::IntSize& aDisplaySize) {
    514  if (aDisplayAspectHeight == 0) {
    515    return Err(NS_ERROR_ILLEGAL_VALUE);
    516  }
    517 
    518  const double aspectRatio =
    519      static_cast<double>(aDisplayAspectWidth) / aDisplayAspectHeight;
    520 
    521  double w = aDisplaySize.width;
    522  double h = aDisplaySize.height;
    523 
    524  if (aspectRatio >= w / h) {
    525    // Adjust w to match the aspect ratio
    526    w = aspectRatio * h;
    527  } else {
    528    // Adjust h to match the aspect ratio
    529    h = w / aspectRatio;
    530  }
    531 
    532  w = std::round(w);
    533  h = std::round(h);
    534  constexpr double MAX = static_cast<double>(
    535      std::numeric_limits<decltype(gfx::IntSize::width)>::max());
    536  if (w > MAX || h > MAX || w < 1.0 || h < 1.0) {
    537    return Err(NS_ERROR_ILLEGAL_VALUE);
    538  }
    539  return gfx::IntSize(static_cast<decltype(gfx::IntSize::width)>(w),
    540                      static_cast<decltype(gfx::IntSize::height)>(h));
    541 }
    542 
    543 // https://w3c.github.io/webcodecs/#create-a-videoframe
    544 static RefPtr<VideoFrame> CreateVideoFrame(
    545    nsIGlobalObject* aGlobalObject, const VideoData* aData, int64_t aTimestamp,
    546    uint64_t aDuration, const Maybe<uint32_t> aDisplayAspectWidth,
    547    const Maybe<uint32_t> aDisplayAspectHeight,
    548    const VideoColorSpaceInternal& aColorSpace) {
    549  MOZ_ASSERT(aGlobalObject);
    550  MOZ_ASSERT(aData);
    551  MOZ_ASSERT((!!aDisplayAspectWidth) == (!!aDisplayAspectHeight));
    552 
    553  Maybe<VideoPixelFormat> format = GuessPixelFormat(aData->mImage.get());
    554  gfx::IntSize displaySize = aData->mDisplay;
    555  if (aDisplayAspectWidth && aDisplayAspectHeight) {
    556    auto r = AdjustDisplaySize(*aDisplayAspectWidth, *aDisplayAspectHeight,
    557                               displaySize);
    558    if (r.isOk()) {
    559      displaySize = r.unwrap();
    560    }
    561  }
    562 
    563  return MakeRefPtr<VideoFrame>(aGlobalObject, aData->mImage, format,
    564                                aData->mImage->GetSize(),
    565                                aData->mImage->GetPictureRect(), displaySize,
    566                                Some(aDuration), aTimestamp, aColorSpace);
    567 }
    568 
    569 /* static */
    570 bool VideoDecoderTraits::IsSupported(
    571    const VideoDecoderConfigInternal& aConfig) {
    572  return CanDecode(aConfig);
    573 }
    574 
    575 /* static */
    576 Result<UniquePtr<TrackInfo>, nsresult> VideoDecoderTraits::CreateTrackInfo(
    577    const VideoDecoderConfigInternal& aConfig) {
    578  LOG("Create a VideoInfo from %s config", aConfig.ToString().get());
    579 
    580  nsTArray<UniquePtr<TrackInfo>> tracks = GetTracksInfo(aConfig);
    581  if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kVideoTrack) {
    582    LOGE("Failed to get TrackInfo");
    583    return Err(NS_ERROR_INVALID_ARG);
    584  }
    585 
    586  UniquePtr<TrackInfo> track(std::move(tracks[0]));
    587  VideoInfo* vi = track->GetAsVideoInfo();
    588  if (!vi) {
    589    LOGE("Failed to get VideoInfo");
    590    return Err(NS_ERROR_INVALID_ARG);
    591  }
    592 
    593  constexpr uint32_t MAX = static_cast<uint32_t>(
    594      std::numeric_limits<decltype(gfx::IntSize::width)>::max());
    595  if (aConfig.mCodedHeight.isSome()) {
    596    if (aConfig.mCodedHeight.value() > MAX) {
    597      LOGE("codedHeight overflows");
    598      return Err(NS_ERROR_INVALID_ARG);
    599    }
    600    vi->mImage.height = static_cast<decltype(gfx::IntSize::height)>(
    601        aConfig.mCodedHeight.value());
    602  }
    603  if (aConfig.mCodedWidth.isSome()) {
    604    if (aConfig.mCodedWidth.value() > MAX) {
    605      LOGE("codedWidth overflows");
    606      return Err(NS_ERROR_INVALID_ARG);
    607    }
    608    vi->mImage.width =
    609        static_cast<decltype(gfx::IntSize::width)>(aConfig.mCodedWidth.value());
    610  }
    611 
    612  if (aConfig.mDisplayAspectHeight.isSome()) {
    613    if (aConfig.mDisplayAspectHeight.value() > MAX) {
    614      LOGE("displayAspectHeight overflows");
    615      return Err(NS_ERROR_INVALID_ARG);
    616    }
    617    vi->mDisplay.height = static_cast<decltype(gfx::IntSize::height)>(
    618        aConfig.mDisplayAspectHeight.value());
    619  }
    620  if (aConfig.mDisplayAspectWidth.isSome()) {
    621    if (aConfig.mDisplayAspectWidth.value() > MAX) {
    622      LOGE("displayAspectWidth overflows");
    623      return Err(NS_ERROR_INVALID_ARG);
    624    }
    625    vi->mDisplay.width = static_cast<decltype(gfx::IntSize::width)>(
    626        aConfig.mDisplayAspectWidth.value());
    627  }
    628 
    629  if (aConfig.mColorSpace.isSome()) {
    630    const VideoColorSpaceInternal& colorSpace(aConfig.mColorSpace.value());
    631    if (colorSpace.mFullRange.isSome()) {
    632      vi->mColorRange = ToColorRange(colorSpace.mFullRange.value());
    633    }
    634    if (colorSpace.mMatrix.isSome()) {
    635      vi->mColorSpace.emplace(ToColorSpace(colorSpace.mMatrix.value()));
    636    }
    637    // Some decoders get their primaries and transfer function from the codec
    638    // string, and it's already set here. This is the case for VP9 decoders.
    639    if (colorSpace.mPrimaries.isSome()) {
    640      auto primaries = ToPrimaries(colorSpace.mPrimaries.value());
    641      if (vi->mColorPrimaries.isSome()) {
    642        if (vi->mColorPrimaries.value() != primaries) {
    643          LOG("Conflict between decoder config and codec string, keeping codec "
    644              "string primaries of %d",
    645              static_cast<int>(primaries));
    646        }
    647      } else {
    648        vi->mColorPrimaries.emplace(primaries);
    649      }
    650    }
    651    if (colorSpace.mTransfer.isSome()) {
    652      auto primaries = ToTransferFunction(colorSpace.mTransfer.value());
    653      if (vi->mTransferFunction.isSome()) {
    654        if (vi->mTransferFunction.value() != primaries) {
    655          LOG("Conflict between decoder config and codec string, keeping codec "
    656              "string transfer function of %d",
    657              static_cast<int>(vi->mTransferFunction.value()));
    658        }
    659      } else {
    660        vi->mTransferFunction.emplace(
    661            ToTransferFunction(colorSpace.mTransfer.value()));
    662      }
    663    }
    664  }
    665 
    666  if (aConfig.mDescription) {
    667    if (!aConfig.mDescription->IsEmpty()) {
    668      LOG("The given config has %zu bytes of description data",
    669          aConfig.mDescription->Length());
    670      if (vi->mExtraData) {
    671        LOGW("The default extra data is overwritten");
    672      }
    673      vi->mExtraData = aConfig.mDescription;
    674    }
    675 
    676    // TODO: Make this utility and replace the similar one in MP4Demuxer.cpp.
    677    if (vi->mExtraData && !vi->mExtraData->IsEmpty() &&
    678        IsH264CodecString(aConfig.mCodec)) {
    679      SPSData spsdata;
    680      if (H264::DecodeSPSFromExtraData(vi->mExtraData.get(), spsdata) &&
    681          spsdata.pic_width > 0 && spsdata.pic_height > 0 &&
    682          H264::EnsureSPSIsSane(spsdata)) {
    683        LOG("H264 sps data - pic size: %d x %d, display size: %d x %d",
    684            spsdata.pic_width, spsdata.pic_height, spsdata.display_width,
    685            spsdata.display_height);
    686 
    687        if (spsdata.pic_width > MAX || spsdata.pic_height > MAX ||
    688            spsdata.display_width > MAX || spsdata.display_height > MAX) {
    689          LOGE("H264 width or height in sps data overflows");
    690          return Err(NS_ERROR_INVALID_ARG);
    691        }
    692 
    693        vi->mImage.width =
    694            static_cast<decltype(gfx::IntSize::width)>(spsdata.pic_width);
    695        vi->mImage.height =
    696            static_cast<decltype(gfx::IntSize::height)>(spsdata.pic_height);
    697        vi->mDisplay.width =
    698            static_cast<decltype(gfx::IntSize::width)>(spsdata.display_width);
    699        vi->mDisplay.height =
    700            static_cast<decltype(gfx::IntSize::height)>(spsdata.display_height);
    701      }
    702    }
    703  } else {
    704    vi->mExtraData = new MediaByteBuffer();
    705  }
    706 
    707  LOG("Created a VideoInfo for decoder - %s", vi->ToString().get());
    708 
    709  return track;
    710 }
    711 
    712 // https://w3c.github.io/webcodecs/#valid-videodecoderconfig
    713 /* static */
    714 bool VideoDecoderTraits::Validate(const VideoDecoderConfig& aConfig,
    715                                  nsCString& aErrorMessage) {
    716  Maybe<nsString> codec = ParseCodecString(aConfig.mCodec);
    717  if (!codec || codec->IsEmpty()) {
    718    aErrorMessage.AssignLiteral("Invalid codec string");
    719    LOGE("%s", aErrorMessage.get());
    720    return false;
    721  }
    722 
    723  if (aConfig.mCodedWidth.WasPassed() != aConfig.mCodedHeight.WasPassed()) {
    724    aErrorMessage.AppendPrintf(
    725        "Missing coded %s",
    726        aConfig.mCodedWidth.WasPassed() ? "height" : "width");
    727    LOGE("%s", aErrorMessage.get());
    728    return false;
    729  }
    730  if (aConfig.mCodedWidth.WasPassed() &&
    731      (aConfig.mCodedWidth.Value() == 0 || aConfig.mCodedHeight.Value() == 0)) {
    732    aErrorMessage.AssignLiteral("codedWidth and/or codedHeight can't be zero");
    733    LOGE("%s", aErrorMessage.get());
    734    return false;
    735  }
    736 
    737  if (aConfig.mDisplayAspectWidth.WasPassed() !=
    738      aConfig.mDisplayAspectHeight.WasPassed()) {
    739    aErrorMessage.AppendPrintf(
    740        "Missing display aspect %s",
    741        aConfig.mDisplayAspectWidth.WasPassed() ? "height" : "width");
    742    LOGE("%s", aErrorMessage.get());
    743    return false;
    744  }
    745  if (aConfig.mDisplayAspectWidth.WasPassed() &&
    746      (aConfig.mDisplayAspectWidth.Value() == 0 ||
    747       aConfig.mDisplayAspectHeight.Value() == 0)) {
    748    aErrorMessage.AssignLiteral(
    749        "display aspect width and height cannot be zero");
    750    LOGE("%s", aErrorMessage.get());
    751    return false;
    752  }
    753 
    754  bool detached =
    755      aConfig.mDescription.WasPassed() &&
    756      (aConfig.mDescription.Value().IsArrayBuffer()
    757           ? JS::ArrayBuffer::fromObject(
    758                 aConfig.mDescription.Value().GetAsArrayBuffer().Obj())
    759                 .isDetached()
    760           : JS::ArrayBufferView::fromObject(
    761                 aConfig.mDescription.Value().GetAsArrayBufferView().Obj())
    762                 .isDetached());
    763 
    764  if (detached) {
    765    aErrorMessage.AssignLiteral("description is detached.");
    766    LOGE("%s", aErrorMessage.get());
    767    return false;
    768  }
    769 
    770  return true;
    771 }
    772 
    773 /* static */
    774 RefPtr<VideoDecoderConfigInternal> VideoDecoderTraits::CreateConfigInternal(
    775    const VideoDecoderConfig& aConfig) {
    776  return VideoDecoderConfigInternal::Create(aConfig);
    777 }
    778 
    779 /* static */
    780 bool VideoDecoderTraits::IsKeyChunk(const EncodedVideoChunk& aInput) {
    781  return aInput.Type() == EncodedVideoChunkType::Key;
    782 }
    783 
    784 /* static */
    785 UniquePtr<EncodedVideoChunkData> VideoDecoderTraits::CreateInputInternal(
    786    const EncodedVideoChunk& aInput) {
    787  return aInput.Clone();
    788 }
    789 
    790 /*
    791 * Below are VideoDecoder implementation
    792 */
    793 
    794 VideoDecoder::VideoDecoder(nsIGlobalObject* aParent,
    795                           RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
    796                           RefPtr<VideoFrameOutputCallback>&& aOutputCallback)
    797    : DecoderTemplate(aParent, std::move(aErrorCallback),
    798                      std::move(aOutputCallback)) {
    799  MOZ_ASSERT(mErrorCallback);
    800  MOZ_ASSERT(mOutputCallback);
    801  LOG("VideoDecoder %p ctor", this);
    802 }
    803 
    804 VideoDecoder::~VideoDecoder() { LOG("VideoDecoder %p dtor", this); }
    805 
    806 JSObject* VideoDecoder::WrapObject(JSContext* aCx,
    807                                   JS::Handle<JSObject*> aGivenProto) {
    808  AssertIsOnOwningThread();
    809 
    810  return VideoDecoder_Binding::Wrap(aCx, this, aGivenProto);
    811 }
    812 
    813 // https://w3c.github.io/webcodecs/#dom-videodecoder-videodecoder
    814 /* static */
    815 already_AddRefed<VideoDecoder> VideoDecoder::Constructor(
    816    const GlobalObject& aGlobal, const VideoDecoderInit& aInit,
    817    ErrorResult& aRv) {
    818  nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
    819  if (!global) {
    820    aRv.Throw(NS_ERROR_FAILURE);
    821    return nullptr;
    822  }
    823 
    824  return MakeAndAddRef<VideoDecoder>(
    825      global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError),
    826      RefPtr<VideoFrameOutputCallback>(aInit.mOutput));
    827 }
    828 
    829 // https://w3c.github.io/webcodecs/#dom-videodecoder-isconfigsupported
    830 /* static */
    831 already_AddRefed<Promise> VideoDecoder::IsConfigSupported(
    832    const GlobalObject& aGlobal, const VideoDecoderConfig& aConfig,
    833    ErrorResult& aRv) {
    834  LOG("VideoDecoder::IsConfigSupported, config: %s",
    835      NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
    836 
    837  nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
    838  if (!global) {
    839    aRv.Throw(NS_ERROR_FAILURE);
    840    return nullptr;
    841  }
    842 
    843  RefPtr<Promise> p = Promise::Create(global.get(), aRv);
    844  if (NS_WARN_IF(aRv.Failed())) {
    845    return p.forget();
    846  }
    847 
    848  nsCString errorMessage;
    849  if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) {
    850    p->MaybeRejectWithTypeError(nsPrintfCString(
    851        "IsConfigSupported: config is invalid: %s", errorMessage.get()));
    852    return p.forget();
    853  }
    854 
    855  RootedDictionary<VideoDecoderConfig> config(aGlobal.Context());
    856  auto r = CloneConfiguration(config, aGlobal.Context(), aConfig, aRv);
    857  if (r.isErr()) {
    858    // This can only be an OOM: all members to clone are known to be valid
    859    // because this is check by ::Validate above.
    860    MOZ_ASSERT(r.inspectErr() == NS_ERROR_OUT_OF_MEMORY &&
    861               aRv.ErrorCodeIs(NS_ERROR_OUT_OF_MEMORY));
    862    return p.forget();
    863  }
    864 
    865  // This is incomplete and will be implemented fully in bug 1967793
    866  auto configInternal = VideoDecoderConfigInternal::Create(aConfig);
    867  ApplyResistFingerprintingIfNeeded(configInternal, global);
    868 
    869  bool canDecode = CanDecode(*configInternal);
    870  RootedDictionary<VideoDecoderSupport> s(aGlobal.Context());
    871  s.mConfig.Construct(std::move(config));
    872  s.mSupported.Construct(canDecode);
    873 
    874  p->MaybeResolve(s);
    875  return p.forget();
    876 }
    877 
    878 already_AddRefed<MediaRawData> VideoDecoder::InputDataToMediaRawData(
    879    UniquePtr<EncodedVideoChunkData>&& aData, TrackInfo& aInfo,
    880    const VideoDecoderConfigInternal& aConfig) {
    881  AssertIsOnOwningThread();
    882  MOZ_ASSERT(aInfo.GetAsVideoInfo());
    883 
    884  if (!aData) {
    885    LOGE("No data for conversion");
    886    return nullptr;
    887  }
    888 
    889  RefPtr<MediaRawData> sample = aData->TakeData();
    890  if (!sample) {
    891    LOGE("Take no data for conversion");
    892    return nullptr;
    893  }
    894 
    895  // aExtraData is either provided by Configure() or a default one created for
    896  // the decoder creation. If it's created for decoder creation only, we don't
    897  // set it to sample.
    898  if (aConfig.mDescription && aInfo.GetAsVideoInfo()->mExtraData) {
    899    sample->mExtraData = aInfo.GetAsVideoInfo()->mExtraData;
    900  }
    901 
    902  LOGV(
    903      "EncodedVideoChunkData %p converted to %zu-byte MediaRawData - time: "
    904      "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64
    905      "us, key-frame: %s, has extra data: %s",
    906      aData.get(), sample->Size(), sample->mTime.ToMicroseconds(),
    907      sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(),
    908      sample->mKeyframe ? "yes" : "no", sample->mExtraData ? "yes" : "no");
    909 
    910  return sample.forget();
    911 }
    912 
    913 nsTArray<RefPtr<VideoFrame>> VideoDecoder::DecodedDataToOutputType(
    914    nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData,
    915    const VideoDecoderConfigInternal& aConfig) {
    916  AssertIsOnOwningThread();
    917 
    918  nsTArray<RefPtr<VideoFrame>> frames;
    919  for (const RefPtr<MediaData>& data : aData) {
    920    MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::VIDEO_DATA);
    921    RefPtr<const VideoData> d(data->As<const VideoData>());
    922    VideoColorSpaceInternal colorSpace;
    923    // Determine which color space to use: prefer the color space as configured
    924    // at the decoder level, if it has one, otherwise look at the underlying
    925    // image and make a guess.
    926    if (aConfig.mColorSpace.isSome() &&
    927        aConfig.mColorSpace->mPrimaries.isSome() &&
    928        aConfig.mColorSpace->mTransfer.isSome() &&
    929        aConfig.mColorSpace->mMatrix.isSome()) {
    930      colorSpace = aConfig.mColorSpace.value();
    931    } else {
    932      colorSpace = GuessColorSpace(d->mImage.get());
    933    }
    934    frames.AppendElement(CreateVideoFrame(
    935        aGlobalObject, d.get(), d->mTime.ToMicroseconds(),
    936        static_cast<uint64_t>(d->mDuration.ToMicroseconds()),
    937        aConfig.mDisplayAspectWidth, aConfig.mDisplayAspectHeight, colorSpace));
    938  }
    939  return frames;
    940 }
    941 
    942 #undef LOG
    943 #undef LOGW
    944 #undef LOGE
    945 #undef LOGV
    946 #undef LOG_INTERNAL
    947 
    948 }  // namespace mozilla::dom
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE