[ tor-browser ].git.dasho

AudioEncoder.cpp (17502B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "mozilla/dom/AudioEncoder.h"
      8 
      9 #include "EncoderConfig.h"
     10 #include "EncoderTraits.h"
     11 #include "EncoderTypes.h"
     12 #include "MediaData.h"
     13 #include "mozilla/Assertions.h"
     14 #include "mozilla/Logging.h"
     15 #include "mozilla/Maybe.h"
     16 #include "mozilla/dom/AudioDataBinding.h"
     17 #include "mozilla/dom/AudioEncoderBinding.h"
     18 #include "mozilla/dom/EncodedAudioChunk.h"
     19 #include "mozilla/dom/EncodedAudioChunkBinding.h"
     20 #include "mozilla/dom/Promise.h"
     21 #include "mozilla/dom/WebCodecsUtils.h"
     22 
     23 extern mozilla::LazyLogModule gWebCodecsLog;
     24 
     25 namespace mozilla::dom {
     26 
     27 #ifdef LOG_INTERNAL
     28 #  undef LOG_INTERNAL
     29 #endif  // LOG_INTERNAL
     30 #define LOG_INTERNAL(level, msg, ...) \
     31  MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
     32 
     33 #ifdef LOG
     34 #  undef LOG
     35 #endif  // LOG
     36 #define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
     37 
     38 #ifdef LOGW
     39 #  undef LOGW
     40 #endif  // LOGW
     41 #define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
     42 
     43 #ifdef LOGE
     44 #  undef LOGE
     45 #endif  // LOGE
     46 #define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
     47 
     48 #ifdef LOGV
     49 #  undef LOGV
     50 #endif  // LOGV
     51 #define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
     52 
     53 NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioEncoder, DOMEventTargetHelper,
     54                                   mErrorCallback, mOutputCallback)
     55 NS_IMPL_ADDREF_INHERITED(AudioEncoder, DOMEventTargetHelper)
     56 NS_IMPL_RELEASE_INHERITED(AudioEncoder, DOMEventTargetHelper)
     57 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioEncoder)
     58 NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
     59 
     60 /*
     61 * Below are helper classes
     62 */
     63 AudioEncoderConfigInternal::AudioEncoderConfigInternal(
     64    const nsAString& aCodec, Maybe<uint32_t> aSampleRate,
     65    Maybe<uint32_t> aNumberOfChannels, Maybe<uint32_t> aBitrate,
     66    BitrateMode aBitrateMode)
     67    : mCodec(aCodec),
     68      mSampleRate(aSampleRate),
     69      mNumberOfChannels(aNumberOfChannels),
     70      mBitrate(aBitrate),
     71      mBitrateMode(aBitrateMode) {}
     72 
     73 AudioEncoderConfigInternal::AudioEncoderConfigInternal(
     74    const AudioEncoderConfig& aConfig)
     75    : AudioEncoderConfigInternal(
     76          aConfig.mCodec, OptionalToMaybe(aConfig.mSampleRate),
     77          OptionalToMaybe(aConfig.mNumberOfChannels),
     78          OptionalToMaybe(aConfig.mBitrate), aConfig.mBitrateMode) {
     79  DebugOnly<nsCString> errorMessage;
     80  if (aConfig.mCodec.EqualsLiteral("opus") && aConfig.mOpus.WasPassed()) {
     81    // All values are in range at this point, the config is known valid.
     82    OpusSpecific specific;
     83    if (aConfig.mOpus.Value().mComplexity.WasPassed()) {
     84      specific.mComplexity = aConfig.mOpus.Value().mComplexity.Value();
     85    } else {
     86      // https://w3c.github.io/webcodecs/opus_codec_registration.html#dom-opusencoderconfig-complexity
     87      // If no value is specificied, the default value is platform-specific:
     88      // User Agents SHOULD set a default of 5 for mobile platforms, and a
     89      // default of 9 for all other platforms.
     90      if (IsOnAndroid()) {
     91        specific.mComplexity = 5;
     92      } else {
     93        specific.mComplexity = 9;
     94      }
     95    }
     96    specific.mApplication = OpusSpecific::Application::Unspecified;
     97    specific.mFrameDuration = aConfig.mOpus.Value().mFrameDuration;
     98    specific.mPacketLossPerc = aConfig.mOpus.Value().mPacketlossperc;
     99    specific.mUseDTX = aConfig.mOpus.Value().mUsedtx;
    100    specific.mUseInBandFEC = aConfig.mOpus.Value().mUseinbandfec;
    101    mSpecific = AsVariant(specific);
    102  }
    103  MOZ_ASSERT(AudioEncoderTraits::Validate(aConfig, errorMessage));
    104 }
    105 
    106 AudioEncoderConfigInternal::AudioEncoderConfigInternal(
    107    const AudioEncoderConfigInternal& aConfig)
    108    : AudioEncoderConfigInternal(aConfig.mCodec, aConfig.mSampleRate,
    109                                 aConfig.mNumberOfChannels, aConfig.mBitrate,
    110                                 aConfig.mBitrateMode) {}
    111 
    112 void AudioEncoderConfigInternal::SetSpecific(
    113    const EncoderConfig::CodecSpecific& aSpecific) {
    114  mSpecific = aSpecific;
    115 }
    116 
    117 /*
    118 * The followings are helpers for AudioEncoder methods
    119 */
    120 
    121 static void CloneConfiguration(RootedDictionary<AudioEncoderConfig>& aDest,
    122                               JSContext* aCx,
    123                               const AudioEncoderConfig& aConfig) {
    124  aDest.mCodec = aConfig.mCodec;
    125 
    126  if (aConfig.mNumberOfChannels.WasPassed()) {
    127    aDest.mNumberOfChannels.Construct(aConfig.mNumberOfChannels.Value());
    128  }
    129  if (aConfig.mSampleRate.WasPassed()) {
    130    aDest.mSampleRate.Construct(aConfig.mSampleRate.Value());
    131  }
    132  if (aConfig.mBitrate.WasPassed()) {
    133    aDest.mBitrate.Construct(aConfig.mBitrate.Value());
    134  }
    135  if (aConfig.mOpus.WasPassed()) {
    136    aDest.mOpus.Construct(aConfig.mOpus.Value());
    137    // Handle the default value manually since it's different on mobile
    138    if (!aConfig.mOpus.Value().mComplexity.WasPassed()) {
    139      if (IsOnAndroid()) {
    140        aDest.mOpus.Value().mComplexity.Construct(5);
    141      } else {
    142        aDest.mOpus.Value().mComplexity.Construct(9);
    143      }
    144    }
    145  }
    146  aDest.mBitrateMode = aConfig.mBitrateMode;
    147 }
    148 
    149 static bool IsAudioEncodeSupported(const nsAString& aCodec) {
    150  LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get());
    151 
    152  return aCodec.EqualsLiteral("opus") || aCodec.EqualsLiteral("vorbis");
    153 }
    154 
    155 static bool CanEncode(const RefPtr<AudioEncoderConfigInternal>& aConfig,
    156                      nsCString& aErrorMessage) {
    157  // TODO: Enable WebCodecs on Android (Bug 1840508)
    158  if (IsOnAndroid()) {
    159    return false;
    160  }
    161  if (!IsAudioEncodeSupported(aConfig->mCodec)) {
    162    return false;
    163  }
    164  if (!IsSupportedAudioCodec(aConfig->mCodec)) {
    165    aErrorMessage.AppendPrintf("%s is not supported",
    166                               NS_ConvertUTF16toUTF8(aConfig->mCodec).get());
    167    return false;
    168  }
    169 
    170  if (aConfig->mNumberOfChannels.value() > 256) {
    171    aErrorMessage.AppendPrintf(
    172        "Invalid number of channels, supported range is between 1 and 256");
    173    return false;
    174  }
    175 
    176  // Somewhat arbitrarily chosen, but reflects real-life and what the rest of
    177  // Gecko does.
    178  if (aConfig->mSampleRate.value() < 3000 ||
    179      aConfig->mSampleRate.value() > 384000) {
    180    aErrorMessage.AppendPrintf(
    181        "Invalid sample-rate of %d, supported range is 3000Hz to 384000Hz",
    182        aConfig->mSampleRate.value());
    183    return false;
    184  }
    185 
    186  return EncoderSupport::Supports(aConfig);
    187 }
    188 
    189 nsCString AudioEncoderConfigInternal::ToString() const {
    190  nsCString rv;
    191 
    192  rv.AppendPrintf("AudioEncoderConfigInternal: %s",
    193                  NS_ConvertUTF16toUTF8(mCodec).get());
    194  if (mSampleRate) {
    195    rv.AppendPrintf(" %" PRIu32 "Hz", mSampleRate.value());
    196  }
    197  if (mNumberOfChannels) {
    198    rv.AppendPrintf(" %" PRIu32 "ch", mNumberOfChannels.value());
    199  }
    200  if (mBitrate) {
    201    rv.AppendPrintf(" %" PRIu32 "bps", mBitrate.value());
    202  }
    203  rv.AppendPrintf(" (%s)", mBitrateMode == mozilla::dom::BitrateMode::Constant
    204                               ? "CRB"
    205                               : "VBR");
    206 
    207  return rv;
    208 }
    209 
    210 EncoderConfig AudioEncoderConfigInternal::ToEncoderConfig() const {
    211  const mozilla::BitrateMode bitrateMode =
    212      mBitrateMode == mozilla::dom::BitrateMode::Constant
    213          ? mozilla::BitrateMode::Constant
    214          : mozilla::BitrateMode::Variable;
    215 
    216  CodecType type = CodecType::Opus;
    217  EncoderConfig::CodecSpecific specific{void_t{}};
    218  if (mCodec.EqualsLiteral("opus")) {
    219    type = CodecType::Opus;
    220    MOZ_ASSERT(mSpecific.is<void_t>() || mSpecific.is<OpusSpecific>());
    221    specific = mSpecific;
    222  } else if (mCodec.EqualsLiteral("vorbis")) {
    223    type = CodecType::Vorbis;
    224  } else if (mCodec.EqualsLiteral("flac")) {
    225    type = CodecType::Flac;
    226  } else if (StringBeginsWith(mCodec, u"pcm-"_ns)) {
    227    type = CodecType::PCM;
    228  } else if (mCodec.EqualsLiteral("ulaw")) {
    229    type = CodecType::PCM;
    230  } else if (mCodec.EqualsLiteral("alaw")) {
    231    type = CodecType::PCM;
    232  } else if (StringBeginsWith(mCodec, u"mp4a."_ns)) {
    233    type = CodecType::AAC;
    234  }
    235 
    236  // This should have been checked ahead of time -- we can't encode without
    237  // knowing the sample-rate or the channel count at the very least.
    238  MOZ_ASSERT(mSampleRate.value());
    239  MOZ_ASSERT(mNumberOfChannels.value());
    240 
    241  return EncoderConfig(type, mNumberOfChannels.value(), bitrateMode,
    242                       AssertedCast<uint32_t>(mSampleRate.value()),
    243                       mBitrate.valueOr(0), specific);
    244 }
    245 
    246 bool AudioEncoderConfigInternal::Equals(
    247    const AudioEncoderConfigInternal& aOther) const {
    248  return false;
    249 }
    250 
    251 bool AudioEncoderConfigInternal::CanReconfigure(
    252    const AudioEncoderConfigInternal& aOther) const {
    253  return false;
    254 }
    255 
    256 already_AddRefed<WebCodecsConfigurationChangeList>
    257 AudioEncoderConfigInternal::Diff(
    258    const AudioEncoderConfigInternal& aOther) const {
    259  return MakeRefPtr<WebCodecsConfigurationChangeList>().forget();
    260 }
    261 
    262 /* static */
    263 bool AudioEncoderTraits::IsSupported(
    264    const AudioEncoderConfigInternal& aConfig) {
    265  nsCString errorMessage;
    266  bool canEncode =
    267      CanEncode(MakeRefPtr<AudioEncoderConfigInternal>(aConfig), errorMessage);
    268  if (!canEncode) {
    269    LOGE("Can't encode configuration %s: %s", aConfig.ToString().get(),
    270         errorMessage.get());
    271  }
    272  return canEncode;
    273 }
    274 
    275 // https://w3c.github.io/webcodecs/#valid-audioencoderconfig
    276 /* static */
    277 bool AudioEncoderTraits::Validate(const AudioEncoderConfig& aConfig,
    278                                  nsCString& aErrorMessage) {
    279  Maybe<nsString> codec = ParseCodecString(aConfig.mCodec);
    280  if (!codec || codec->IsEmpty()) {
    281    LOGE("Validating AudioEncoderConfig: invalid codec string");
    282    return false;
    283  }
    284 
    285  if (!aConfig.mNumberOfChannels.WasPassed()) {
    286    aErrorMessage.AppendPrintf("Channel count required");
    287    return false;
    288  }
    289  if (aConfig.mNumberOfChannels.Value() == 0) {
    290    aErrorMessage.AppendPrintf(
    291        "Invalid number of channels, supported range is between 1 and 256");
    292    return false;
    293  }
    294  if (!aConfig.mSampleRate.WasPassed()) {
    295    aErrorMessage.AppendPrintf("Sample-rate required");
    296    return false;
    297  }
    298  if (aConfig.mSampleRate.Value() == 0) {
    299    aErrorMessage.AppendPrintf("Invalid sample-rate of 0");
    300    return false;
    301  }
    302 
    303  if (aConfig.mBitrate.WasPassed() &&
    304      aConfig.mBitrate.Value() > std::numeric_limits<int>::max()) {
    305    aErrorMessage.AppendPrintf("Invalid config: bitrate value too large");
    306    return false;
    307  }
    308 
    309  // https://github.com/w3c/webcodecs/issues/816
    310  if ((aConfig.mBitrate.WasPassed() && aConfig.mBitrate.Value() == 0)) {
    311    aErrorMessage.AssignLiteral(
    312        "Invalid AudioEncoderConfig: bitrate equal to 0");
    313    LOGE("%s", aErrorMessage.get());
    314    return false;
    315  }
    316 
    317  if (codec->EqualsLiteral("opus")) {
    318    // This comes from
    319    // https://w3c.github.io/webcodecs/opus_codec_registration.html#opus-encoder-config
    320    if (aConfig.mBitrate.WasPassed() && (aConfig.mBitrate.Value() < 6000 ||
    321                                         aConfig.mBitrate.Value() > 510000)) {
    322      aErrorMessage.AppendPrintf(
    323          "Invalid config: bitrate value outside of [6k, 510k] for opus");
    324      return false;
    325    }
    326    if (aConfig.mOpus.WasPassed()) {
    327      // Verify value ranges
    328      const std::array validFrameDurationUs = {2500,  5000,  10000,
    329                                               20000, 40000, 60000};
    330      if (std::find(validFrameDurationUs.begin(), validFrameDurationUs.end(),
    331                    aConfig.mOpus.Value().mFrameDuration) ==
    332          validFrameDurationUs.end()) {
    333        aErrorMessage.AppendPrintf("Invalid config: invalid frame duration");
    334        return false;
    335      }
    336      if (aConfig.mOpus.Value().mComplexity.WasPassed() &&
    337          aConfig.mOpus.Value().mComplexity.Value() > 10) {
    338        aErrorMessage.AppendPrintf(
    339            "Invalid config: Opus complexity greater than 10");
    340        return false;
    341      }
    342      if (aConfig.mOpus.Value().mPacketlossperc > 100) {
    343        aErrorMessage.AppendPrintf(
    344            "Invalid config: Opus packet loss percentage greater than 100");
    345        return false;
    346      }
    347    }
    348  }
    349 
    350  return true;
    351 }
    352 
    353 /* static */
    354 RefPtr<AudioEncoderConfigInternal> AudioEncoderTraits::CreateConfigInternal(
    355    const AudioEncoderConfig& aConfig) {
    356  nsCString errorMessage;
    357  if (!AudioEncoderTraits::Validate(aConfig, errorMessage)) {
    358    return nullptr;
    359  }
    360  return MakeRefPtr<AudioEncoderConfigInternal>(aConfig);
    361 }
    362 
    363 /* static */
    364 RefPtr<mozilla::AudioData> AudioEncoderTraits::CreateInputInternal(
    365    const dom::AudioData& aInput,
    366    const dom::VideoEncoderEncodeOptions& /* unused */) {
    367  return aInput.ToAudioData();
    368 }
    369 
    370 /*
    371 * Below are AudioEncoder implementation
    372 */
    373 
    374 AudioEncoder::AudioEncoder(
    375    nsIGlobalObject* aParent, RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
    376    RefPtr<EncodedAudioChunkOutputCallback>&& aOutputCallback)
    377    : EncoderTemplate(aParent, std::move(aErrorCallback),
    378                      std::move(aOutputCallback)) {
    379  MOZ_ASSERT(mErrorCallback);
    380  MOZ_ASSERT(mOutputCallback);
    381  LOG("AudioEncoder %p ctor", this);
    382 }
    383 
    384 AudioEncoder::~AudioEncoder() {
    385  LOG("AudioEncoder %p dtor", this);
    386  (void)ResetInternal(NS_ERROR_DOM_ABORT_ERR);
    387 }
    388 
    389 JSObject* AudioEncoder::WrapObject(JSContext* aCx,
    390                                   JS::Handle<JSObject*> aGivenProto) {
    391  AssertIsOnOwningThread();
    392 
    393  return AudioEncoder_Binding::Wrap(aCx, this, aGivenProto);
    394 }
    395 
    396 // https://w3c.github.io/webcodecs/#dom-audioencoder-audioencoder
    397 /* static */
    398 already_AddRefed<AudioEncoder> AudioEncoder::Constructor(
    399    const GlobalObject& aGlobal, const AudioEncoderInit& aInit,
    400    ErrorResult& aRv) {
    401  nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
    402  if (!global) {
    403    aRv.Throw(NS_ERROR_FAILURE);
    404    return nullptr;
    405  }
    406 
    407  return MakeAndAddRef<AudioEncoder>(
    408      global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError),
    409      RefPtr<EncodedAudioChunkOutputCallback>(aInit.mOutput));
    410 }
    411 
    412 // https://w3c.github.io/webcodecs/#dom-audioencoder-isconfigsupported
    413 /* static */
    414 already_AddRefed<Promise> AudioEncoder::IsConfigSupported(
    415    const GlobalObject& aGlobal, const AudioEncoderConfig& aConfig,
    416    ErrorResult& aRv) {
    417  LOG("AudioEncoder::IsConfigSupported, config: %s",
    418      NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
    419 
    420  nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
    421  if (!global) {
    422    aRv.Throw(NS_ERROR_FAILURE);
    423    return nullptr;
    424  }
    425 
    426  RefPtr<Promise> p = Promise::Create(global.get(), aRv);
    427  if (NS_WARN_IF(aRv.Failed())) {
    428    return p.forget();
    429  }
    430 
    431  nsCString errorMessage;
    432  if (!AudioEncoderTraits::Validate(aConfig, errorMessage)) {
    433    p->MaybeRejectWithTypeError(errorMessage);
    434    return p.forget();
    435  }
    436 
    437  // TODO: Move the following works to another thread to unblock the current
    438  // thread, as what spec suggests.
    439 
    440  RootedDictionary<AudioEncoderConfig> config(aGlobal.Context());
    441  CloneConfiguration(config, aGlobal.Context(), aConfig);
    442 
    443  auto configInternal = MakeRefPtr<AudioEncoderConfigInternal>(aConfig);
    444  bool canEncode = CanEncode(configInternal, errorMessage);
    445  if (!canEncode) {
    446    LOG("CanEncode failed: %s", errorMessage.get());
    447  }
    448  RootedDictionary<AudioEncoderSupport> s(aGlobal.Context());
    449  s.mConfig.Construct(std::move(config));
    450  s.mSupported.Construct(canEncode);
    451 
    452  p->MaybeResolve(s);
    453  return p.forget();
    454 }
    455 
    456 RefPtr<EncodedAudioChunk> AudioEncoder::EncodedDataToOutputType(
    457    nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData) {
    458  AssertIsOnOwningThread();
    459 
    460  // Package into an EncodedAudioChunk
    461  auto buffer =
    462      MakeRefPtr<MediaAlignedByteBuffer>(aData->Data(), aData->Size());
    463  auto encoded = MakeRefPtr<EncodedAudioChunk>(
    464      aGlobalObject, buffer.forget(), EncodedAudioChunkType::Key,
    465      aData->mTime.ToMicroseconds(),
    466      aData->mDuration.IsZero() ? Nothing()
    467                                : Some(aData->mDuration.ToMicroseconds()));
    468  return encoded;
    469 }
    470 
    471 void AudioEncoder::EncoderConfigToDecoderConfig(
    472    JSContext* aCx, const RefPtr<MediaRawData>& aRawData,
    473    const AudioEncoderConfigInternal& aSrcConfig,
    474    AudioDecoderConfig& aDestConfig) const {
    475  MOZ_ASSERT(aCx);
    476  MOZ_ASSERT(aSrcConfig.mSampleRate.isSome());
    477  MOZ_ASSERT(aSrcConfig.mNumberOfChannels.isSome());
    478 
    479  uint32_t sampleRate = aSrcConfig.mSampleRate.value();
    480  uint32_t channelCount = aSrcConfig.mNumberOfChannels.value();
    481  // Check if the encoder had to modify the settings because of codec
    482  // constraints. e.g. FFmpegAudioEncoder can encode any sample-rate, but if the
    483  // codec is Opus, then it will resample the audio one of the specific rates
    484  // supported by the encoder.
    485  if (aRawData->mConfig) {
    486    sampleRate = aRawData->mConfig->mSampleRate;
    487    channelCount = aRawData->mConfig->mNumberOfChannels;
    488  }
    489 
    490  aDestConfig.mCodec = aSrcConfig.mCodec;
    491  aDestConfig.mNumberOfChannels = channelCount;
    492  aDestConfig.mSampleRate = sampleRate;
    493 
    494  if (aRawData->mExtraData && !aRawData->mExtraData->IsEmpty()) {
    495    Span<const uint8_t> description(aRawData->mExtraData->Elements(),
    496                                    aRawData->mExtraData->Length());
    497    if (!CopyExtradataToDescription(aCx, description,
    498                                    aDestConfig.mDescription.Construct())) {
    499      LOGE("Failed to copy extra data");
    500    }
    501  }
    502 }
    503 
    504 #undef LOG
    505 #undef LOGW
    506 #undef LOGE
    507 #undef LOGV
    508 #undef LOG_INTERNAL
    509 
    510 }  // namespace mozilla::dom
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE