[ tor-browser ].git.dasho

MediaBufferDecoder.cpp (25808B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "MediaBufferDecoder.h"
      8 
      9 #include <speex/speex_resampler.h>
     10 
     11 #include "AudioBuffer.h"
     12 #include "AudioContext.h"
     13 #include "AudioNodeEngine.h"
     14 #include "BufferMediaResource.h"
     15 #include "DecoderTraits.h"
     16 #include "MediaContainerType.h"
     17 #include "MediaDataDecoderProxy.h"
     18 #include "MediaDataDemuxer.h"
     19 #include "MediaQueue.h"
     20 #include "PDMFactory.h"
     21 #include "VideoUtils.h"
     22 #include "WebAudioUtils.h"
     23 #include "js/MemoryFunctions.h"
     24 #include "mozilla/AbstractThread.h"
     25 #include "mozilla/Logging.h"
     26 #include "mozilla/StaticPrefs_media.h"
     27 #include "mozilla/dom/AudioContextBinding.h"
     28 #include "mozilla/dom/BaseAudioContextBinding.h"
     29 #include "mozilla/dom/DOMException.h"
     30 #include "mozilla/dom/Promise.h"
     31 #include "nsContentUtils.h"
     32 #include "nsGlobalWindowInner.h"
     33 #include "nsMimeTypes.h"
     34 
     35 namespace mozilla {
     36 
     37 extern LazyLogModule gMediaDecoderLog;
     38 
     39 #define LOG(x, ...) \
     40  MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (x, ##__VA_ARGS__))
     41 
     42 using namespace dom;
     43 
     44 class ReportResultTask final : public Runnable {
     45 public:
     46  ReportResultTask(WebAudioDecodeJob& aDecodeJob,
     47                   WebAudioDecodeJob::ResultFn aFunction,
     48                   WebAudioDecodeJob::ErrorCode aErrorCode)
     49      : Runnable("ReportResultTask"),
     50        mDecodeJob(aDecodeJob),
     51        mFunction(aFunction),
     52        mErrorCode(aErrorCode) {
     53    MOZ_ASSERT(aFunction);
     54  }
     55 
     56  NS_IMETHOD Run() override {
     57    MOZ_ASSERT(NS_IsMainThread());
     58 
     59    (mDecodeJob.*mFunction)(mErrorCode);
     60 
     61    return NS_OK;
     62  }
     63 
     64 private:
     65  // Note that the mDecodeJob member will probably die when mFunction is run.
     66  // Therefore, it is not safe to do anything fancy with it in this class.
     67  // Really, this class is only used because nsRunnableMethod doesn't support
     68  // methods accepting arguments.
     69  WebAudioDecodeJob& mDecodeJob;
     70  WebAudioDecodeJob::ResultFn mFunction;
     71  WebAudioDecodeJob::ErrorCode mErrorCode;
     72 };
     73 
     74 enum class PhaseEnum : int { Decode, AllocateBuffer, Done };
     75 
     76 class MediaDecodeTask final : public Runnable {
     77 public:
     78  MediaDecodeTask(const MediaContainerType& aContainerType, uint8_t* aBuffer,
     79                  uint32_t aLength, WebAudioDecodeJob& aDecodeJob)
     80      : Runnable("MediaDecodeTask"),
     81        mContainerType(aContainerType),
     82        mBuffer(aBuffer),
     83        mLength(aLength),
     84        mBatchSize(StaticPrefs::media_rdd_webaudio_batch_size()),
     85        mDecodeJob(aDecodeJob),
     86        mPhase(PhaseEnum::Decode) {
     87    MOZ_ASSERT(aBuffer);
     88    MOZ_ASSERT(NS_IsMainThread());
     89  }
     90 
     91  // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.  See
     92  // bug 1535398.
     93  MOZ_CAN_RUN_SCRIPT_BOUNDARY
     94  NS_IMETHOD Run() override;
     95  bool Init();
     96  TaskQueue* PSupervisorTaskQueue() { return mPSupervisorTaskQueue; }
     97  bool OnPSupervisorTaskQueue() const {
     98    return mPSupervisorTaskQueue->IsCurrentThreadIn();
     99  }
    100 
    101 private:
    102  MOZ_CAN_RUN_SCRIPT
    103  void ReportFailureOnMainThread(WebAudioDecodeJob::ErrorCode aErrorCode) {
    104    if (NS_IsMainThread()) {
    105      Cleanup();
    106      mDecodeJob.OnFailure(aErrorCode);
    107    } else {
    108      // Take extra care to cleanup on the main thread
    109      mMainThread->Dispatch(NewRunnableMethod("MediaDecodeTask::Cleanup", this,
    110                                              &MediaDecodeTask::Cleanup));
    111 
    112      nsCOMPtr<nsIRunnable> event = new ReportResultTask(
    113          mDecodeJob, &WebAudioDecodeJob::OnFailure, aErrorCode);
    114      mMainThread->Dispatch(event.forget());
    115    }
    116  }
    117 
    118  void Decode();
    119 
    120  void OnCreateDecoderCompleted(RefPtr<MediaDataDecoder> aDecoder);
    121  MOZ_CAN_RUN_SCRIPT void OnCreateDecoderFailed(const MediaResult& aError);
    122 
    123  MOZ_CAN_RUN_SCRIPT void OnInitDemuxerCompleted();
    124  MOZ_CAN_RUN_SCRIPT void OnInitDemuxerFailed(const MediaResult& aError);
    125 
    126  void InitDecoder();
    127  void OnInitDecoderCompleted();
    128  MOZ_CAN_RUN_SCRIPT void OnInitDecoderFailed();
    129 
    130  void DoDemux();
    131  void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
    132  MOZ_CAN_RUN_SCRIPT void OnAudioDemuxFailed(const MediaResult& aError);
    133 
    134  void DoDecode();
    135  void OnAudioDecodeCompleted(MediaDataDecoder::DecodedData&& aResults);
    136  MOZ_CAN_RUN_SCRIPT void OnAudioDecodeFailed(const MediaResult& aError);
    137 
    138  void DoDrain();
    139  MOZ_CAN_RUN_SCRIPT void OnAudioDrainCompleted(
    140      MediaDataDecoder::DecodedData&& aResults);
    141  MOZ_CAN_RUN_SCRIPT void OnAudioDrainFailed(const MediaResult& aError);
    142 
    143  void ShutdownDecoder();
    144 
    145  MOZ_CAN_RUN_SCRIPT void FinishDecode();
    146  MOZ_CAN_RUN_SCRIPT void AllocateBuffer();
    147  MOZ_CAN_RUN_SCRIPT void CallbackTheResult();
    148 
    149  void Cleanup() {
    150    MOZ_ASSERT(NS_IsMainThread());
    151    JS_free(nullptr, mBuffer);
    152    if (mTrackDemuxer) {
    153      mTrackDemuxer->BreakCycles();
    154    }
    155    mTrackDemuxer = nullptr;
    156    mDemuxer = nullptr;
    157    mPSupervisorTaskQueue = nullptr;
    158    mPDecoderTaskQueue = nullptr;
    159  }
    160 
    161 private:
    162  MediaContainerType mContainerType;
    163  uint8_t* mBuffer;
    164  const uint32_t mLength;
    165  const uint32_t mBatchSize;
    166  WebAudioDecodeJob& mDecodeJob;
    167  PhaseEnum mPhase;
    168  RefPtr<TaskQueue> mPSupervisorTaskQueue;
    169  RefPtr<TaskQueue> mPDecoderTaskQueue;
    170  RefPtr<MediaDataDemuxer> mDemuxer;
    171  RefPtr<MediaTrackDemuxer> mTrackDemuxer;
    172  RefPtr<MediaDataDecoder> mDecoder;
    173  nsTArray<RefPtr<MediaRawData>> mRawSamples;
    174  MediaInfo mMediaInfo;
    175  MediaQueue<AudioData> mAudioQueue;
    176  RefPtr<AbstractThread> mMainThread;
    177 };
    178 
    179 NS_IMETHODIMP
    180 MediaDecodeTask::Run() {
    181  switch (mPhase) {
    182    case PhaseEnum::Decode:
    183      Decode();
    184      break;
    185    case PhaseEnum::AllocateBuffer:
    186      AllocateBuffer();
    187      break;
    188    case PhaseEnum::Done:
    189      break;
    190  }
    191 
    192  return NS_OK;
    193 }
    194 
    195 bool MediaDecodeTask::Init() {
    196  MOZ_ASSERT(NS_IsMainThread());
    197 
    198  RefPtr<BufferMediaResource> resource =
    199      new BufferMediaResource(static_cast<uint8_t*>(mBuffer), mLength);
    200 
    201  mMainThread = AbstractThread::MainThread();
    202 
    203  mPSupervisorTaskQueue =
    204      TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR),
    205                        "MediaBufferDecoder::mPSupervisorTaskQueue");
    206  mPDecoderTaskQueue =
    207      TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
    208                        "MediaBufferDecoder::mPDecoderTaskQueue");
    209 
    210  // If you change this list to add support for new decoders, please consider
    211  // updating HTMLMediaElement::CreateDecoder as well.
    212  mDemuxer = DecoderTraits::CreateDemuxer(mContainerType, resource);
    213  if (!mDemuxer) {
    214    return false;
    215  }
    216 
    217  return true;
    218 }
    219 
    220 class AutoResampler final {
    221 public:
    222  AutoResampler() : mResampler(nullptr) {}
    223  ~AutoResampler() {
    224    if (mResampler) {
    225      speex_resampler_destroy(mResampler);
    226    }
    227  }
    228  operator SpeexResamplerState*() const {
    229    MOZ_ASSERT(mResampler);
    230    return mResampler;
    231  }
    232  void operator=(SpeexResamplerState* aResampler) { mResampler = aResampler; }
    233 
    234 private:
    235  SpeexResamplerState* mResampler;
    236 };
    237 
    238 void MediaDecodeTask::Decode() {
    239  MOZ_ASSERT(OnPSupervisorTaskQueue());
    240 
    241  mDemuxer->Init()->Then(PSupervisorTaskQueue(), __func__, this,
    242                         &MediaDecodeTask::OnInitDemuxerCompleted,
    243                         &MediaDecodeTask::OnInitDemuxerFailed);
    244 }
    245 
    246 void MediaDecodeTask::OnInitDemuxerCompleted() {
    247  MOZ_ASSERT(OnPSupervisorTaskQueue());
    248 
    249  if (!!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack)) {
    250    mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
    251    if (!mTrackDemuxer) {
    252      LOG("MediaDecodeTask: Could not get a track demuxer.");
    253      ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
    254      return;
    255    }
    256 
    257    RefPtr<PDMFactory> platform = new PDMFactory();
    258    UniquePtr<TrackInfo> audioInfo = mTrackDemuxer->GetInfo();
    259    // We actively ignore audio tracks that we know we can't play.
    260    if (audioInfo && audioInfo->IsValid() &&
    261        !platform->SupportsMimeType(audioInfo->mMimeType).isEmpty()) {
    262      mMediaInfo.mAudio = *audioInfo->GetAsAudioInfo();
    263    }
    264  }
    265 
    266  RefPtr<PDMFactory> pdm = new PDMFactory();
    267  pdm->CreateDecoder(
    268         {*mMediaInfo.mAudio.GetAsAudioInfo(), TrackInfo::kAudioTrack})
    269      ->Then(PSupervisorTaskQueue(), __func__, this,
    270             &MediaDecodeTask::OnCreateDecoderCompleted,
    271             &MediaDecodeTask::OnCreateDecoderFailed);
    272 }
    273 
    274 void MediaDecodeTask::OnCreateDecoderCompleted(
    275    RefPtr<MediaDataDecoder> aDecoder) {
    276  MOZ_ASSERT(OnPSupervisorTaskQueue());
    277 
    278  mDecoder = new MediaDataDecoderProxy(aDecoder.forget(),
    279                                       do_AddRef(mPDecoderTaskQueue.get()));
    280  InitDecoder();
    281 }
    282 
    283 void MediaDecodeTask::OnCreateDecoderFailed(const MediaResult& aError) {
    284  MOZ_ASSERT(OnPSupervisorTaskQueue());
    285 
    286  LOG("MediaDecodeTask: Could not create a decoder.");
    287  ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
    288 }
    289 
    290 void MediaDecodeTask::OnInitDemuxerFailed(const MediaResult& aError) {
    291  MOZ_ASSERT(OnPSupervisorTaskQueue());
    292 
    293  LOG("MediaDecodeTask: Could not initialize the demuxer.");
    294  ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
    295 }
    296 
    297 void MediaDecodeTask::InitDecoder() {
    298  MOZ_ASSERT(OnPSupervisorTaskQueue());
    299 
    300  mDecoder->Init()->Then(PSupervisorTaskQueue(), __func__, this,
    301                         &MediaDecodeTask::OnInitDecoderCompleted,
    302                         &MediaDecodeTask::OnInitDecoderFailed);
    303 }
    304 
    305 void MediaDecodeTask::OnInitDecoderCompleted() {
    306  MOZ_ASSERT(OnPSupervisorTaskQueue());
    307 
    308  DoDemux();
    309 }
    310 
    311 void MediaDecodeTask::OnInitDecoderFailed() {
    312  MOZ_ASSERT(OnPSupervisorTaskQueue());
    313 
    314  ShutdownDecoder();
    315  LOG("MediaDecodeTask: Could not initialize the decoder");
    316  ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
    317 }
    318 
    319 void MediaDecodeTask::DoDemux() {
    320  MOZ_ASSERT(OnPSupervisorTaskQueue());
    321 
    322  mTrackDemuxer->GetSamples(mBatchSize)
    323      ->Then(PSupervisorTaskQueue(), __func__, this,
    324             &MediaDecodeTask::OnAudioDemuxCompleted,
    325             &MediaDecodeTask::OnAudioDemuxFailed);
    326 }
    327 
    328 void MediaDecodeTask::OnAudioDemuxCompleted(
    329    RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
    330  MOZ_ASSERT(OnPSupervisorTaskQueue());
    331 
    332  mRawSamples.AppendElements(aSamples->GetSamples());
    333 
    334  DoDemux();
    335 }
    336 
    337 void MediaDecodeTask::OnAudioDemuxFailed(const MediaResult& aError) {
    338  MOZ_ASSERT(OnPSupervisorTaskQueue());
    339 
    340  if (aError.Code() == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
    341    DoDecode();
    342  } else {
    343    ShutdownDecoder();
    344    LOG("MediaDecodeTask: Audio demux failed");
    345    ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
    346  }
    347 }
    348 
    349 void MediaDecodeTask::DoDecode() {
    350  MOZ_ASSERT(OnPSupervisorTaskQueue());
    351 
    352  if (mRawSamples.IsEmpty()) {
    353    DoDrain();
    354    return;
    355  }
    356 
    357  if (mBatchSize > 1 && mDecoder->CanDecodeBatch()) {
    358    nsTArray<RefPtr<MediaRawData>> rawSampleBatch;
    359    const int batchSize = std::min((unsigned long)mBatchSize,
    360                                   (unsigned long)mRawSamples.Length());
    361    for (int i = 0; i < batchSize; ++i) {
    362      rawSampleBatch.AppendElement(std::move(mRawSamples[i]));
    363    }
    364 
    365    mDecoder->DecodeBatch(std::move(rawSampleBatch))
    366        ->Then(PSupervisorTaskQueue(), __func__, this,
    367               &MediaDecodeTask::OnAudioDecodeCompleted,
    368               &MediaDecodeTask::OnAudioDecodeFailed);
    369 
    370    mRawSamples.RemoveElementsAt(0, batchSize);
    371  } else {
    372    RefPtr<MediaRawData> sample = std::move(mRawSamples[0]);
    373 
    374    mDecoder->Decode(sample)->Then(PSupervisorTaskQueue(), __func__, this,
    375                                   &MediaDecodeTask::OnAudioDecodeCompleted,
    376                                   &MediaDecodeTask::OnAudioDecodeFailed);
    377 
    378    mRawSamples.RemoveElementAt(0);
    379  }
    380 }
    381 
    382 void MediaDecodeTask::OnAudioDecodeCompleted(
    383    MediaDataDecoder::DecodedData&& aResults) {
    384  MOZ_ASSERT(OnPSupervisorTaskQueue());
    385 
    386  for (auto&& sample : aResults) {
    387    MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
    388    RefPtr<AudioData> audioData = sample->As<AudioData>();
    389 
    390    mMediaInfo.mAudio.mRate = audioData->mRate;
    391    mMediaInfo.mAudio.mChannels = audioData->mChannels;
    392 
    393    mAudioQueue.Push(audioData.forget());
    394  }
    395 
    396  DoDecode();
    397 }
    398 
    399 void MediaDecodeTask::OnAudioDecodeFailed(const MediaResult& aError) {
    400  MOZ_ASSERT(OnPSupervisorTaskQueue());
    401 
    402  ShutdownDecoder();
    403  LOG("MediaDecodeTask: decode audio failed.");
    404  ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
    405 }
    406 
    407 void MediaDecodeTask::DoDrain() {
    408  MOZ_ASSERT(OnPSupervisorTaskQueue());
    409 
    410  mDecoder->Drain()->Then(PSupervisorTaskQueue(), __func__, this,
    411                          &MediaDecodeTask::OnAudioDrainCompleted,
    412                          &MediaDecodeTask::OnAudioDrainFailed);
    413 }
    414 
    415 void MediaDecodeTask::OnAudioDrainCompleted(
    416    MediaDataDecoder::DecodedData&& aResults) {
    417  MOZ_ASSERT(OnPSupervisorTaskQueue());
    418 
    419  if (aResults.IsEmpty()) {
    420    FinishDecode();
    421    return;
    422  }
    423 
    424  for (auto&& sample : aResults) {
    425    MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
    426    RefPtr<AudioData> audioData = sample->As<AudioData>();
    427 
    428    mAudioQueue.Push(audioData.forget());
    429  }
    430  DoDrain();
    431 }
    432 
    433 void MediaDecodeTask::OnAudioDrainFailed(const MediaResult& aError) {
    434  MOZ_ASSERT(OnPSupervisorTaskQueue());
    435 
    436  ShutdownDecoder();
    437  LOG("MediaDecodeTask: Drain audio failed");
    438  ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
    439 }
    440 
    441 void MediaDecodeTask::ShutdownDecoder() {
    442  MOZ_ASSERT(OnPSupervisorTaskQueue());
    443 
    444  if (!mDecoder) {
    445    return;
    446  }
    447 
    448  RefPtr<MediaDecodeTask> self = this;
    449  mDecoder->Shutdown();
    450  mDecoder = nullptr;
    451 }
    452 
    453 static void UpmixPreviousData(
    454    const RefPtr<ThreadSharedFloatArrayBufferList>& aOldBuffers,
    455    const RefPtr<ThreadSharedFloatArrayBufferList>& aNewBuffers,
    456    const uint32_t aCopyCount, const uint32_t aPrevChannelCount,
    457    const uint32_t aChannelCount) {
    458  if (aCopyCount > 0) {
    459    // Copy all existing buffers
    460    for (uint32_t i = 0; i < aPrevChannelCount; ++i) {
    461      const float* src = aOldBuffers->GetData(i);
    462      float* dst = aNewBuffers->GetDataForWrite(i);
    463      AudioBufferCopyWithScale(src, 1.0, dst, aCopyCount);
    464    }
    465 
    466    // Upmix from channel 0 to create new channels if needed
    467    for (uint32_t i = aPrevChannelCount; i < aChannelCount; ++i) {
    468      const float* src = aOldBuffers->GetData(0);
    469      float* dst = aNewBuffers->GetDataForWrite(i);
    470      AudioBufferCopyWithScale(src, 1.0, dst, aCopyCount);
    471    }
    472  }
    473 }
    474 
    475 static RefPtr<ThreadSharedFloatArrayBufferList> CreateChannelBuffers(
    476    const uint32_t aChannelCount, const uint32_t aResampledFrames) {
    477  // This buffer has separate channel arrays that could be transferred to
    478  // JS::NewArrayBufferWithContents(), but AudioBuffer::RestoreJSChannelData()
    479  // does not yet take advantage of this.
    480  RefPtr<ThreadSharedFloatArrayBufferList> buffer =
    481      ThreadSharedFloatArrayBufferList::Create(aChannelCount, aResampledFrames,
    482                                               fallible);
    483  if (!buffer) {
    484    LOG("MediaDecodeTask: Could not create final buffer (f32)");
    485    return nullptr;
    486  }
    487  return buffer;
    488 }
    489 void MediaDecodeTask::FinishDecode() {
    490  MOZ_ASSERT(OnPSupervisorTaskQueue());
    491 
    492  ShutdownDecoder();
    493 
    494  uint32_t frameCount = mAudioQueue.AudioFramesCount();
    495  uint32_t channelCount = mMediaInfo.mAudio.mChannels;
    496  uint32_t sampleRate = mMediaInfo.mAudio.mRate;
    497 
    498  if (!frameCount || !channelCount || !sampleRate) {
    499    LOG("MediaDecodeTask: invalid content frame count, channel count or "
    500        "sample-rate");
    501    ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
    502    return;
    503  }
    504 
    505  const uint32_t destSampleRate = mDecodeJob.mContext->SampleRate();
    506  AutoResampler resampler;
    507 
    508  uint32_t resampledFrames = frameCount;
    509  if (sampleRate != destSampleRate) {
    510    resampledFrames = static_cast<uint32_t>(
    511        static_cast<uint64_t>(destSampleRate) *
    512        static_cast<uint64_t>(frameCount) / static_cast<uint64_t>(sampleRate));
    513 
    514    resampler = speex_resampler_init(channelCount, sampleRate, destSampleRate,
    515                                     SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr);
    516    speex_resampler_skip_zeros(resampler);
    517    resampledFrames += speex_resampler_get_output_latency(resampler);
    518  }
    519 
    520  // Allocate contiguous channel buffers.  Note that if we end up resampling,
    521  // we may write fewer bytes than mResampledFrames to the output buffer, in
    522  // which case writeIndex will tell us how many valid samples we have.
    523  auto newBuffers = CreateChannelBuffers(channelCount, resampledFrames);
    524  if (!newBuffers) {
    525    ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
    526    return;
    527  }
    528  mDecodeJob.mBuffer.mChannelData.SetLength(channelCount);
    529  for (uint32_t i = 0; i < channelCount; ++i) {
    530    mDecodeJob.mBuffer.mChannelData[i] = newBuffers->GetData(i);
    531  }
    532 
    533  mDecodeJob.mBuffer.mBuffer = std::move(newBuffers);
    534  mDecodeJob.mBuffer.mVolume = 1.0f;
    535  mDecodeJob.mBuffer.mBufferFormat = AUDIO_OUTPUT_FORMAT;
    536 
    537  uint32_t writeIndex = 0;
    538  RefPtr<AudioData> audioData;
    539  while ((audioData = mAudioQueue.PopFront())) {
    540    if (!audioData->Frames()) {
    541      // The packet contains no audio frames, skip it.
    542      continue;
    543    }
    544 
    545    audioData->EnsureAudioBuffer();  // could lead to a copy :(
    546 
    547    // Edge case - incoming packet has more channels than we've allocated
    548    // memory for. Allocate additional channel buffers and upmix.
    549    if (channelCount < audioData->mChannels) {
    550      LOG("MediaDecodeTask: Expected %u channels, found %u. Adding channels.",
    551          channelCount, audioData->mChannels);
    552      newBuffers = CreateChannelBuffers(audioData->mChannels, resampledFrames);
    553      if (!newBuffers) {
    554        ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
    555        return;
    556      }
    557      RefPtr<ThreadSharedFloatArrayBufferList> oldBuffers =
    558          mDecodeJob.mBuffer.mBuffer->AsThreadSharedFloatArrayBufferList();
    559      UpmixPreviousData(oldBuffers, newBuffers, writeIndex, channelCount,
    560                        audioData->mChannels);
    561      mDecodeJob.mBuffer.mChannelData.SetLength(audioData->mChannels);
    562      for (uint32_t i = 0; i < audioData->mChannels; ++i) {
    563        mDecodeJob.mBuffer.mChannelData[i] = newBuffers->GetData(i);
    564      }
    565      mDecodeJob.mBuffer.mBuffer = std::move(newBuffers);
    566      channelCount = audioData->mChannels;
    567    }
    568 
    569    const AudioDataValue* bufferData =
    570        static_cast<AudioDataValue*>(audioData->mAudioBuffer->Data());
    571 
    572    if (sampleRate != destSampleRate) {
    573      const uint32_t maxOutSamples = resampledFrames - writeIndex;
    574 
    575      for (uint32_t i = 0; i < audioData->mChannels; ++i) {
    576        uint32_t inSamples = audioData->Frames();
    577        uint32_t outSamples = maxOutSamples;
    578        AudioDataValue* outData =
    579            mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
    580            writeIndex;
    581 
    582        WebAudioUtils::SpeexResamplerProcess(
    583            resampler, i, &bufferData[i * audioData->Frames()], &inSamples,
    584            outData, &outSamples);
    585 
    586        if (i == audioData->mChannels - 1) {
    587          writeIndex += outSamples;
    588          MOZ_ASSERT(writeIndex <= resampledFrames);
    589          MOZ_ASSERT(inSamples == audioData->Frames());
    590        }
    591      }
    592    } else {
    593      for (uint32_t i = 0; i < audioData->mChannels; ++i) {
    594        AudioDataValue* outData =
    595            mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
    596            writeIndex;
    597        PodCopy(outData, &bufferData[i * audioData->Frames()],
    598                audioData->Frames());
    599 
    600        if (i == audioData->mChannels - 1) {
    601          writeIndex += audioData->Frames();
    602        }
    603      }
    604    }
    605  }
    606 
    607  if (sampleRate != destSampleRate) {
    608    uint32_t inputLatency = speex_resampler_get_input_latency(resampler);
    609    const uint32_t maxOutSamples = resampledFrames - writeIndex;
    610    for (uint32_t i = 0; i < channelCount; ++i) {
    611      uint32_t inSamples = inputLatency;
    612      uint32_t outSamples = maxOutSamples;
    613      AudioDataValue* outData =
    614          mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
    615          writeIndex;
    616 
    617      WebAudioUtils::SpeexResamplerProcess(resampler, i,
    618                                           (AudioDataValue*)nullptr, &inSamples,
    619                                           outData, &outSamples);
    620 
    621      if (i == channelCount - 1) {
    622        writeIndex += outSamples;
    623        MOZ_ASSERT(writeIndex <= resampledFrames);
    624        MOZ_ASSERT(inSamples == inputLatency);
    625      }
    626    }
    627  }
    628 
    629  mDecodeJob.mBuffer.mDuration = writeIndex;
    630  mPhase = PhaseEnum::AllocateBuffer;
    631  mMainThread->Dispatch(do_AddRef(this));
    632 }
    633 
    634 void MediaDecodeTask::AllocateBuffer() {
    635  MOZ_ASSERT(NS_IsMainThread());
    636 
    637  if (!mDecodeJob.AllocateBuffer()) {
    638    LOG("MediaDecodeTask: Could not allocate final buffer");
    639    ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
    640    return;
    641  }
    642 
    643  mPhase = PhaseEnum::Done;
    644  CallbackTheResult();
    645 }
    646 
    647 void MediaDecodeTask::CallbackTheResult() {
    648  MOZ_ASSERT(NS_IsMainThread());
    649 
    650  Cleanup();
    651 
    652  // Now, we're ready to call the script back with the resulting buffer
    653  mDecodeJob.OnSuccess(WebAudioDecodeJob::NoError);
    654 }
    655 
    656 bool WebAudioDecodeJob::AllocateBuffer() {
    657  MOZ_ASSERT(!mOutput);
    658  MOZ_ASSERT(NS_IsMainThread());
    659 
    660  // Now create the AudioBuffer
    661  mOutput = AudioBuffer::Create(mContext->GetOwnerWindow(),
    662                                mContext->SampleRate(), std::move(mBuffer));
    663  return mOutput != nullptr;
    664 }
    665 
    666 void AsyncDecodeWebAudio(const char* aContentType, uint8_t* aBuffer,
    667                         uint32_t aLength, WebAudioDecodeJob& aDecodeJob) {
    668  Maybe<MediaContainerType> containerType =
    669      MakeMediaContainerType(aContentType);
    670  // Do not attempt to decode the media if we were not successful at sniffing
    671  // the container type.
    672  if (!*aContentType || strcmp(aContentType, APPLICATION_OCTET_STREAM) == 0 ||
    673      !containerType) {
    674    nsCOMPtr<nsIRunnable> event =
    675        new ReportResultTask(aDecodeJob, &WebAudioDecodeJob::OnFailure,
    676                             WebAudioDecodeJob::UnknownContent);
    677    JS_free(nullptr, aBuffer);
    678    aDecodeJob.mContext->Dispatch(event.forget());
    679    return;
    680  }
    681 
    682  RefPtr<MediaDecodeTask> task =
    683      new MediaDecodeTask(*containerType, aBuffer, aLength, aDecodeJob);
    684  if (!task->Init()) {
    685    nsCOMPtr<nsIRunnable> event =
    686        new ReportResultTask(aDecodeJob, &WebAudioDecodeJob::OnFailure,
    687                             WebAudioDecodeJob::UnknownError);
    688    aDecodeJob.mContext->Dispatch(event.forget());
    689  } else {
    690    nsresult rv = task->PSupervisorTaskQueue()->Dispatch(task.forget());
    691    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
    692    (void)rv;
    693  }
    694 }
    695 
    696 WebAudioDecodeJob::WebAudioDecodeJob(AudioContext* aContext, Promise* aPromise,
    697                                     DecodeSuccessCallback* aSuccessCallback,
    698                                     DecodeErrorCallback* aFailureCallback)
    699    : mContext(aContext),
    700      mPromise(aPromise),
    701      mSuccessCallback(aSuccessCallback),
    702      mFailureCallback(aFailureCallback) {
    703  MOZ_ASSERT(aContext);
    704  MOZ_ASSERT(NS_IsMainThread());
    705  MOZ_COUNT_CTOR(WebAudioDecodeJob);
    706 }
    707 
    708 WebAudioDecodeJob::~WebAudioDecodeJob() {
    709  MOZ_ASSERT(NS_IsMainThread());
    710  MOZ_COUNT_DTOR(WebAudioDecodeJob);
    711 }
    712 
    713 void WebAudioDecodeJob::OnSuccess(ErrorCode aErrorCode) {
    714  MOZ_ASSERT(NS_IsMainThread());
    715  MOZ_ASSERT(aErrorCode == NoError);
    716 
    717  RefPtr<AudioBuffer> output(mOutput);
    718  if (mSuccessCallback) {
    719    RefPtr<DecodeSuccessCallback> callback(mSuccessCallback);
    720    // Ignore errors in calling the callback, since there is not much that we
    721    // can do about it here.
    722    callback->Call(*output);
    723  }
    724  mPromise->MaybeResolve(output);
    725 
    726  mContext->RemoveFromDecodeQueue(this);
    727 }
    728 
    729 void WebAudioDecodeJob::OnFailure(ErrorCode aErrorCode) {
    730  MOZ_ASSERT(NS_IsMainThread());
    731 
    732  const char* errorMessage;
    733  switch (aErrorCode) {
    734    case UnknownContent:
    735      errorMessage =
    736          "The buffer passed to decodeAudioData contains an unknown content "
    737          "type.";
    738      break;
    739    case InvalidContent:
    740      errorMessage =
    741          "The buffer passed to decodeAudioData contains invalid content which "
    742          "cannot be decoded successfully.";
    743      break;
    744    case NoAudio:
    745      errorMessage =
    746          "The buffer passed to decodeAudioData does not contain any audio.";
    747      break;
    748    case NoError:
    749      MOZ_FALLTHROUGH_ASSERT("Who passed NoError to OnFailure?");
    750      // Fall through to get some sort of a sane error message if this actually
    751      // happens at runtime.
    752    case UnknownError:
    753      [[fallthrough]];
    754    default:
    755      errorMessage =
    756          "An unknown error occurred while processing decodeAudioData.";
    757      break;
    758  }
    759 
    760  // Ignore errors in calling the callback, since there is not much that we can
    761  // do about it here.
    762  nsAutoCString errorString(errorMessage);
    763  if (mFailureCallback) {
    764    RefPtr<DOMException> exception = DOMException::Create(
    765        NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR, errorString);
    766    RefPtr<DecodeErrorCallback> callback(mFailureCallback);
    767    callback->Call(*exception);
    768  }
    769 
    770  mPromise->MaybeRejectWithEncodingError(errorString);
    771 
    772  mContext->RemoveFromDecodeQueue(this);
    773 }
    774 
    775 size_t WebAudioDecodeJob::SizeOfExcludingThis(
    776    MallocSizeOf aMallocSizeOf) const {
    777  size_t amount = 0;
    778  if (mSuccessCallback) {
    779    amount += mSuccessCallback->SizeOfIncludingThis(aMallocSizeOf);
    780  }
    781  if (mFailureCallback) {
    782    amount += mFailureCallback->SizeOfIncludingThis(aMallocSizeOf);
    783  }
    784  if (mOutput) {
    785    amount += mOutput->SizeOfIncludingThis(aMallocSizeOf);
    786  }
    787  amount += mBuffer.SizeOfExcludingThis(aMallocSizeOf, false);
    788  return amount;
    789 }
    790 
    791 size_t WebAudioDecodeJob::SizeOfIncludingThis(
    792    MallocSizeOf aMallocSizeOf) const {
    793  return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
    794 }
    795 
    796 }  // namespace mozilla
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE