[ tor-browser ].git.dasho

OmxDataDecoder.cpp (35467B)
      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "OmxDataDecoder.h"
      8 
      9 #include "OMX_Audio.h"
     10 #include "OMX_Component.h"
     11 #include "OMX_Types.h"
     12 #include "OmxPlatformLayer.h"
     13 #include "mozilla/IntegerPrintfMacros.h"
     14 
     15 #ifdef LOG
     16 #  undef LOG
     17 #  undef LOGL
     18 #endif
     19 
     20 #define LOG(arg, ...)                                                  \
     21  DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
     22            ##__VA_ARGS__)
     23 
     24 #define LOGL(arg, ...)                                                     \
     25  DDMOZ_LOGEX(self.get(), sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, \
     26              __func__, ##__VA_ARGS__)
     27 
     28 #define CHECK_OMX_ERR(err)      \
     29  if (err != OMX_ErrorNone) {   \
     30    NotifyError(err, __func__); \
     31    return;                     \
     32  }
     33 
     34 namespace mozilla {
     35 
     36 using namespace gfx;
     37 
     38 static const char* StateTypeToStr(OMX_STATETYPE aType) {
     39  MOZ_ASSERT(aType == OMX_StateLoaded || aType == OMX_StateIdle ||
     40             aType == OMX_StateExecuting || aType == OMX_StatePause ||
     41             aType == OMX_StateWaitForResources || aType == OMX_StateInvalid);
     42 
     43  switch (aType) {
     44    case OMX_StateLoaded:
     45      return "OMX_StateLoaded";
     46    case OMX_StateIdle:
     47      return "OMX_StateIdle";
     48    case OMX_StateExecuting:
     49      return "OMX_StateExecuting";
     50    case OMX_StatePause:
     51      return "OMX_StatePause";
     52    case OMX_StateWaitForResources:
     53      return "OMX_StateWaitForResources";
     54    case OMX_StateInvalid:
     55      return "OMX_StateInvalid";
     56    default:
     57      return "Unknown";
     58  }
     59 }
     60 
     61 // A helper class to retrieve AudioData or VideoData.
     62 class MediaDataHelper {
     63 protected:
     64  virtual ~MediaDataHelper() = default;
     65 
     66 public:
     67  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataHelper)
     68 
     69  MediaDataHelper(const TrackInfo* aTrackInfo,
     70                  layers::ImageContainer* aImageContainer,
     71                  OmxPromiseLayer* aOmxLayer);
     72 
     73  already_AddRefed<MediaData> GetMediaData(BufferData* aBufferData,
     74                                           bool& aPlatformDepenentData);
     75 
     76 protected:
     77  already_AddRefed<AudioData> CreateAudioData(BufferData* aBufferData);
     78 
     79  already_AddRefed<VideoData> CreateYUV420VideoData(BufferData* aBufferData);
     80 
     81  const TrackInfo* mTrackInfo;
     82 
     83  OMX_PARAM_PORTDEFINITIONTYPE mOutputPortDef;
     84 
     85  // audio output
     86  MediaQueue<AudioData> mAudioQueue;
     87 
     88  AudioCompactor mAudioCompactor;
     89 
     90  // video output
     91  RefPtr<layers::ImageContainer> mImageContainer;
     92 };
     93 
     94 OmxDataDecoder::OmxDataDecoder(const TrackInfo& aTrackInfo,
     95                               layers::ImageContainer* aImageContainer,
     96                               Maybe<TrackingId> aTrackingId)
     97    : mOmxTaskQueue(
     98          CreateMediaDecodeTaskQueue("OmxDataDecoder::mOmxTaskQueue")),
     99      mImageContainer(aImageContainer),
    100      mWatchManager(this, mOmxTaskQueue),
    101      mOmxState(OMX_STATETYPE::OMX_StateInvalid, "OmxDataDecoder::mOmxState"),
    102      mTrackInfo(aTrackInfo.Clone()),
    103      mFlushing(false),
    104      mShuttingDown(false),
    105      mCheckingInputExhausted(false),
    106      mPortSettingsChanged(-1, "OmxDataDecoder::mPortSettingsChanged"),
    107      mTrackingId(std::move(aTrackingId)) {
    108  LOG("");
    109  mOmxLayer = new OmxPromiseLayer(mOmxTaskQueue, this, aImageContainer);
    110 }
    111 
    112 OmxDataDecoder::~OmxDataDecoder() { LOG(""); }
    113 
    114 void OmxDataDecoder::InitializationTask() {
    115  mWatchManager.Watch(mOmxState, &OmxDataDecoder::OmxStateRunner);
    116  mWatchManager.Watch(mPortSettingsChanged,
    117                      &OmxDataDecoder::PortSettingsChanged);
    118 }
    119 
    120 void OmxDataDecoder::EndOfStream() {
    121  LOG("");
    122  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    123 
    124  RefPtr<OmxDataDecoder> self = this;
    125  mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
    126      ->Then(mOmxTaskQueue, __func__,
    127             [self, this](OmxCommandPromise::ResolveOrRejectValue&& aValue) {
    128               mDrainPromise.ResolveIfExists(std::move(mDecodedData), __func__);
    129               mDecodedData = DecodedData();
    130             });
    131 }
    132 
    133 RefPtr<MediaDataDecoder::InitPromise> OmxDataDecoder::Init() {
    134  LOG("");
    135 
    136  mThread = GetCurrentSerialEventTarget();
    137  RefPtr<OmxDataDecoder> self = this;
    138  return InvokeAsync(mOmxTaskQueue, __func__, [self, this]() {
    139    InitializationTask();
    140 
    141    RefPtr<InitPromise> p = mInitPromise.Ensure(__func__);
    142    mOmxLayer->Init(mTrackInfo.get())
    143        ->Then(
    144            mOmxTaskQueue, __func__,
    145            [self, this]() {
    146              // Omx state should be OMX_StateIdle.
    147              mOmxState = mOmxLayer->GetState();
    148              MOZ_ASSERT(mOmxState != OMX_StateIdle);
    149            },
    150            [self, this]() {
    151              RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    152            });
    153    return p;
    154  });
    155 }
    156 
    157 RefPtr<MediaDataDecoder::DecodePromise> OmxDataDecoder::Decode(
    158    MediaRawData* aSample) {
    159  LOG("sample %p", aSample);
    160  MOZ_ASSERT(mThread->IsOnCurrentThread());
    161  MOZ_ASSERT(mInitPromise.IsEmpty());
    162 
    163  RefPtr<OmxDataDecoder> self = this;
    164  RefPtr<MediaRawData> sample = aSample;
    165  return InvokeAsync(mOmxTaskQueue, __func__, [self, this, sample]() {
    166    RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
    167 
    168    mTrackingId.apply([&](const auto& aId) {
    169      MediaInfoFlag flag = MediaInfoFlag::None;
    170      flag |= (sample->mKeyframe ? MediaInfoFlag::KeyFrame
    171                                 : MediaInfoFlag::NonKeyFrame);
    172 
    173      mPerformanceRecorder.Start(sample->mTimecode.ToMicroseconds(),
    174                                 "OmxDataDecoder"_ns, aId, flag);
    175    });
    176    mMediaRawDatas.AppendElement(std::move(sample));
    177 
    178    // Start to fill/empty buffers.
    179    if (mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting) {
    180      FillAndEmptyBuffers();
    181    }
    182    return p;
    183  });
    184 }
    185 
    186 RefPtr<MediaDataDecoder::FlushPromise> OmxDataDecoder::Flush() {
    187  LOG("");
    188  MOZ_ASSERT(mThread->IsOnCurrentThread());
    189 
    190  mFlushing = true;
    191 
    192  return InvokeAsync(mOmxTaskQueue, this, __func__, &OmxDataDecoder::DoFlush);
    193 }
    194 
    195 RefPtr<MediaDataDecoder::DecodePromise> OmxDataDecoder::Drain() {
    196  LOG("");
    197  MOZ_ASSERT(mThread->IsOnCurrentThread());
    198 
    199  RefPtr<OmxDataDecoder> self = this;
    200  return InvokeAsync(mOmxTaskQueue, __func__, [self]() {
    201    RefPtr<DecodePromise> p = self->mDrainPromise.Ensure(__func__);
    202    self->SendEosBuffer();
    203    return p;
    204  });
    205 }
    206 
    207 RefPtr<ShutdownPromise> OmxDataDecoder::Shutdown() {
    208  LOG("");
    209  // mThread may not be set if Init hasn't been called first.
    210  MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread());
    211 
    212  mShuttingDown = true;
    213 
    214  return InvokeAsync(mOmxTaskQueue, this, __func__,
    215                     &OmxDataDecoder::DoAsyncShutdown);
    216 }
    217 
    218 RefPtr<ShutdownPromise> OmxDataDecoder::DoAsyncShutdown() {
    219  LOG("");
    220  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    221  MOZ_ASSERT(!mFlushing);
    222 
    223  mWatchManager.Unwatch(mOmxState, &OmxDataDecoder::OmxStateRunner);
    224  mWatchManager.Unwatch(mPortSettingsChanged,
    225                        &OmxDataDecoder::PortSettingsChanged);
    226 
    227  // Flush to all ports, so all buffers can be returned from component.
    228  RefPtr<OmxDataDecoder> self = this;
    229  mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
    230      ->Then(
    231          mOmxTaskQueue, __func__,
    232          [self]() -> RefPtr<OmxCommandPromise> {
    233            LOGL("DoAsyncShutdown: flush complete");
    234            return self->mOmxLayer->SendCommand(OMX_CommandStateSet,
    235                                                OMX_StateIdle, nullptr);
    236          },
    237          [self](const OmxCommandFailureHolder& aError) {
    238            self->mOmxLayer->Shutdown();
    239            return OmxCommandPromise::CreateAndReject(aError, __func__);
    240          })
    241      ->Then(
    242          mOmxTaskQueue, __func__,
    243          [self]() -> RefPtr<OmxCommandPromise> {
    244            RefPtr<OmxCommandPromise> p = self->mOmxLayer->SendCommand(
    245                OMX_CommandStateSet, OMX_StateLoaded, nullptr);
    246 
    247            // According to spec 3.1.1.2.2.1:
    248            // OMX_StateLoaded needs to be sent before releasing buffers.
    249            // And state transition from OMX_StateIdle to OMX_StateLoaded
    250            // is completed when all of the buffers have been removed
    251            // from the component.
    252            // Here the buffer promises are not resolved due to displaying
    253            // in layer, it needs to wait before the layer returns the
    254            // buffers.
    255            LOGL("DoAsyncShutdown: releasing buffers...");
    256            self->ReleaseBuffers(OMX_DirInput);
    257            self->ReleaseBuffers(OMX_DirOutput);
    258 
    259            return p;
    260          },
    261          [self](const OmxCommandFailureHolder& aError) {
    262            self->mOmxLayer->Shutdown();
    263            return OmxCommandPromise::CreateAndReject(aError, __func__);
    264          })
    265      ->Then(
    266          mOmxTaskQueue, __func__,
    267          [self]() -> RefPtr<ShutdownPromise> {
    268            LOGL(
    269                "DoAsyncShutdown: OMX_StateLoaded, it is safe to shutdown omx");
    270            self->mOmxLayer->Shutdown();
    271            self->mWatchManager.Shutdown();
    272            self->mOmxLayer = nullptr;
    273            self->mMediaDataHelper = nullptr;
    274            self->mShuttingDown = false;
    275            return ShutdownPromise::CreateAndResolve(true, __func__);
    276          },
    277          [self]() -> RefPtr<ShutdownPromise> {
    278            self->mOmxLayer->Shutdown();
    279            self->mWatchManager.Shutdown();
    280            self->mOmxLayer = nullptr;
    281            self->mMediaDataHelper = nullptr;
    282            return ShutdownPromise::CreateAndReject(false, __func__);
    283          })
    284      ->Then(
    285          mThread, __func__,
    286          [self]() {
    287            self->mOmxTaskQueue->BeginShutdown();
    288            self->mOmxTaskQueue->AwaitShutdownAndIdle();
    289            self->mShutdownPromise.Resolve(true, __func__);
    290          },
    291          [self]() {
    292            self->mOmxTaskQueue->BeginShutdown();
    293            self->mOmxTaskQueue->AwaitShutdownAndIdle();
    294            self->mShutdownPromise.Resolve(true, __func__);
    295          });
    296  return mShutdownPromise.Ensure(__func__);
    297 }
    298 
    299 void OmxDataDecoder::FillBufferDone(BufferData* aData) {
    300  MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT);
    301 
    302  // Don't output sample when flush or shutting down, especially for video
    303  // decoded frame. Because video decoded frame can have a promise in
    304  // BufferData waiting for layer to resolve it via recycle callback, if other
    305  // module doesn't send it to layer, it will cause a unresolved promise and
    306  // waiting for resolve infinitely.
    307  if (mFlushing || mShuttingDown) {
    308    LOG("mFlush or mShuttingDown, drop data");
    309    aData->mStatus = BufferData::BufferStatus::FREE;
    310    return;
    311  }
    312 
    313  if (aData->mBuffer->nFlags & OMX_BUFFERFLAG_EOS) {
    314    // Reach eos, it's an empty data so it doesn't need to output.
    315    EndOfStream();
    316    aData->mStatus = BufferData::BufferStatus::FREE;
    317  } else {
    318    Output(aData);
    319    FillAndEmptyBuffers();
    320  }
    321 }
    322 
    323 void OmxDataDecoder::Output(BufferData* aData) {
    324  if (!mMediaDataHelper) {
    325    mMediaDataHelper =
    326        new MediaDataHelper(mTrackInfo.get(), mImageContainer, mOmxLayer);
    327  }
    328 
    329  bool isPlatformData = false;
    330  RefPtr<MediaData> data =
    331      mMediaDataHelper->GetMediaData(aData, isPlatformData);
    332  if (!data) {
    333    aData->mStatus = BufferData::BufferStatus::FREE;
    334    return;
    335  }
    336 
    337  if (isPlatformData) {
    338    // If the MediaData is platform dependnet data, it's mostly a kind of
    339    // limited resource, so we use promise to notify when the resource is free.
    340    aData->mStatus = BufferData::BufferStatus::OMX_CLIENT_OUTPUT;
    341 
    342    MOZ_RELEASE_ASSERT(aData->mPromise.IsEmpty());
    343    RefPtr<OmxBufferPromise> p = aData->mPromise.Ensure(__func__);
    344 
    345    RefPtr<OmxDataDecoder> self = this;
    346    RefPtr<BufferData> buffer = aData;
    347    p->Then(
    348        mOmxTaskQueue, __func__,
    349        [self, buffer]() {
    350          MOZ_RELEASE_ASSERT(buffer->mStatus ==
    351                             BufferData::BufferStatus::OMX_CLIENT_OUTPUT);
    352          buffer->mStatus = BufferData::BufferStatus::FREE;
    353          self->FillAndEmptyBuffers();
    354        },
    355        [buffer]() {
    356          MOZ_RELEASE_ASSERT(buffer->mStatus ==
    357                             BufferData::BufferStatus::OMX_CLIENT_OUTPUT);
    358          buffer->mStatus = BufferData::BufferStatus::FREE;
    359        });
    360  } else {
    361    aData->mStatus = BufferData::BufferStatus::FREE;
    362  }
    363 
    364  if (mTrackInfo->IsVideo()) {
    365    mPerformanceRecorder.Record(
    366        aData->mRawData->mTimecode.ToMicroseconds(), [&](DecodeStage& aStage) {
    367          const auto& image = data->As<VideoData>()->mImage;
    368          aStage.SetResolution(image->GetSize().Width(),
    369                               image->GetSize().Height());
    370          aStage.SetImageFormat(DecodeStage::YUV420P);
    371          aStage.SetColorDepth(image->GetColorDepth());
    372        });
    373  }
    374 
    375  mDecodedData.AppendElement(std::move(data));
    376 }
    377 
    378 void OmxDataDecoder::FillBufferFailure(OmxBufferFailureHolder aFailureHolder) {
    379  NotifyError(aFailureHolder.mError, __func__);
    380 }
    381 
    382 void OmxDataDecoder::EmptyBufferDone(BufferData* aData) {
    383  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    384  MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT);
    385 
    386  // Nothing to do when status of input buffer is OMX_CLIENT.
    387  aData->mStatus = BufferData::BufferStatus::FREE;
    388  FillAndEmptyBuffers();
    389 
    390  // There is no way to know if component gets enough raw samples to generate
    391  // output, especially for video decoding. So here it needs to request raw
    392  // samples aggressively.
    393  if (!mCheckingInputExhausted && !mMediaRawDatas.Length()) {
    394    mCheckingInputExhausted = true;
    395 
    396    RefPtr<OmxDataDecoder> self = this;
    397    nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
    398        "OmxDataDecoder::EmptyBufferDone", [self, this]() {
    399          mCheckingInputExhausted = false;
    400 
    401          if (mMediaRawDatas.Length()) {
    402            return;
    403          }
    404 
    405          mDecodePromise.ResolveIfExists(std::move(mDecodedData), __func__);
    406          mDecodedData = DecodedData();
    407        });
    408 
    409    nsresult rv = mOmxTaskQueue->Dispatch(r.forget());
    410    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
    411    (void)rv;
    412  }
    413 }
    414 
    415 void OmxDataDecoder::EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder) {
    416  NotifyError(aFailureHolder.mError, __func__);
    417 }
    418 
    419 void OmxDataDecoder::NotifyError(OMX_ERRORTYPE aOmxError, const char* aLine,
    420                                 const MediaResult& aError) {
    421  LOG("NotifyError %d (%s) at %s", static_cast<int>(aOmxError),
    422      aError.ErrorName().get(), aLine);
    423  mDecodedData = DecodedData();
    424  mDecodePromise.RejectIfExists(aError, __func__);
    425  mDrainPromise.RejectIfExists(aError, __func__);
    426  mFlushPromise.RejectIfExists(aError, __func__);
    427 }
    428 
    429 void OmxDataDecoder::FillAndEmptyBuffers() {
    430  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    431  MOZ_ASSERT(mOmxState == OMX_StateExecuting);
    432 
    433  // During the port setting changed, it is forbidden to do any buffer
    434  // operation.
    435  if (mPortSettingsChanged != -1 || mShuttingDown || mFlushing) {
    436    return;
    437  }
    438 
    439  // Trigger input port.
    440  while (!!mMediaRawDatas.Length()) {
    441    // input buffer must be used by component if there is data available.
    442    RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput);
    443    if (!inbuf) {
    444      LOG("no input buffer!");
    445      break;
    446    }
    447 
    448    RefPtr<MediaRawData> data = mMediaRawDatas[0];
    449    // Buffer size should large enough for raw data.
    450    MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= data->Size());
    451 
    452    memcpy(inbuf->mBuffer->pBuffer, data->Data(), data->Size());
    453    inbuf->mBuffer->nFilledLen = data->Size();
    454    inbuf->mBuffer->nOffset = 0;
    455    inbuf->mBuffer->nFlags = inbuf->mBuffer->nAllocLen > data->Size()
    456                                 ? OMX_BUFFERFLAG_ENDOFFRAME
    457                                 : 0;
    458    inbuf->mBuffer->nTimeStamp = data->mTime.ToMicroseconds();
    459    if (data->Size()) {
    460      inbuf->mRawData = mMediaRawDatas[0];
    461    } else {
    462      LOG("send EOS buffer");
    463      inbuf->mBuffer->nFlags |= OMX_BUFFERFLAG_EOS;
    464    }
    465 
    466    LOG("feed sample %p to omx component, len %ld, flag %lX", data.get(),
    467        inbuf->mBuffer->nFilledLen, inbuf->mBuffer->nFlags);
    468    mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this,
    469                                        &OmxDataDecoder::EmptyBufferDone,
    470                                        &OmxDataDecoder::EmptyBufferFailure);
    471    mMediaRawDatas.RemoveElementAt(0);
    472  }
    473 
    474  // Trigger output port.
    475  while (true) {
    476    RefPtr<BufferData> outbuf = FindAvailableBuffer(OMX_DirOutput);
    477    if (!outbuf) {
    478      break;
    479    }
    480 
    481    mOmxLayer->FillBuffer(outbuf)->Then(mOmxTaskQueue, __func__, this,
    482                                        &OmxDataDecoder::FillBufferDone,
    483                                        &OmxDataDecoder::FillBufferFailure);
    484  }
    485 }
    486 
    487 OmxPromiseLayer::BufferData* OmxDataDecoder::FindAvailableBuffer(
    488    OMX_DIRTYPE aType) {
    489  BUFFERLIST* buffers = GetBuffers(aType);
    490 
    491  for (uint32_t i = 0; i < buffers->Length(); i++) {
    492    BufferData* buf = buffers->ElementAt(i);
    493    if (buf->mStatus == BufferData::BufferStatus::FREE) {
    494      return buf;
    495    }
    496  }
    497 
    498  return nullptr;
    499 }
    500 
    501 nsresult OmxDataDecoder::AllocateBuffers(OMX_DIRTYPE aType) {
    502  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    503 
    504  return mOmxLayer->AllocateOmxBuffer(aType, GetBuffers(aType));
    505 }
    506 
    507 nsresult OmxDataDecoder::ReleaseBuffers(OMX_DIRTYPE aType) {
    508  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    509 
    510  return mOmxLayer->ReleaseOmxBuffer(aType, GetBuffers(aType));
    511 }
    512 
    513 nsTArray<RefPtr<OmxPromiseLayer::BufferData>>* OmxDataDecoder::GetBuffers(
    514    OMX_DIRTYPE aType) {
    515  MOZ_ASSERT(aType == OMX_DIRTYPE::OMX_DirInput ||
    516             aType == OMX_DIRTYPE::OMX_DirOutput);
    517 
    518  if (aType == OMX_DIRTYPE::OMX_DirInput) {
    519    return &mInPortBuffers;
    520  }
    521  return &mOutPortBuffers;
    522 }
    523 
    524 void OmxDataDecoder::ResolveInitPromise(StaticString aMethodName) {
    525  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    526  LOG("called from %s", aMethodName.get());
    527  mInitPromise.ResolveIfExists(mTrackInfo->GetType(), aMethodName);
    528 }
    529 
    530 void OmxDataDecoder::RejectInitPromise(MediaResult aError,
    531                                       StaticString aMethodName) {
    532  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    533  mInitPromise.RejectIfExists(aError, aMethodName);
    534 }
    535 
    536 void OmxDataDecoder::OmxStateRunner() {
    537  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    538  LOG("OMX state: %s", StateTypeToStr(mOmxState));
    539 
    540  // TODO: maybe it'd be better to use promise CompletionPromise() to replace
    541  //       this state machine.
    542  if (mOmxState == OMX_StateLoaded) {
    543    ConfigCodec();
    544 
    545    // Send OpenMax state command to OMX_StateIdle.
    546    RefPtr<OmxDataDecoder> self = this;
    547    mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr)
    548        ->Then(
    549            mOmxTaskQueue, __func__,
    550            [self]() {
    551              // Current state should be OMX_StateIdle.
    552              self->mOmxState = self->mOmxLayer->GetState();
    553              MOZ_ASSERT(self->mOmxState == OMX_StateIdle);
    554            },
    555            [self]() {
    556              self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    557            });
    558 
    559    // Allocate input and output buffers.
    560    OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput,
    561                           OMX_DIRTYPE::OMX_DirOutput};
    562    for (const auto id : types) {
    563      if (NS_FAILED(AllocateBuffers(id))) {
    564        LOG("Failed to allocate buffer on port %d", id);
    565        RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    566        break;
    567      }
    568    }
    569  } else if (mOmxState == OMX_StateIdle) {
    570    RefPtr<OmxDataDecoder> self = this;
    571    mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateExecuting, nullptr)
    572        ->Then(
    573            mOmxTaskQueue, __func__,
    574            [self]() {
    575              self->mOmxState = self->mOmxLayer->GetState();
    576              MOZ_ASSERT(self->mOmxState == OMX_StateExecuting);
    577 
    578              self->ResolveInitPromise(__func__);
    579            },
    580            [self]() {
    581              self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    582            });
    583  } else if (mOmxState == OMX_StateExecuting) {
    584    // Configure codec once it gets OMX_StateExecuting state.
    585    FillCodecConfigDataToOmx();
    586  } else {
    587    MOZ_ASSERT(0);
    588  }
    589 }
    590 
    591 void OmxDataDecoder::ConfigCodec() {
    592  OMX_ERRORTYPE err = mOmxLayer->Config();
    593  CHECK_OMX_ERR(err);
    594 }
    595 
    596 void OmxDataDecoder::FillCodecConfigDataToOmx() {
    597  // Codec configure data should be the first sample running on Omx TaskQueue.
    598  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    599  MOZ_ASSERT(!mMediaRawDatas.Length());
    600  MOZ_ASSERT(mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting);
    601 
    602  RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput);
    603  RefPtr<MediaByteBuffer> csc;
    604  if (mTrackInfo->IsAudio()) {
    605    // It would be nice to instead use more specific information here, but
    606    // we force a byte buffer for now since this handles arbitrary codecs.
    607    // TODO(bug 1768566): implement further type checking for codec data.
    608    csc = ForceGetAudioCodecSpecificBlob(
    609        mTrackInfo->GetAsAudioInfo()->mCodecSpecificConfig);
    610  } else if (mTrackInfo->IsVideo()) {
    611    csc = mTrackInfo->GetAsVideoInfo()->mExtraData;
    612  }
    613 
    614  MOZ_RELEASE_ASSERT(csc);
    615 
    616  // Some codecs like h264, its codec specific data is at the first packet, not
    617  // in container.
    618  if (csc->Length()) {
    619    // Buffer size should large enough for raw data.
    620    MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= csc->Length());
    621 
    622    memcpy(inbuf->mBuffer->pBuffer, csc->Elements(), csc->Length());
    623    inbuf->mBuffer->nFilledLen = csc->Length();
    624    inbuf->mBuffer->nOffset = 0;
    625    inbuf->mBuffer->nFlags =
    626        (OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_CODECCONFIG);
    627 
    628    LOG("Feed codec configure data to OMX component");
    629    mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this,
    630                                        &OmxDataDecoder::EmptyBufferDone,
    631                                        &OmxDataDecoder::EmptyBufferFailure);
    632  }
    633 }
    634 
    635 bool OmxDataDecoder::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1,
    636                           OMX_U32 aData2) {
    637  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    638 
    639  if (mOmxLayer->Event(aEvent, aData1, aData2)) {
    640    return true;
    641  }
    642 
    643  switch (aEvent) {
    644    case OMX_EventPortSettingsChanged: {
    645      // Don't always disable port. See bug 1235340.
    646      if (aData2 == 0 || aData2 == OMX_IndexParamPortDefinition) {
    647        // According to spec: "To prevent the loss of any input data, the
    648        // component issuing the OMX_EventPortSettingsChanged event on its input
    649        // port should buffer all input port data that arrives between the
    650        // emission of the OMX_EventPortSettingsChanged event and the arrival of
    651        // the command to disable the input port."
    652        //
    653        // So client needs to disable port and reallocate buffers.
    654        MOZ_ASSERT(mPortSettingsChanged == -1);
    655        mPortSettingsChanged = AssertedCast<int32_t>(aData1);
    656      }
    657      LOG("Got OMX_EventPortSettingsChanged event");
    658      break;
    659    }
    660    default: {
    661      // Got error during decoding, send msg to MFR skipping to next key frame.
    662      if (aEvent == OMX_EventError && mOmxState == OMX_StateExecuting) {
    663        NotifyError((OMX_ERRORTYPE)aData1, __func__,
    664                    MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__));
    665        return true;
    666      }
    667      LOG("WARNING: got none handle event: %d, aData1: %lu, aData2: %lu",
    668          aEvent, aData1, aData2);
    669      return false;
    670    }
    671  }
    672 
    673  return true;
    674 }
    675 
    676 bool OmxDataDecoder::BuffersCanBeReleased(OMX_DIRTYPE aType) {
    677  BUFFERLIST* buffers = GetBuffers(aType);
    678  uint32_t len = buffers->Length();
    679  for (uint32_t i = 0; i < len; i++) {
    680    BufferData::BufferStatus buf_status = buffers->ElementAt(i)->mStatus;
    681    if (buf_status == BufferData::BufferStatus::OMX_COMPONENT ||
    682        buf_status == BufferData::BufferStatus::OMX_CLIENT_OUTPUT) {
    683      return false;
    684    }
    685  }
    686  return true;
    687 }
    688 
    689 OMX_DIRTYPE
    690 OmxDataDecoder::GetPortDirection(uint32_t aPortIndex) {
    691  OMX_PARAM_PORTDEFINITIONTYPE def;
    692  InitOmxParameter(&def);
    693  def.nPortIndex = mPortSettingsChanged;
    694 
    695  OMX_ERRORTYPE err =
    696      mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def));
    697  if (err != OMX_ErrorNone) {
    698    return OMX_DirMax;
    699  }
    700  return def.eDir;
    701 }
    702 
    703 RefPtr<OmxPromiseLayer::OmxBufferPromise::AllPromiseType>
    704 OmxDataDecoder::CollectBufferPromises(OMX_DIRTYPE aType) {
    705  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    706 
    707  nsTArray<RefPtr<OmxBufferPromise>> promises;
    708  OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput};
    709  for (const auto type : types) {
    710    if ((aType == type) || (aType == OMX_DirMax)) {
    711      // find the buffer which has promise.
    712      BUFFERLIST* buffers = GetBuffers(type);
    713 
    714      for (uint32_t i = 0; i < buffers->Length(); i++) {
    715        BufferData* buf = buffers->ElementAt(i);
    716        if (!buf->mPromise.IsEmpty()) {
    717          // OmxBufferPromise is not exclusive, it can be multiple "Then"s, so
    718          // it is safe to call "Ensure" here.
    719          promises.AppendElement(buf->mPromise.Ensure(__func__));
    720        }
    721      }
    722    }
    723  }
    724 
    725  LOG("CollectBufferPromises: type %d, total %zu promiese", aType,
    726      promises.Length());
    727  if (promises.Length()) {
    728    return OmxBufferPromise::All(mOmxTaskQueue, promises);
    729  }
    730 
    731  return OmxBufferPromise::AllPromiseType::CreateAndResolve(
    732      nsTArray<BufferData*>(), __func__);
    733 }
    734 
    735 void OmxDataDecoder::PortSettingsChanged() {
    736  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    737 
    738  if (mPortSettingsChanged == -1 ||
    739      mOmxState == OMX_STATETYPE::OMX_StateInvalid) {
    740    return;
    741  }
    742 
    743  // The PortSettingsChanged algorithm:
    744  //
    745  //   1. disable port.
    746  //   2. wait for port buffers return to client and then release these buffers.
    747  //   3. enable port.
    748  //   4. allocate port buffers.
    749  //
    750 
    751  // Disable port. Get port definition if the target port is enable.
    752  OMX_PARAM_PORTDEFINITIONTYPE def;
    753  InitOmxParameter(&def);
    754  def.nPortIndex = mPortSettingsChanged;
    755 
    756  OMX_ERRORTYPE err =
    757      mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def));
    758  CHECK_OMX_ERR(err);
    759 
    760  RefPtr<OmxDataDecoder> self = this;
    761  if (def.bEnabled) {
    762    // 1. disable port.
    763    LOG("PortSettingsChanged: disable port %lu", def.nPortIndex);
    764    mOmxLayer
    765        ->SendCommand(OMX_CommandPortDisable, mPortSettingsChanged, nullptr)
    766        ->Then(
    767            mOmxTaskQueue, __func__,
    768            [self, def]() -> RefPtr<OmxCommandPromise> {
    769              // 3. enable port.
    770              // Send enable port command.
    771              RefPtr<OmxCommandPromise> p = self->mOmxLayer->SendCommand(
    772                  OMX_CommandPortEnable, self->mPortSettingsChanged, nullptr);
    773 
    774              // 4. allocate port buffers.
    775              // Allocate new port buffers.
    776              nsresult rv = self->AllocateBuffers(def.eDir);
    777              if (NS_FAILED(rv)) {
    778                self->NotifyError(OMX_ErrorUndefined, __func__);
    779              }
    780 
    781              return p;
    782            },
    783            [self](const OmxCommandFailureHolder& aError) {
    784              self->NotifyError(OMX_ErrorUndefined, __func__);
    785              return OmxCommandPromise::CreateAndReject(aError, __func__);
    786            })
    787        ->Then(
    788            mOmxTaskQueue, __func__,
    789            [self]() {
    790              LOGL("PortSettingsChanged: port settings changed complete");
    791              // finish port setting changed.
    792              self->mPortSettingsChanged = -1;
    793              self->FillAndEmptyBuffers();
    794            },
    795            [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); });
    796 
    797    // 2. wait for port buffers return to client and then release these buffers.
    798    //
    799    // Port buffers will be returned to client soon once OMX_CommandPortDisable
    800    // command is sent. Then releasing these buffers.
    801    CollectBufferPromises(def.eDir)->Then(
    802        mOmxTaskQueue, __func__,
    803        [self, def]() {
    804          MOZ_ASSERT(self->BuffersCanBeReleased(def.eDir));
    805          nsresult rv = self->ReleaseBuffers(def.eDir);
    806          if (NS_FAILED(rv)) {
    807            MOZ_RELEASE_ASSERT(0);
    808            self->NotifyError(OMX_ErrorUndefined, __func__);
    809          }
    810        },
    811        [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); });
    812  }
    813 }
    814 
    815 void OmxDataDecoder::SendEosBuffer() {
    816  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    817 
    818  // There is no 'Drain' API in OpenMax, so it needs to wait for output sample
    819  // with EOS flag. However, MediaRawData doesn't provide EOS information,
    820  // so here it generates an empty BufferData with eos OMX_BUFFERFLAG_EOS in
    821  // queue. This behaviour should be compliant with spec, I think...
    822  RefPtr<MediaRawData> eos_data = new MediaRawData();
    823  mMediaRawDatas.AppendElement(eos_data);
    824  FillAndEmptyBuffers();
    825 }
    826 
    827 RefPtr<MediaDataDecoder::FlushPromise> OmxDataDecoder::DoFlush() {
    828  MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
    829 
    830  mDecodedData = DecodedData();
    831  mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    832  mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
    833  mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
    834 
    835  RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__);
    836 
    837  // 1. Call OMX command OMX_CommandFlush in Omx TaskQueue.
    838  // 2. Remove all elements in mMediaRawDatas when flush is completed.
    839  mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
    840      ->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::FlushComplete,
    841             &OmxDataDecoder::FlushFailure);
    842 
    843  return p;
    844 }
    845 
    846 void OmxDataDecoder::FlushComplete(OMX_COMMANDTYPE aCommandType) {
    847  mMediaRawDatas.Clear();
    848  mFlushing = false;
    849 
    850  LOG("Flush complete");
    851  mFlushPromise.ResolveIfExists(true, __func__);
    852 }
    853 
    854 void OmxDataDecoder::FlushFailure(OmxCommandFailureHolder aFailureHolder) {
    855  mFlushing = false;
    856  mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
    857 }
    858 
    859 MediaDataHelper::MediaDataHelper(const TrackInfo* aTrackInfo,
    860                                 layers::ImageContainer* aImageContainer,
    861                                 OmxPromiseLayer* aOmxLayer)
    862    : mTrackInfo(aTrackInfo),
    863      mAudioCompactor(mAudioQueue),
    864      mImageContainer(aImageContainer) {
    865  InitOmxParameter(&mOutputPortDef);
    866  mOutputPortDef.nPortIndex = aOmxLayer->OutputPortIndex();
    867  aOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &mOutputPortDef,
    868                          sizeof(mOutputPortDef));
    869 }
    870 
    871 already_AddRefed<MediaData> MediaDataHelper::GetMediaData(
    872    BufferData* aBufferData, bool& aPlatformDepenentData) {
    873  aPlatformDepenentData = false;
    874  RefPtr<MediaData> data;
    875 
    876  if (mTrackInfo->IsAudio()) {
    877    if (!aBufferData->mBuffer->nFilledLen) {
    878      return nullptr;
    879    }
    880    data = CreateAudioData(aBufferData);
    881  } else if (mTrackInfo->IsVideo()) {
    882    data = aBufferData->GetPlatformMediaData();
    883    if (data) {
    884      aPlatformDepenentData = true;
    885    } else {
    886      if (!aBufferData->mBuffer->nFilledLen) {
    887        return nullptr;
    888      }
    889      // Get YUV VideoData, it uses more CPU, in most cases, on software codec.
    890      data = CreateYUV420VideoData(aBufferData);
    891    }
    892 
    893    // Update video time code, duration... from the raw data.
    894    VideoData* video(data->As<VideoData>());
    895    if (aBufferData->mRawData) {
    896      video->mTime = aBufferData->mRawData->mTime;
    897      video->mTimecode = aBufferData->mRawData->mTimecode;
    898      video->mOffset = aBufferData->mRawData->mOffset;
    899      video->mDuration = aBufferData->mRawData->mDuration;
    900      video->mKeyframe = aBufferData->mRawData->mKeyframe;
    901    }
    902  }
    903 
    904  return data.forget();
    905 }
    906 
    907 already_AddRefed<AudioData> MediaDataHelper::CreateAudioData(
    908    BufferData* aBufferData) {
    909  RefPtr<AudioData> audio;
    910  OMX_BUFFERHEADERTYPE* buf = aBufferData->mBuffer;
    911  const AudioInfo* info = mTrackInfo->GetAsAudioInfo();
    912  if (buf->nFilledLen) {
    913    uint64_t offset = 0;
    914    uint32_t frames = buf->nFilledLen / (2 * info->mChannels);
    915    if (aBufferData->mRawData) {
    916      offset = aBufferData->mRawData->mOffset;
    917    }
    918    typedef AudioCompactor::NativeCopy OmxCopy;
    919    mAudioCompactor.Push(
    920        offset, buf->nTimeStamp, info->mRate, frames, info->mChannels,
    921        OmxCopy(buf->pBuffer + buf->nOffset, buf->nFilledLen, info->mChannels));
    922    audio = mAudioQueue.PopFront();
    923  }
    924 
    925  return audio.forget();
    926 }
    927 
    928 already_AddRefed<VideoData> MediaDataHelper::CreateYUV420VideoData(
    929    BufferData* aBufferData) {
    930  uint8_t* yuv420p_buffer = (uint8_t*)aBufferData->mBuffer->pBuffer;
    931  int32_t stride = mOutputPortDef.format.video.nStride;
    932  uint32_t slice_height = mOutputPortDef.format.video.nSliceHeight;
    933  int32_t width = mTrackInfo->GetAsVideoInfo()->mImage.width;
    934  int32_t height = mTrackInfo->GetAsVideoInfo()->mImage.height;
    935 
    936  // TODO: convert other formats to YUV420.
    937  if (mOutputPortDef.format.video.eColorFormat !=
    938      OMX_COLOR_FormatYUV420Planar) {
    939    return nullptr;
    940  }
    941 
    942  size_t yuv420p_y_size = stride * slice_height;
    943  size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2);
    944  uint8_t* yuv420p_y = yuv420p_buffer;
    945  uint8_t* yuv420p_u = yuv420p_y + yuv420p_y_size;
    946  uint8_t* yuv420p_v = yuv420p_u + yuv420p_u_size;
    947 
    948  VideoData::YCbCrBuffer b;
    949  b.mPlanes[0].mData = yuv420p_y;
    950  b.mPlanes[0].mWidth = width;
    951  b.mPlanes[0].mHeight = height;
    952  b.mPlanes[0].mStride = stride;
    953  b.mPlanes[0].mSkip = 0;
    954 
    955  b.mPlanes[1].mData = yuv420p_u;
    956  b.mPlanes[1].mWidth = (width + 1) / 2;
    957  b.mPlanes[1].mHeight = (height + 1) / 2;
    958  b.mPlanes[1].mStride = (stride + 1) / 2;
    959  b.mPlanes[1].mSkip = 0;
    960 
    961  b.mPlanes[2].mData = yuv420p_v;
    962  b.mPlanes[2].mWidth = (width + 1) / 2;
    963  b.mPlanes[2].mHeight = (height + 1) / 2;
    964  b.mPlanes[2].mStride = (stride + 1) / 2;
    965  b.mPlanes[2].mSkip = 0;
    966 
    967  b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
    968 
    969  VideoInfo info(*mTrackInfo->GetAsVideoInfo());
    970 
    971  auto maybeColorSpace = info.mColorSpace;
    972  if (!maybeColorSpace) {
    973    maybeColorSpace = Some(DefaultColorSpace({width, height}));
    974  }
    975  b.mYUVColorSpace = *maybeColorSpace;
    976 
    977  auto maybeColorPrimaries = info.mColorPrimaries;
    978  if (!maybeColorPrimaries) {
    979    maybeColorPrimaries = Some(gfx::ColorSpace2::BT709);
    980  }
    981  b.mColorPrimaries = *maybeColorPrimaries;
    982 
    983  Result<already_AddRefed<VideoData>, MediaResult> result =
    984      VideoData::CreateAndCopyData(
    985          info, mImageContainer,
    986          0,                                     // Filled later by caller.
    987          media::TimeUnit::Zero(),               // Filled later by caller.
    988          media::TimeUnit::FromMicroseconds(1),  // We don't know the duration.
    989          b,
    990          false,  // Filled later by caller.
    991          media::TimeUnit::FromMicroseconds(-1), info.ImageRect(), nullptr);
    992 
    993  if (result.isErr()) {
    994    MediaResult r = result.unwrapErr();
    995    MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug,
    996            ("Failed to create a YUV420 VideoData - %s: %s",
    997             r.ErrorName().get(), r.Message().get()));
    998    return nullptr;
    999  }
   1000 
   1001  RefPtr<VideoData> data = result.unwrap();
   1002  MOZ_ASSERT(data);
   1003  MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug,
   1004          ("YUV420 VideoData: disp width %d, height %d, pic width %d, height "
   1005           "%d, time %lld",
   1006           info.mDisplay.width, info.mDisplay.height, info.mImage.width,
   1007           info.mImage.height, aBufferData->mBuffer->nTimeStamp));
   1008 
   1009  return data.forget();
   1010 }
   1011 
   1012 }  // namespace mozilla
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE