tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

RTCRtpScriptTransformer.cpp (15546B)

      1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim: set ts=2 et sw=2 tw=80: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
      6 
      7 #include "RTCRtpScriptTransformer.h"
      8 
      9 #include <stdint.h>
     10 
     11 #include <memory>
     12 #include <string>
     13 #include <utility>
     14 
     15 #include "ErrorList.h"
     16 #include "api/frame_transformer_interface.h"
     17 #include "js/CallArgs.h"
     18 #include "js/RootingAPI.h"
     19 #include "js/Value.h"
     20 #include "libwebrtcglue/FrameTransformerProxy.h"
     21 #include "mozilla/AlreadyAddRefed.h"
     22 #include "mozilla/Assertions.h"
     23 #include "mozilla/ErrorResult.h"
     24 #include "mozilla/HoldDropJSObjects.h"
     25 #include "mozilla/Likely.h"
     26 #include "mozilla/Logging.h"
     27 #include "mozilla/Maybe.h"
     28 #include "mozilla/RefPtr.h"
     29 #include "mozilla/dom/BindingDeclarations.h"
     30 #include "mozilla/dom/BindingUtils.h"
     31 #include "mozilla/dom/Promise-inl.h"
     32 #include "mozilla/dom/Promise.h"
     33 #include "mozilla/dom/PrototypeList.h"
     34 #include "mozilla/dom/RTCEncodedAudioFrame.h"
     35 #include "mozilla/dom/RTCEncodedVideoFrame.h"
     36 #include "mozilla/dom/RTCRtpScriptTransformerBinding.h"
     37 #include "mozilla/dom/ReadableStream.h"
     38 #include "mozilla/dom/ReadableStreamControllerBase.h"
     39 #include "mozilla/dom/ToJSValue.h"
     40 #include "mozilla/dom/UnderlyingSinkCallbackHelpers.h"
     41 #include "mozilla/dom/UnderlyingSourceCallbackHelpers.h"
     42 #include "mozilla/dom/WorkerRef.h"
     43 #include "mozilla/dom/WritableStream.h"
     44 #include "mozilla/dom/WritableStreamDefaultController.h"
     45 #include "nsCOMPtr.h"
     46 #include "nsContentUtils.h"
     47 #include "nsCycleCollectionParticipant.h"
     48 #include "nsCycleCollectionTraversalCallback.h"
     49 #include "nsDebug.h"
     50 #include "nsIGlobalObject.h"
     51 #include "nsISupports.h"
     52 #include "nsLiteralString.h"
     53 #include "nsString.h"
     54 #include "nsStringFwd.h"
     55 #include "nsTArray.h"
     56 #include "nsWrapperCache.h"
     57 #include "sdp/SdpAttribute.h"  // CheckRidValidity
     58 
     59 namespace mozilla::dom {
     60 
     61 LazyLogModule gScriptTransformerLog("RTCRtpScriptTransformer");
     62 
     63 NS_IMPL_CYCLE_COLLECTION_INHERITED(nsISupportsStreamSource,
     64                                   UnderlyingSourceAlgorithmsWrapper, mStream,
     65                                   mThingQueuedPromise, mQueue)
     66 NS_IMPL_ADDREF_INHERITED(nsISupportsStreamSource,
     67                         UnderlyingSourceAlgorithmsWrapper)
     68 NS_IMPL_RELEASE_INHERITED(nsISupportsStreamSource,
     69                          UnderlyingSourceAlgorithmsWrapper)
     70 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsISupportsStreamSource)
     71 NS_INTERFACE_MAP_END_INHERITING(UnderlyingSourceAlgorithmsWrapper)
     72 
     73 nsISupportsStreamSource::nsISupportsStreamSource() = default;
     74 
     75 nsISupportsStreamSource::~nsISupportsStreamSource() = default;
     76 
     77 void nsISupportsStreamSource::Init(ReadableStream* aStream) {
     78  mStream = aStream;
     79 }
     80 
     81 void nsISupportsStreamSource::Enqueue(nsISupports* aThing) {
     82  if (!mThingQueuedPromise) {
     83    mQueue.AppendElement(aThing);
     84    return;
     85  }
     86 
     87  // Maybe put a limit here? Or at least some sort of logging if this gets
     88  // unreasonably long?
     89  AutoJSAPI jsapi;
     90  if (NS_WARN_IF(!jsapi.Init(mStream->GetParentObject()))) {
     91    return;
     92  }
     93 
     94  EnqueueToStream(jsapi.cx(), aThing);
     95  mThingQueuedPromise->MaybeResolveWithUndefined();
     96  mThingQueuedPromise = nullptr;
     97 }
     98 
     99 already_AddRefed<Promise> nsISupportsStreamSource::PullCallbackImpl(
    100    JSContext* aCx, ReadableStreamControllerBase& aController,
    101    ErrorResult& aRv) {
    102  if (!mQueue.IsEmpty()) {
    103    EnqueueOneThingFromQueue(aCx);
    104    return nullptr;
    105  }
    106 
    107  RefPtr<nsISupportsStreamSource> self(this);
    108  mThingQueuedPromise = Promise::CreateInfallible(mStream->GetParentObject());
    109  return do_AddRef(mThingQueuedPromise);
    110 }
    111 
    112 void nsISupportsStreamSource::EnqueueToStream(JSContext* aCx,
    113                                              nsISupports* aThing) {
    114  JS::Rooted<JS::Value> jsThing(aCx);
    115  if (NS_WARN_IF(MOZ_UNLIKELY(!ToJSValue(aCx, *aThing, &jsThing)))) {
    116    // Do we want to add error handling for this?
    117    return;
    118  }
    119  IgnoredErrorResult rv;
    120  // EnqueueNative is CAN_RUN_SCRIPT. Need a strong-ref temporary.
    121  auto stream = mStream;
    122  stream->EnqueueNative(aCx, jsThing, rv);
    123 }
    124 
    125 void nsISupportsStreamSource::EnqueueOneThingFromQueue(JSContext* aCx) {
    126  if (!mQueue.IsEmpty()) {
    127    RefPtr<nsISupports> thing = mQueue[0];
    128    mQueue.RemoveElementAt(0);
    129    EnqueueToStream(aCx, thing);
    130  }
    131 }
    132 
    133 NS_IMPL_CYCLE_COLLECTION_INHERITED(WritableStreamRTCFrameSink,
    134                                   UnderlyingSinkAlgorithmsWrapper,
    135                                   mTransformer)
    136 NS_IMPL_ADDREF_INHERITED(WritableStreamRTCFrameSink,
    137                         UnderlyingSinkAlgorithmsWrapper)
    138 NS_IMPL_RELEASE_INHERITED(WritableStreamRTCFrameSink,
    139                          UnderlyingSinkAlgorithmsWrapper)
    140 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(WritableStreamRTCFrameSink)
    141 NS_INTERFACE_MAP_END_INHERITING(UnderlyingSinkAlgorithmsWrapper)
    142 
    143 WritableStreamRTCFrameSink::WritableStreamRTCFrameSink(
    144    RTCRtpScriptTransformer* aTransformer)
    145    : mTransformer(aTransformer) {}
    146 
    147 WritableStreamRTCFrameSink::~WritableStreamRTCFrameSink() = default;
    148 
    149 already_AddRefed<Promise> WritableStreamRTCFrameSink::WriteCallbackImpl(
    150    JSContext* aCx, JS::Handle<JS::Value> aChunk,
    151    WritableStreamDefaultController& aController, ErrorResult& aError) {
    152  // Spec does not say to do this right now. Might be a spec bug, needs
    153  // clarification.
    154  // https://github.com/w3c/webrtc-encoded-transform/issues/191
    155  if (NS_WARN_IF(!aChunk.isObject())) {
    156    aError.ThrowTypeError(
    157        "Wrong type for RTCRtpScriptTransformer.[[writeable]]: Not an object");
    158    return nullptr;
    159  }
    160 
    161  // Lame. But, without a webidl base class, this is the only way.
    162  RefPtr<RTCEncodedVideoFrame> video;
    163  UNWRAP_OBJECT(RTCEncodedVideoFrame, &aChunk.toObject(), video);
    164  RefPtr<RTCEncodedAudioFrame> audio;
    165  UNWRAP_OBJECT(RTCEncodedAudioFrame, &aChunk.toObject(), audio);
    166 
    167  RefPtr<RTCEncodedFrameBase> frame;
    168  if (video) {
    169    frame = video;
    170  } else if (audio) {
    171    frame = audio;
    172  }
    173 
    174  if (NS_WARN_IF(!frame)) {
    175    aError.ThrowTypeError(
    176        "Wrong type for RTCRtpScriptTransformer.[[writeable]]: Not an "
    177        "RTCEncodedAudioFrame or RTCEncodedVideoFrame");
    178    return nullptr;
    179  }
    180 
    181  return mTransformer->OnTransformedFrame(frame, aError);
    182 }
    183 
    184 // There is not presently an implementation of these for nsTHashMap :(
    185 inline void ImplCycleCollectionUnlink(
    186    RTCRtpScriptTransformer::GenerateKeyFramePromises& aMap) {
    187  for (auto& tableEntry : aMap) {
    188    ImplCycleCollectionUnlink(*tableEntry.GetModifiableData());
    189  }
    190  aMap.Clear();
    191 }
    192 
    193 inline void ImplCycleCollectionTraverse(
    194    nsCycleCollectionTraversalCallback& aCallback,
    195    RTCRtpScriptTransformer::GenerateKeyFramePromises& aMap, const char* aName,
    196    uint32_t aFlags = 0) {
    197  for (auto& tableEntry : aMap) {
    198    ImplCycleCollectionTraverse(aCallback, *tableEntry.GetModifiableData(),
    199                                aName, aFlags);
    200  }
    201 }
    202 
    203 NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_WITH_JS_MEMBERS(
    204    RTCRtpScriptTransformer,
    205    (mGlobal, mReadableSource, mReadable, mWritable, mWritableSink,
    206     mKeyFrameRequestPromises, mGenerateKeyFramePromises),
    207    (mOptions))
    208 
    209 NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpScriptTransformer)
    210 NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCRtpScriptTransformer)
    211 
    212 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCRtpScriptTransformer)
    213  NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
    214  NS_INTERFACE_MAP_ENTRY(nsISupports)
    215 NS_INTERFACE_MAP_END
    216 
    217 RTCRtpScriptTransformer::RTCRtpScriptTransformer(nsIGlobalObject* aGlobal)
    218    : mGlobal(aGlobal),
    219      mReadableSource(new nsISupportsStreamSource),
    220      mWritableSink(new WritableStreamRTCFrameSink(this)),
    221      mOptions(JS::UndefinedHandleValue) {
    222  mozilla::HoldJSObjects(this);
    223 }
    224 
    225 RTCRtpScriptTransformer::~RTCRtpScriptTransformer() {
    226  mozilla::DropJSObjects(this);
    227 }
    228 
    229 nsresult RTCRtpScriptTransformer::Init(JSContext* aCx,
    230                                       JS::Handle<JS::Value> aOptions,
    231                                       WorkerPrivate* aWorkerPrivate,
    232                                       FrameTransformerProxy* aProxy) {
    233  ErrorResult rv;
    234  RefPtr<nsIGlobalObject> global(mGlobal);
    235  auto source = mReadableSource;
    236  auto sink = mWritableSink;
    237 
    238  // NOTE: We do not transfer these streams from mainthread, as the spec says,
    239  // because there's no JS observable reason to. The spec is likely to change
    240  // here, because it is overspecifying implementation details.
    241  mReadable = ReadableStream::CreateNative(aCx, global, *source, Some(1.0),
    242                                           nullptr, rv);
    243  if (rv.Failed()) {
    244    return rv.StealNSResult();
    245  }
    246  mReadableSource->Init(mReadable);
    247 
    248  // WritableStream::CreateNative takes a nsIGlobalObject&, but
    249  // ReadableStream::CreateNative takes a nsIGlobalObject*?
    250  mWritable =
    251      WritableStream::CreateNative(aCx, *global, *sink, Nothing(), nullptr, rv);
    252  if (rv.Failed()) {
    253    return rv.StealNSResult();
    254  }
    255 
    256  mOptions = aOptions;
    257  mProxy = aProxy;
    258  // This will return null if the worker is already shutting down.
    259  // A call to ReleaseScriptTransformer will eventually result in a call to
    260  // NotifyReleased.
    261  mWorkerRef = StrongWorkerRef::Create(
    262      aWorkerPrivate, "RTCRtpScriptTransformer",
    263      [this, self = RefPtr(this)]() { mProxy->ReleaseScriptTransformer(); });
    264  if (mWorkerRef) {
    265    mProxy->SetScriptTransformer(*this);
    266  }
    267  return NS_OK;
    268 }
    269 
    270 void RTCRtpScriptTransformer::NotifyReleased() {
    271  RejectPendingPromises();
    272  mWorkerRef = nullptr;
    273  mProxy = nullptr;
    274 }
    275 
    276 void RTCRtpScriptTransformer::RejectPendingPromises() {
    277  for (const auto& promise : mKeyFrameRequestPromises) {
    278    ErrorResult rv;
    279    rv.ThrowInvalidStateError(
    280        "RTCRtpScriptTransformer is not associated with a receiver");
    281    promise->MaybeReject(std::move(rv));
    282  }
    283  mKeyFrameRequestPromises.Clear();
    284 
    285  // GenerateKeyFrame promises are indexed by rid
    286  for (auto& ridAndPromises : mGenerateKeyFramePromises) {
    287    for (const auto& promise : ridAndPromises.GetData()) {
    288      ErrorResult rv;
    289      rv.ThrowInvalidStateError(
    290          "RTCRtpScriptTransformer is not associated with a sender");
    291      promise->MaybeReject(std::move(rv));
    292    }
    293  }
    294  mGenerateKeyFramePromises.Clear();
    295 }
    296 
    297 void RTCRtpScriptTransformer::TransformFrame(
    298    std::unique_ptr<webrtc::TransformableFrameInterface> aFrame) {
    299  if (!mVideo.isSome()) {
    300    // First frame. mProxy will know whether it's video or not by now.
    301    mVideo = mProxy->IsVideo();
    302    MOZ_ASSERT(mVideo.isSome());
    303  }
    304 
    305  RefPtr<RTCEncodedFrameBase> domFrame;
    306  if (*mVideo) {
    307    // If this is a send video keyframe, resolve any pending GenerateKeyFrame
    308    // promises for its rid.
    309    if (aFrame->GetDirection() ==
    310        webrtc::TransformableFrameInterface::Direction::kSender) {
    311      auto* videoFrame =
    312          static_cast<webrtc::TransformableVideoFrameInterface*>(aFrame.get());
    313      if (videoFrame->IsKeyFrame()) {
    314        MOZ_ASSERT(videoFrame->Rid().has_value());
    315        ResolveGenerateKeyFramePromises(
    316            videoFrame->Rid().value_or(std::string()),
    317            videoFrame->GetTimestamp());
    318        if (videoFrame->Rid().has_value() && !videoFrame->Rid()->empty() &&
    319            videoFrame->Metadata().GetSimulcastIdx() == 0) {
    320          ResolveGenerateKeyFramePromises("", videoFrame->GetTimestamp());
    321        }
    322      }
    323    }
    324    domFrame = new RTCEncodedVideoFrame(mGlobal, std::move(aFrame),
    325                                        ++mLastEnqueuedFrameCounter, this);
    326  } else {
    327    domFrame = new RTCEncodedAudioFrame(mGlobal, std::move(aFrame),
    328                                        ++mLastEnqueuedFrameCounter, this);
    329  }
    330  mReadableSource->Enqueue(domFrame);
    331 }
    332 
    333 void RTCRtpScriptTransformer::GetOptions(JSContext* aCx,
    334                                         JS::MutableHandle<JS::Value> aVal,
    335                                         ErrorResult& aError) {
    336  if (!ToJSValue(aCx, mOptions, aVal)) {
    337    aError.NoteJSContextException(aCx);
    338  }
    339 }
    340 
    341 already_AddRefed<Promise> RTCRtpScriptTransformer::GenerateKeyFrame(
    342    const Optional<nsAString>& aRid) {
    343  Maybe<std::string> utf8Rid;
    344  if (aRid.WasPassed()) {
    345    utf8Rid = Some(NS_ConvertUTF16toUTF8(aRid.Value()).get());
    346    std::string error;
    347    if (!SdpRidAttributeList::CheckRidValidity(*utf8Rid, &error)) {
    348      ErrorResult rv;
    349      nsCString nsError(error.c_str());
    350      rv.ThrowNotAllowedError(nsError);
    351      return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv);
    352    }
    353  }
    354 
    355  nsCString key;
    356  if (utf8Rid.isSome()) {
    357    key.Assign(utf8Rid->data(), utf8Rid->size());
    358  }
    359 
    360  nsTArray<RefPtr<Promise>>& promises =
    361      mGenerateKeyFramePromises.LookupOrInsert(key);
    362  if (!promises.Length()) {
    363    // No pending keyframe generation request for this rid. Make one.
    364    if (mProxy && !mProxy->GenerateKeyFrame(utf8Rid)) {
    365      ErrorResult rv;
    366      rv.ThrowInvalidStateError(
    367          "RTCRtpScriptTransformer is not associated with a video sender");
    368      return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv);
    369    }
    370  }
    371  RefPtr<Promise> promise = Promise::CreateInfallible(GetParentObject());
    372  promises.AppendElement(promise);
    373  return promise.forget();
    374 }
    375 
    376 void RTCRtpScriptTransformer::ResolveGenerateKeyFramePromises(
    377    const std::string& aRid, uint64_t aTimestamp) {
    378  nsCString key(aRid.data(), aRid.size());
    379  nsTArray<RefPtr<Promise>> promises;
    380  mGenerateKeyFramePromises.Remove(key, &promises);
    381  for (auto& promise : promises) {
    382    promise->MaybeResolve(aTimestamp);
    383  }
    384 }
    385 
    386 void RTCRtpScriptTransformer::GenerateKeyFrameError(
    387    const Maybe<std::string>& aRid, const CopyableErrorResult& aResult) {
    388  nsCString key;
    389  if (aRid.isSome()) {
    390    key.Assign(aRid->data(), aRid->size());
    391  }
    392  nsTArray<RefPtr<Promise>> promises;
    393  mGenerateKeyFramePromises.Remove(key, &promises);
    394  for (auto& promise : promises) {
    395    CopyableErrorResult rv(aResult);
    396    promise->MaybeReject(std::move(rv));
    397  }
    398 }
    399 
    400 already_AddRefed<Promise> RTCRtpScriptTransformer::SendKeyFrameRequest() {
    401  if (!mKeyFrameRequestPromises.Length()) {
    402    if (mProxy && !mProxy->RequestKeyFrame()) {
    403      ErrorResult rv;
    404      rv.ThrowInvalidStateError(
    405          "RTCRtpScriptTransformer is not associated with a video receiver");
    406      return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv);
    407    }
    408  }
    409  RefPtr<Promise> promise = Promise::CreateInfallible(GetParentObject());
    410  mKeyFrameRequestPromises.AppendElement(promise);
    411  return promise.forget();
    412 }
    413 
    414 void RTCRtpScriptTransformer::KeyFrameRequestDone() {
    415  auto promises = std::move(mKeyFrameRequestPromises);
    416  for (const auto& promise : promises) {
    417    promise->MaybeResolveWithUndefined();
    418  }
    419 }
    420 
    421 JSObject* RTCRtpScriptTransformer::WrapObject(
    422    JSContext* aCx, JS::Handle<JSObject*> aGivenProto) {
    423  return RTCRtpScriptTransformer_Binding::Wrap(aCx, this, aGivenProto);
    424 }
    425 
    426 already_AddRefed<Promise> RTCRtpScriptTransformer::OnTransformedFrame(
    427    RTCEncodedFrameBase* aFrame, ErrorResult& aError) {
    428  // Spec says to skip frames that are out of order or have wrong owner
    429  if (aFrame->GetCounter() > mLastReceivedFrameCounter &&
    430      aFrame->CheckOwner(this) && mProxy) {
    431    mLastReceivedFrameCounter = aFrame->GetCounter();
    432    // also skip if frame has been detached (transferred away)
    433    if (auto frame = aFrame->TakeFrame()) {
    434      mProxy->OnTransformedFrame(std::move(frame));
    435    }
    436  }
    437 
    438  return Promise::CreateResolvedWithUndefined(GetParentObject(), aError);
    439 }
    440 
    441 }  // namespace mozilla::dom
    442 
    443 #undef LOGTAG