[ tor-browser ].git.dasho

TextTrackManager.cpp (29173B)
      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
      5 * You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "mozilla/dom/TextTrackManager.h"
      8 
      9 #include "mozilla/ClearOnShutdown.h"
     10 #include "mozilla/CycleCollectedJSContext.h"
     11 #include "mozilla/Maybe.h"
     12 #include "mozilla/dom/Document.h"
     13 #include "mozilla/dom/Event.h"
     14 #include "mozilla/dom/HTMLMediaElement.h"
     15 #include "mozilla/dom/HTMLTrackElement.h"
     16 #include "mozilla/dom/HTMLVideoElement.h"
     17 #include "mozilla/dom/TextTrack.h"
     18 #include "mozilla/dom/TextTrackCue.h"
     19 #include "mozilla/nsVideoFrame.h"
     20 #include "nsComponentManagerUtils.h"
     21 #include "nsGlobalWindowInner.h"
     22 #include "nsIFrame.h"
     23 #include "nsIWebVTTParserWrapper.h"
     24 #include "nsVariant.h"
     25 
     26 mozilla::LazyLogModule gTextTrackLog("WebVTT");
     27 
     28 #define WEBVTT_LOG(msg, ...)              \
     29  MOZ_LOG(gTextTrackLog, LogLevel::Debug, \
     30          ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
     31 #define WEBVTT_LOGV(msg, ...)               \
     32  MOZ_LOG(gTextTrackLog, LogLevel::Verbose, \
     33          ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
     34 
     35 namespace mozilla::dom {
     36 
     37 NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver);
     38 
     39 void TextTrackManager::ShutdownObserverProxy::Unregister() {
     40  nsContentUtils::UnregisterShutdownObserver(this);
     41  mManager = nullptr;
     42 }
     43 
     44 CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement) {
     45  mMediaElement = aMediaElement;
     46 }
     47 
     48 Maybe<uint32_t> CompareTextTracks::TrackChildPosition(
     49    TextTrack* aTextTrack) const {
     50  MOZ_DIAGNOSTIC_ASSERT(aTextTrack);
     51  HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
     52  if (!trackElement) {
     53    return Nothing();
     54  }
     55  return mMediaElement->ComputeIndexOf(trackElement);
     56 }
     57 
     58 bool CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const {
     59  // Two tracks can never be equal. If they have corresponding TrackElements
     60  // they would need to occupy the same tree position (impossible) and in the
     61  // case of tracks coming from AddTextTrack source we put the newest at the
     62  // last position, so they won't be equal as well.
     63  return false;
     64 }
     65 
     66 bool CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const {
     67  // Protect against nullptr TextTrack objects; treat them as
     68  // sorting toward the end.
     69  if (!aOne) {
     70    return false;
     71  }
     72  if (!aTwo) {
     73    return true;
     74  }
     75  TextTrackSource sourceOne = aOne->GetTextTrackSource();
     76  TextTrackSource sourceTwo = aTwo->GetTextTrackSource();
     77  if (sourceOne != sourceTwo) {
     78    return sourceOne == TextTrackSource::Track ||
     79           (sourceOne == TextTrackSource::AddTextTrack &&
     80            sourceTwo == TextTrackSource::MediaResourceSpecific);
     81  }
     82  switch (sourceOne) {
     83    case TextTrackSource::Track: {
     84      Maybe<uint32_t> positionOne = TrackChildPosition(aOne);
     85      Maybe<uint32_t> positionTwo = TrackChildPosition(aTwo);
     86      // If either position one or positiontwo are Nothing then something has
     87      // gone wrong. In this case we should just put them at the back of the
     88      // list.
     89      return positionOne.isSome() && positionTwo.isSome() &&
     90             *positionOne < *positionTwo;
     91    }
     92    case TextTrackSource::AddTextTrack:
     93      // For AddTextTrack sources the tracks will already be in the correct
     94      // relative order in the source array. Assume we're called in iteration
     95      // order and can therefore always report aOne < aTwo to maintain the
     96      // original temporal ordering.
     97      return true;
     98    case TextTrackSource::MediaResourceSpecific:
     99      // No rules for Media Resource Specific tracks yet.
    100      break;
    101  }
    102  return true;
    103 }
    104 
    105 NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
    106                         mPendingTextTracks, mNewCues)
    107 
    108 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
    109  NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
    110 NS_INTERFACE_MAP_END
    111 
    112 NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
    113 NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
    114 
    115 StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
    116 
    117 TextTrackManager::TextTrackManager(HTMLMediaElement* aMediaElement)
    118    : mMediaElement(aMediaElement),
    119      mHasSeeked(false),
    120      mLastTimeMarchesOnCalled(media::TimeUnit::Zero()),
    121      mTimeMarchesOnDispatched(false),
    122      mUpdateCueDisplayDispatched(false),
    123      performedTrackSelection(false),
    124      mShutdown(false) {
    125  nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
    126 
    127  NS_ENSURE_TRUE_VOID(parentObject);
    128  WEBVTT_LOG("Create TextTrackManager");
    129  nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
    130  mNewCues = new TextTrackCueList(window);
    131  mTextTracks = new TextTrackList(window, this);
    132  mPendingTextTracks = new TextTrackList(window, this);
    133 
    134  if (!sParserWrapper) {
    135    nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
    136        do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
    137    MOZ_ASSERT(parserWrapper, "Can't create nsIWebVTTParserWrapper");
    138    sParserWrapper = parserWrapper;
    139    ClearOnShutdown(&sParserWrapper);
    140  }
    141  mShutdownProxy = new ShutdownObserverProxy(this);
    142 }
    143 
    144 TextTrackManager::~TextTrackManager() {
    145  WEBVTT_LOG("~TextTrackManager");
    146  mShutdownProxy->Unregister();
    147 }
    148 
    149 TextTrackList* TextTrackManager::GetTextTracks() const { return mTextTracks; }
    150 
    151 already_AddRefed<TextTrack> TextTrackManager::AddTextTrack(
    152    TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage,
    153    TextTrackMode aMode, TextTrackReadyState aReadyState,
    154    TextTrackSource aTextTrackSource) {
    155  if (!mMediaElement || !mTextTracks) {
    156    return nullptr;
    157  }
    158  RefPtr<TextTrack> track = mTextTracks->AddTextTrack(
    159      aKind, aLabel, aLanguage, aMode, aReadyState, aTextTrackSource,
    160      CompareTextTracks(mMediaElement));
    161  WEBVTT_LOG("AddTextTrack %p kind %" PRIu32 " Label %s Language %s",
    162             track.get(), static_cast<uint32_t>(aKind),
    163             NS_ConvertUTF16toUTF8(aLabel).get(),
    164             NS_ConvertUTF16toUTF8(aLanguage).get());
    165  AddCues(track);
    166 
    167  if (aTextTrackSource == TextTrackSource::Track) {
    168    RefPtr<nsIRunnable> task = NewRunnableMethod(
    169        "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
    170        &TextTrackManager::HonorUserPreferencesForTrackSelection);
    171    NS_DispatchToMainThread(task.forget());
    172  }
    173 
    174  return track.forget();
    175 }
    176 
    177 void TextTrackManager::AddTextTrack(TextTrack* aTextTrack) {
    178  if (!mMediaElement || !mTextTracks) {
    179    return;
    180  }
    181  WEBVTT_LOG("AddTextTrack TextTrack %p", aTextTrack);
    182  mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement));
    183  AddCues(aTextTrack);
    184 
    185  if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) {
    186    RefPtr<nsIRunnable> task = NewRunnableMethod(
    187        "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
    188        &TextTrackManager::HonorUserPreferencesForTrackSelection);
    189    NS_DispatchToMainThread(task.forget());
    190  }
    191 }
    192 
    193 void TextTrackManager::AddCues(TextTrack* aTextTrack) {
    194  if (!mNewCues) {
    195    WEBVTT_LOG("AddCues mNewCues is null");
    196    return;
    197  }
    198 
    199  TextTrackCueList* cueList = aTextTrack->GetCues();
    200  if (cueList) {
    201    bool dummy;
    202    WEBVTT_LOGV("AddCues, CuesNum=%d", cueList->Length());
    203    for (uint32_t i = 0; i < cueList->Length(); ++i) {
    204      mNewCues->AddCue(*cueList->IndexedGetter(i, dummy));
    205    }
    206    MaybeRunTimeMarchesOn();
    207  }
    208 }
    209 
    210 void TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack,
    211                                       bool aPendingListOnly) {
    212  if (!mPendingTextTracks || !mTextTracks) {
    213    return;
    214  }
    215 
    216  WEBVTT_LOG("RemoveTextTrack TextTrack %p", aTextTrack);
    217  mPendingTextTracks->RemoveTextTrack(aTextTrack);
    218  if (aPendingListOnly) {
    219    return;
    220  }
    221 
    222  mTextTracks->RemoveTextTrack(aTextTrack);
    223  // Remove the cues in mNewCues belong to aTextTrack.
    224  TextTrackCueList* removeCueList = aTextTrack->GetCues();
    225  if (removeCueList) {
    226    WEBVTT_LOGV("RemoveTextTrack removeCuesNum=%d", removeCueList->Length());
    227    for (uint32_t i = 0; i < removeCueList->Length(); ++i) {
    228      mNewCues->RemoveCue(*((*removeCueList)[i]));
    229    }
    230    MaybeRunTimeMarchesOn();
    231  }
    232 }
    233 
    234 void TextTrackManager::DidSeek() {
    235  WEBVTT_LOG("DidSeek");
    236  mHasSeeked = true;
    237 }
    238 
    239 void TextTrackManager::UpdateCueDisplay() {
    240  WEBVTT_LOG("UpdateCueDisplay");
    241  mUpdateCueDisplayDispatched = false;
    242 
    243  if (!mMediaElement || !mTextTracks || IsShutdown()) {
    244    WEBVTT_LOG("Abort UpdateCueDisplay.");
    245    return;
    246  }
    247 
    248  nsIFrame* frame = mMediaElement->GetPrimaryFrame();
    249  nsVideoFrame* videoFrame = do_QueryFrame(frame);
    250  if (!videoFrame) {
    251    WEBVTT_LOG("Abort UpdateCueDisplay, because of no video frame.");
    252    return;
    253  }
    254 
    255  nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay();
    256  if (!overlay) {
    257    WEBVTT_LOG("Abort UpdateCueDisplay, because of no overlay.");
    258    return;
    259  }
    260 
    261  RefPtr<nsPIDOMWindowInner> window =
    262      mMediaElement->OwnerDoc()->GetInnerWindow();
    263  if (!window) {
    264    WEBVTT_LOG("Abort UpdateCueDisplay, because of no window.");
    265  }
    266 
    267  nsTArray<RefPtr<TextTrackCue>> showingCues;
    268  mTextTracks->GetShowingCues(showingCues);
    269 
    270  WEBVTT_LOG("UpdateCueDisplay, processCues, showingCuesNum=%zu",
    271             showingCues.Length());
    272  RefPtr<nsVariantCC> jsCues = new nsVariantCC();
    273  jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE, &NS_GET_IID(EventTarget),
    274                     showingCues.Length(),
    275                     static_cast<void*>(showingCues.Elements()));
    276  nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls();
    277 
    278  nsContentUtils::AddScriptRunner(NS_NewRunnableFunction(
    279      "TextTrackManager::UpdateCueDisplay",
    280      [window, jsCues, overlay, controls]() {
    281        if (sParserWrapper) {
    282          sParserWrapper->ProcessCues(window, jsCues, overlay, controls);
    283        }
    284      }));
    285 }
    286 
    287 void TextTrackManager::NotifyCueAdded(TextTrackCue& aCue) {
    288  WEBVTT_LOG("NotifyCueAdded, cue=%p", &aCue);
    289  if (mNewCues) {
    290    mNewCues->AddCue(aCue);
    291  }
    292  MaybeRunTimeMarchesOn();
    293 }
    294 
    295 void TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue) {
    296  WEBVTT_LOG("NotifyCueRemoved, cue=%p", &aCue);
    297  if (mNewCues) {
    298    mNewCues->RemoveCue(aCue);
    299  }
    300  MaybeRunTimeMarchesOn();
    301  DispatchUpdateCueDisplay();
    302 }
    303 
    304 void TextTrackManager::PopulatePendingList() {
    305  if (!mTextTracks || !mPendingTextTracks || !mMediaElement) {
    306    return;
    307  }
    308  uint32_t len = mTextTracks->Length();
    309  bool dummy;
    310  for (uint32_t index = 0; index < len; ++index) {
    311    TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
    312    if (ttrack && ttrack->Mode() != TextTrackMode::Disabled &&
    313        ttrack->ReadyState() == TextTrackReadyState::Loading) {
    314      mPendingTextTracks->AddTextTrack(ttrack,
    315                                       CompareTextTracks(mMediaElement));
    316    }
    317  }
    318 }
    319 
    320 void TextTrackManager::AddListeners() {
    321  if (mMediaElement) {
    322    mMediaElement->AddEventListener(u"resizecaption"_ns, this, false, false);
    323    mMediaElement->AddEventListener(u"resizevideocontrols"_ns, this, false,
    324                                    false);
    325    mMediaElement->AddEventListener(u"seeked"_ns, this, false, false);
    326    mMediaElement->AddEventListener(u"controlbarchange"_ns, this, false, true);
    327  }
    328 }
    329 
    330 void TextTrackManager::HonorUserPreferencesForTrackSelection() {
    331  if (performedTrackSelection || !mTextTracks) {
    332    return;
    333  }
    334  WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
    335  TextTrackKind ttKinds[] = {TextTrackKind::Captions, TextTrackKind::Subtitles};
    336 
    337  // Steps 1 - 3: Perform automatic track selection for different TextTrack
    338  // Kinds.
    339  PerformTrackSelection(ttKinds, std::size(ttKinds));
    340  PerformTrackSelection(TextTrackKind::Descriptions);
    341  PerformTrackSelection(TextTrackKind::Chapters);
    342 
    343  // Step 4: Set all TextTracks with a kind of metadata that are disabled
    344  // to hidden.
    345  for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
    346    TextTrack* track = (*mTextTracks)[i];
    347    if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) &&
    348        track->Mode() == TextTrackMode::Disabled) {
    349      track->SetMode(TextTrackMode::Hidden);
    350    }
    351  }
    352 
    353  performedTrackSelection = true;
    354 }
    355 
    356 bool TextTrackManager::TrackIsDefault(TextTrack* aTextTrack) {
    357  HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
    358  if (!trackElement) {
    359    return false;
    360  }
    361  return trackElement->Default();
    362 }
    363 
    364 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind) {
    365  TextTrackKind ttKinds[] = {aTextTrackKind};
    366  PerformTrackSelection(ttKinds, std::size(ttKinds));
    367 }
    368 
    369 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[],
    370                                             uint32_t size) {
    371  nsTArray<TextTrack*> candidates;
    372  GetTextTracksOfKinds(aTextTrackKinds, size, candidates);
    373 
    374  // Step 3: If any TextTracks in candidates are showing then abort these steps.
    375  for (uint32_t i = 0; i < candidates.Length(); i++) {
    376    if (candidates[i]->Mode() == TextTrackMode::Showing) {
    377      WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",
    378                  static_cast<int>(candidates[i]->Kind()));
    379      return;
    380    }
    381  }
    382 
    383  // Step 4: Honor user preferences for track selection, otherwise, set the
    384  // first TextTrack in candidates with a default attribute to showing.
    385  // TODO: Bug 981691 - Honor user preferences for text track selection.
    386  for (uint32_t i = 0; i < candidates.Length(); i++) {
    387    if (TrackIsDefault(candidates[i]) &&
    388        candidates[i]->Mode() == TextTrackMode::Disabled) {
    389      candidates[i]->SetMode(TextTrackMode::Showing);
    390      WEBVTT_LOGV("PerformTrackSelection set Showing kind %d",
    391                  static_cast<int>(candidates[i]->Kind()));
    392      return;
    393    }
    394  }
    395 }
    396 
    397 void TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],
    398                                            uint32_t size,
    399                                            nsTArray<TextTrack*>& aTextTracks) {
    400  for (uint32_t i = 0; i < size; i++) {
    401    GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks);
    402  }
    403 }
    404 
    405 void TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind,
    406                                           nsTArray<TextTrack*>& aTextTracks) {
    407  if (!mTextTracks) {
    408    return;
    409  }
    410  for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
    411    TextTrack* textTrack = (*mTextTracks)[i];
    412    if (textTrack->Kind() == aTextTrackKind) {
    413      aTextTracks.AppendElement(textTrack);
    414    }
    415  }
    416 }
    417 
    418 NS_IMETHODIMP
    419 TextTrackManager::HandleEvent(Event* aEvent) {
    420  if (!mTextTracks) {
    421    return NS_OK;
    422  }
    423 
    424  nsAutoString type;
    425  aEvent->GetType(type);
    426  WEBVTT_LOG("Handle event %s", NS_ConvertUTF16toUTF8(type).get());
    427 
    428  const bool setDirty = type.EqualsLiteral("seeked") ||
    429                        type.EqualsLiteral("resizecaption") ||
    430                        type.EqualsLiteral("resizevideocontrols");
    431  const bool updateDisplay = type.EqualsLiteral("controlbarchange") ||
    432                             type.EqualsLiteral("resizecaption");
    433 
    434  if (setDirty) {
    435    for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
    436      ((*mTextTracks)[i])->SetCuesDirty();
    437    }
    438  }
    439  if (updateDisplay) {
    440    UpdateCueDisplay();
    441  }
    442 
    443  return NS_OK;
    444 }
    445 
    446 class SimpleTextTrackEvent : public Runnable {
    447 public:
    448  friend class CompareSimpleTextTrackEvents;
    449  SimpleTextTrackEvent(const nsAString& aEventName, double aTime,
    450                       TextTrack* aTrack, TextTrackCue* aCue)
    451      : Runnable("dom::SimpleTextTrackEvent"),
    452        mName(aEventName),
    453        mTime(aTime),
    454        mTrack(aTrack),
    455        mCue(aCue) {}
    456 
    457  NS_IMETHOD Run() override {
    458    WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf", mCue.get(),
    459                NS_ConvertUTF16toUTF8(mName).get(), mTime);
    460    mCue->DispatchTrustedEvent(mName);
    461    return NS_OK;
    462  }
    463 
    464  void Dispatch() {
    465    if (nsCOMPtr<nsIGlobalObject> global = mCue->GetOwnerGlobal()) {
    466      global->Dispatch(do_AddRef(this));
    467    } else {
    468      NS_DispatchToMainThread(do_AddRef(this));
    469    }
    470  }
    471 
    472 private:
    473  nsString mName;
    474  double mTime;
    475  TextTrack* mTrack;
    476  RefPtr<TextTrackCue> mCue;
    477 };
    478 
    479 class CompareSimpleTextTrackEvents {
    480 private:
    481  Maybe<uint32_t> TrackChildPosition(SimpleTextTrackEvent* aEvent) const {
    482    if (aEvent->mTrack) {
    483      HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement();
    484      if (trackElement) {
    485        return mMediaElement->ComputeIndexOf(trackElement);
    486      }
    487    }
    488    return Nothing();
    489  }
    490  HTMLMediaElement* mMediaElement;
    491 
    492 public:
    493  explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement) {
    494    mMediaElement = aMediaElement;
    495  }
    496 
    497  bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
    498    return false;
    499  }
    500 
    501  bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
    502    // TimeMarchesOn step 13.1.
    503    if (aOne->mTime < aTwo->mTime) {
    504      return true;
    505    }
    506    if (aOne->mTime > aTwo->mTime) {
    507      return false;
    508    }
    509 
    510    // TimeMarchesOn step 13.2 text track cue order.
    511    // TextTrack position in TextTrackList
    512    TextTrack* t1 = aOne->mTrack;
    513    TextTrack* t2 = aTwo->mTrack;
    514    MOZ_ASSERT(t1, "CompareSimpleTextTrackEvents t1 is null");
    515    MOZ_ASSERT(t2, "CompareSimpleTextTrackEvents t2 is null");
    516    if (t1 != t2) {
    517      TextTrackList* tList = t1->GetTextTrackList();
    518      MOZ_ASSERT(tList, "CompareSimpleTextTrackEvents tList is null");
    519      nsTArray<RefPtr<TextTrack>>& textTracks = tList->GetTextTrackArray();
    520      auto index1 = textTracks.IndexOf(t1);
    521      auto index2 = textTracks.IndexOf(t2);
    522      if (index1 < index2) {
    523        return true;
    524      }
    525      if (index1 > index2) {
    526        return false;
    527      }
    528    }
    529 
    530    MOZ_ASSERT(t1 == t2, "CompareSimpleTextTrackEvents t1 != t2");
    531    // c1 and c2 are both belongs to t1.
    532    TextTrackCue* c1 = aOne->mCue;
    533    TextTrackCue* c2 = aTwo->mCue;
    534    if (c1 != c2) {
    535      if (c1->StartTime() < c2->StartTime()) {
    536        return true;
    537      }
    538      if (c1->StartTime() > c2->StartTime()) {
    539        return false;
    540      }
    541      if (c1->EndTime() < c2->EndTime()) {
    542        return true;
    543      }
    544      if (c1->EndTime() > c2->EndTime()) {
    545        return false;
    546      }
    547 
    548      TextTrackCueList* cueList = t1->GetCues();
    549      MOZ_ASSERT(cueList);
    550      nsTArray<RefPtr<TextTrackCue>>& cues = cueList->GetCuesArray();
    551      auto index1 = cues.IndexOf(c1);
    552      auto index2 = cues.IndexOf(c2);
    553      if (index1 < index2) {
    554        return true;
    555      }
    556      if (index1 > index2) {
    557        return false;
    558      }
    559    }
    560 
    561    // TimeMarchesOn step 13.3.
    562    if (aOne->mName.EqualsLiteral("enter") ||
    563        aTwo->mName.EqualsLiteral("exit")) {
    564      return true;
    565    }
    566    return false;
    567  }
    568 };
    569 
    570 class TextTrackListInternal {
    571 public:
    572  void AddTextTrack(TextTrack* aTextTrack,
    573                    const CompareTextTracks& aCompareTT) {
    574    if (!mTextTracks.Contains(aTextTrack)) {
    575      mTextTracks.InsertElementSorted(aTextTrack, aCompareTT);
    576    }
    577  }
    578  uint32_t Length() const { return mTextTracks.Length(); }
    579  TextTrack* operator[](uint32_t aIndex) {
    580    return mTextTracks.SafeElementAt(aIndex, nullptr);
    581  }
    582 
    583 private:
    584  nsTArray<RefPtr<TextTrack>> mTextTracks;
    585 };
    586 
    587 void TextTrackManager::DispatchUpdateCueDisplay() {
    588  if (!mUpdateCueDisplayDispatched && !IsShutdown()) {
    589    WEBVTT_LOG("DispatchUpdateCueDisplay");
    590    if (nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow()) {
    591      nsGlobalWindowInner::Cast(win)->Dispatch(
    592          NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay", this,
    593                            &TextTrackManager::UpdateCueDisplay));
    594      mUpdateCueDisplayDispatched = true;
    595    }
    596  }
    597 }
    598 
    599 void TextTrackManager::DispatchTimeMarchesOn() {
    600  // Run the algorithm if no previous instance is still running, otherwise
    601  // enqueue the current playback position and whether only that changed
    602  // through its usual monotonic increase during normal playback; current
    603  // executing call upon completion will check queue for further 'work'.
    604  if (!mTimeMarchesOnDispatched && !IsShutdown()) {
    605    WEBVTT_LOG("DispatchTimeMarchesOn");
    606    if (nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow()) {
    607      nsGlobalWindowInner::Cast(win)->Dispatch(
    608          NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn", this,
    609                            &TextTrackManager::TimeMarchesOn));
    610      mTimeMarchesOnDispatched = true;
    611    }
    612  }
    613 }
    614 
    615 // https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
    616 void TextTrackManager::TimeMarchesOn() {
    617  NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
    618  mTimeMarchesOnDispatched = false;
    619 
    620  CycleCollectedJSContext* context = CycleCollectedJSContext::Get();
    621  if (context && context->IsInStableOrMetaStableState()) {
    622    // FireTimeUpdate can be called while at stable state following a
    623    // current position change which triggered a state watcher in MediaDecoder
    624    // (see bug 1443429).
    625    // TimeMarchesOn() will modify JS attributes which is forbidden while in
    626    // stable state. So we dispatch a task to perform such operation later
    627    // instead.
    628    DispatchTimeMarchesOn();
    629    return;
    630  }
    631  WEBVTT_LOG("TimeMarchesOn");
    632 
    633  // Early return if we don't have any TextTracks or shutting down.
    634  if (!mTextTracks || mTextTracks->Length() == 0 || IsShutdown() ||
    635      !mMediaElement) {
    636    return;
    637  }
    638 
    639  if (mMediaElement->ReadyState() == HTMLMediaElement_Binding::HAVE_NOTHING) {
    640    WEBVTT_LOG(
    641        "TimeMarchesOn return because media doesn't contain any data yet");
    642    return;
    643  }
    644 
    645  if (mMediaElement->Seeking()) {
    646    WEBVTT_LOG("TimeMarchesOn return during seeking");
    647    return;
    648  }
    649 
    650  // Step 1, 2, 3, 4
    651  using CueBuckets = TextTrack::CueBuckets;
    652  CueBuckets currentCues;
    653  CueBuckets otherCues;
    654  CueBuckets missedCues;
    655  auto currentPlaybackTime =
    656      media::TimeUnit::FromSeconds(mMediaElement->CurrentTime());
    657  bool hasNormalPlayback = !mHasSeeked;
    658  mHasSeeked = false;
    659  WEBVTT_LOG(
    660      "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf "
    661      "hasNormalPlayback %d",
    662      mLastTimeMarchesOnCalled.ToSeconds(), currentPlaybackTime.ToSeconds(),
    663      hasNormalPlayback);
    664 
    665  // The reason we collect other cues is (1) to change active cues to inactive,
    666  // (2) find missing cues, so we actually no need to process all cues. We just
    667  // need to handle cues which are in the time interval [lastTime:currentTime]
    668  // or [currentTime:lastTime] (seeking forward). That can help us to reduce the
    669  // size of other cues, which can improve execution time.
    670  auto start = std::min(mLastTimeMarchesOnCalled, currentPlaybackTime);
    671  auto end = std::max(mLastTimeMarchesOnCalled, currentPlaybackTime);
    672  media::TimeInterval interval(start, end);
    673  WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start.ToSeconds(),
    674              end.ToSeconds());
    675  for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
    676    TextTrack* track = (*mTextTracks)[idx];
    677    if (track) {
    678      track->GetOverlappingCurrentOtherAndMissCues(
    679          &currentCues, &otherCues, &missedCues, interval,
    680          hasNormalPlayback ? Some(mLastTimeMarchesOnCalled.ToSeconds())
    681                            : Nothing());
    682    }
    683  }
    684 
    685  WEBVTT_LOGV("TimeMarchesOn currentCues %zu", currentCues.AllCues().Length());
    686  WEBVTT_LOGV("TimeMarchesOn otherCues %zu", otherCues.AllCues().Length());
    687  WEBVTT_LOGV("TimeMarchesOn missedCues %zu", missedCues.AllCues().Length());
    688  // Step 5. Empty now.
    689  // TODO: Step 6: fire timeupdate?
    690 
    691  // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
    692  // 1. All of the cues in current cues have their active flag set.
    693  // 2. None of the cues in other cues have their active flag set.
    694  // 3. Missed cues is empty.
    695  const bool hasOnlyActiveCurrentCues = currentCues.InactiveCues().IsEmpty();
    696  const bool hasNoActiveOtherCues = otherCues.ActiveCues().IsEmpty();
    697  const bool hasNoMissedCues = missedCues.AllCues().IsEmpty();
    698  if (hasOnlyActiveCurrentCues && hasNoActiveOtherCues && hasNoMissedCues) {
    699    mLastTimeMarchesOnCalled = currentPlaybackTime;
    700    WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf",
    701               mLastTimeMarchesOnCalled.ToSeconds());
    702    return;
    703  }
    704 
    705  // Step 8. Respect PauseOnExit flag if not seek.
    706  if (hasNormalPlayback) {
    707    if (otherCues.HasPauseOnExit(TextTrack::CueActivityState::Active) ||
    708        missedCues.HasPauseOnExit(TextTrack::CueActivityState::All)) {
    709      WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
    710      mMediaElement->Pause();
    711    }
    712  }
    713 
    714  // Step 15.
    715  // Sort text tracks in the same order as the text tracks appear
    716  // in the media element's list of text tracks, and remove
    717  // duplicates.
    718  TextTrackListInternal affectedTracks;
    719  // Step 13, 14.
    720  nsTArray<RefPtr<SimpleTextTrackEvent>> eventList;
    721  // Step 9, 10.
    722  // For each text track cue in missed cues, prepare an event named
    723  // enter for the TextTrackCue object with the cue start time.
    724  for (const auto& cue : missedCues.AllCues()) {
    725    WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in missing cues",
    726               cue.get(), cue->StartTime(), cue->EndTime());
    727    SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
    728        u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
    729    eventList.InsertElementSorted(event,
    730                                  CompareSimpleTextTrackEvents(mMediaElement));
    731    affectedTracks.AddTextTrack(cue->GetTrack(),
    732                                CompareTextTracks(mMediaElement));
    733  }
    734 
    735  // Step 11, 17.
    736  nsTArray<RefPtr<TextTrackCue>> cuesShouldDispatchExit;
    737  for (const auto& cue : otherCues.AllCues()) {
    738    if (cue->GetActive() || missedCues.AllCues().Contains(cue)) {
    739      double time =
    740          cue->StartTime() > cue->EndTime() ? cue->StartTime() : cue->EndTime();
    741      WEBVTT_LOG("Prepare 'exit' event for cue %p [%f, %f] in other cues",
    742                 cue.get(), cue->StartTime(), cue->EndTime());
    743      SimpleTextTrackEvent* event =
    744          new SimpleTextTrackEvent(u"exit"_ns, time, cue->GetTrack(), cue);
    745      eventList.InsertElementSorted(
    746          event, CompareSimpleTextTrackEvents(mMediaElement));
    747      affectedTracks.AddTextTrack(cue->GetTrack(),
    748                                  CompareTextTracks(mMediaElement));
    749    }
    750    cue->SetActive(false);
    751  }
    752 
    753  // Step 12, 17.
    754  for (const auto& cue : currentCues.InactiveCues()) {
    755    WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in current cues",
    756               cue.get(), cue->StartTime(), cue->EndTime());
    757    SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
    758        u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
    759    eventList.InsertElementSorted(event,
    760                                  CompareSimpleTextTrackEvents(mMediaElement));
    761    affectedTracks.AddTextTrack(cue->GetTrack(),
    762                                CompareTextTracks(mMediaElement));
    763    cue->SetActive(true);
    764  }
    765 
    766  // Fire the eventList
    767  for (uint32_t i = 0; i < eventList.Length(); ++i) {
    768    eventList[i]->Dispatch();
    769  }
    770 
    771  // Step 16.
    772  for (uint32_t i = 0; i < affectedTracks.Length(); ++i) {
    773    TextTrack* ttrack = affectedTracks[i];
    774    if (ttrack) {
    775      ttrack->DispatchAsyncTrustedEvent(u"cuechange"_ns);
    776      HTMLTrackElement* trackElement = ttrack->GetTrackElement();
    777      if (trackElement) {
    778        trackElement->DispatchTrackRunnable(u"cuechange"_ns);
    779      }
    780    }
    781  }
    782 
    783  mLastTimeMarchesOnCalled = currentPlaybackTime;
    784 
    785  // Step 18.
    786  UpdateCueDisplay();
    787 }
    788 
    789 void TextTrackManager::NotifyCueUpdated(TextTrackCue* aCue) {
    790  // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
    791  WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue);
    792  MaybeRunTimeMarchesOn();
    793  // For the case "Texttrack.mode = hidden/showing", if the mode
    794  // changing between showing and hidden, TimeMarchesOn
    795  // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
    796  DispatchUpdateCueDisplay();
    797 }
    798 
    799 void TextTrackManager::NotifyReset() {
    800  // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
    801  // This will unset all cues' active flag and update the cue display.
    802  WEBVTT_LOG("NotifyReset");
    803  mLastTimeMarchesOnCalled = media::TimeUnit::Zero();
    804  for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
    805    (*mTextTracks)[idx]->SetCuesInactive();
    806  }
    807  UpdateCueDisplay();
    808 }
    809 
    810 bool TextTrackManager::IsLoaded() {
    811  return mTextTracks ? mTextTracks->AreTextTracksLoaded() : true;
    812 }
    813 
    814 bool TextTrackManager::IsShutdown() const {
    815  return (mShutdown || !sParserWrapper);
    816 }
    817 
    818 void TextTrackManager::MaybeRunTimeMarchesOn() {
    819  MOZ_ASSERT(mMediaElement);
    820  // According to spec, we should check media element's show poster flag before
    821  // running `TimeMarchesOn` in following situations, (1) add cue (2) remove cue
    822  // (3) cue's start time changes (4) cues's end time changes
    823  // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:time-marches-on
    824  // https://html.spec.whatwg.org/multipage/media.html#text-track-api:time-marches-on
    825  if (mMediaElement->GetShowPosterFlag()) {
    826    return;
    827  }
    828  TimeMarchesOn();
    829 }
    830 
    831 }  // namespace mozilla::dom
    832 
    833 #undef WEBVTT_LOG
    834 #undef WEBVTT_LOGV
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE