TrackBuffersManager.cpp (138868B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "TrackBuffersManager.h" 8 9 #include "ContainerParser.h" 10 #include "MP4Demuxer.h" 11 #include "MediaInfo.h" 12 #include "MediaSourceDemuxer.h" 13 #include "MediaSourceUtils.h" 14 #include "SourceBuffer.h" 15 #include "SourceBufferResource.h" 16 #include "SourceBufferTask.h" 17 #include "WebMDemuxer.h" 18 #include "mozilla/ErrorResult.h" 19 #include "mozilla/Preferences.h" 20 #include "mozilla/ProfilerLabels.h" 21 #include "mozilla/ProfilerMarkers.h" 22 #include "mozilla/StaticPrefs_media.h" 23 #include "nsMimeTypes.h" 24 25 extern mozilla::LogModule* GetMediaSourceLog(); 26 27 #define MSE_DEBUG(arg, ...) \ 28 DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, "::%s: " arg, \ 29 __func__, ##__VA_ARGS__) 30 #define MSE_DEBUGV(arg, ...) \ 31 DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, "::%s: " arg, \ 32 __func__, ##__VA_ARGS__) 33 34 mozilla::LogModule* GetMediaSourceSamplesLog() { 35 static mozilla::LazyLogModule sLogModule("MediaSourceSamples"); 36 return sLogModule; 37 } 38 #define SAMPLE_DEBUG(arg, ...) \ 39 DDMOZ_LOG(GetMediaSourceSamplesLog(), mozilla::LogLevel::Debug, \ 40 "::%s: " arg, __func__, ##__VA_ARGS__) 41 #define SAMPLE_DEBUGV(arg, ...) \ 42 DDMOZ_LOG(GetMediaSourceSamplesLog(), mozilla::LogLevel::Verbose, \ 43 "::%s: " arg, __func__, ##__VA_ARGS__) 44 45 namespace mozilla { 46 47 using dom::SourceBufferAppendMode; 48 using media::Interval; 49 using media::TimeInterval; 50 using media::TimeIntervals; 51 using media::TimeUnit; 52 using AppendBufferResult = SourceBufferTask::AppendBufferResult; 53 using AppendState = SourceBufferAttributes::AppendState; 54 55 static Atomic<uint32_t> sStreamSourceID(0u); 56 57 class DispatchKeyNeededEvent : public Runnable { 58 public: 59 DispatchKeyNeededEvent(MediaSourceDecoder* aDecoder, 60 const nsTArray<uint8_t>& aInitData, 61 const nsString& aInitDataType) 62 : Runnable("DispatchKeyNeededEvent"), 63 mDecoder(aDecoder), 64 mInitData(aInitData.Clone()), 65 mInitDataType(aInitDataType) {} 66 NS_IMETHOD Run() override { 67 // Note: Null check the owner, as the decoder could have been shutdown 68 // since this event was dispatched. 69 MediaDecoderOwner* owner = mDecoder->GetOwner(); 70 if (owner) { 71 owner->DispatchEncrypted(mInitData, mInitDataType); 72 } 73 mDecoder = nullptr; 74 return NS_OK; 75 } 76 77 private: 78 RefPtr<MediaSourceDecoder> mDecoder; 79 nsTArray<uint8_t> mInitData; 80 nsString mInitDataType; 81 }; 82 83 TrackBuffersManager::TrackBuffersManager(MediaSourceDecoder* aParentDecoder, 84 const MediaContainerType& aType) 85 : mBufferFull(false), 86 mFirstInitializationSegmentReceived(false), 87 mChangeTypeReceived(false), 88 mNewMediaSegmentStarted(false), 89 mActiveTrack(false), 90 mType(aType), 91 mParser(ContainerParser::CreateForMIMEType(aType)), 92 mProcessedInput(0), 93 mParentDecoder(new nsMainThreadPtrHolder<MediaSourceDecoder>( 94 "TrackBuffersManager::mParentDecoder", aParentDecoder, 95 false /* strict */)), 96 mAbstractMainThread(aParentDecoder->AbstractMainThread()), 97 mVideoEvictionThreshold(Preferences::GetUint( 98 "media.mediasource.eviction_threshold.video", 150 * 1024 * 1024)), 99 mAudioEvictionThreshold(Preferences::GetUint( 100 "media.mediasource.eviction_threshold.audio", 20 * 1024 * 1024)), 101 mEvictionBufferWatermarkRatio(0.9), 102 mEvictionState(EvictionState::NO_EVICTION_NEEDED), 103 mMutex("TrackBuffersManager"), 104 mTaskQueue(aParentDecoder->GetDemuxer()->GetTaskQueue()), 105 mTaskQueueCapability(Some(EventTargetCapability{mTaskQueue.get()})) { 106 MOZ_ASSERT(NS_IsMainThread(), "Must be instanciated on the main thread"); 107 DDLINKCHILD("parser", mParser.get()); 108 } 109 110 TrackBuffersManager::~TrackBuffersManager() { ShutdownDemuxers(); } 111 112 RefPtr<TrackBuffersManager::AppendPromise> TrackBuffersManager::AppendData( 113 already_AddRefed<MediaByteBuffer> aData, 114 const SourceBufferAttributes& aAttributes) { 115 MOZ_ASSERT(NS_IsMainThread()); 116 RefPtr<MediaByteBuffer> data(aData); 117 MSE_DEBUG("Appending %zu bytes", data->Length()); 118 119 Reopen(); 120 121 return InvokeAsync(static_cast<AbstractThread*>(GetTaskQueueSafe().get()), 122 this, __func__, &TrackBuffersManager::DoAppendData, 123 data.forget(), aAttributes); 124 } 125 126 RefPtr<TrackBuffersManager::AppendPromise> TrackBuffersManager::DoAppendData( 127 already_AddRefed<MediaByteBuffer> aData, 128 const SourceBufferAttributes& aAttributes) { 129 RefPtr<AppendBufferTask> task = 130 new AppendBufferTask(std::move(aData), aAttributes); 131 RefPtr<AppendPromise> p = task->mPromise.Ensure(__func__); 132 QueueTask(task); 133 134 return p; 135 } 136 137 void TrackBuffersManager::QueueTask(SourceBufferTask* aTask) { 138 // The source buffer is a wrapped native, it would be unlinked twice and so 139 // the TrackBuffersManager::Detach() would also be called twice. Since the 140 // detach task has been done before, we could ignore this task. 141 RefPtr<TaskQueue> taskQueue = GetTaskQueueSafe(); 142 if (!taskQueue) { 143 MOZ_ASSERT(aTask->GetType() == SourceBufferTask::Type::Detach, 144 "only detach task could happen here!"); 145 MSE_DEBUG("Could not queue the task '%s' without task queue", 146 aTask->GetTypeName()); 147 return; 148 } 149 150 if (!taskQueue->IsCurrentThreadIn()) { 151 nsresult rv = 152 taskQueue->Dispatch(NewRunnableMethod<RefPtr<SourceBufferTask>>( 153 "TrackBuffersManager::QueueTask", this, 154 &TrackBuffersManager::QueueTask, aTask)); 155 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); 156 (void)rv; 157 return; 158 } 159 mQueue.Push(aTask); 160 ProcessTasks(); 161 } 162 163 void TrackBuffersManager::ProcessTasks() { 164 // ProcessTask is always called OnTaskQueue, however it is possible that it is 165 // called once again after a first Detach task has run, in which case 166 // mTaskQueue would be null. 167 // This can happen under two conditions: 168 // 1- Two Detach tasks were queued in a row due to a double cycle collection. 169 // 2- An call to ProcessTasks() had queued another run of ProcessTasks while 170 // a Detach task is pending. 171 // We handle these two cases by aborting early. 172 // A second Detach task was queued, prior the first one running, ignore it. 173 if (!mTaskQueue) { 174 RefPtr<SourceBufferTask> task = mQueue.Pop(); 175 if (!task) { 176 return; 177 } 178 MOZ_RELEASE_ASSERT(task->GetType() == SourceBufferTask::Type::Detach, 179 "only detach task could happen here!"); 180 MSE_DEBUG("Could not process the task '%s' after detached", 181 task->GetTypeName()); 182 return; 183 } 184 185 mTaskQueueCapability->AssertOnCurrentThread(); 186 typedef SourceBufferTask::Type Type; 187 188 if (mCurrentTask) { 189 // Already have a task pending. ProcessTask will be scheduled once the 190 // current task complete. 191 return; 192 } 193 RefPtr<SourceBufferTask> task = mQueue.Pop(); 194 if (!task) { 195 // nothing to do. 196 return; 197 } 198 199 MSE_DEBUG("Process task '%s'", task->GetTypeName()); 200 switch (task->GetType()) { 201 case Type::AppendBuffer: 202 mCurrentTask = task; 203 if (!mInputBuffer || mInputBuffer->IsEmpty()) { 204 // Note: we reset mInputBuffer here to ensure it doesn't grow unbounded. 205 mInputBuffer.reset(); 206 mInputBuffer = Some(MediaSpan(task->As<AppendBufferTask>()->mBuffer)); 207 } else { 208 // mInputBuffer wasn't empty, so we can't just reset it, but we move 209 // the data into a new buffer to clear out data no longer in the span. 210 MSE_DEBUG( 211 "mInputBuffer not empty during append -- data will be copied to " 212 "new buffer. mInputBuffer->Length()=%zu " 213 "mInputBuffer->Buffer()->Length()=%zu", 214 mInputBuffer->Length(), mInputBuffer->Buffer()->Length()); 215 const RefPtr<MediaByteBuffer> newBuffer{new MediaByteBuffer()}; 216 // Set capacity outside of ctor to let us explicitly handle OOM. 217 const size_t newCapacity = 218 mInputBuffer->Length() + 219 task->As<AppendBufferTask>()->mBuffer->Length(); 220 if (!newBuffer->SetCapacity(newCapacity, fallible)) { 221 RejectAppend(NS_ERROR_OUT_OF_MEMORY, __func__); 222 return; 223 } 224 // Use infallible appends as we've already set capacity above. 225 newBuffer->AppendElements(mInputBuffer->Elements(), 226 mInputBuffer->Length()); 227 newBuffer->AppendElements(*task->As<AppendBufferTask>()->mBuffer); 228 mInputBuffer = Some(MediaSpan(newBuffer)); 229 } 230 mSourceBufferAttributes = MakeUnique<SourceBufferAttributes>( 231 task->As<AppendBufferTask>()->mAttributes); 232 mAppendWindow = 233 Interval<double>(mSourceBufferAttributes->GetAppendWindowStart(), 234 mSourceBufferAttributes->GetAppendWindowEnd()); 235 ScheduleSegmentParserLoop(); 236 break; 237 case Type::RangeRemoval: { 238 bool rv = CodedFrameRemoval(task->As<RangeRemovalTask>()->mRange); 239 task->As<RangeRemovalTask>()->mPromise.Resolve(rv, __func__); 240 break; 241 } 242 case Type::EvictData: 243 DoEvictData(task->As<EvictDataTask>()->mPlaybackTime, 244 task->As<EvictDataTask>()->mSizeToEvict); 245 break; 246 case Type::Abort: 247 // not handled yet, and probably never. 248 break; 249 case Type::Reset: 250 CompleteResetParserState(); 251 break; 252 case Type::Detach: 253 mCurrentInputBuffer = nullptr; 254 MOZ_DIAGNOSTIC_ASSERT(mQueue.Length() == 0, 255 "Detach task must be the last"); 256 mVideoTracks.Reset(); 257 mAudioTracks.Reset(); 258 ShutdownDemuxers(); 259 ResetTaskQueue(); 260 return; 261 case Type::ChangeType: 262 MOZ_RELEASE_ASSERT(!mCurrentTask); 263 MSE_DEBUG("Processing type change from %s -> %s", 264 mType.OriginalString().get(), 265 task->As<ChangeTypeTask>()->mType.OriginalString().get()); 266 mType = task->As<ChangeTypeTask>()->mType; 267 mChangeTypeReceived = true; 268 mInitData = nullptr; 269 // A new input buffer will be created once we receive a new init segment. 270 // The first segment received after a changeType call must be an init 271 // segment. 272 mCurrentInputBuffer = nullptr; 273 CompleteResetParserState(); 274 break; 275 default: 276 NS_WARNING("Invalid Task"); 277 } 278 TaskQueueFromTaskQueue()->Dispatch( 279 NewRunnableMethod("TrackBuffersManager::ProcessTasks", this, 280 &TrackBuffersManager::ProcessTasks)); 281 } 282 283 // The MSE spec requires that we abort the current SegmentParserLoop 284 // which is then followed by a call to ResetParserState. 285 // However due to our asynchronous design this causes inherent difficulties. 286 // As the spec behaviour is non deterministic anyway, we instead process all 287 // pending frames found in the input buffer. 288 void TrackBuffersManager::AbortAppendData() { 289 MOZ_ASSERT(NS_IsMainThread()); 290 MSE_DEBUG(""); 291 292 QueueTask(new AbortTask()); 293 } 294 295 void TrackBuffersManager::ResetParserState( 296 SourceBufferAttributes& aAttributes) { 297 MOZ_ASSERT(NS_IsMainThread()); 298 MSE_DEBUG(""); 299 300 // Spec states: 301 // 1. If the append state equals PARSING_MEDIA_SEGMENT and the input buffer 302 // contains some complete coded frames, then run the coded frame processing 303 // algorithm until all of these complete coded frames have been processed. 304 // However, we will wait until all coded frames have been processed regardless 305 // of the value of append state. 306 QueueTask(new ResetTask()); 307 308 // ResetParserState has some synchronous steps that much be performed now. 309 // The remaining steps will be performed once the ResetTask gets executed. 310 311 // 6. If the mode attribute equals "sequence", then set the group start 312 // timestamp to the group end timestamp 313 if (aAttributes.GetAppendMode() == SourceBufferAppendMode::Sequence) { 314 aAttributes.SetGroupStartTimestamp(aAttributes.GetGroupEndTimestamp()); 315 } 316 // 8. Set append state to WAITING_FOR_SEGMENT. 317 aAttributes.SetAppendState(AppendState::WAITING_FOR_SEGMENT); 318 } 319 320 RefPtr<TrackBuffersManager::RangeRemovalPromise> 321 TrackBuffersManager::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) { 322 MOZ_ASSERT(NS_IsMainThread()); 323 MSE_DEBUG("From %.2f to %.2f", aStart.ToSeconds(), aEnd.ToSeconds()); 324 325 Reopen(); 326 327 return InvokeAsync(static_cast<AbstractThread*>(GetTaskQueueSafe().get()), 328 this, __func__, 329 &TrackBuffersManager::CodedFrameRemovalWithPromise, 330 TimeInterval(aStart, aEnd)); 331 } 332 333 TrackBuffersManager::EvictDataResult TrackBuffersManager::EvictData( 334 const TimeUnit& aPlaybackTime, int64_t aSize, TrackType aType) { 335 MOZ_ASSERT(NS_IsMainThread()); 336 337 if (aSize > EvictionThreshold(aType)) { 338 // We're adding more data than we can hold. 339 return EvictDataResult::BUFFER_FULL; 340 } 341 const int64_t toEvict = GetSize() + aSize - EvictionThreshold(aType); 342 343 const uint32_t canEvict = 344 Evictable(HasVideo() ? TrackInfo::kVideoTrack : TrackInfo::kAudioTrack); 345 346 MSE_DEBUG("currentTime=%" PRId64 " buffered=%" PRId64 347 "kB, eviction threshold=%" PRId64 348 "kB, " 349 "evict=%" PRId64 "kB canevict=%" PRIu32 "kB", 350 aPlaybackTime.ToMicroseconds(), GetSize() / 1024, 351 EvictionThreshold(aType) / 1024, toEvict / 1024, canEvict / 1024); 352 353 if (toEvict <= 0) { 354 mEvictionState = EvictionState::NO_EVICTION_NEEDED; 355 return EvictDataResult::NO_DATA_EVICTED; 356 } 357 358 EvictDataResult result; 359 360 if (mBufferFull && mEvictionState == EvictionState::EVICTION_COMPLETED && 361 canEvict < uint32_t(toEvict)) { 362 // Our buffer is currently full. We will make another eviction attempt. 363 // However, the current appendBuffer will fail as we can't know ahead of 364 // time if the eviction will later succeed. 365 result = EvictDataResult::BUFFER_FULL; 366 } else { 367 mEvictionState = EvictionState::EVICTION_NEEDED; 368 result = EvictDataResult::NO_DATA_EVICTED; 369 } 370 MSE_DEBUG("Reached our size limit, schedule eviction of %" PRId64 371 " bytes (%s)", 372 toEvict, 373 result == EvictDataResult::BUFFER_FULL ? "buffer full" 374 : "no data evicted"); 375 QueueTask(new EvictDataTask(aPlaybackTime, toEvict)); 376 377 return result; 378 } 379 380 void TrackBuffersManager::EvictDataWithoutSize(TrackType aType, 381 const media::TimeUnit& aTarget) { 382 MOZ_ASSERT(OnTaskQueue()); 383 auto& track = GetTracksData(aType); 384 const auto bufferedSz = track.mSizeBuffer; 385 const auto evictionSize = EvictionThreshold(aType); 386 const double watermarkRatio = bufferedSz / (double)(evictionSize); // can > 1 387 388 MSE_DEBUG( 389 "EvictDataWithoutSize, track=%s, buffered=%u" 390 "kB, eviction threshold=%" PRId64 "kB, wRatio=%f, target=%" PRId64 391 ", bufferedRange=%s", 392 TrackTypeToStr(aType), bufferedSz / 1024, evictionSize / 1024, 393 watermarkRatio, aTarget.ToMicroseconds(), 394 DumpTimeRanges(track.mBufferedRanges).get()); 395 396 // This type of eviction MUST only be used when the target is not in the 397 // current buffered range, which means all data are evictable. Otherwise, we 398 // have to calculate the amount of evictable data. 399 MOZ_ASSERT(track.mBufferedRanges.Find(aTarget) == TimeIntervals::NoIndex); 400 401 // This type of eviction is introduced to mitigate the pressure of the buffer 402 // nearly being full. If the buffer is still far from being full, we do 403 // nothing. 404 if (watermarkRatio < mEvictionBufferWatermarkRatio) { 405 return; 406 } 407 MSE_DEBUG("Queued EvictDataTask to evict size automatically"); 408 QueueTask(new EvictDataTask(aTarget)); 409 } 410 411 void TrackBuffersManager::ChangeType(const MediaContainerType& aType) { 412 MOZ_ASSERT(NS_IsMainThread()); 413 414 QueueTask(new ChangeTypeTask(aType)); 415 } 416 417 TimeIntervals TrackBuffersManager::Buffered() const { 418 MSE_DEBUG(""); 419 420 // http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered 421 422 MutexAutoLock mut(mMutex); 423 nsTArray<const TimeIntervals*> tracks; 424 if (HasVideo()) { 425 tracks.AppendElement(&mVideoBufferedRanges); 426 } 427 if (HasAudio()) { 428 tracks.AppendElement(&mAudioBufferedRanges); 429 } 430 431 // 2. Let highest end time be the largest track buffer ranges end time across 432 // all the track buffers managed by this SourceBuffer object. 433 TimeUnit highestEndTime = HighestEndTime(tracks); 434 435 // 3. Let intersection ranges equal a TimeRange object containing a single 436 // range from 0 to highest end time. 437 TimeIntervals intersection{ 438 TimeInterval(TimeUnit::FromSeconds(0), highestEndTime)}; 439 440 // 4. For each track buffer managed by this SourceBuffer, run the following 441 // steps: 442 // 1. Let track ranges equal the track buffer ranges for the current track 443 // buffer. 444 for (const TimeIntervals* trackRanges : tracks) { 445 // 2. If readyState is "ended", then set the end time on the last range in 446 // track ranges to highest end time. 447 // 3. Let new intersection ranges equal the intersection between the 448 // intersection ranges and the track ranges. 449 if (mEnded) { 450 TimeIntervals tR = *trackRanges; 451 tR.Add(TimeInterval(tR.GetEnd(), highestEndTime)); 452 intersection.Intersection(tR); 453 } else { 454 intersection.Intersection(*trackRanges); 455 } 456 } 457 return intersection; 458 } 459 460 int64_t TrackBuffersManager::GetSize() const { return mSizeSourceBuffer; } 461 462 void TrackBuffersManager::SetEnded( 463 const dom::Optional<dom::MediaSourceEndOfStreamError>& aError) { 464 MOZ_ASSERT(NS_IsMainThread()); 465 MutexAutoLock lock(mMutex); 466 if (!aError.WasPassed()) { 467 // error is not set. 468 // Notify the media element that it now has all of the media data. 469 // https://w3c.github.io/media-source/#dfn-end-of-stream 470 mHaveAllData = true; 471 } 472 mEnded = true; 473 } 474 475 void TrackBuffersManager::Reopen() { 476 MOZ_ASSERT(NS_IsMainThread()); 477 MutexAutoLock lock(mMutex); 478 mHaveAllData = false; 479 mEnded = false; 480 } 481 482 void TrackBuffersManager::Detach() { 483 MOZ_ASSERT(NS_IsMainThread()); 484 MSE_DEBUG(""); 485 QueueTask(new DetachTask()); 486 } 487 488 void TrackBuffersManager::CompleteResetParserState() { 489 mTaskQueueCapability->AssertOnCurrentThread(); 490 AUTO_PROFILER_LABEL("TrackBuffersManager::CompleteResetParserState", 491 MEDIA_PLAYBACK); 492 MSE_DEBUG(""); 493 494 // We shouldn't change mInputDemuxer while a demuxer init/reset request is 495 // being processed. See bug 1239983. 496 MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists(), 497 "Previous AppendBuffer didn't complete"); 498 499 for (auto& track : GetTracksList()) { 500 // 2. Unset the last decode timestamp on all track buffers. 501 // 3. Unset the last frame duration on all track buffers. 502 // 4. Unset the highest end timestamp on all track buffers. 503 // 5. Set the need random access point flag on all track buffers to true. 504 track->ResetAppendState(); 505 506 // if we have been aborted, we may have pending frames that we are going 507 // to discard now. 508 track->mQueuedSamples.Clear(); 509 } 510 511 // 7. Remove all bytes from the input buffer. 512 mPendingInputBuffer.reset(); 513 mInputBuffer.reset(); 514 if (mCurrentInputBuffer) { 515 mCurrentInputBuffer->EvictAll(); 516 // The demuxer will be recreated during the next run of SegmentParserLoop. 517 // As such we don't need to notify it that data has been removed. 518 mCurrentInputBuffer = new SourceBufferResource(); 519 } 520 521 // We could be left with a demuxer in an unusable state. It needs to be 522 // recreated. Unless we have a pending changeType operation, we store in the 523 // InputBuffer an init segment which will be parsed during the next Segment 524 // Parser Loop and a new demuxer will be created and initialized. 525 // If we are in the middle of a changeType operation, then we do not have an 526 // init segment yet. The next appendBuffer operation will need to provide such 527 // init segment. 528 if (mFirstInitializationSegmentReceived && !mChangeTypeReceived) { 529 MOZ_ASSERT(mInitData && mInitData->Length(), 530 "we must have an init segment"); 531 // The aim here is really to destroy our current demuxer. 532 CreateDemuxerforMIMEType(); 533 // Recreate our input buffer. We can't directly assign the initData buffer 534 // to mInputBuffer as it will get modified in the Segment Parser Loop. 535 mInputBuffer = Some(MediaSpan::WithCopyOf(mInitData)); 536 RecreateParser(true); 537 } else { 538 RecreateParser(false); 539 } 540 } 541 542 int64_t TrackBuffersManager::EvictionThreshold( 543 TrackInfo::TrackType aType) const { 544 MOZ_ASSERT(aType != TrackInfo::kTextTrack); 545 if (aType == TrackInfo::kVideoTrack || 546 (aType == TrackInfo::kUndefinedTrack && HasVideo())) { 547 return mVideoEvictionThreshold; 548 } 549 return mAudioEvictionThreshold; 550 } 551 552 void TrackBuffersManager::DoEvictData(const TimeUnit& aPlaybackTime, 553 Maybe<int64_t> aSizeToEvict) { 554 mTaskQueueCapability->AssertOnCurrentThread(); 555 AUTO_PROFILER_LABEL("TrackBuffersManager::DoEvictData", MEDIA_PLAYBACK); 556 MSE_DEBUG("DoEvictData, time=%" PRId64, aPlaybackTime.ToMicroseconds()); 557 558 mEvictionState = EvictionState::EVICTION_COMPLETED; 559 560 // Video is what takes the most space, only evict there if we have video. 561 auto& track = HasVideo() ? mVideoTracks : mAudioTracks; 562 const auto& buffer = track.GetTrackBuffer(); 563 if (buffer.IsEmpty()) { 564 // Buffer has been emptied while the eviction was queued, nothing to do. 565 return; 566 } 567 if (track.mBufferedRanges.IsEmpty()) { 568 MSE_DEBUG( 569 "DoEvictData running with no buffered ranges. 0 duration data likely " 570 "present in our buffer(s). Evicting all data!"); 571 // We have no buffered ranges, but may still have data. This happens if the 572 // buffer is full of 0 duration data. Normal removal procedures don't clear 573 // 0 duration data, so blow away all our data. 574 RemoveAllCodedFrames(); 575 return; 576 } 577 578 // The targeted playback time isn't in the buffered range yet, we're waiting 579 // for the data so all data in current buffered range are evictable. 580 if (!aSizeToEvict) { 581 // If the targeted time has been appended to our buffer range, then we need 582 // to recalculate which part of data is evictable. We will only perform 583 // following eviction when all data in the buffer range are evictable. 584 if (track.mBufferedRanges.Find(aPlaybackTime) != TimeIntervals::NoIndex) { 585 return; 586 } 587 // Evict data until the size falls below the threshold we set. 588 const int64_t sizeToEvict = 589 GetSize() - static_cast<int64_t>(EvictionThreshold() * 590 mEvictionBufferWatermarkRatio); 591 // Another eviction or frame removal has been executed before this task. 592 if (sizeToEvict <= 0) { 593 return; 594 } 595 int64_t toEvict = sizeToEvict; 596 597 // We need to evict data from a place which is the furthest from the 598 // playback time, otherwise we might incorrectly evict the data which should 599 // have stayed in the buffer in order to decode the frame at the targeted 600 // playback time. Eg. targeted time is X, and we might need a key frame from 601 // X-5. Therefore, we should evict data from a place which is far from X 602 // (maybe X±100) 603 const TimeUnit start = track.mBufferedRanges.GetStart(); 604 const TimeUnit end = track.mBufferedRanges.GetEnd(); 605 MSE_DEBUG("PlaybackTime=%" PRId64 ", extents=[%" PRId64 ", %" PRId64 "]", 606 aPlaybackTime.ToMicroseconds(), start.ToMicroseconds(), 607 end.ToMicroseconds()); 608 if (end - aPlaybackTime > aPlaybackTime - start) { 609 size_t evictedFramesStartIndex = buffer.Length(); 610 while (evictedFramesStartIndex > 0 && toEvict > 0) { 611 --evictedFramesStartIndex; 612 toEvict -= AssertedCast<int64_t>( 613 buffer[evictedFramesStartIndex]->ComputedSizeOfIncludingThis()); 614 } 615 MSE_DEBUG("Auto evicting %" PRId64 " bytes [%" PRId64 ", inf] from tail", 616 sizeToEvict - toEvict, 617 buffer[evictedFramesStartIndex]->mTime.ToMicroseconds()); 618 CodedFrameRemoval(TimeInterval(buffer[evictedFramesStartIndex]->mTime, 619 TimeUnit::FromInfinity())); 620 } else { 621 uint32_t lastKeyFrameIndex = 0; 622 int64_t partialEvict = 0; 623 for (uint32_t i = 0; i < buffer.Length(); i++) { 624 const auto& frame = buffer[i]; 625 if (frame->mKeyframe) { 626 lastKeyFrameIndex = i; 627 toEvict -= partialEvict; 628 if (toEvict <= 0) { 629 break; 630 } 631 partialEvict = 0; 632 } 633 partialEvict += 634 AssertedCast<int64_t>(frame->ComputedSizeOfIncludingThis()); 635 } 636 TimeUnit start = track.mBufferedRanges[0].mStart; 637 TimeUnit end = 638 buffer[lastKeyFrameIndex]->mTime - TimeUnit::FromMicroseconds(1); 639 MSE_DEBUG("Auto evicting %" PRId64 " bytes [%" PRId64 ", %" PRId64 640 "] from head", 641 sizeToEvict - toEvict, start.ToMicroseconds(), 642 end.ToMicroseconds()); 643 if (end > start) { 644 CodedFrameRemoval(TimeInterval(start, end)); 645 } 646 } 647 return; 648 } 649 650 // Remove any data we've already played, or before the next sample to be 651 // demuxed whichever is lowest. 652 TimeUnit lowerLimit = std::min(track.mNextSampleTime, aPlaybackTime); 653 uint32_t lastKeyFrameIndex = 0; 654 int64_t sizeToEvict = *aSizeToEvict; 655 int64_t toEvict = sizeToEvict; 656 int64_t partialEvict = 0; 657 for (uint32_t i = 0; i < buffer.Length(); i++) { 658 const auto& frame = buffer[i]; 659 if (frame->mKeyframe) { 660 lastKeyFrameIndex = i; 661 toEvict -= partialEvict; 662 if (toEvict <= 0) { 663 break; 664 } 665 partialEvict = 0; 666 } 667 if (frame->GetEndTime() >= lowerLimit) { 668 break; 669 } 670 partialEvict += AssertedCast<int64_t>(frame->ComputedSizeOfIncludingThis()); 671 } 672 673 const int64_t finalSize = mSizeSourceBuffer - sizeToEvict; 674 675 if (lastKeyFrameIndex > 0) { 676 MSE_DEBUG("Step1. Evicting %" PRId64 " bytes prior currentTime", 677 sizeToEvict - toEvict); 678 TimeUnit start = track.mBufferedRanges[0].mStart; 679 TimeUnit end = 680 buffer[lastKeyFrameIndex]->mTime - TimeUnit::FromMicroseconds(1); 681 if (end > start) { 682 CodedFrameRemoval(TimeInterval(start, end)); 683 } 684 } 685 686 if (mSizeSourceBuffer <= finalSize) { 687 MSE_DEBUG("Total buffer size is already smaller than final size"); 688 return; 689 } 690 691 toEvict = mSizeSourceBuffer - finalSize; 692 693 // See if we can evict data into the future. 694 // We do not evict data from the currently used buffered interval. 695 696 TimeUnit currentPosition = std::max(aPlaybackTime, track.mNextSampleTime); 697 TimeIntervals futureBuffered( 698 TimeInterval(currentPosition, TimeUnit::FromInfinity())); 699 futureBuffered.Intersection(track.mBufferedRanges); 700 futureBuffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); 701 if (futureBuffered.Length() <= 1) { 702 // We have one continuous segment ahead of us: 703 MSE_DEBUG("Nothing in future can be evicted"); 704 return; 705 } 706 707 // Don't evict before the end of the current segment 708 TimeUnit upperLimit = futureBuffered[0].mEnd; 709 uint32_t evictedFramesStartIndex = buffer.Length(); 710 for (uint32_t i = buffer.Length(); i-- > 0;) { 711 const auto& frame = buffer[i]; 712 if (frame->mTime <= upperLimit || toEvict <= 0) { 713 // We've reached a frame that shouldn't be evicted -> Evict after it -> 714 // i+1. Or the previous loop reached the eviction threshold -> Evict from 715 // it -> i+1. 716 evictedFramesStartIndex = i + 1; 717 break; 718 } 719 toEvict -= AssertedCast<int64_t>(frame->ComputedSizeOfIncludingThis()); 720 } 721 if (evictedFramesStartIndex < buffer.Length()) { 722 MSE_DEBUG("Step2. Evicting %" PRId64 " bytes from trailing data", 723 mSizeSourceBuffer - finalSize - toEvict); 724 CodedFrameRemoval(TimeInterval(buffer[evictedFramesStartIndex]->mTime, 725 TimeUnit::FromInfinity())); 726 } 727 } 728 729 RefPtr<TrackBuffersManager::RangeRemovalPromise> 730 TrackBuffersManager::CodedFrameRemovalWithPromise( 731 const TimeInterval& aInterval) { 732 mTaskQueueCapability->AssertOnCurrentThread(); 733 734 RefPtr<RangeRemovalTask> task = new RangeRemovalTask(aInterval); 735 RefPtr<RangeRemovalPromise> p = task->mPromise.Ensure(__func__); 736 QueueTask(task); 737 738 return p; 739 } 740 741 bool TrackBuffersManager::CodedFrameRemoval(const TimeInterval& aInterval) { 742 MOZ_ASSERT(OnTaskQueue()); 743 AUTO_PROFILER_LABEL("TrackBuffersManager::CodedFrameRemoval", MEDIA_PLAYBACK); 744 MSE_DEBUG("From %.2fs to %.2f", aInterval.mStart.ToSeconds(), 745 aInterval.mEnd.ToSeconds()); 746 747 #if DEBUG 748 if (HasVideo()) { 749 MSE_DEBUG("before video ranges=%s", 750 DumpTimeRangesRaw(mVideoTracks.mBufferedRanges).get()); 751 } 752 if (HasAudio()) { 753 MSE_DEBUG("before audio ranges=%s", 754 DumpTimeRangesRaw(mAudioTracks.mBufferedRanges).get()); 755 } 756 #endif 757 758 // 1. Let start be the starting presentation timestamp for the removal range. 759 TimeUnit start = aInterval.mStart; 760 // 2. Let end be the end presentation timestamp for the removal range. 761 TimeUnit end = aInterval.mEnd; 762 763 bool dataRemoved = false; 764 765 // 3. For each track buffer in this source buffer, run the following steps: 766 for (auto* track : GetTracksList()) { 767 MSE_DEBUGV("Processing %s track", track->mInfo->mMimeType.get()); 768 // 1. Let remove end timestamp be the current value of duration 769 // See bug: https://www.w3.org/Bugs/Public/show_bug.cgi?id=28727 770 // At worse we will remove all frames until the end, unless a key frame is 771 // found between the current interval's end and the trackbuffer's end. 772 TimeUnit removeEndTimestamp = track->mBufferedRanges.GetEnd(); 773 774 if (start > removeEndTimestamp) { 775 // Nothing to remove. 776 continue; 777 } 778 779 // 2. If this track buffer has a random access point timestamp that is 780 // greater than or equal to end, then update remove end timestamp to that 781 // random access point timestamp. 782 if (end < track->mBufferedRanges.GetEnd()) { 783 for (auto& frame : track->GetTrackBuffer()) { 784 if (frame->mKeyframe && frame->mTime >= end) { 785 removeEndTimestamp = frame->mTime; 786 break; 787 } 788 } 789 } 790 791 // 3. Remove all media data, from this track buffer, that contain starting 792 // timestamps greater than or equal to start and less than the remove end 793 // timestamp. 794 // 4. Remove decoding dependencies of the coded frames removed in the 795 // previous step: Remove all coded frames between the coded frames removed 796 // in the previous step and the next random access point after those removed 797 // frames. 798 TimeIntervals removedInterval{TimeInterval(start, removeEndTimestamp)}; 799 RemoveFrames(removedInterval, *track, 0, RemovalMode::kRemoveFrame); 800 801 // 5. If this object is in activeSourceBuffers, the current playback 802 // position is greater than or equal to start and less than the remove end 803 // timestamp, and HTMLMediaElement.readyState is greater than HAVE_METADATA, 804 // then set the HTMLMediaElement.readyState attribute to HAVE_METADATA and 805 // stall playback. This will be done by the MDSM during playback. 806 // TODO properly, so it works even if paused. 807 } 808 809 UpdateBufferedRanges(); 810 811 // Update our reported total size. 812 mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer; 813 814 // 4. If buffer full flag equals true and this object is ready to accept more 815 // bytes, then set the buffer full flag to false. 816 if (mBufferFull && mSizeSourceBuffer < EvictionThreshold()) { 817 mBufferFull = false; 818 } 819 820 return dataRemoved; 821 } 822 823 void TrackBuffersManager::RemoveAllCodedFrames() { 824 // This is similar to RemoveCodedFrames, but will attempt to remove ALL 825 // the frames. This is not to spec, as explained below at step 3.1. Steps 826 // below coincide with Remove Coded Frames algorithm from the spec. 827 MSE_DEBUG("RemoveAllCodedFrames called."); 828 MOZ_ASSERT(OnTaskQueue()); 829 AUTO_PROFILER_LABEL("TrackBuffersManager::RemoveAllCodedFrames", 830 MEDIA_PLAYBACK); 831 832 // 1. Let start be the starting presentation timestamp for the removal range. 833 TimeUnit start{}; 834 // 2. Let end be the end presentation timestamp for the removal range. 835 TimeUnit end = TimeUnit::FromMicroseconds(1); 836 // Find an end time such that our range will include every frame in every 837 // track. We do this by setting the end of our interval to the largest end 838 // time seen + 1 microsecond. 839 for (TrackData* track : GetTracksList()) { 840 for (auto& frame : track->GetTrackBuffer()) { 841 MOZ_ASSERT(frame->mTime >= start, 842 "Shouldn't have frame at negative time!"); 843 TimeUnit frameEnd = frame->mTime + frame->mDuration; 844 if (frameEnd > end) { 845 end = frameEnd + TimeUnit::FromMicroseconds(1); 846 } 847 } 848 } 849 850 // 3. For each track buffer in this source buffer, run the following steps: 851 TimeIntervals removedInterval{TimeInterval(start, end)}; 852 for (TrackData* track : GetTracksList()) { 853 // 1. Let remove end timestamp be the current value of duration 854 // ^ It's off spec, but we ignore this in order to clear 0 duration frames. 855 // If we don't ignore this rule and our buffer is full of 0 duration frames 856 // at timestamp n, we get an eviction range of [0, n). When we get to step 857 // 3.3 below, the 0 duration frames will not be evicted because their 858 // timestamp is not less than remove end timestamp -- it will in fact be 859 // equal to remove end timestamp. 860 // 861 // 2. If this track buffer has a random access point timestamp that is 862 // greater than or equal to end, then update remove end timestamp to that 863 // random access point timestamp. 864 // ^ We've made sure end > any sample's timestamp, so can skip this. 865 // 866 // 3. Remove all media data, from this track buffer, that contain starting 867 // timestamps greater than or equal to start and less than the remove end 868 // timestamp. 869 // 4. Remove decoding dependencies of the coded frames removed in the 870 // previous step: Remove all coded frames between the coded frames removed 871 // in the previous step and the next random access point after those removed 872 // frames. 873 874 // This should remove every frame in the track because removedInterval was 875 // constructed such that every frame in any track falls into that interval. 876 RemoveFrames(removedInterval, *track, 0, RemovalMode::kRemoveFrame); 877 878 // 5. If this object is in activeSourceBuffers, the current playback 879 // position is greater than or equal to start and less than the remove end 880 // timestamp, and HTMLMediaElement.readyState is greater than HAVE_METADATA, 881 // then set the HTMLMediaElement.readyState attribute to HAVE_METADATA and 882 // stall playback. This will be done by the MDSM during playback. 883 // TODO properly, so it works even if paused. 884 } 885 886 UpdateBufferedRanges(); 887 #ifdef DEBUG 888 { 889 MutexAutoLock lock(mMutex); 890 MOZ_ASSERT( 891 mAudioBufferedRanges.IsEmpty(), 892 "Should have no buffered video ranges after evicting everything."); 893 MOZ_ASSERT( 894 mVideoBufferedRanges.IsEmpty(), 895 "Should have no buffered video ranges after evicting everything."); 896 } 897 #endif 898 mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer; 899 MOZ_ASSERT(mSizeSourceBuffer == 0, 900 "Buffer should be empty after evicting everything!"); 901 if (mBufferFull && mSizeSourceBuffer < EvictionThreshold()) { 902 mBufferFull = false; 903 } 904 } 905 906 void TrackBuffersManager::UpdateBufferedRanges() { 907 MutexAutoLock mut(mMutex); 908 909 mVideoBufferedRanges = mVideoTracks.mSanitizedBufferedRanges; 910 mAudioBufferedRanges = mAudioTracks.mSanitizedBufferedRanges; 911 912 #if DEBUG 913 if (HasVideo()) { 914 MSE_DEBUG("after video ranges=%s", 915 DumpTimeRangesRaw(mVideoTracks.mBufferedRanges).get()); 916 } 917 if (HasAudio()) { 918 MSE_DEBUG("after audio ranges=%s", 919 DumpTimeRangesRaw(mAudioTracks.mBufferedRanges).get()); 920 } 921 #endif 922 if (profiler_thread_is_being_profiled_for_markers()) { 923 nsPrintfCString msg("buffered, "); 924 if (HasVideo()) { 925 msg += "video="_ns; 926 msg += DumpTimeRangesRaw(mVideoTracks.mBufferedRanges); 927 } 928 if (HasAudio()) { 929 msg += "audio="_ns; 930 msg += DumpTimeRangesRaw(mAudioTracks.mBufferedRanges); 931 } 932 PROFILER_MARKER_TEXT("UpdateBufferedRanges", MEDIA_PLAYBACK, {}, msg); 933 } 934 } 935 936 void TrackBuffersManager::SegmentParserLoop() { 937 MOZ_ASSERT(OnTaskQueue()); 938 AUTO_PROFILER_LABEL("TrackBuffersManager::SegmentParserLoop", MEDIA_PLAYBACK); 939 940 while (true) { 941 // 1. If the input buffer is empty, then jump to the need more data step 942 // below. 943 if (!mInputBuffer || mInputBuffer->IsEmpty()) { 944 NeedMoreData(); 945 return; 946 } 947 // 2. If the input buffer contains bytes that violate the SourceBuffer 948 // byte stream format specification, then run the append error algorithm 949 // with the decode error parameter set to true and abort this algorithm. 950 // TODO 951 952 // 3. Remove any bytes that the byte stream format specifications say must 953 // be ignored from the start of the input buffer. We do not remove bytes 954 // from our input buffer. Instead we enforce that our ContainerParser is 955 // able to skip over all data that is supposed to be ignored. 956 957 // 4. If the append state equals WAITING_FOR_SEGMENT, then run the following 958 // steps: 959 if (mSourceBufferAttributes->GetAppendState() == 960 AppendState::WAITING_FOR_SEGMENT) { 961 MediaResult haveInitSegment = 962 mParser->IsInitSegmentPresent(*mInputBuffer); 963 if (NS_SUCCEEDED(haveInitSegment)) { 964 SetAppendState(AppendState::PARSING_INIT_SEGMENT); 965 if (mFirstInitializationSegmentReceived && !mChangeTypeReceived) { 966 // This is a new initialization segment. Obsolete the old one. 967 RecreateParser(false); 968 } 969 continue; 970 } 971 MediaResult haveMediaSegment = 972 mParser->IsMediaSegmentPresent(*mInputBuffer); 973 if (NS_SUCCEEDED(haveMediaSegment)) { 974 SetAppendState(AppendState::PARSING_MEDIA_SEGMENT); 975 mNewMediaSegmentStarted = true; 976 continue; 977 } 978 // We have neither an init segment nor a media segment. 979 // Check if it was invalid data. 980 if (haveInitSegment != NS_ERROR_NOT_AVAILABLE) { 981 MSE_DEBUG("Found invalid data."); 982 RejectAppend(haveInitSegment, __func__); 983 return; 984 } 985 if (haveMediaSegment != NS_ERROR_NOT_AVAILABLE) { 986 MSE_DEBUG("Found invalid data."); 987 RejectAppend(haveMediaSegment, __func__); 988 return; 989 } 990 MSE_DEBUG("Found incomplete data."); 991 NeedMoreData(); 992 return; 993 } 994 995 MOZ_ASSERT(mSourceBufferAttributes->GetAppendState() == 996 AppendState::PARSING_INIT_SEGMENT || 997 mSourceBufferAttributes->GetAppendState() == 998 AppendState::PARSING_MEDIA_SEGMENT); 999 1000 TimeUnit start, end; 1001 MediaResult newData = NS_ERROR_NOT_AVAILABLE; 1002 1003 if (mSourceBufferAttributes->GetAppendState() == 1004 AppendState::PARSING_INIT_SEGMENT || 1005 (mSourceBufferAttributes->GetAppendState() == 1006 AppendState::PARSING_MEDIA_SEGMENT && 1007 mFirstInitializationSegmentReceived && !mChangeTypeReceived)) { 1008 newData = mParser->ParseStartAndEndTimestamps(*mInputBuffer, start, end); 1009 if (NS_FAILED(newData) && newData.Code() != NS_ERROR_NOT_AVAILABLE) { 1010 RejectAppend(newData, __func__); 1011 return; 1012 } 1013 mProcessedInput += mInputBuffer->Length(); 1014 } 1015 1016 // 5. If the append state equals PARSING_INIT_SEGMENT, then run the 1017 // following steps: 1018 if (mSourceBufferAttributes->GetAppendState() == 1019 AppendState::PARSING_INIT_SEGMENT) { 1020 if (mParser->InitSegmentRange().IsEmpty()) { 1021 mInputBuffer.reset(); 1022 NeedMoreData(); 1023 return; 1024 } 1025 InitializationSegmentReceived(); 1026 return; 1027 } 1028 if (mSourceBufferAttributes->GetAppendState() == 1029 AppendState::PARSING_MEDIA_SEGMENT) { 1030 // 1. If the first initialization segment received flag is false, then run 1031 // the append error algorithm with the decode error parameter set to 1032 // true and abort this algorithm. 1033 // Or we are in the process of changeType, in which case we must first 1034 // get an init segment before getting a media segment. 1035 if (!mFirstInitializationSegmentReceived || mChangeTypeReceived) { 1036 RejectAppend(NS_ERROR_FAILURE, __func__); 1037 return; 1038 } 1039 1040 // We can't feed some demuxers (WebMDemuxer) with data that do not have 1041 // monotonizally increasing timestamps. So we check if we have a 1042 // discontinuity from the previous segment parsed. 1043 // If so, recreate a new demuxer to ensure that the demuxer is only fed 1044 // monotonically increasing data. 1045 if (mNewMediaSegmentStarted) { 1046 if (NS_SUCCEEDED(newData) && mLastParsedEndTime.isSome() && 1047 start < mLastParsedEndTime.ref()) { 1048 nsPrintfCString msg( 1049 "Re-creating demuxer, new start (%" PRId64 1050 ") is smaller than last parsed end time (%" PRId64 ")", 1051 start.ToMicroseconds(), mLastParsedEndTime->ToMicroseconds()); 1052 if (profiler_thread_is_being_profiled_for_markers()) { 1053 PROFILER_MARKER_TEXT("Re-create demuxer", MEDIA_PLAYBACK, {}, msg); 1054 } 1055 MSE_DEBUG("%s", msg.get()); 1056 mFrameEndTimeBeforeRecreateDemuxer = Some(end); 1057 ResetDemuxingState(); 1058 return; 1059 } 1060 if (NS_SUCCEEDED(newData) || !mParser->MediaSegmentRange().IsEmpty()) { 1061 if (mPendingInputBuffer) { 1062 // We now have a complete media segment header. We can resume 1063 // parsing the data. 1064 AppendDataToCurrentInputBuffer(*mPendingInputBuffer); 1065 mPendingInputBuffer.reset(); 1066 } 1067 mNewMediaSegmentStarted = false; 1068 } else { 1069 // We don't have any data to demux yet, stash aside the data. 1070 // This also handles the case: 1071 // 2. If the input buffer does not contain a complete media segment 1072 // header yet, then jump to the need more data step below. 1073 if (!mPendingInputBuffer) { 1074 mPendingInputBuffer = Some(MediaSpan(*mInputBuffer)); 1075 } else { 1076 // Note we reset mInputBuffer below, so this won't end up appending 1077 // the contents of mInputBuffer to itself. 1078 mPendingInputBuffer->Append(*mInputBuffer); 1079 } 1080 1081 mInputBuffer.reset(); 1082 NeedMoreData(); 1083 return; 1084 } 1085 } 1086 1087 // 3. If the input buffer contains one or more complete coded frames, then 1088 // run the coded frame processing algorithm. 1089 RefPtr<TrackBuffersManager> self = this; 1090 CodedFrameProcessing() 1091 ->Then( 1092 TaskQueueFromTaskQueue(), __func__, 1093 [self](bool aNeedMoreData) { 1094 self->mTaskQueueCapability->AssertOnCurrentThread(); 1095 self->mProcessingRequest.Complete(); 1096 if (aNeedMoreData) { 1097 self->NeedMoreData(); 1098 } else { 1099 self->ScheduleSegmentParserLoop(); 1100 } 1101 }, 1102 [self](const MediaResult& aRejectValue) { 1103 self->mTaskQueueCapability->AssertOnCurrentThread(); 1104 self->mProcessingRequest.Complete(); 1105 self->RejectAppend(aRejectValue, __func__); 1106 }) 1107 ->Track(mProcessingRequest); 1108 return; 1109 } 1110 } 1111 } 1112 1113 void TrackBuffersManager::NeedMoreData() { 1114 MSE_DEBUG(""); 1115 MOZ_DIAGNOSTIC_ASSERT(mCurrentTask && 1116 mCurrentTask->GetType() == 1117 SourceBufferTask::Type::AppendBuffer); 1118 MOZ_DIAGNOSTIC_ASSERT(mSourceBufferAttributes); 1119 1120 mCurrentTask->As<AppendBufferTask>()->mPromise.Resolve( 1121 SourceBufferTask::AppendBufferResult(mActiveTrack, 1122 *mSourceBufferAttributes), 1123 __func__); 1124 mSourceBufferAttributes = nullptr; 1125 mCurrentTask = nullptr; 1126 ProcessTasks(); 1127 } 1128 1129 void TrackBuffersManager::RejectAppend(const MediaResult& aRejectValue, 1130 const char* aName) { 1131 MSE_DEBUG("rv=%" PRIu32, static_cast<uint32_t>(aRejectValue.Code())); 1132 MOZ_DIAGNOSTIC_ASSERT(mCurrentTask && 1133 mCurrentTask->GetType() == 1134 SourceBufferTask::Type::AppendBuffer); 1135 1136 mCurrentTask->As<AppendBufferTask>()->mPromise.Reject(aRejectValue, __func__); 1137 mSourceBufferAttributes = nullptr; 1138 mCurrentTask = nullptr; 1139 ProcessTasks(); 1140 } 1141 1142 void TrackBuffersManager::ScheduleSegmentParserLoop() { 1143 MOZ_ASSERT(OnTaskQueue()); 1144 TaskQueueFromTaskQueue()->Dispatch( 1145 NewRunnableMethod("TrackBuffersManager::SegmentParserLoop", this, 1146 &TrackBuffersManager::SegmentParserLoop)); 1147 } 1148 1149 void TrackBuffersManager::ShutdownDemuxers() { 1150 if (profiler_thread_is_being_profiled_for_markers()) { 1151 PROFILER_MARKER_UNTYPED("ShutdownDemuxers", MEDIA_PLAYBACK); 1152 } 1153 if (mVideoTracks.mDemuxer) { 1154 mVideoTracks.mDemuxer->BreakCycles(); 1155 mVideoTracks.mDemuxer = nullptr; 1156 } 1157 if (mAudioTracks.mDemuxer) { 1158 mAudioTracks.mDemuxer->BreakCycles(); 1159 mAudioTracks.mDemuxer = nullptr; 1160 } 1161 // We shouldn't change mInputDemuxer while a demuxer init/reset request is 1162 // being processed. See bug 1239983. 1163 MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists()); 1164 mInputDemuxer = nullptr; 1165 mLastParsedEndTime.reset(); 1166 } 1167 1168 void TrackBuffersManager::CreateDemuxerforMIMEType() { 1169 mTaskQueueCapability->AssertOnCurrentThread(); 1170 MSE_DEBUG("mType.OriginalString=%s", mType.OriginalString().get()); 1171 ShutdownDemuxers(); 1172 1173 if (mType.Type() == MEDIAMIMETYPE(VIDEO_WEBM) || 1174 mType.Type() == MEDIAMIMETYPE(AUDIO_WEBM)) { 1175 if (mFrameEndTimeBeforeRecreateDemuxer) { 1176 MSE_DEBUG( 1177 "CreateDemuxerFromMimeType: " 1178 "mFrameEndTimeBeforeRecreateDemuxer=%" PRId64, 1179 mFrameEndTimeBeforeRecreateDemuxer->ToMicroseconds()); 1180 } 1181 mInputDemuxer = new WebMDemuxer(mCurrentInputBuffer, true, 1182 mFrameEndTimeBeforeRecreateDemuxer); 1183 mFrameEndTimeBeforeRecreateDemuxer.reset(); 1184 DDLINKCHILD("demuxer", mInputDemuxer.get()); 1185 return; 1186 } 1187 1188 if (mType.Type() == MEDIAMIMETYPE(VIDEO_MP4) || 1189 mType.Type() == MEDIAMIMETYPE(AUDIO_MP4)) { 1190 mInputDemuxer = new MP4Demuxer(mCurrentInputBuffer); 1191 mFrameEndTimeBeforeRecreateDemuxer.reset(); 1192 DDLINKCHILD("demuxer", mInputDemuxer.get()); 1193 return; 1194 } 1195 NS_WARNING("Not supported (yet)"); 1196 } 1197 1198 // We reset the demuxer by creating a new one and initializing it. 1199 void TrackBuffersManager::ResetDemuxingState() { 1200 MOZ_ASSERT(OnTaskQueue()); 1201 MOZ_ASSERT(mParser && mParser->HasInitData()); 1202 AUTO_PROFILER_LABEL("TrackBuffersManager::ResetDemuxingState", 1203 MEDIA_PLAYBACK); 1204 if (profiler_thread_is_being_profiled_for_markers()) { 1205 PROFILER_MARKER_UNTYPED("ResetDemuxingState", MEDIA_PLAYBACK); 1206 } 1207 1208 RecreateParser(true); 1209 mCurrentInputBuffer = new SourceBufferResource(); 1210 // The demuxer isn't initialized yet ; we don't want to notify it 1211 // that data has been appended yet ; so we simply append the init segment 1212 // to the resource. 1213 mCurrentInputBuffer->AppendData(mParser->InitData()); 1214 CreateDemuxerforMIMEType(); 1215 if (!mInputDemuxer) { 1216 RejectAppend(NS_ERROR_FAILURE, __func__); 1217 return; 1218 } 1219 mInputDemuxer->Init() 1220 ->Then(TaskQueueFromTaskQueue(), __func__, this, 1221 &TrackBuffersManager::OnDemuxerResetDone, 1222 &TrackBuffersManager::OnDemuxerInitFailed) 1223 ->Track(mDemuxerInitRequest); 1224 } 1225 1226 void TrackBuffersManager::OnDemuxerResetDone(const MediaResult& aResult) { 1227 MOZ_ASSERT(OnTaskQueue()); 1228 mDemuxerInitRequest.Complete(); 1229 1230 if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) { 1231 RejectAppend(aResult, __func__); 1232 return; 1233 } 1234 1235 // mInputDemuxer shouldn't have been destroyed while a demuxer init/reset 1236 // request was being processed. See bug 1239983. 1237 MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer); 1238 1239 if (aResult != NS_OK && mParentDecoder) { 1240 RefPtr<TrackBuffersManager> self = this; 1241 mAbstractMainThread->Dispatch(NS_NewRunnableFunction( 1242 "TrackBuffersManager::OnDemuxerResetDone", [self, aResult]() { 1243 if (self->mParentDecoder && self->mParentDecoder->GetOwner()) { 1244 self->mParentDecoder->GetOwner()->DecodeWarning(aResult); 1245 } 1246 })); 1247 } 1248 1249 // Recreate track demuxers. 1250 uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack); 1251 if (numVideos) { 1252 // We currently only handle the first video track. 1253 mVideoTracks.mDemuxer = 1254 mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); 1255 MOZ_ASSERT(mVideoTracks.mDemuxer); 1256 DDLINKCHILD("video demuxer", mVideoTracks.mDemuxer.get()); 1257 } 1258 1259 uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack); 1260 if (numAudios) { 1261 // We currently only handle the first audio track. 1262 mAudioTracks.mDemuxer = 1263 mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); 1264 MOZ_ASSERT(mAudioTracks.mDemuxer); 1265 DDLINKCHILD("audio demuxer", mAudioTracks.mDemuxer.get()); 1266 } 1267 1268 if (mPendingInputBuffer) { 1269 // We had a partial media segment header stashed aside. 1270 // Reparse its content so we can continue parsing the current input buffer. 1271 TimeUnit start, end; 1272 mParser->ParseStartAndEndTimestamps(*mPendingInputBuffer, start, end); 1273 mProcessedInput += mPendingInputBuffer->Length(); 1274 } 1275 1276 SegmentParserLoop(); 1277 } 1278 1279 void TrackBuffersManager::AppendDataToCurrentInputBuffer( 1280 const MediaSpan& aData) { 1281 MOZ_ASSERT(mCurrentInputBuffer); 1282 mCurrentInputBuffer->AppendData(aData); 1283 mInputDemuxer->NotifyDataArrived(); 1284 } 1285 1286 void TrackBuffersManager::InitializationSegmentReceived() { 1287 MOZ_ASSERT(OnTaskQueue()); 1288 MOZ_ASSERT(mParser->HasCompleteInitData()); 1289 AUTO_PROFILER_LABEL("TrackBuffersManager::InitializationSegmentReceived", 1290 MEDIA_PLAYBACK); 1291 if (profiler_thread_is_being_profiled_for_markers()) { 1292 PROFILER_MARKER_UNTYPED("InitializationSegmentReceived", MEDIA_PLAYBACK); 1293 } 1294 1295 int64_t endInit = mParser->InitSegmentRange().mEnd; 1296 if (mInputBuffer->Length() > mProcessedInput || 1297 int64_t(mProcessedInput - mInputBuffer->Length()) > endInit) { 1298 // Something is not quite right with the data appended. Refuse it. 1299 RejectAppend(MediaResult(NS_ERROR_FAILURE, 1300 "Invalid state following initialization segment"), 1301 __func__); 1302 return; 1303 } 1304 1305 mCurrentInputBuffer = new SourceBufferResource(); 1306 // The demuxer isn't initialized yet ; we don't want to notify it 1307 // that data has been appended yet ; so we simply append the init segment 1308 // to the resource. 1309 mCurrentInputBuffer->AppendData(mParser->InitData()); 1310 uint32_t length = endInit - (mProcessedInput - mInputBuffer->Length()); 1311 MOZ_RELEASE_ASSERT(length <= mInputBuffer->Length()); 1312 mInputBuffer->RemoveFront(length); 1313 CreateDemuxerforMIMEType(); 1314 if (!mInputDemuxer) { 1315 NS_WARNING("TODO type not supported"); 1316 RejectAppend(NS_ERROR_DOM_NOT_SUPPORTED_ERR, __func__); 1317 return; 1318 } 1319 mInputDemuxer->Init() 1320 ->Then(TaskQueueFromTaskQueue(), __func__, this, 1321 &TrackBuffersManager::OnDemuxerInitDone, 1322 &TrackBuffersManager::OnDemuxerInitFailed) 1323 ->Track(mDemuxerInitRequest); 1324 } 1325 1326 bool TrackBuffersManager::IsRepeatInitData( 1327 const MediaInfo& aNewMediaInfo) const { 1328 MOZ_ASSERT(OnTaskQueue()); 1329 if (!mInitData) { 1330 // There is no previous init data, so this cannot be a repeat. 1331 return false; 1332 } 1333 1334 if (mChangeTypeReceived) { 1335 // If we're received change type we want to reprocess init data. 1336 return false; 1337 } 1338 1339 MOZ_DIAGNOSTIC_ASSERT(mInitData, "Init data should be non-null"); 1340 if (*mInitData == *mParser->InitData()) { 1341 // We have previous init data, and it's the same binary data as we've just 1342 // parsed. 1343 return true; 1344 } 1345 1346 // At this point the binary data doesn't match, but it's possible to have the 1347 // different binary representations for the same logical init data. These 1348 // checks can be revised as we encounter such cases in the wild. 1349 1350 bool audioInfoIsRepeat = false; 1351 if (aNewMediaInfo.HasAudio()) { 1352 if (!mAudioTracks.mLastInfo) { 1353 // There is no old audio info, so this can't be a repeat. 1354 return false; 1355 } 1356 audioInfoIsRepeat = 1357 *mAudioTracks.mLastInfo->GetAsAudioInfo() == aNewMediaInfo.mAudio; 1358 if (!aNewMediaInfo.HasVideo()) { 1359 // Only have audio. 1360 return audioInfoIsRepeat; 1361 } 1362 } 1363 1364 bool videoInfoIsRepeat = false; 1365 if (aNewMediaInfo.HasVideo()) { 1366 if (!mVideoTracks.mLastInfo) { 1367 // There is no old video info, so this can't be a repeat. 1368 return false; 1369 } 1370 videoInfoIsRepeat = 1371 *mVideoTracks.mLastInfo->GetAsVideoInfo() == aNewMediaInfo.mVideo; 1372 if (!aNewMediaInfo.HasAudio()) { 1373 // Only have video. 1374 return videoInfoIsRepeat; 1375 } 1376 } 1377 1378 if (audioInfoIsRepeat && videoInfoIsRepeat) { 1379 MOZ_DIAGNOSTIC_ASSERT( 1380 aNewMediaInfo.HasVideo() && aNewMediaInfo.HasAudio(), 1381 "This should only be reachable if audio and video are present"); 1382 // Video + audio are present and both have the same init data. 1383 return true; 1384 } 1385 1386 return false; 1387 } 1388 1389 void TrackBuffersManager::OnDemuxerInitDone(const MediaResult& aResult) { 1390 mTaskQueueCapability->AssertOnCurrentThread(); 1391 MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer, "mInputDemuxer has been destroyed"); 1392 AUTO_PROFILER_LABEL("TrackBuffersManager::OnDemuxerInitDone", MEDIA_PLAYBACK); 1393 1394 mDemuxerInitRequest.Complete(); 1395 1396 if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) { 1397 RejectAppend(aResult, __func__); 1398 return; 1399 } 1400 1401 MediaInfo info; 1402 1403 uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack); 1404 if (numVideos) { 1405 // We currently only handle the first video track. 1406 mVideoTracks.mDemuxer = 1407 mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); 1408 MOZ_ASSERT(mVideoTracks.mDemuxer); 1409 DDLINKCHILD("video demuxer", mVideoTracks.mDemuxer.get()); 1410 info.mVideo = *mVideoTracks.mDemuxer->GetInfo()->GetAsVideoInfo(); 1411 info.mVideo.mTrackId = 2; 1412 } 1413 1414 uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack); 1415 if (numAudios) { 1416 // We currently only handle the first audio track. 1417 mAudioTracks.mDemuxer = 1418 mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); 1419 MOZ_ASSERT(mAudioTracks.mDemuxer); 1420 DDLINKCHILD("audio demuxer", mAudioTracks.mDemuxer.get()); 1421 info.mAudio = *mAudioTracks.mDemuxer->GetInfo()->GetAsAudioInfo(); 1422 info.mAudio.mTrackId = 1; 1423 } 1424 1425 TimeUnit videoDuration = numVideos ? info.mVideo.mDuration : TimeUnit::Zero(); 1426 TimeUnit audioDuration = numAudios ? info.mAudio.mDuration : TimeUnit::Zero(); 1427 1428 TimeUnit duration = std::max(videoDuration, audioDuration); 1429 // 1. Update the duration attribute if it currently equals NaN. 1430 // Those steps are performed by the MediaSourceDecoder::SetInitialDuration 1431 mAbstractMainThread->Dispatch(NewRunnableMethod<TimeUnit>( 1432 "MediaSourceDecoder::SetInitialDuration", mParentDecoder.get(), 1433 &MediaSourceDecoder::SetInitialDuration, 1434 !duration.IsZero() ? duration : TimeUnit::FromInfinity())); 1435 1436 // 2. If the initialization segment has no audio, video, or text tracks, then 1437 // run the append error algorithm with the decode error parameter set to true 1438 // and abort these steps. 1439 if (!numVideos && !numAudios) { 1440 RejectAppend(NS_ERROR_FAILURE, __func__); 1441 return; 1442 } 1443 1444 // 3. If the first initialization segment received flag is true, then run the 1445 // following steps: 1446 if (mFirstInitializationSegmentReceived) { 1447 if (numVideos != mVideoTracks.mNumTracks || 1448 numAudios != mAudioTracks.mNumTracks) { 1449 RejectAppend(NS_ERROR_FAILURE, __func__); 1450 return; 1451 } 1452 // 1. If more than one track for a single type are present (ie 2 audio 1453 // tracks), then the Track IDs match the ones in the first initialization 1454 // segment. 1455 // TODO 1456 // 2. Add the appropriate track descriptions from this initialization 1457 // segment to each of the track buffers. 1458 // TODO 1459 // 3. Set the need random access point flag on all track buffers to true. 1460 mVideoTracks.mNeedRandomAccessPoint = true; 1461 mAudioTracks.mNeedRandomAccessPoint = true; 1462 } 1463 1464 // Check if we've received the same init data again. Some streams will 1465 // resend the same data. In these cases we don't need to change the stream 1466 // id as it's the same stream. Doing so would recreate decoders, possibly 1467 // leading to gaps in audio and/or video (see bug 1450952). 1468 bool isRepeatInitData = IsRepeatInitData(info); 1469 1470 MOZ_ASSERT(mFirstInitializationSegmentReceived || !isRepeatInitData, 1471 "Should never detect repeat init data for first segment!"); 1472 1473 // If we have new init data we configure and set track info as needed. If we 1474 // have repeat init data we carry forward our existing track info. 1475 if (!isRepeatInitData) { 1476 // Increase our stream id. 1477 uint32_t streamID = sStreamSourceID++; 1478 1479 // 4. Let active track flag equal false. 1480 bool activeTrack = false; 1481 1482 // 5. If the first initialization segment received flag is false, then run 1483 // the following steps: 1484 if (!mFirstInitializationSegmentReceived) { 1485 MSE_DEBUG("Get first init data"); 1486 mAudioTracks.mNumTracks = numAudios; 1487 // TODO: 1488 // 1. If the initialization segment contains tracks with codecs the user 1489 // agent does not support, then run the append error algorithm with the 1490 // decode error parameter set to true and abort these steps. 1491 1492 // 2. For each audio track in the initialization segment, run following 1493 // steps: for (uint32_t i = 0; i < numAudios; i++) { 1494 if (numAudios) { 1495 // 1. Let audio byte stream track ID be the Track ID for the current 1496 // track being processed. 1497 // 2. Let audio language be a BCP 47 language tag for the language 1498 // specified in the initialization segment for this track or an empty 1499 // string if no language info is present. 1500 // 3. If audio language equals an empty string or the 'und' BCP 47 1501 // value, then run the default track language algorithm with 1502 // byteStreamTrackID set to audio byte stream track ID and type set to 1503 // "audio" and assign the value returned by the algorithm to audio 1504 // language. 1505 // 4. Let audio label be a label specified in the initialization segment 1506 // for this track or an empty string if no label info is present. 1507 // 5. If audio label equals an empty string, then run the default track 1508 // label algorithm with byteStreamTrackID set to audio byte stream track 1509 // ID and type set to "audio" and assign the value returned by the 1510 // algorithm to audio label. 1511 // 6. Let audio kinds be an array of kind strings specified in the 1512 // initialization segment for this track or an empty array if no kind 1513 // information is provided. 1514 // 7. If audio kinds equals an empty array, then run the default track 1515 // kinds algorithm with byteStreamTrackID set to audio byte stream track 1516 // ID and type set to "audio" and assign the value returned by the 1517 // algorithm to audio kinds. 1518 // 8. For each value in audio kinds, run the following steps: 1519 // 1. Let current audio kind equal the value from audio kinds for this 1520 // iteration of the loop. 1521 // 2. Let new audio track be a new AudioTrack object. 1522 // 3. Generate a unique ID and assign it to the id property on new 1523 // audio track. 1524 // 4. Assign audio language to the language property on new audio 1525 // track. 1526 // 5. Assign audio label to the label property on new audio track. 1527 // 6. Assign current audio kind to the kind property on new audio 1528 // track. 1529 // 7. If audioTracks.length equals 0, then run the following steps: 1530 // 1. Set the enabled property on new audio track to true. 1531 // 2. Set active track flag to true. 1532 activeTrack = true; 1533 // 8. Add new audio track to the audioTracks attribute on this 1534 // SourceBuffer object. 1535 // 9. Queue a task to fire a trusted event named addtrack, that does 1536 // not bubble and is not cancelable, and that uses the TrackEvent 1537 // interface, at the AudioTrackList object referenced by the 1538 // audioTracks attribute on this SourceBuffer object. 1539 // 10. Add new audio track to the audioTracks attribute on the 1540 // HTMLMediaElement. 1541 // 11. Queue a task to fire a trusted event named addtrack, that does 1542 // not bubble and is not cancelable, and that uses the TrackEvent 1543 // interface, at the AudioTrackList object referenced by the 1544 // audioTracks attribute on the HTMLMediaElement. 1545 mAudioTracks.mBuffers.AppendElement(TrackBuffer()); 1546 // 10. Add the track description for this track to the track buffer. 1547 mAudioTracks.mInfo = new TrackInfoSharedPtr(info.mAudio, streamID); 1548 mAudioTracks.mLastInfo = mAudioTracks.mInfo; 1549 } 1550 1551 mVideoTracks.mNumTracks = numVideos; 1552 // 3. For each video track in the initialization segment, run following 1553 // steps: for (uint32_t i = 0; i < numVideos; i++) { 1554 if (numVideos) { 1555 // 1. Let video byte stream track ID be the Track ID for the current 1556 // track being processed. 1557 // 2. Let video language be a BCP 47 language tag for the language 1558 // specified in the initialization segment for this track or an empty 1559 // string if no language info is present. 1560 // 3. If video language equals an empty string or the 'und' BCP 47 1561 // value, then run the default track language algorithm with 1562 // byteStreamTrackID set to video byte stream track ID and type set to 1563 // "video" and assign the value returned by the algorithm to video 1564 // language. 1565 // 4. Let video label be a label specified in the initialization segment 1566 // for this track or an empty string if no label info is present. 1567 // 5. If video label equals an empty string, then run the default track 1568 // label algorithm with byteStreamTrackID set to video byte stream track 1569 // ID and type set to "video" and assign the value returned by the 1570 // algorithm to video label. 1571 // 6. Let video kinds be an array of kind strings specified in the 1572 // initialization segment for this track or an empty array if no kind 1573 // information is provided. 1574 // 7. If video kinds equals an empty array, then run the default track 1575 // kinds algorithm with byteStreamTrackID set to video byte stream track 1576 // ID and type set to "video" and assign the value returned by the 1577 // algorithm to video kinds. 1578 // 8. For each value in video kinds, run the following steps: 1579 // 1. Let current video kind equal the value from video kinds for this 1580 // iteration of the loop. 1581 // 2. Let new video track be a new VideoTrack object. 1582 // 3. Generate a unique ID and assign it to the id property on new 1583 // video track. 1584 // 4. Assign video language to the language property on new video 1585 // track. 1586 // 5. Assign video label to the label property on new video track. 1587 // 6. Assign current video kind to the kind property on new video 1588 // track. 1589 // 7. If videoTracks.length equals 0, then run the following steps: 1590 // 1. Set the selected property on new video track to true. 1591 // 2. Set active track flag to true. 1592 activeTrack = true; 1593 // 8. Add new video track to the videoTracks attribute on this 1594 // SourceBuffer object. 1595 // 9. Queue a task to fire a trusted event named addtrack, that does 1596 // not bubble and is not cancelable, and that uses the TrackEvent 1597 // interface, at the VideoTrackList object referenced by the 1598 // videoTracks attribute on this SourceBuffer object. 1599 // 10. Add new video track to the videoTracks attribute on the 1600 // HTMLMediaElement. 1601 // 11. Queue a task to fire a trusted event named addtrack, that does 1602 // not bubble and is not cancelable, and that uses the TrackEvent 1603 // interface, at the VideoTrackList object referenced by the 1604 // videoTracks attribute on the HTMLMediaElement. 1605 mVideoTracks.mBuffers.AppendElement(TrackBuffer()); 1606 // 10. Add the track description for this track to the track buffer. 1607 mVideoTracks.mInfo = new TrackInfoSharedPtr(info.mVideo, streamID); 1608 mVideoTracks.mLastInfo = mVideoTracks.mInfo; 1609 } 1610 // 4. For each text track in the initialization segment, run following 1611 // steps: 1612 // 5. If active track flag equals true, then run the following steps: 1613 // This is handled by SourceBuffer once the promise is resolved. 1614 if (activeTrack) { 1615 mActiveTrack = true; 1616 } 1617 1618 // 6. Set first initialization segment received flag to true. 1619 mFirstInitializationSegmentReceived = true; 1620 } else { 1621 MSE_DEBUG("Get new init data"); 1622 mAudioTracks.mLastInfo = new TrackInfoSharedPtr(info.mAudio, streamID); 1623 mVideoTracks.mLastInfo = new TrackInfoSharedPtr(info.mVideo, streamID); 1624 } 1625 1626 UniquePtr<EncryptionInfo> crypto = mInputDemuxer->GetCrypto(); 1627 if (crypto && crypto->IsEncrypted()) { 1628 // Try and dispatch 'encrypted'. Won't go if ready state still 1629 // HAVE_NOTHING. 1630 for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) { 1631 nsCOMPtr<nsIRunnable> r = new DispatchKeyNeededEvent( 1632 mParentDecoder, crypto->mInitDatas[i].mInitData, 1633 crypto->mInitDatas[i].mType); 1634 mAbstractMainThread->Dispatch(r.forget()); 1635 } 1636 info.mCrypto = *crypto; 1637 // We clear our crypto init data array, so the MediaFormatReader will 1638 // not emit an encrypted event for the same init data again. 1639 info.mCrypto.mInitDatas.Clear(); 1640 } 1641 1642 { 1643 MutexAutoLock mut(mMutex); 1644 mInfo = info; 1645 } 1646 } 1647 // We now have a valid init data ; we can store it for later use. 1648 mInitData = mParser->InitData(); 1649 1650 // We have now completed the changeType operation. 1651 mChangeTypeReceived = false; 1652 1653 // 3. Remove the initialization segment bytes from the beginning of the input 1654 // buffer. This step has already been done in InitializationSegmentReceived 1655 // when we transferred the content into mCurrentInputBuffer. 1656 mCurrentInputBuffer->EvictAll(); 1657 mInputDemuxer->NotifyDataRemoved(); 1658 RecreateParser(true); 1659 1660 // 4. Set append state to WAITING_FOR_SEGMENT. 1661 SetAppendState(AppendState::WAITING_FOR_SEGMENT); 1662 // 5. Jump to the loop top step above. 1663 ScheduleSegmentParserLoop(); 1664 1665 if (aResult != NS_OK && mParentDecoder) { 1666 RefPtr<TrackBuffersManager> self = this; 1667 mAbstractMainThread->Dispatch(NS_NewRunnableFunction( 1668 "TrackBuffersManager::OnDemuxerInitDone", [self, aResult]() { 1669 if (self->mParentDecoder && self->mParentDecoder->GetOwner()) { 1670 self->mParentDecoder->GetOwner()->DecodeWarning(aResult); 1671 } 1672 })); 1673 } 1674 } 1675 1676 void TrackBuffersManager::OnDemuxerInitFailed(const MediaResult& aError) { 1677 mTaskQueueCapability->AssertOnCurrentThread(); 1678 MSE_DEBUG(""); 1679 MOZ_ASSERT(aError != NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); 1680 mDemuxerInitRequest.Complete(); 1681 1682 RejectAppend(aError, __func__); 1683 } 1684 1685 RefPtr<TrackBuffersManager::CodedFrameProcessingPromise> 1686 TrackBuffersManager::CodedFrameProcessing() { 1687 MOZ_ASSERT(OnTaskQueue()); 1688 MOZ_ASSERT(mProcessingPromise.IsEmpty()); 1689 AUTO_PROFILER_LABEL("TrackBuffersManager::CodedFrameProcessing", 1690 MEDIA_PLAYBACK); 1691 1692 MediaByteRange mediaRange = mParser->MediaSegmentRange(); 1693 if (mediaRange.IsEmpty()) { 1694 AppendDataToCurrentInputBuffer(*mInputBuffer); 1695 mInputBuffer.reset(); 1696 } else { 1697 MOZ_ASSERT(mProcessedInput >= mInputBuffer->Length()); 1698 if (int64_t(mProcessedInput - mInputBuffer->Length()) > mediaRange.mEnd) { 1699 // Something is not quite right with the data appended. Refuse it. 1700 // This would typically happen if the previous media segment was partial 1701 // yet a new complete media segment was added. 1702 return CodedFrameProcessingPromise::CreateAndReject(NS_ERROR_FAILURE, 1703 __func__); 1704 } 1705 // The mediaRange is offset by the init segment position previously added. 1706 uint32_t length = 1707 mediaRange.mEnd - (mProcessedInput - mInputBuffer->Length()); 1708 if (!length) { 1709 // We've completed our earlier media segment and no new data is to be 1710 // processed. This happens with some containers that can't detect that a 1711 // media segment is ending until a new one starts. 1712 RefPtr<CodedFrameProcessingPromise> p = 1713 mProcessingPromise.Ensure(__func__); 1714 CompleteCodedFrameProcessing(); 1715 return p; 1716 } 1717 AppendDataToCurrentInputBuffer(mInputBuffer->To(length)); 1718 mInputBuffer->RemoveFront(length); 1719 } 1720 1721 RefPtr<CodedFrameProcessingPromise> p = mProcessingPromise.Ensure(__func__); 1722 1723 DoDemuxVideo(); 1724 1725 return p; 1726 } 1727 1728 void TrackBuffersManager::OnDemuxFailed(TrackType aTrack, 1729 const MediaResult& aError) { 1730 MOZ_ASSERT(OnTaskQueue()); 1731 MSE_DEBUG("Failed to demux %s, failure:%s", 1732 aTrack == TrackType::kVideoTrack ? "video" : "audio", 1733 aError.ErrorName().get()); 1734 switch (aError.Code()) { 1735 case NS_ERROR_DOM_MEDIA_END_OF_STREAM: 1736 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: 1737 if (aTrack == TrackType::kVideoTrack) { 1738 DoDemuxAudio(); 1739 } else { 1740 CompleteCodedFrameProcessing(); 1741 } 1742 break; 1743 default: 1744 // https://w3c.github.io/media-source/#sourcebuffer-segment-parser-loop 1745 // 2. If the [[input buffer]] contains bytes that violate the 1746 // SourceBuffer byte stream format specification, then run the append 1747 // error algorithm and abort this algorithm. 1748 RejectProcessing(aError, __func__); 1749 break; 1750 } 1751 } 1752 1753 void TrackBuffersManager::DoDemuxVideo() { 1754 MOZ_ASSERT(OnTaskQueue()); 1755 if (!HasVideo()) { 1756 DoDemuxAudio(); 1757 return; 1758 } 1759 mVideoTracks.mDemuxer->GetSamples(-1) 1760 ->Then(TaskQueueFromTaskQueue(), __func__, this, 1761 &TrackBuffersManager::OnVideoDemuxCompleted, 1762 &TrackBuffersManager::OnVideoDemuxFailed) 1763 ->Track(mVideoTracks.mDemuxRequest); 1764 } 1765 1766 void TrackBuffersManager::MaybeDispatchEncryptedEvent( 1767 const nsTArray<RefPtr<MediaRawData>>& aSamples) { 1768 // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING. 1769 for (const RefPtr<MediaRawData>& sample : aSamples) { 1770 for (const nsTArray<uint8_t>& initData : sample->mCrypto.mInitDatas) { 1771 nsCOMPtr<nsIRunnable> r = new DispatchKeyNeededEvent( 1772 mParentDecoder, initData, sample->mCrypto.mInitDataType); 1773 mAbstractMainThread->Dispatch(r.forget()); 1774 } 1775 } 1776 } 1777 1778 void TrackBuffersManager::OnVideoDemuxCompleted( 1779 const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples) { 1780 mTaskQueueCapability->AssertOnCurrentThread(); 1781 mVideoTracks.mDemuxRequest.Complete(); 1782 mVideoTracks.mQueuedSamples.AppendElements(aSamples->GetSamples()); 1783 MSE_DEBUG("%zu video samples demuxed, queued-sz=%zu", 1784 aSamples->GetSamples().Length(), 1785 mVideoTracks.mQueuedSamples.Length()); 1786 1787 MaybeDispatchEncryptedEvent(aSamples->GetSamples()); 1788 DoDemuxAudio(); 1789 } 1790 1791 void TrackBuffersManager::DoDemuxAudio() { 1792 MOZ_ASSERT(OnTaskQueue()); 1793 if (!HasAudio()) { 1794 CompleteCodedFrameProcessing(); 1795 return; 1796 } 1797 mAudioTracks.mDemuxer->GetSamples(-1) 1798 ->Then(TaskQueueFromTaskQueue(), __func__, this, 1799 &TrackBuffersManager::OnAudioDemuxCompleted, 1800 &TrackBuffersManager::OnAudioDemuxFailed) 1801 ->Track(mAudioTracks.mDemuxRequest); 1802 } 1803 1804 void TrackBuffersManager::OnAudioDemuxCompleted( 1805 const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples) { 1806 mTaskQueueCapability->AssertOnCurrentThread(); 1807 MSE_DEBUG("%zu audio samples demuxed", aSamples->GetSamples().Length()); 1808 // When using MSE, it's possible for each fragments to have their own 1809 // duration, with a duration that is incorrectly rounded. Ignore the trimming 1810 // information set by the demuxer to ensure a continous playback. 1811 for (const auto& sample : aSamples->GetSamples()) { 1812 sample->mOriginalPresentationWindow = Nothing(); 1813 } 1814 mAudioTracks.mDemuxRequest.Complete(); 1815 mAudioTracks.mQueuedSamples.AppendElements(aSamples->GetSamples()); 1816 CompleteCodedFrameProcessing(); 1817 1818 MaybeDispatchEncryptedEvent(aSamples->GetSamples()); 1819 } 1820 1821 void TrackBuffersManager::CompleteCodedFrameProcessing() { 1822 MOZ_ASSERT(OnTaskQueue()); 1823 AUTO_PROFILER_LABEL("TrackBuffersManager::CompleteCodedFrameProcessing", 1824 MEDIA_PLAYBACK); 1825 1826 // 1. For each coded frame in the media segment run the following steps: 1827 // Coded Frame Processing steps 1.1 to 1.21. 1828 1829 if (mSourceBufferAttributes->GetAppendMode() == 1830 SourceBufferAppendMode::Sequence && 1831 mVideoTracks.mQueuedSamples.Length() && 1832 mAudioTracks.mQueuedSamples.Length()) { 1833 // When we are in sequence mode, the order in which we process the frames is 1834 // important as it determines the future value of timestampOffset. 1835 // So we process the earliest sample first. See bug 1293576. 1836 TimeInterval videoInterval = 1837 PresentationInterval(mVideoTracks.mQueuedSamples); 1838 TimeInterval audioInterval = 1839 PresentationInterval(mAudioTracks.mQueuedSamples); 1840 if (audioInterval.mStart < videoInterval.mStart) { 1841 ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks); 1842 ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks); 1843 } else { 1844 ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks); 1845 ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks); 1846 } 1847 } else { 1848 ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks); 1849 ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks); 1850 } 1851 1852 #if defined(DEBUG) 1853 if (HasVideo()) { 1854 const auto& track = mVideoTracks.GetTrackBuffer(); 1855 MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe); 1856 for (uint32_t i = 1; i < track.Length(); i++) { 1857 MOZ_ASSERT( 1858 (track[i - 1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() && 1859 track[i - 1]->mTimecode <= track[i]->mTimecode) || 1860 track[i]->mKeyframe); 1861 } 1862 } 1863 if (HasAudio()) { 1864 const auto& track = mAudioTracks.GetTrackBuffer(); 1865 MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe); 1866 for (uint32_t i = 1; i < track.Length(); i++) { 1867 MOZ_ASSERT( 1868 (track[i - 1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() && 1869 track[i - 1]->mTimecode <= track[i]->mTimecode) || 1870 track[i]->mKeyframe); 1871 } 1872 } 1873 #endif 1874 1875 mVideoTracks.mQueuedSamples.Clear(); 1876 mAudioTracks.mQueuedSamples.Clear(); 1877 1878 UpdateBufferedRanges(); 1879 1880 // Update our reported total size. 1881 mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer; 1882 1883 // Return to step 6.4 of Segment Parser Loop algorithm 1884 // 4. If this SourceBuffer is full and cannot accept more media data, then set 1885 // the buffer full flag to true. 1886 if (mSizeSourceBuffer >= EvictionThreshold()) { 1887 mBufferFull = true; 1888 } 1889 1890 // 5. If the input buffer does not contain a complete media segment, then jump 1891 // to the need more data step below. 1892 if (mParser->MediaSegmentRange().IsEmpty()) { 1893 ResolveProcessing(true, __func__); 1894 return; 1895 } 1896 1897 mLastParsedEndTime = Some(std::max(mAudioTracks.mLastParsedEndTime, 1898 mVideoTracks.mLastParsedEndTime)); 1899 1900 // 6. Remove the media segment bytes from the beginning of the input buffer. 1901 // Clear our demuxer from any already processed data. 1902 int64_t safeToEvict = 1903 std::min(HasVideo() ? mVideoTracks.mDemuxer->GetEvictionOffset( 1904 mVideoTracks.mLastParsedEndTime) 1905 : INT64_MAX, 1906 HasAudio() ? mAudioTracks.mDemuxer->GetEvictionOffset( 1907 mAudioTracks.mLastParsedEndTime) 1908 : INT64_MAX); 1909 mCurrentInputBuffer->EvictBefore(safeToEvict); 1910 1911 mInputDemuxer->NotifyDataRemoved(); 1912 RecreateParser(true); 1913 1914 // 7. Set append state to WAITING_FOR_SEGMENT. 1915 SetAppendState(AppendState::WAITING_FOR_SEGMENT); 1916 1917 // 8. Jump to the loop top step above. 1918 ResolveProcessing(false, __func__); 1919 } 1920 1921 void TrackBuffersManager::RejectProcessing(const MediaResult& aRejectValue, 1922 const char* aName) { 1923 mProcessingPromise.RejectIfExists(aRejectValue, __func__); 1924 } 1925 1926 void TrackBuffersManager::ResolveProcessing(bool aResolveValue, 1927 const char* aName) { 1928 mProcessingPromise.ResolveIfExists(aResolveValue, __func__); 1929 } 1930 1931 void TrackBuffersManager::CheckSequenceDiscontinuity( 1932 const TimeUnit& aPresentationTime) { 1933 if (mSourceBufferAttributes->GetAppendMode() == 1934 SourceBufferAppendMode::Sequence && 1935 mSourceBufferAttributes->HaveGroupStartTimestamp()) { 1936 mSourceBufferAttributes->SetTimestampOffset( 1937 mSourceBufferAttributes->GetGroupStartTimestamp() - aPresentationTime); 1938 mSourceBufferAttributes->SetGroupEndTimestamp( 1939 mSourceBufferAttributes->GetGroupStartTimestamp()); 1940 mVideoTracks.mNeedRandomAccessPoint = true; 1941 mAudioTracks.mNeedRandomAccessPoint = true; 1942 mSourceBufferAttributes->ResetGroupStartTimestamp(); 1943 } 1944 } 1945 1946 TimeInterval TrackBuffersManager::PresentationInterval( 1947 const TrackBuffer& aSamples) const { 1948 TimeInterval presentationInterval = 1949 TimeInterval(aSamples[0]->mTime, aSamples[0]->GetEndTime()); 1950 1951 for (uint32_t i = 1; i < aSamples.Length(); i++) { 1952 const auto& sample = aSamples[i]; 1953 presentationInterval = presentationInterval.Span( 1954 TimeInterval(sample->mTime, sample->GetEndTime())); 1955 } 1956 return presentationInterval; 1957 } 1958 1959 void TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, 1960 TrackData& aTrackData) { 1961 AUTO_PROFILER_LABEL("TrackBuffersManager::ProcessFrames", MEDIA_PLAYBACK); 1962 if (!aSamples.Length()) { 1963 return; 1964 } 1965 1966 // 1. If generate timestamps flag equals true 1967 // Let presentation timestamp equal 0. 1968 // Otherwise 1969 // Let presentation timestamp be a double precision floating point 1970 // representation of the coded frame's presentation timestamp in seconds. 1971 TimeUnit presentationTimestamp = mSourceBufferAttributes->mGenerateTimestamps 1972 ? TimeUnit::Zero() 1973 : aSamples[0]->mTime; 1974 1975 // 3. If mode equals "sequence" and group start timestamp is set, then run the 1976 // following steps: 1977 CheckSequenceDiscontinuity(presentationTimestamp); 1978 1979 // 5. Let track buffer equal the track buffer that the coded frame will be 1980 // added to. 1981 auto& trackBuffer = aTrackData; 1982 1983 TimeIntervals samplesRange; 1984 uint32_t sizeNewSamples = 0; 1985 TrackBuffer samples; // array that will contain the frames to be added 1986 // to our track buffer. 1987 1988 // We assume that no frames are contiguous within a media segment and as such 1989 // don't need to check for discontinuity except for the first frame and should 1990 // a frame be ignored due to the target window. 1991 bool needDiscontinuityCheck = true; 1992 1993 // Highest presentation time seen in samples block. 1994 TimeUnit highestSampleTime; 1995 1996 if (aSamples.Length()) { 1997 aTrackData.mLastParsedEndTime = TimeUnit(); 1998 } 1999 2000 auto addToSamples = [&](MediaRawData* aSample, 2001 const TimeInterval& aInterval) { 2002 aSample->mTime = aInterval.mStart; 2003 aSample->mDuration = aInterval.Length(); 2004 aSample->mTrackInfo = trackBuffer.mLastInfo; 2005 SAMPLE_DEBUGV( 2006 "Add sample [%" PRId64 "%s,%" PRId64 "%s] by interval %s", 2007 aSample->mTime.ToMicroseconds(), aSample->mTime.ToString().get(), 2008 aSample->GetEndTime().ToMicroseconds(), 2009 aSample->GetEndTime().ToString().get(), aInterval.ToString().get()); 2010 MOZ_DIAGNOSTIC_ASSERT(aSample->HasValidTime()); 2011 MOZ_DIAGNOSTIC_ASSERT(TimeInterval(aSample->mTime, aSample->GetEndTime()) == 2012 aInterval); 2013 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED 2014 auto oldRangeEnd = samplesRange.GetEnd(); 2015 #endif 2016 samplesRange += aInterval; 2017 // For debug purpose, if the sample range grows, it should match the 2018 // sample's end time. 2019 MOZ_DIAGNOSTIC_ASSERT_IF(samplesRange.GetEnd() > oldRangeEnd, 2020 samplesRange.GetEnd() == aSample->GetEndTime()); 2021 sizeNewSamples += aSample->ComputedSizeOfIncludingThis(); 2022 samples.AppendElement(aSample); 2023 }; 2024 2025 // Will be set to the last frame dropped due to being outside mAppendWindow. 2026 // It will be added prior the first following frame which can be added to the 2027 // track buffer. 2028 // This sample will be set with a duration of only 1us which will cause it to 2029 // be dropped once returned by the decoder. 2030 // This sample is required to "prime" the decoder so that the following frame 2031 // can be fully decoded. 2032 RefPtr<MediaRawData> previouslyDroppedSample; 2033 for (auto& sample : aSamples) { 2034 const TimeUnit sampleEndTime = sample->GetEndTime(); 2035 if (sampleEndTime > aTrackData.mLastParsedEndTime) { 2036 aTrackData.mLastParsedEndTime = sampleEndTime; 2037 } 2038 2039 // We perform step 10 right away as we can't do anything should a keyframe 2040 // be needed until we have one. 2041 2042 // 10. If the need random access point flag on track buffer equals true, 2043 // then run the following steps: 2044 if (trackBuffer.mNeedRandomAccessPoint) { 2045 // 1. If the coded frame is not a random access point, then drop the coded 2046 // frame and jump to the top of the loop to start processing the next 2047 // coded frame. 2048 if (!sample->mKeyframe) { 2049 previouslyDroppedSample = nullptr; 2050 nsPrintfCString msg("skipping sample [%" PRId64 ",%" PRId64 "]", 2051 sample->mTime.ToMicroseconds(), 2052 sample->GetEndTime().ToMicroseconds()); 2053 if (profiler_thread_is_being_profiled_for_markers()) { 2054 PROFILER_MARKER_TEXT("Skipping Frame", MEDIA_PLAYBACK, {}, msg); 2055 } 2056 SAMPLE_DEBUGV("%s", msg.get()); 2057 continue; 2058 } 2059 // 2. Set the need random access point flag on track buffer to false. 2060 trackBuffer.mNeedRandomAccessPoint = false; 2061 } 2062 2063 // We perform step 1,2 and 4 at once: 2064 // 1. If generate timestamps flag equals true: 2065 // Let presentation timestamp equal 0. 2066 // Let decode timestamp equal 0. 2067 // Otherwise: 2068 // Let presentation timestamp be a double precision floating point 2069 // representation of the coded frame's presentation timestamp in seconds. 2070 // Let decode timestamp be a double precision floating point 2071 // representation of the coded frame's decode timestamp in seconds. 2072 2073 // 2. Let frame duration be a double precision floating point representation 2074 // of the coded frame's duration in seconds. Step 3 is performed earlier or 2075 // when a discontinuity has been detected. 2076 // 4. If timestampOffset is not 0, then run the following steps: 2077 2078 TimeUnit sampleTime = sample->mTime; 2079 TimeUnit sampleTimecode = sample->mTimecode; 2080 TimeUnit sampleDuration = sample->mDuration; 2081 // Keep the timestamp, set by js, in the time base of the container. 2082 TimeUnit timestampOffset = 2083 mSourceBufferAttributes->GetTimestampOffset().ToBase(sample->mTime); 2084 2085 TimeInterval sampleInterval = 2086 mSourceBufferAttributes->mGenerateTimestamps 2087 ? TimeInterval(timestampOffset, timestampOffset + sampleDuration) 2088 : TimeInterval(timestampOffset + sampleTime, 2089 timestampOffset + sampleTime + sampleDuration); 2090 TimeUnit decodeTimestamp = mSourceBufferAttributes->mGenerateTimestamps 2091 ? timestampOffset 2092 : timestampOffset + sampleTimecode; 2093 2094 SAMPLE_DEBUG( 2095 "Processing %s frame [%" PRId64 "%s,%" PRId64 "%s] (adjusted:[%" PRId64 2096 "%s,%" PRId64 "%s]), dts:%" PRId64 ", duration:%" PRId64 ", kf:%d)", 2097 aTrackData.mInfo->mMimeType.get(), sample->mTime.ToMicroseconds(), 2098 sample->mTime.ToString().get(), sample->GetEndTime().ToMicroseconds(), 2099 sample->GetEndTime().ToString().get(), 2100 sampleInterval.mStart.ToMicroseconds(), 2101 sampleInterval.mStart.ToString().get(), 2102 sampleInterval.mEnd.ToMicroseconds(), 2103 sampleInterval.mEnd.ToString().get(), 2104 sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(), 2105 sample->mKeyframe); 2106 2107 // 6. If last decode timestamp for track buffer is set and decode timestamp 2108 // is less than last decode timestamp: OR If last decode timestamp for track 2109 // buffer is set and the difference between decode timestamp and last decode 2110 // timestamp is greater than 2 times last frame duration: 2111 if (needDiscontinuityCheck && trackBuffer.mLastDecodeTimestamp.isSome() && 2112 (decodeTimestamp < trackBuffer.mLastDecodeTimestamp.ref() || 2113 (decodeTimestamp - trackBuffer.mLastDecodeTimestamp.ref() > 2114 trackBuffer.mLongestFrameDuration * 2))) { 2115 if (profiler_thread_is_being_profiled_for_markers()) { 2116 PROFILER_MARKER_UNTYPED("Discontinuity detected", MEDIA_PLAYBACK); 2117 } 2118 MSE_DEBUG("Discontinuity detected."); 2119 SourceBufferAppendMode appendMode = 2120 mSourceBufferAttributes->GetAppendMode(); 2121 2122 // 1a. If mode equals "segments": 2123 if (appendMode == SourceBufferAppendMode::Segments) { 2124 // Set group end timestamp to presentation timestamp. 2125 mSourceBufferAttributes->SetGroupEndTimestamp(sampleInterval.mStart); 2126 } 2127 // 1b. If mode equals "sequence": 2128 if (appendMode == SourceBufferAppendMode::Sequence) { 2129 // Set group start timestamp equal to the group end timestamp. 2130 mSourceBufferAttributes->SetGroupStartTimestamp( 2131 mSourceBufferAttributes->GetGroupEndTimestamp()); 2132 } 2133 for (auto& track : GetTracksList()) { 2134 // 2. Unset the last decode timestamp on all track buffers. 2135 // 3. Unset the last frame duration on all track buffers. 2136 // 4. Unset the highest end timestamp on all track buffers. 2137 // 5. Set the need random access point flag on all track buffers to 2138 // true. 2139 MSE_DEBUG("Resetting append state"); 2140 track->ResetAppendState(); 2141 } 2142 // 6. Jump to the Loop Top step above to restart processing of the current 2143 // coded frame. Rather that restarting the process for the frame, we run 2144 // the first steps again instead. 2145 // 3. If mode equals "sequence" and group start timestamp is set, then run 2146 // the following steps: 2147 TimeUnit presentationTimestamp = 2148 mSourceBufferAttributes->mGenerateTimestamps ? TimeUnit() 2149 : sampleTime; 2150 CheckSequenceDiscontinuity(presentationTimestamp); 2151 2152 if (!sample->mKeyframe) { 2153 previouslyDroppedSample = nullptr; 2154 continue; 2155 } 2156 if (appendMode == SourceBufferAppendMode::Sequence) { 2157 // mSourceBufferAttributes->GetTimestampOffset() was modified during 2158 // CheckSequenceDiscontinuity. We need to update our variables. 2159 timestampOffset = 2160 mSourceBufferAttributes->GetTimestampOffset().ToBase(sample->mTime); 2161 sampleInterval = 2162 mSourceBufferAttributes->mGenerateTimestamps 2163 ? TimeInterval(timestampOffset, 2164 timestampOffset + sampleDuration) 2165 : TimeInterval(timestampOffset + sampleTime, 2166 timestampOffset + sampleTime + sampleDuration); 2167 decodeTimestamp = mSourceBufferAttributes->mGenerateTimestamps 2168 ? timestampOffset 2169 : timestampOffset + sampleTimecode; 2170 } 2171 trackBuffer.mNeedRandomAccessPoint = false; 2172 needDiscontinuityCheck = false; 2173 } 2174 2175 // 7. Let frame end timestamp equal the sum of presentation timestamp and 2176 // frame duration. 2177 // "presentation timestamp" and "frame duration" are double precision 2178 // representations. 2179 Interval<double> frameStartAndEnd( 2180 sampleInterval.mStart.ToSeconds(), 2181 sampleInterval.mStart.ToSeconds() + sampleDuration.ToSeconds()); 2182 // 8. If presentation timestamp is less than appendWindowStart, then set the 2183 // need random access point flag to true, drop the coded frame, and jump to 2184 // the top of the loop to start processing the next coded frame. 2185 // 9. If frame end timestamp is greater than appendWindowEnd, then set the 2186 // need random access point flag to true, drop the coded frame, and jump to 2187 // the top of the loop to start processing the next coded frame. 2188 if (!mAppendWindow.ContainsStrict(frameStartAndEnd)) { 2189 // Calculate the intersection using the same denominator as will be used 2190 // for the sample time and duration so that a truncated sample is 2191 // collected only if its truncated duration would be non-zero. 2192 TimeInterval appendWindow( 2193 TimeUnit::FromSecondsWithBaseOf(mAppendWindow.mStart, sampleTime), 2194 TimeUnit::FromSecondsWithBaseOf(mAppendWindow.mEnd, sampleTime)); 2195 if (appendWindow.IntersectsStrict(sampleInterval)) { 2196 // 8. Note: Some implementations MAY choose to collect some of these 2197 // coded frames with presentation timestamp less than 2198 // appendWindowStart and use them to generate a splice at the first 2199 // coded frame that has a presentation timestamp greater than or 2200 // equal to appendWindowStart even if that frame is not a random 2201 // access point. Supporting this requires multiple decoders or faster 2202 // than real-time decoding so for now this behavior will not be a 2203 // normative requirement. 2204 // 9. Note: Some implementations MAY choose to collect coded frames with 2205 // presentation timestamp less than appendWindowEnd and frame end 2206 // timestamp greater than appendWindowEnd and use them to generate a 2207 // splice across the portion of the collected coded frames within the 2208 // append window at time of collection, and the beginning portion of 2209 // later processed frames which only partially overlap the end of the 2210 // collected coded frames. Supporting this requires multiple decoders 2211 // or faster than real-time decoding so for now this behavior will 2212 // not be a normative requirement. In conjunction with collecting 2213 // coded frames that span appendWindowStart, implementations MAY thus 2214 // support gapless audio splicing. 2215 TimeInterval intersection = appendWindow.Intersection(sampleInterval); 2216 sample->mOriginalPresentationWindow = Some(sampleInterval); 2217 MSE_DEBUGV("will truncate frame from [%" PRId64 "%s,%" PRId64 2218 "%s] to [%" PRId64 "%s,%" PRId64 "%s]", 2219 sampleInterval.mStart.ToMicroseconds(), 2220 sampleInterval.mStart.ToString().get(), 2221 sampleInterval.mEnd.ToMicroseconds(), 2222 sampleInterval.mEnd.ToString().get(), 2223 intersection.mStart.ToMicroseconds(), 2224 intersection.mStart.ToString().get(), 2225 intersection.mEnd.ToMicroseconds(), 2226 intersection.mEnd.ToString().get()); 2227 sampleInterval = intersection; 2228 } else { 2229 sample->mOriginalPresentationWindow = Some(sampleInterval); 2230 sample->mTimecode = decodeTimestamp; 2231 previouslyDroppedSample = sample; 2232 MSE_DEBUGV("frame [%" PRId64 "%s,%" PRId64 2233 "%s] outside appendWindow [%f.6%s,%f.6%s] dropping", 2234 sampleInterval.mStart.ToMicroseconds(), 2235 sampleInterval.mStart.ToString().get(), 2236 sampleInterval.mEnd.ToMicroseconds(), 2237 sampleInterval.mEnd.ToString().get(), mAppendWindow.mStart, 2238 appendWindow.mStart.ToString().get(), mAppendWindow.mEnd, 2239 appendWindow.mEnd.ToString().get()); 2240 if (samples.Length()) { 2241 // We are creating a discontinuity in the samples. 2242 // Insert the samples processed so far. 2243 InsertFrames(samples, samplesRange, trackBuffer); 2244 samples.Clear(); 2245 samplesRange = TimeIntervals(); 2246 trackBuffer.mSizeBuffer += sizeNewSamples; 2247 sizeNewSamples = 0; 2248 UpdateHighestTimestamp(trackBuffer, highestSampleTime); 2249 } 2250 trackBuffer.mNeedRandomAccessPoint = true; 2251 needDiscontinuityCheck = true; 2252 continue; 2253 } 2254 } 2255 if (previouslyDroppedSample) { 2256 MSE_DEBUGV("Adding silent frame"); 2257 // This "silent" sample will be added so that it starts exactly before the 2258 // first usable one. The duration of the actual sample will be adjusted so 2259 // that the total duration stay the same. This sample will be dropped 2260 // after decoding by the AudioTrimmer (if audio). 2261 TimeInterval previouslyDroppedSampleInterval = 2262 TimeInterval(sampleInterval.mStart, sampleInterval.mStart); 2263 addToSamples(previouslyDroppedSample, previouslyDroppedSampleInterval); 2264 previouslyDroppedSample = nullptr; 2265 sampleInterval.mStart += previouslyDroppedSampleInterval.Length(); 2266 } 2267 2268 sample->mTimecode = decodeTimestamp; 2269 addToSamples(sample, sampleInterval); 2270 2271 // Steps 11,12,13,14, 15 and 16 will be done in one block in InsertFrames. 2272 2273 trackBuffer.mLongestFrameDuration = 2274 trackBuffer.mLastFrameDuration.isSome() 2275 ? sample->mKeyframe 2276 ? sampleDuration 2277 : std::max(sampleDuration, trackBuffer.mLongestFrameDuration) 2278 : sampleDuration; 2279 2280 // 17. Set last decode timestamp for track buffer to decode timestamp. 2281 trackBuffer.mLastDecodeTimestamp = Some(decodeTimestamp); 2282 // 18. Set last frame duration for track buffer to frame duration. 2283 trackBuffer.mLastFrameDuration = Some(sampleDuration); 2284 2285 // 19. If highest end timestamp for track buffer is unset or frame end 2286 // timestamp is greater than highest end timestamp, then set highest end 2287 // timestamp for track buffer to frame end timestamp. 2288 if (trackBuffer.mHighestEndTimestamp.isNothing() || 2289 sampleInterval.mEnd > trackBuffer.mHighestEndTimestamp.ref()) { 2290 trackBuffer.mHighestEndTimestamp = Some(sampleInterval.mEnd); 2291 } 2292 if (sampleInterval.mStart > highestSampleTime) { 2293 highestSampleTime = sampleInterval.mStart; 2294 } 2295 // 20. If frame end timestamp is greater than group end timestamp, then set 2296 // group end timestamp equal to frame end timestamp. 2297 if (sampleInterval.mEnd > mSourceBufferAttributes->GetGroupEndTimestamp()) { 2298 mSourceBufferAttributes->SetGroupEndTimestamp(sampleInterval.mEnd); 2299 } 2300 // 21. If generate timestamps flag equals true, then set timestampOffset 2301 // equal to frame end timestamp. 2302 if (mSourceBufferAttributes->mGenerateTimestamps) { 2303 mSourceBufferAttributes->SetTimestampOffset(sampleInterval.mEnd); 2304 } 2305 } 2306 2307 if (samples.Length()) { 2308 InsertFrames(samples, samplesRange, trackBuffer); 2309 trackBuffer.mSizeBuffer += sizeNewSamples; 2310 UpdateHighestTimestamp(trackBuffer, highestSampleTime); 2311 } 2312 } 2313 2314 bool TrackBuffersManager::CheckNextInsertionIndex(TrackData& aTrackData, 2315 const TimeUnit& aSampleTime) { 2316 if (aTrackData.mNextInsertionIndex.isSome()) { 2317 return true; 2318 } 2319 2320 const TrackBuffer& data = aTrackData.GetTrackBuffer(); 2321 2322 if (data.IsEmpty() || aSampleTime < aTrackData.mBufferedRanges.GetStart()) { 2323 aTrackData.mNextInsertionIndex = Some(0u); 2324 return true; 2325 } 2326 2327 // Find which discontinuity we should insert the frame before. 2328 TimeInterval target; 2329 for (const auto& interval : aTrackData.mBufferedRanges) { 2330 if (aSampleTime < interval.mStart) { 2331 target = interval; 2332 break; 2333 } 2334 } 2335 if (target.IsEmpty()) { 2336 // No target found, it will be added at the end of the track buffer. 2337 aTrackData.mNextInsertionIndex = Some(uint32_t(data.Length())); 2338 return true; 2339 } 2340 // We now need to find the first frame of the searched interval. 2341 // We will insert our new frames right before. 2342 for (uint32_t i = 0; i < data.Length(); i++) { 2343 const RefPtr<MediaRawData>& sample = data[i]; 2344 if (sample->mTime >= target.mStart || 2345 sample->GetEndTime() > target.mStart) { 2346 aTrackData.mNextInsertionIndex = Some(i); 2347 return true; 2348 } 2349 } 2350 NS_ASSERTION(false, "Insertion Index Not Found"); 2351 return false; 2352 } 2353 2354 void TrackBuffersManager::InsertFrames(TrackBuffer& aSamples, 2355 const TimeIntervals& aIntervals, 2356 TrackData& aTrackData) { 2357 AUTO_PROFILER_LABEL("TrackBuffersManager::InsertFrames", MEDIA_PLAYBACK); 2358 // 5. Let track buffer equal the track buffer that the coded frame will be 2359 // added to. 2360 auto& trackBuffer = aTrackData; 2361 2362 MSE_DEBUGV("Processing %zu %s frames(start:%" PRId64 " end:%" PRId64 ")", 2363 aSamples.Length(), aTrackData.mInfo->mMimeType.get(), 2364 aIntervals.GetStart().ToMicroseconds(), 2365 aIntervals.GetEnd().ToMicroseconds()); 2366 PROFILER_MARKER_FMT("InsertFrames", MEDIA_PLAYBACK, {}, 2367 "Processing {} {}frames(start:{} end:{})", 2368 aSamples.Length(), aTrackData.mInfo->mMimeType.get(), 2369 aIntervals.GetStart().ToMicroseconds(), 2370 aIntervals.GetEnd().ToMicroseconds()); 2371 2372 // 11. Let spliced audio frame be an unset variable for holding audio splice 2373 // information 2374 // 12. Let spliced timed text frame be an unset variable for holding timed 2375 // text splice information 2376 2377 // 13. If last decode timestamp for track buffer is unset and presentation 2378 // timestamp falls within the presentation interval of a coded frame in track 2379 // buffer,then run the following steps: For now we only handle replacing 2380 // existing frames with the new ones. So we skip this step. 2381 2382 // 14. Remove existing coded frames in track buffer: 2383 // a) If highest end timestamp for track buffer is not set: 2384 // Remove all coded frames from track buffer that have a presentation 2385 // timestamp greater than or equal to presentation timestamp and less 2386 // than frame end timestamp. 2387 // b) If highest end timestamp for track buffer is set and less than or 2388 // equal to presentation timestamp: 2389 // Remove all coded frames from track buffer that have a presentation 2390 // timestamp greater than or equal to highest end timestamp and less than 2391 // frame end timestamp 2392 2393 // There is an ambiguity on how to remove frames, which was lodged with: 2394 // https://www.w3.org/Bugs/Public/show_bug.cgi?id=28710, implementing as per 2395 // bug description. 2396 2397 // 15. Remove decoding dependencies of the coded frames removed in the 2398 // previous step: Remove all coded frames between the coded frames removed in 2399 // the previous step and the next random access point after those removed 2400 // frames. 2401 2402 if (trackBuffer.mBufferedRanges.IntersectsStrict(aIntervals)) { 2403 if (aSamples[0]->mKeyframe && 2404 (mType.Type() == MEDIAMIMETYPE("video/webm") || 2405 mType.Type() == MEDIAMIMETYPE("audio/webm"))) { 2406 // We are starting a new GOP, we do not have to worry about breaking an 2407 // existing current coded frame group. Reset the next insertion index 2408 // so the search for when to start our frames removal can be exhaustive. 2409 // This is a workaround for bug 1276184 and only until either bug 1277733 2410 // or bug 1209386 is fixed. 2411 // With the webm container, we can't always properly determine the 2412 // duration of the last frame, which may cause the last frame of a cluster 2413 // to overlap the following frame. 2414 trackBuffer.mNextInsertionIndex.reset(); 2415 } 2416 uint32_t index = RemoveFrames(aIntervals, trackBuffer, 2417 trackBuffer.mNextInsertionIndex.refOr(0), 2418 RemovalMode::kTruncateFrame); 2419 if (index) { 2420 trackBuffer.mNextInsertionIndex = Some(index); 2421 } 2422 } 2423 2424 // 16. Add the coded frame with the presentation timestamp, decode timestamp, 2425 // and frame duration to the track buffer. 2426 if (!CheckNextInsertionIndex(aTrackData, aSamples[0]->mTime)) { 2427 RejectProcessing(NS_ERROR_FAILURE, __func__); 2428 return; 2429 } 2430 2431 // Adjust our demuxing index if necessary. 2432 if (trackBuffer.mNextGetSampleIndex.isSome()) { 2433 if (trackBuffer.mNextInsertionIndex.ref() == 2434 trackBuffer.mNextGetSampleIndex.ref() && 2435 aIntervals.GetEnd() >= trackBuffer.mNextSampleTime) { 2436 MSE_DEBUG("Next sample to be played got overwritten"); 2437 trackBuffer.mNextGetSampleIndex.reset(); 2438 ResetEvictionIndex(trackBuffer); 2439 } else if (trackBuffer.mNextInsertionIndex.ref() <= 2440 trackBuffer.mNextGetSampleIndex.ref()) { 2441 trackBuffer.mNextGetSampleIndex.ref() += aSamples.Length(); 2442 // We could adjust the eviction index so that the new data gets added to 2443 // the evictable amount (as it is prior currentTime). However, considering 2444 // new data is being added prior the current playback, it's likely that 2445 // this data will be played next, and as such we probably don't want to 2446 // have it evicted too early. So instead reset the eviction index instead. 2447 ResetEvictionIndex(trackBuffer); 2448 } 2449 } 2450 2451 TrackBuffer& data = trackBuffer.GetTrackBuffer(); 2452 data.InsertElementsAt(trackBuffer.mNextInsertionIndex.ref(), aSamples); 2453 trackBuffer.mNextInsertionIndex.ref() += aSamples.Length(); 2454 2455 // Update our buffered range with new sample interval. 2456 trackBuffer.mBufferedRanges += aIntervals; 2457 2458 MSE_DEBUG("Inserted %s frame:%s, buffered-range:%s, mHighestEndTimestamp=%s", 2459 aTrackData.mInfo->mMimeType.get(), DumpTimeRanges(aIntervals).get(), 2460 DumpTimeRanges(trackBuffer.mBufferedRanges).get(), 2461 trackBuffer.mHighestEndTimestamp 2462 ? trackBuffer.mHighestEndTimestamp->ToString().get() 2463 : "none"); 2464 // We allow a fuzz factor in our interval of half a frame length, 2465 // as fuzz is +/- value, giving an effective leeway of a full frame 2466 // length. 2467 if (!aIntervals.IsEmpty()) { 2468 TimeIntervals range(aIntervals); 2469 range.SetFuzz(trackBuffer.mLongestFrameDuration / 2); 2470 trackBuffer.mSanitizedBufferedRanges += range; 2471 } 2472 } 2473 2474 void TrackBuffersManager::UpdateHighestTimestamp( 2475 TrackData& aTrackData, const media::TimeUnit& aHighestTime) { 2476 if (aHighestTime > aTrackData.mHighestStartTimestamp) { 2477 MutexAutoLock mut(mMutex); 2478 aTrackData.mHighestStartTimestamp = aHighestTime; 2479 } 2480 } 2481 2482 uint32_t TrackBuffersManager::RemoveFrames(const TimeIntervals& aIntervals, 2483 TrackData& aTrackData, 2484 uint32_t aStartIndex, 2485 RemovalMode aMode) { 2486 AUTO_PROFILER_LABEL("TrackBuffersManager::RemoveFrames", MEDIA_PLAYBACK); 2487 TrackBuffer& data = aTrackData.GetTrackBuffer(); 2488 Maybe<uint32_t> firstRemovedIndex; 2489 uint32_t lastRemovedIndex = 0; 2490 2491 TimeIntervals intervals = 2492 aIntervals.ToBase(aTrackData.mHighestStartTimestamp); 2493 2494 // We loop from aStartIndex to avoid removing frames that we inserted earlier 2495 // and part of the current coded frame group. This is allows to handle step 2496 // 14 of the coded frame processing algorithm without having to check the 2497 // value of highest end timestamp: "Remove existing coded frames in track 2498 // buffer: 2499 // If highest end timestamp for track buffer is not set: 2500 // Remove all coded frames from track buffer that have a presentation 2501 // timestamp greater than or equal to presentation timestamp and less than 2502 // frame end timestamp. 2503 // If highest end timestamp for track buffer is set and less than or equal to 2504 // presentation timestamp: 2505 // Remove all coded frames from track buffer that have a presentation 2506 // timestamp greater than or equal to highest end timestamp and less than 2507 // frame end timestamp. 2508 TimeUnit intervalsEnd = intervals.GetEnd(); 2509 for (uint32_t i = aStartIndex; i < data.Length(); i++) { 2510 RefPtr<MediaRawData>& sample = data[i]; 2511 if (intervals.ContainsStrict(sample->mTime)) { 2512 // The start of this existing frame will be overwritten, we drop that 2513 // entire frame. 2514 MSE_DEBUGV("overriding start of frame [%" PRId64 ",%" PRId64 2515 "] with [%" PRId64 ",%" PRId64 "] dropping", 2516 sample->mTime.ToMicroseconds(), 2517 sample->GetEndTime().ToMicroseconds(), 2518 intervals.GetStart().ToMicroseconds(), 2519 intervals.GetEnd().ToMicroseconds()); 2520 if (firstRemovedIndex.isNothing()) { 2521 firstRemovedIndex = Some(i); 2522 } 2523 lastRemovedIndex = i; 2524 continue; 2525 } 2526 TimeInterval sampleInterval(sample->mTime, sample->GetEndTime()); 2527 if (aMode == RemovalMode::kTruncateFrame && 2528 intervals.IntersectsStrict(sampleInterval)) { 2529 // The sample to be overwritten is only partially covered. 2530 TimeIntervals intersection = 2531 Intersection(intervals, TimeIntervals(sampleInterval)); 2532 bool found = false; 2533 TimeUnit startTime = intersection.GetStart(&found); 2534 MOZ_DIAGNOSTIC_ASSERT(found, "Must intersect with added coded frames"); 2535 (void)found; 2536 // Signal that this frame should be truncated when decoded. 2537 if (!sample->mOriginalPresentationWindow) { 2538 sample->mOriginalPresentationWindow = Some(sampleInterval); 2539 } 2540 MOZ_ASSERT(startTime > sample->mTime); 2541 sample->mDuration = startTime - sample->mTime; 2542 MOZ_DIAGNOSTIC_ASSERT(sample->mDuration.IsValid()); 2543 MSE_DEBUGV("partial overwrite of frame [%" PRId64 ",%" PRId64 2544 "] with [%" PRId64 ",%" PRId64 2545 "] trim to " 2546 "[%" PRId64 ",%" PRId64 "]", 2547 sampleInterval.mStart.ToMicroseconds(), 2548 sampleInterval.mEnd.ToMicroseconds(), 2549 intervals.GetStart().ToMicroseconds(), 2550 intervals.GetEnd().ToMicroseconds(), 2551 sample->mTime.ToMicroseconds(), 2552 sample->GetEndTime().ToMicroseconds()); 2553 continue; 2554 } 2555 2556 if (sample->mTime >= intervalsEnd) { 2557 // We can break the loop now. All frames up to the next keyframe will be 2558 // removed during the next step. 2559 break; 2560 } 2561 } 2562 2563 if (firstRemovedIndex.isNothing()) { 2564 return 0; 2565 } 2566 2567 // Remove decoding dependencies of the coded frames removed in the previous 2568 // step: Remove all coded frames between the coded frames removed in the 2569 // previous step and the next random access point after those removed frames. 2570 for (uint32_t i = lastRemovedIndex + 1; i < data.Length(); i++) { 2571 const RefPtr<MediaRawData>& sample = data[i]; 2572 if (sample->mKeyframe) { 2573 break; 2574 } 2575 lastRemovedIndex = i; 2576 } 2577 2578 uint32_t sizeRemoved = 0; 2579 TimeIntervals removedIntervals; 2580 for (uint32_t i = firstRemovedIndex.ref(); i <= lastRemovedIndex; i++) { 2581 const RefPtr<MediaRawData> sample = data[i]; 2582 TimeInterval sampleInterval = 2583 TimeInterval(sample->mTime, sample->GetEndTime()); 2584 removedIntervals += sampleInterval; 2585 sizeRemoved += sample->ComputedSizeOfIncludingThis(); 2586 } 2587 aTrackData.mSizeBuffer -= sizeRemoved; 2588 2589 nsPrintfCString msg("Removing frames from:%u for %s (frames:%u) ([%f, %f))", 2590 firstRemovedIndex.ref(), 2591 aTrackData.mInfo->mMimeType.get(), 2592 lastRemovedIndex - firstRemovedIndex.ref() + 1, 2593 removedIntervals.GetStart().ToSeconds(), 2594 removedIntervals.GetEnd().ToSeconds()); 2595 MSE_DEBUG("%s", msg.get()); 2596 if (profiler_thread_is_being_profiled_for_markers()) { 2597 PROFILER_MARKER_TEXT("RemoveFrames", MEDIA_PLAYBACK, {}, msg); 2598 } 2599 2600 if (aTrackData.mNextGetSampleIndex.isSome()) { 2601 if (aTrackData.mNextGetSampleIndex.ref() >= firstRemovedIndex.ref() && 2602 aTrackData.mNextGetSampleIndex.ref() <= lastRemovedIndex) { 2603 MSE_DEBUG("Next sample to be played got evicted"); 2604 aTrackData.mNextGetSampleIndex.reset(); 2605 ResetEvictionIndex(aTrackData); 2606 } else if (aTrackData.mNextGetSampleIndex.ref() > lastRemovedIndex) { 2607 uint32_t samplesRemoved = lastRemovedIndex - firstRemovedIndex.ref() + 1; 2608 aTrackData.mNextGetSampleIndex.ref() -= samplesRemoved; 2609 if (aTrackData.mEvictionIndex.mLastIndex > lastRemovedIndex) { 2610 MOZ_DIAGNOSTIC_ASSERT( 2611 aTrackData.mEvictionIndex.mLastIndex >= samplesRemoved && 2612 aTrackData.mEvictionIndex.mEvictable >= sizeRemoved, 2613 "Invalid eviction index"); 2614 MutexAutoLock mut(mMutex); 2615 aTrackData.mEvictionIndex.mLastIndex -= samplesRemoved; 2616 aTrackData.mEvictionIndex.mEvictable -= sizeRemoved; 2617 } else { 2618 ResetEvictionIndex(aTrackData); 2619 } 2620 } 2621 } 2622 2623 if (aTrackData.mNextInsertionIndex.isSome()) { 2624 if (aTrackData.mNextInsertionIndex.ref() > firstRemovedIndex.ref() && 2625 aTrackData.mNextInsertionIndex.ref() <= lastRemovedIndex + 1) { 2626 aTrackData.ResetAppendState(); 2627 MSE_DEBUG("NextInsertionIndex got reset."); 2628 } else if (aTrackData.mNextInsertionIndex.ref() > lastRemovedIndex + 1) { 2629 aTrackData.mNextInsertionIndex.ref() -= 2630 lastRemovedIndex - firstRemovedIndex.ref() + 1; 2631 } 2632 } 2633 2634 // Update our buffered range to exclude the range just removed. 2635 MSE_DEBUG("Removing %s from bufferedRange %s", 2636 DumpTimeRanges(removedIntervals).get(), 2637 DumpTimeRanges(aTrackData.mBufferedRanges).get()); 2638 aTrackData.mBufferedRanges -= removedIntervals; 2639 2640 // Update sanitized buffered ranges. As well as subtracting intervals for 2641 // the removed samples, adjacent gaps between samples that were filled in 2642 // through Interval fuzz also need to be removed. Calculate the complete 2643 // gaps left due to frame removal by comparing the remaining buffered 2644 // intervals with the removed intervals, and then subtract these gaps from 2645 // mSanitizedBufferedRanges. 2646 TimeIntervals gaps; 2647 // If the earliest buffered interval was removed then a leading interval will 2648 // be removed from mSanitizedBufferedRanges. 2649 Maybe<TimeUnit> gapStart = 2650 Some(aTrackData.mSanitizedBufferedRanges.GetStart()); 2651 for (auto removedInterval = removedIntervals.cbegin(), 2652 bufferedInterval = aTrackData.mBufferedRanges.cbegin(); 2653 removedInterval < removedIntervals.cend();) { 2654 if (bufferedInterval == aTrackData.mBufferedRanges.cend()) { 2655 // No more buffered intervals. The rest has been removed. 2656 // gapStart has always been set here, either before the loop or when 2657 // bufferedInterval was incremented. 2658 gaps += TimeInterval(gapStart.value(), 2659 aTrackData.mSanitizedBufferedRanges.GetEnd()); 2660 break; 2661 } 2662 if (bufferedInterval->mEnd <= removedInterval->mStart) { 2663 gapStart = Some(bufferedInterval->mEnd); 2664 ++bufferedInterval; 2665 continue; 2666 } 2667 MOZ_ASSERT(removedInterval->mEnd <= bufferedInterval->mStart); 2668 // If gapStart is not set then gaps already includes the gap up to 2669 // bufferedInterval. 2670 if (gapStart) { 2671 gaps += TimeInterval(gapStart.value(), bufferedInterval->mStart); 2672 gapStart.reset(); 2673 } 2674 ++removedInterval; 2675 } 2676 MSE_DEBUG("Removing %s from mSanitizedBufferedRanges %s", 2677 DumpTimeRanges(gaps).get(), 2678 DumpTimeRanges(aTrackData.mSanitizedBufferedRanges).get()); 2679 aTrackData.mSanitizedBufferedRanges -= gaps; 2680 2681 data.RemoveElementsAt(firstRemovedIndex.ref(), 2682 lastRemovedIndex - firstRemovedIndex.ref() + 1); 2683 2684 if (removedIntervals.GetEnd() >= aTrackData.mHighestStartTimestamp && 2685 removedIntervals.GetStart() <= aTrackData.mHighestStartTimestamp) { 2686 // The sample with the highest presentation time got removed. 2687 // Rescan the trackbuffer to determine the new one. 2688 TimeUnit highestStartTime; 2689 for (const auto& sample : data) { 2690 if (sample->mTime > highestStartTime) { 2691 highestStartTime = sample->mTime; 2692 } 2693 } 2694 MutexAutoLock mut(mMutex); 2695 aTrackData.mHighestStartTimestamp = highestStartTime; 2696 } 2697 2698 MSE_DEBUG( 2699 "After removing frames, %s data sz=%zu, highestStartTimestamp=% " PRId64 2700 ", bufferedRange=%s, sanitizedBufferedRanges=%s", 2701 aTrackData.mInfo->mMimeType.get(), data.Length(), 2702 aTrackData.mHighestStartTimestamp.ToMicroseconds(), 2703 DumpTimeRanges(aTrackData.mBufferedRanges).get(), 2704 DumpTimeRanges(aTrackData.mSanitizedBufferedRanges).get()); 2705 2706 // If all frames are removed, both buffer and buffered range should be empty. 2707 if (data.IsEmpty()) { 2708 MOZ_ASSERT(aTrackData.mBufferedRanges.IsEmpty()); 2709 // We still can't figure out why above assertion would fail, so we keep it 2710 // on debug build, and do a workaround for other builds to ensure that 2711 // buffered range should match the data. 2712 if (!aTrackData.mBufferedRanges.IsEmpty()) { 2713 NS_WARNING( 2714 nsPrintfCString("Empty data but has non-empty buffered range %s ?!", 2715 DumpTimeRanges(aTrackData.mBufferedRanges).get()) 2716 .get()); 2717 aTrackData.mBufferedRanges.Clear(); 2718 } 2719 } 2720 if (aTrackData.mBufferedRanges.IsEmpty()) { 2721 TimeIntervals sampleIntervals; 2722 for (const auto& sample : data) { 2723 sampleIntervals += TimeInterval(sample->mTime, sample->GetEndTime()); 2724 } 2725 MOZ_ASSERT(sampleIntervals.IsEmpty()); 2726 // We still can't figure out why above assertion would fail, so we keep it 2727 // on debug build, and do a workaround for other builds to ensure that 2728 // buffered range should match the data. 2729 if (!sampleIntervals.IsEmpty()) { 2730 NS_WARNING( 2731 nsPrintfCString( 2732 "Empty buffer range but has non-empty sample intervals %s ?!", 2733 DumpTimeRanges(sampleIntervals).get()) 2734 .get()); 2735 aTrackData.mBufferedRanges += sampleIntervals; 2736 TimeIntervals range(sampleIntervals); 2737 range.SetFuzz(aTrackData.mLongestFrameDuration / 2); 2738 aTrackData.mSanitizedBufferedRanges += range; 2739 } 2740 } 2741 2742 return firstRemovedIndex.ref(); 2743 } 2744 2745 void TrackBuffersManager::RecreateParser(bool aReuseInitData) { 2746 MOZ_ASSERT(OnTaskQueue()); 2747 // Recreate our parser for only the data remaining. This is required 2748 // as it has parsed the entire InputBuffer provided. 2749 // Once the old TrackBuffer/MediaSource implementation is removed 2750 // we can optimize this part. TODO 2751 if (mParser) { 2752 DDUNLINKCHILD(mParser.get()); 2753 } 2754 mParser = ContainerParser::CreateForMIMEType(mType); 2755 DDLINKCHILD("parser", mParser.get()); 2756 if (aReuseInitData && mInitData) { 2757 MSE_DEBUG("Using existing init data to reset parser"); 2758 TimeUnit start, end; 2759 mParser->ParseStartAndEndTimestamps(MediaSpan(mInitData), start, end); 2760 mProcessedInput = mInitData->Length(); 2761 } else { 2762 MSE_DEBUG("Resetting parser, not reusing init data"); 2763 mProcessedInput = 0; 2764 } 2765 } 2766 2767 nsTArray<TrackBuffersManager::TrackData*> TrackBuffersManager::GetTracksList() { 2768 nsTArray<TrackData*> tracks; 2769 if (HasVideo()) { 2770 tracks.AppendElement(&mVideoTracks); 2771 } 2772 if (HasAudio()) { 2773 tracks.AppendElement(&mAudioTracks); 2774 } 2775 return tracks; 2776 } 2777 2778 nsTArray<const TrackBuffersManager::TrackData*> 2779 TrackBuffersManager::GetTracksList() const { 2780 nsTArray<const TrackData*> tracks; 2781 if (HasVideo()) { 2782 tracks.AppendElement(&mVideoTracks); 2783 } 2784 if (HasAudio()) { 2785 tracks.AppendElement(&mAudioTracks); 2786 } 2787 return tracks; 2788 } 2789 2790 void TrackBuffersManager::SetAppendState(AppendState aAppendState) { 2791 MSE_DEBUG("AppendState changed from %s to %s", 2792 SourceBufferAttributes::EnumValueToString( 2793 mSourceBufferAttributes->GetAppendState()), 2794 SourceBufferAttributes::EnumValueToString(aAppendState)); 2795 mSourceBufferAttributes->SetAppendState(aAppendState); 2796 } 2797 2798 MediaInfo TrackBuffersManager::GetMetadata() const { 2799 MutexAutoLock mut(mMutex); 2800 return mInfo; 2801 } 2802 2803 const TimeIntervals& TrackBuffersManager::Buffered( 2804 TrackInfo::TrackType aTrack) const { 2805 MOZ_ASSERT(OnTaskQueue()); 2806 return GetTracksData(aTrack).mBufferedRanges; 2807 } 2808 2809 const media::TimeUnit& TrackBuffersManager::HighestStartTime( 2810 TrackInfo::TrackType aTrack) const { 2811 MOZ_ASSERT(OnTaskQueue()); 2812 return GetTracksData(aTrack).mHighestStartTimestamp; 2813 } 2814 2815 TimeIntervals TrackBuffersManager::SafeBuffered( 2816 TrackInfo::TrackType aTrack) const { 2817 MutexAutoLock mut(mMutex); 2818 return aTrack == TrackInfo::kVideoTrack ? mVideoBufferedRanges 2819 : mAudioBufferedRanges; 2820 } 2821 2822 TimeUnit TrackBuffersManager::HighestStartTime() const { 2823 MutexAutoLock mut(mMutex); 2824 TimeUnit highestStartTime; 2825 for (auto& track : GetTracksList()) { 2826 highestStartTime = 2827 std::max(track->mHighestStartTimestamp, highestStartTime); 2828 } 2829 return highestStartTime; 2830 } 2831 2832 TimeUnit TrackBuffersManager::HighestEndTime() const { 2833 MutexAutoLock mut(mMutex); 2834 2835 nsTArray<const TimeIntervals*> tracks; 2836 if (HasVideo()) { 2837 tracks.AppendElement(&mVideoBufferedRanges); 2838 } 2839 if (HasAudio()) { 2840 tracks.AppendElement(&mAudioBufferedRanges); 2841 } 2842 return HighestEndTime(tracks); 2843 } 2844 2845 TimeUnit TrackBuffersManager::HighestEndTime( 2846 nsTArray<const TimeIntervals*>& aTracks) const { 2847 mMutex.AssertCurrentThreadOwns(); 2848 2849 TimeUnit highestEndTime; 2850 2851 for (const auto& trackRanges : aTracks) { 2852 highestEndTime = std::max(trackRanges->GetEnd(), highestEndTime); 2853 } 2854 return highestEndTime; 2855 } 2856 2857 void TrackBuffersManager::ResetEvictionIndex(TrackData& aTrackData) { 2858 MutexAutoLock mut(mMutex); 2859 MSE_DEBUG("ResetEvictionIndex for %s", aTrackData.mInfo->mMimeType.get()); 2860 aTrackData.mEvictionIndex.Reset(); 2861 } 2862 2863 void TrackBuffersManager::UpdateEvictionIndex(TrackData& aTrackData, 2864 uint32_t currentIndex) { 2865 uint32_t evictable = 0; 2866 TrackBuffer& data = aTrackData.GetTrackBuffer(); 2867 MOZ_DIAGNOSTIC_ASSERT(currentIndex >= aTrackData.mEvictionIndex.mLastIndex, 2868 "Invalid call"); 2869 MOZ_DIAGNOSTIC_ASSERT( 2870 currentIndex == data.Length() || data[currentIndex]->mKeyframe, 2871 "Must stop at keyframe"); 2872 2873 for (uint32_t i = aTrackData.mEvictionIndex.mLastIndex; i < currentIndex; 2874 i++) { 2875 evictable += data[i]->ComputedSizeOfIncludingThis(); 2876 } 2877 aTrackData.mEvictionIndex.mLastIndex = currentIndex; 2878 MutexAutoLock mut(mMutex); 2879 aTrackData.mEvictionIndex.mEvictable += evictable; 2880 MSE_DEBUG("UpdateEvictionIndex for %s (idx=%u, evictable=%u)", 2881 aTrackData.mInfo->mMimeType.get(), 2882 aTrackData.mEvictionIndex.mLastIndex, 2883 aTrackData.mEvictionIndex.mEvictable); 2884 } 2885 2886 const TrackBuffersManager::TrackBuffer& TrackBuffersManager::GetTrackBuffer( 2887 TrackInfo::TrackType aTrack) const { 2888 MOZ_ASSERT(OnTaskQueue()); 2889 return GetTracksData(aTrack).GetTrackBuffer(); 2890 } 2891 2892 uint32_t TrackBuffersManager::FindSampleIndex(const TrackBuffer& aTrackBuffer, 2893 const TimeInterval& aInterval) { 2894 TimeUnit target = aInterval.mStart - aInterval.mFuzz; 2895 2896 for (uint32_t i = 0; i < aTrackBuffer.Length(); i++) { 2897 const RefPtr<MediaRawData>& sample = aTrackBuffer[i]; 2898 if (sample->mTime >= target || sample->GetEndTime() > target) { 2899 return i; 2900 } 2901 } 2902 MOZ_ASSERT(false, "FindSampleIndex called with invalid arguments"); 2903 2904 return 0; 2905 } 2906 2907 TimeUnit TrackBuffersManager::Seek(TrackInfo::TrackType aTrack, 2908 const TimeUnit& aTime, 2909 const TimeUnit& aFuzz) { 2910 MOZ_ASSERT(OnTaskQueue()); 2911 AUTO_PROFILER_LABEL("TrackBuffersManager::Seek", MEDIA_PLAYBACK); 2912 auto& trackBuffer = GetTracksData(aTrack); 2913 const TrackBuffersManager::TrackBuffer& track = GetTrackBuffer(aTrack); 2914 MSE_DEBUG("Seek, track=%s, target=%" PRId64, TrackTypeToStr(aTrack), 2915 aTime.ToMicroseconds()); 2916 2917 if (!track.Length()) { 2918 // This a reset. It will be followed by another valid seek. 2919 trackBuffer.mNextGetSampleIndex = Some(uint32_t(0)); 2920 trackBuffer.mNextSampleTimecode = TimeUnit(); 2921 trackBuffer.mNextSampleTime = TimeUnit(); 2922 ResetEvictionIndex(trackBuffer); 2923 return TimeUnit(); 2924 } 2925 2926 uint32_t i = 0; 2927 2928 if (aTime != TimeUnit()) { 2929 // Determine the interval of samples we're attempting to seek to. 2930 TimeIntervals buffered = trackBuffer.mBufferedRanges; 2931 // Fuzz factor is +/- aFuzz; as we want to only eliminate gaps 2932 // that are less than aFuzz wide, we set a fuzz factor aFuzz/2. 2933 buffered.SetFuzz(aFuzz / 2); 2934 TimeIntervals::IndexType index = buffered.Find(aTime); 2935 MOZ_ASSERT(index != TimeIntervals::NoIndex, 2936 "We shouldn't be called if aTime isn't buffered"); 2937 TimeInterval target = buffered[index]; 2938 target.mFuzz = aFuzz; 2939 i = FindSampleIndex(track, target); 2940 } 2941 2942 Maybe<TimeUnit> lastKeyFrameTime; 2943 TimeUnit lastKeyFrameTimecode; 2944 uint32_t lastKeyFrameIndex = 0; 2945 for (; i < track.Length(); i++) { 2946 const RefPtr<MediaRawData>& sample = track[i]; 2947 TimeUnit sampleTime = sample->mTime; 2948 if (sampleTime > aTime && lastKeyFrameTime.isSome()) { 2949 break; 2950 } 2951 if (sample->mKeyframe) { 2952 lastKeyFrameTimecode = sample->mTimecode; 2953 lastKeyFrameTime = Some(sampleTime); 2954 lastKeyFrameIndex = i; 2955 } 2956 if (sampleTime == aTime || 2957 (sampleTime > aTime && lastKeyFrameTime.isSome())) { 2958 break; 2959 } 2960 } 2961 MSE_DEBUG("Keyframe %s found at %" PRId64 " @ %u", 2962 lastKeyFrameTime.isSome() ? "" : "not", 2963 lastKeyFrameTime.refOr(TimeUnit()).ToMicroseconds(), 2964 lastKeyFrameIndex); 2965 2966 trackBuffer.mNextGetSampleIndex = Some(lastKeyFrameIndex); 2967 trackBuffer.mNextSampleTimecode = lastKeyFrameTimecode; 2968 trackBuffer.mNextSampleTime = lastKeyFrameTime.refOr(TimeUnit()); 2969 ResetEvictionIndex(trackBuffer); 2970 UpdateEvictionIndex(trackBuffer, lastKeyFrameIndex); 2971 2972 return lastKeyFrameTime.refOr(TimeUnit()); 2973 } 2974 2975 uint32_t TrackBuffersManager::SkipToNextRandomAccessPoint( 2976 TrackInfo::TrackType aTrack, const TimeUnit& aTimeThreadshold, 2977 const media::TimeUnit& aFuzz, bool& aFound) { 2978 mTaskQueueCapability->AssertOnCurrentThread(); 2979 AUTO_PROFILER_LABEL("TrackBuffersManager::SkipToNextRandomAccessPoint", 2980 MEDIA_PLAYBACK); 2981 uint32_t parsed = 0; 2982 auto& trackData = GetTracksData(aTrack); 2983 const TrackBuffer& track = GetTrackBuffer(aTrack); 2984 aFound = false; 2985 2986 // SkipToNextRandomAccessPoint can only be called if aTimeThreadshold is known 2987 // to be buffered. 2988 2989 if (NS_FAILED(SetNextGetSampleIndexIfNeeded(aTrack, aFuzz))) { 2990 return 0; 2991 } 2992 2993 TimeUnit nextSampleTimecode = trackData.mNextSampleTimecode; 2994 TimeUnit nextSampleTime = trackData.mNextSampleTime; 2995 uint32_t i = trackData.mNextGetSampleIndex.ref(); 2996 uint32_t originalPos = i; 2997 2998 for (; i < track.Length(); i++) { 2999 const MediaRawData* sample = 3000 GetSample(aTrack, i, nextSampleTimecode, nextSampleTime, aFuzz); 3001 if (!sample) { 3002 break; 3003 } 3004 if (sample->mKeyframe && sample->mTime >= aTimeThreadshold) { 3005 aFound = true; 3006 break; 3007 } 3008 nextSampleTimecode = sample->GetEndTimecode(); 3009 nextSampleTime = sample->GetEndTime(); 3010 parsed++; 3011 } 3012 3013 // Adjust the next demux time and index so that the next call to 3014 // SkipToNextRandomAccessPoint will not count again the parsed sample as 3015 // skipped. 3016 if (aFound) { 3017 trackData.mNextSampleTimecode = track[i]->mTimecode; 3018 trackData.mNextSampleTime = track[i]->mTime; 3019 trackData.mNextGetSampleIndex = Some(i); 3020 } else if (i > 0) { 3021 // Go back to the previous keyframe or the original position so the next 3022 // demux can succeed and be decoded. 3023 for (uint32_t j = i; j-- > originalPos;) { 3024 const RefPtr<MediaRawData>& sample = track[j]; 3025 if (sample->mKeyframe) { 3026 trackData.mNextSampleTimecode = sample->mTimecode; 3027 trackData.mNextSampleTime = sample->mTime; 3028 trackData.mNextGetSampleIndex = Some(uint32_t(j)); 3029 // We are unable to skip to a keyframe past aTimeThreshold, however 3030 // we are speeding up decoding by dropping the unplayable frames. 3031 // So we can mark aFound as true. 3032 aFound = true; 3033 break; 3034 } 3035 parsed--; 3036 } 3037 } 3038 3039 if (aFound) { 3040 UpdateEvictionIndex(trackData, trackData.mNextGetSampleIndex.ref()); 3041 } 3042 3043 return parsed; 3044 } 3045 3046 const MediaRawData* TrackBuffersManager::GetSample(TrackInfo::TrackType aTrack, 3047 uint32_t aIndex, 3048 const TimeUnit& aExpectedDts, 3049 const TimeUnit& aExpectedPts, 3050 const TimeUnit& aFuzz) { 3051 MOZ_ASSERT(OnTaskQueue()); 3052 const TrackBuffer& track = GetTrackBuffer(aTrack); 3053 3054 if (aIndex >= track.Length()) { 3055 MSE_DEBUGV( 3056 "Can't get sample due to reaching to the end, index=%u, " 3057 "length=%zu", 3058 aIndex, track.Length()); 3059 // reached the end. 3060 return nullptr; 3061 } 3062 3063 if (!(aExpectedDts + aFuzz).IsValid() || !(aExpectedPts + aFuzz).IsValid()) { 3064 // Time overflow, it seems like we also reached the end. 3065 MSE_DEBUGV("Can't get sample due to time overflow, expectedPts=%" PRId64 3066 ", aExpectedDts=%" PRId64 ", fuzz=%" PRId64, 3067 aExpectedPts.ToMicroseconds(), aExpectedPts.ToMicroseconds(), 3068 aFuzz.ToMicroseconds()); 3069 return nullptr; 3070 } 3071 3072 const RefPtr<MediaRawData>& sample = track[aIndex]; 3073 if (!aIndex || sample->mTimecode <= aExpectedDts + aFuzz || 3074 sample->mTime <= aExpectedPts + aFuzz) { 3075 MOZ_DIAGNOSTIC_ASSERT(sample->HasValidTime()); 3076 return sample; 3077 } 3078 3079 MSE_DEBUGV("Can't get sample due to big gap, sample=%" PRId64 3080 ", expectedPts=%" PRId64 ", aExpectedDts=%" PRId64 3081 ", fuzz=%" PRId64, 3082 sample->mTime.ToMicroseconds(), aExpectedPts.ToMicroseconds(), 3083 aExpectedPts.ToMicroseconds(), aFuzz.ToMicroseconds()); 3084 3085 // Gap is too big. End of Stream or Waiting for Data. 3086 // TODO, check that we have continuous data based on the sanitized buffered 3087 // range instead. 3088 return nullptr; 3089 } 3090 3091 already_AddRefed<MediaRawData> TrackBuffersManager::GetSample( 3092 TrackInfo::TrackType aTrack, const TimeUnit& aFuzz, MediaResult& aResult) { 3093 mTaskQueueCapability->AssertOnCurrentThread(); 3094 AUTO_PROFILER_LABEL("TrackBuffersManager::GetSample", MEDIA_PLAYBACK); 3095 auto& trackData = GetTracksData(aTrack); 3096 const TrackBuffer& track = GetTrackBuffer(aTrack); 3097 3098 aResult = NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA; 3099 3100 if (trackData.mNextGetSampleIndex.isSome()) { 3101 if (trackData.mNextGetSampleIndex.ref() >= track.Length()) { 3102 aResult = NS_ERROR_DOM_MEDIA_END_OF_STREAM; 3103 return nullptr; 3104 } 3105 const MediaRawData* sample = GetSample( 3106 aTrack, trackData.mNextGetSampleIndex.ref(), 3107 trackData.mNextSampleTimecode, trackData.mNextSampleTime, aFuzz); 3108 if (!sample) { 3109 return nullptr; 3110 } 3111 3112 RefPtr<MediaRawData> p = sample->Clone(); 3113 if (!p) { 3114 aResult = MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__); 3115 return nullptr; 3116 } 3117 if (p->mKeyframe) { 3118 UpdateEvictionIndex(trackData, trackData.mNextGetSampleIndex.ref()); 3119 } 3120 trackData.mNextGetSampleIndex.ref()++; 3121 // Estimate decode timestamp and timestamp of the next sample. 3122 TimeUnit nextSampleTimecode = sample->GetEndTimecode(); 3123 TimeUnit nextSampleTime = sample->GetEndTime(); 3124 const MediaRawData* nextSample = 3125 GetSample(aTrack, trackData.mNextGetSampleIndex.ref(), 3126 nextSampleTimecode, nextSampleTime, aFuzz); 3127 if (nextSample) { 3128 // We have a valid next sample, can use exact values. 3129 trackData.mNextSampleTimecode = nextSample->mTimecode; 3130 trackData.mNextSampleTime = nextSample->mTime; 3131 } else { 3132 // Next sample isn't available yet. Use estimates. 3133 trackData.mNextSampleTimecode = nextSampleTimecode; 3134 trackData.mNextSampleTime = nextSampleTime; 3135 } 3136 aResult = NS_OK; 3137 return p.forget(); 3138 } 3139 3140 aResult = SetNextGetSampleIndexIfNeeded(aTrack, aFuzz); 3141 3142 if (NS_FAILED(aResult)) { 3143 return nullptr; 3144 } 3145 3146 MOZ_RELEASE_ASSERT(trackData.mNextGetSampleIndex.isSome() && 3147 trackData.mNextGetSampleIndex.ref() < track.Length()); 3148 const RefPtr<MediaRawData>& sample = 3149 track[trackData.mNextGetSampleIndex.ref()]; 3150 RefPtr<MediaRawData> p = sample->Clone(); 3151 if (!p) { 3152 // OOM 3153 aResult = MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__); 3154 return nullptr; 3155 } 3156 MOZ_DIAGNOSTIC_ASSERT(p->HasValidTime()); 3157 3158 // Find the previous keyframe to calculate the evictable amount. 3159 uint32_t i = trackData.mNextGetSampleIndex.ref(); 3160 for (; !track[i]->mKeyframe; i--) { 3161 } 3162 UpdateEvictionIndex(trackData, i); 3163 3164 trackData.mNextGetSampleIndex.ref()++; 3165 trackData.mNextSampleTimecode = sample->GetEndTimecode(); 3166 trackData.mNextSampleTime = sample->GetEndTime(); 3167 return p.forget(); 3168 } 3169 3170 int32_t TrackBuffersManager::FindCurrentPosition(TrackInfo::TrackType aTrack, 3171 const TimeUnit& aFuzz) const { 3172 MOZ_ASSERT(OnTaskQueue()); 3173 const auto& trackData = GetTracksData(aTrack); 3174 const TrackBuffer& track = GetTrackBuffer(aTrack); 3175 int32_t trackLength = AssertedCast<int32_t>(track.Length()); 3176 3177 // Perform an exact search first. 3178 for (int32_t i = 0; i < trackLength; i++) { 3179 const RefPtr<MediaRawData>& sample = track[i]; 3180 TimeInterval sampleInterval{sample->mTimecode, sample->GetEndTimecode()}; 3181 3182 if (sampleInterval.ContainsStrict(trackData.mNextSampleTimecode)) { 3183 return i; 3184 } 3185 if (sampleInterval.mStart > trackData.mNextSampleTimecode) { 3186 // Samples are ordered by timecode. There's no need to search 3187 // any further. 3188 break; 3189 } 3190 } 3191 3192 for (int32_t i = 0; i < trackLength; i++) { 3193 const RefPtr<MediaRawData>& sample = track[i]; 3194 TimeInterval sampleInterval{sample->mTimecode, sample->GetEndTimecode(), 3195 aFuzz}; 3196 3197 if (sampleInterval.ContainsWithStrictEnd(trackData.mNextSampleTimecode)) { 3198 return i; 3199 } 3200 if (sampleInterval.mStart - aFuzz > trackData.mNextSampleTimecode) { 3201 // Samples are ordered by timecode. There's no need to search 3202 // any further. 3203 break; 3204 } 3205 } 3206 3207 // We couldn't find our sample by decode timestamp. Attempt to find it using 3208 // presentation timestamp. There will likely be small jerkiness. 3209 for (int32_t i = 0; i < trackLength; i++) { 3210 const RefPtr<MediaRawData>& sample = track[i]; 3211 TimeInterval sampleInterval{sample->mTime, sample->GetEndTime(), aFuzz}; 3212 3213 if (sampleInterval.ContainsWithStrictEnd(trackData.mNextSampleTimecode)) { 3214 return i; 3215 } 3216 } 3217 3218 // Still not found. 3219 return -1; 3220 } 3221 3222 uint32_t TrackBuffersManager::Evictable(TrackInfo::TrackType aTrack) const { 3223 MutexAutoLock mut(mMutex); 3224 return GetTracksData(aTrack).mEvictionIndex.mEvictable; 3225 } 3226 3227 TimeUnit TrackBuffersManager::GetNextRandomAccessPoint( 3228 TrackInfo::TrackType aTrack, const TimeUnit& aFuzz) { 3229 mTaskQueueCapability->AssertOnCurrentThread(); 3230 3231 // So first determine the current position in the track buffer if necessary. 3232 if (NS_FAILED(SetNextGetSampleIndexIfNeeded(aTrack, aFuzz))) { 3233 return TimeUnit::FromInfinity(); 3234 } 3235 3236 auto& trackData = GetTracksData(aTrack); 3237 const TrackBuffersManager::TrackBuffer& track = GetTrackBuffer(aTrack); 3238 3239 uint32_t i = trackData.mNextGetSampleIndex.ref(); 3240 TimeUnit nextSampleTimecode = trackData.mNextSampleTimecode; 3241 TimeUnit nextSampleTime = trackData.mNextSampleTime; 3242 3243 for (; i < track.Length(); i++) { 3244 const MediaRawData* sample = 3245 GetSample(aTrack, i, nextSampleTimecode, nextSampleTime, aFuzz); 3246 if (!sample) { 3247 break; 3248 } 3249 if (sample->mKeyframe) { 3250 return sample->mTime; 3251 } 3252 nextSampleTimecode = sample->GetEndTimecode(); 3253 nextSampleTime = sample->GetEndTime(); 3254 } 3255 return TimeUnit::FromInfinity(); 3256 } 3257 3258 nsresult TrackBuffersManager::SetNextGetSampleIndexIfNeeded( 3259 TrackInfo::TrackType aTrack, const TimeUnit& aFuzz) { 3260 MOZ_ASSERT(OnTaskQueue()); 3261 auto& trackData = GetTracksData(aTrack); 3262 const TrackBuffer& track = GetTrackBuffer(aTrack); 3263 3264 if (trackData.mNextGetSampleIndex.isSome()) { 3265 // We already know the next GetSample index. 3266 return NS_OK; 3267 } 3268 3269 if (!track.Length()) { 3270 // There's nothing to find yet. 3271 return NS_ERROR_DOM_MEDIA_END_OF_STREAM; 3272 } 3273 3274 if (trackData.mNextSampleTimecode == TimeUnit()) { 3275 // First demux, get first sample. 3276 trackData.mNextGetSampleIndex = Some(0u); 3277 return NS_OK; 3278 } 3279 3280 if (trackData.mNextSampleTimecode > track.LastElement()->GetEndTimecode()) { 3281 // The next element is past our last sample. We're done. 3282 trackData.mNextGetSampleIndex = Some(uint32_t(track.Length())); 3283 return NS_ERROR_DOM_MEDIA_END_OF_STREAM; 3284 } 3285 3286 int32_t pos = FindCurrentPosition(aTrack, aFuzz); 3287 if (pos < 0) { 3288 // Not found, must wait for more data. 3289 MSE_DEBUG("Couldn't find sample (pts:%" PRId64 " dts:%" PRId64 ")", 3290 trackData.mNextSampleTime.ToMicroseconds(), 3291 trackData.mNextSampleTimecode.ToMicroseconds()); 3292 return NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA; 3293 } 3294 trackData.mNextGetSampleIndex = Some(uint32_t(pos)); 3295 return NS_OK; 3296 } 3297 3298 void TrackBuffersManager::TrackData::AddSizeOfResources( 3299 MediaSourceDecoder::ResourceSizes* aSizes) const { 3300 for (const TrackBuffer& buffer : mBuffers) { 3301 for (const MediaRawData* data : buffer) { 3302 aSizes->mByteSize += data->SizeOfIncludingThis(aSizes->mMallocSizeOf); 3303 } 3304 } 3305 } 3306 3307 RefPtr<GenericPromise> TrackBuffersManager::RequestDebugInfo( 3308 dom::TrackBuffersManagerDebugInfo& aInfo) const { 3309 const RefPtr<TaskQueue> taskQueue = GetTaskQueueSafe(); 3310 if (!taskQueue) { 3311 return GenericPromise::CreateAndResolve(true, __func__); 3312 } 3313 if (!taskQueue->IsCurrentThreadIn()) { 3314 // Run the request on the task queue if it's not already. 3315 return InvokeAsync(taskQueue.get(), __func__, 3316 [this, self = RefPtr{this}, &aInfo] { 3317 return RequestDebugInfo(aInfo); 3318 }); 3319 } 3320 mTaskQueueCapability->AssertOnCurrentThread(); 3321 GetDebugInfo(aInfo); 3322 return GenericPromise::CreateAndResolve(true, __func__); 3323 } 3324 3325 void TrackBuffersManager::GetDebugInfo( 3326 dom::TrackBuffersManagerDebugInfo& aInfo) const { 3327 MOZ_ASSERT(OnTaskQueue(), 3328 "This shouldn't be called off the task queue because we're about " 3329 "to touch a lot of data that is used on the task queue"); 3330 CopyUTF8toUTF16(mType.Type().AsString(), aInfo.mType); 3331 3332 if (HasAudio()) { 3333 aInfo.mNextSampleTime = mAudioTracks.mNextSampleTime.ToSeconds(); 3334 aInfo.mNumSamples = 3335 AssertedCast<int32_t>(mAudioTracks.mBuffers[0].Length()); 3336 aInfo.mBufferSize = AssertedCast<int32_t>(mAudioTracks.mSizeBuffer); 3337 aInfo.mEvictable = AssertedCast<int32_t>(Evictable(TrackInfo::kAudioTrack)); 3338 aInfo.mNextGetSampleIndex = 3339 AssertedCast<int32_t>(mAudioTracks.mNextGetSampleIndex.valueOr(-1)); 3340 aInfo.mNextInsertionIndex = 3341 AssertedCast<int32_t>(mAudioTracks.mNextInsertionIndex.valueOr(-1)); 3342 media::TimeIntervals ranges = SafeBuffered(TrackInfo::kAudioTrack); 3343 dom::Sequence<dom::BufferRange> items; 3344 for (uint32_t i = 0; i < ranges.Length(); ++i) { 3345 // dom::Sequence is a FallibleTArray 3346 dom::BufferRange* range = items.AppendElement(fallible); 3347 if (!range) { 3348 break; 3349 } 3350 range->mStart = ranges.Start(i).ToSeconds(); 3351 range->mEnd = ranges.End(i).ToSeconds(); 3352 } 3353 aInfo.mRanges = std::move(items); 3354 } else if (HasVideo()) { 3355 aInfo.mNextSampleTime = mVideoTracks.mNextSampleTime.ToSeconds(); 3356 aInfo.mNumSamples = 3357 AssertedCast<int32_t>(mVideoTracks.mBuffers[0].Length()); 3358 aInfo.mBufferSize = AssertedCast<int32_t>(mVideoTracks.mSizeBuffer); 3359 aInfo.mEvictable = AssertedCast<int32_t>(Evictable(TrackInfo::kVideoTrack)); 3360 aInfo.mNextGetSampleIndex = 3361 AssertedCast<int32_t>(mVideoTracks.mNextGetSampleIndex.valueOr(-1)); 3362 aInfo.mNextInsertionIndex = 3363 AssertedCast<int32_t>(mVideoTracks.mNextInsertionIndex.valueOr(-1)); 3364 media::TimeIntervals ranges = SafeBuffered(TrackInfo::kVideoTrack); 3365 dom::Sequence<dom::BufferRange> items; 3366 for (uint32_t i = 0; i < ranges.Length(); ++i) { 3367 // dom::Sequence is a FallibleTArray 3368 dom::BufferRange* range = items.AppendElement(fallible); 3369 if (!range) { 3370 break; 3371 } 3372 range->mStart = ranges.Start(i).ToSeconds(); 3373 range->mEnd = ranges.End(i).ToSeconds(); 3374 } 3375 aInfo.mRanges = std::move(items); 3376 } 3377 } 3378 3379 void TrackBuffersManager::AddSizeOfResources( 3380 MediaSourceDecoder::ResourceSizes* aSizes) const { 3381 mTaskQueueCapability->AssertOnCurrentThread(); 3382 3383 if (mInputBuffer.isSome() && mInputBuffer->Buffer()) { 3384 // mInputBuffer should be the sole owner of the underlying buffer, so this 3385 // won't double count. 3386 aSizes->mByteSize += mInputBuffer->Buffer()->ShallowSizeOfIncludingThis( 3387 aSizes->mMallocSizeOf); 3388 } 3389 if (mInitData) { 3390 aSizes->mByteSize += 3391 mInitData->ShallowSizeOfIncludingThis(aSizes->mMallocSizeOf); 3392 } 3393 if (mPendingInputBuffer.isSome() && mPendingInputBuffer->Buffer()) { 3394 // mPendingInputBuffer should be the sole owner of the underlying buffer, so 3395 // this won't double count. 3396 aSizes->mByteSize += 3397 mPendingInputBuffer->Buffer()->ShallowSizeOfIncludingThis( 3398 aSizes->mMallocSizeOf); 3399 } 3400 3401 mVideoTracks.AddSizeOfResources(aSizes); 3402 mAudioTracks.AddSizeOfResources(aSizes); 3403 } 3404 3405 } // namespace mozilla 3406 #undef MSE_DEBUG 3407 #undef MSE_DEBUGV 3408 #undef SAMPLE_DEBUG 3409 #undef SAMPLE_DEBUGV