tor-browser

TestVideoSink.cpp (6971B)

---

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "AudioDeviceInfo.h"
#include "AudioSink.h"
#include "AudioSinkWrapper.h"
#include "ImageContainer.h"
#include "MockCubeb.h"
#include "MockMediaDecoderOwner.h"
#include "TimeUnits.h"
#include "VideoFrameContainer.h"
#include "VideoSink.h"
#include "gtest/gtest.h"
#include "mozilla/gtest/WaitFor.h"

using namespace mozilla;
using namespace mozilla::layers;

using media::TimeUnit;

TEST(TestVideoSink, FrameThrottling)
{
 MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual);
 CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext());

 MediaInfo info;
 info.EnableAudio();  // to control the advance of time through MockCubeb
 info.EnableVideo();

 MediaQueue<AudioData> audioQueue;
 auto audioSinkCreator = [&]() {
   return UniquePtr<AudioSink>{new AudioSink(AbstractThread::GetCurrent(),
                                             audioQueue, info.mAudio,
                                             /*resistFingerprinting*/ false)};
 };
 RefPtr wrapper = new AudioSinkWrapper(
     AbstractThread::GetCurrent(), audioQueue, std::move(audioSinkCreator),
     /*initialVolume*/ 1.0, /*playbackRate*/ 1.0, /*preservesPitch*/ true,
     /*sinkDevice*/ nullptr);

 auto owner = std::make_unique<MockMediaDecoderOwner>();
 RefPtr container = new VideoFrameContainer(
     owner.get(),
     MakeAndAddRef<ImageContainer>(ImageUsageType::VideoFrameContainer,
#ifdef MOZ_WIDGET_ANDROID
                                   // Work around bug 1922144
                                   ImageContainer::SYNCHRONOUS
#else
                                   ImageContainer::ASYNCHRONOUS
#endif
                                   ));

 MediaQueue<VideoData> videoQueue;
 RefPtr frameStatistics = new FrameStatistics();
 RefPtr videoSink = new VideoSink(AbstractThread::GetCurrent(), wrapper,
                                  videoQueue, container, *frameStatistics,
                                  /*aVQueueSentToCompositerSize*/ 9999);
 auto initPromise = TakeN(cubeb->StreamInitEvent(), 1);
 videoSink->Start(TimeUnit::Zero(), info);
 auto [stream] = WaitFor(initPromise).unwrap()[0];
 uint32_t audioRate = stream->SampleRate();

 // Enough audio data that it does not underrun, which would stop the clock.
 size_t audioFrameCount = 1000 * info.mAudio.mRate / audioRate;
 AlignedAudioBuffer samples(audioFrameCount * info.mAudio.mChannels);
 RefPtr audioData = new AudioData(
     /*aOffset*/ 0, /*aTime*/ TimeUnit(0, info.mAudio.mRate),
     std::move(samples), info.mAudio.mChannels, info.mAudio.mRate);
 audioQueue.Push(audioData);

 auto image = container->GetImageContainer()->CreatePlanarYCbCrImage();
 static uint8_t pixel[] = {0x00};
 PlanarYCbCrData imageData;
 imageData.mYChannel = imageData.mCbChannel = imageData.mCrChannel = pixel;
 imageData.mYStride = imageData.mCbCrStride = 1;
 imageData.mPictureRect = gfx::IntRect(0, 0, 1, 1);
 image->CopyData(imageData);

 TimeUnit nextFrameTime = TimeUnit(0, audioRate);
 auto PushVideoFrame = [&](const gfx::IntSize& aSize,
                           const TimeUnit& aDuration) {
   static bool isKeyFrame = true;
   RefPtr frame = VideoData::CreateFromImage(aSize, /*aOffset*/ 0,
                                             /*aTime*/ nextFrameTime,
                                             aDuration, image, isKeyFrame,
                                             /*aTimecode*/ nextFrameTime);
   frame->mFrameID = container->NewFrameID();
   videoQueue.Push(frame);
   nextFrameTime = frame->GetEndTime();
   isKeyFrame = false;
 };

 gfx::IntSize size1{1, 1};
 PushVideoFrame(size1, TimeUnit(1, audioRate));
 gfx::IntSize size2{1, 2};
 PushVideoFrame(size2, TimeUnit(1, audioRate));
 // UpdateRenderedVideoFrames() will keep scheduling additional events in
 // antipication of the audio clock advancing for the second frame, so wait
 // for only the initial size from the first frame.
 SpinEventLoopUntil("the intrinsic size receives an initial value"_ns, [&] {
   return container->CurrentIntrinsicSize().isSome();
 });
 EXPECT_EQ(container->CurrentIntrinsicSize().value(), size1);

 // Advance time to expire both frames.
 stream->ManualDataCallback(nextFrameTime.ToTicksAtRate(audioRate) + 1);
 // Run UpdateRenderedVideoFramesByTimer(), which is scheduled on TimeStamp's
 // clock, which we don't control.
 SpinEventLoopUntil(
     "the intrinsic size is updated to that of frame 2"_ns,
     [&] { return container->CurrentIntrinsicSize().value() == size2; });

 // The next frame is overdue but has not yet expired.
 gfx::IntSize size3{1, 3};
 PushVideoFrame(size3, TimeUnit(2, audioRate));
 gfx::IntSize size4{1, 4};
 PushVideoFrame(size4, TimeUnit(1, audioRate));
 // Run UpdateRenderedVideoFrames() via OnVideoQueuePushed().
 NS_ProcessPendingEvents(nullptr);
 EXPECT_EQ(container->CurrentIntrinsicSize().value(), size3);
 EXPECT_EQ(frameStatistics->GetDroppedSinkFrames(), 0u);

 // Advance time to expire the two frames in the queue and the next three.
 stream->ManualDataCallback(static_cast<long>(
     nextFrameTime.ToTicksAtRate(audioRate) + 11 - stream->Position()));
 // This frame has a longer duration and is late.
 gfx::IntSize size5{1, 5};
 PushVideoFrame(size5, TimeUnit(8, audioRate));
 // The most recent frame was late, and so is not rendered yet because it may
 // be dropped.
 //
 // OnVideoQueuePushed() uses TryUpdateRenderedVideoFrames(), which no-ops if
 // an update is already scheduled.  Wait for the update scheduled for
 // frame 4.
 SpinEventLoopUntil(
     "the intrinsic size is updated to that of frame 4"_ns,
     [&] { return container->CurrentIntrinsicSize().value() == size4; });

 // This frame is also late.
 gfx::IntSize size6{1, 6};
 PushVideoFrame(size6, TimeUnit(1, audioRate));
 NS_ProcessPendingEvents(nullptr);
 // One frame was dropped, but the most recent frame was rendered because its
 // lateness was less than the duration of the dropped frame.
 EXPECT_EQ(frameStatistics->GetDroppedSinkFrames(), 1u);
 EXPECT_EQ(container->CurrentIntrinsicSize().value(), size6);

 gfx::IntSize size7{1, 7};
 PushVideoFrame(size7, TimeUnit(1, audioRate));
 NS_ProcessPendingEvents(nullptr);
 // The most recent frame was late, and so is not rendered yet because it may
 // be dropped.
 EXPECT_EQ(container->CurrentIntrinsicSize().value(), size6);

 // On playback pause, the most recent frame is rendered.
 videoSink->SetPlaying(false);
 EXPECT_EQ(container->CurrentIntrinsicSize().value(), size7);
 EXPECT_EQ(frameStatistics->GetDroppedSinkFrames(), 1u);
 videoSink->Stop();
 videoSink->Shutdown();
}