commit ebf720220fa8a697e23e59685c07bbe2d33af511 parent ebec12a46cd39406e07ff3d17e5d5829b00ff0fe Author: Dan Baker <dbaker@mozilla.com> Date: Mon, 1 Dec 2025 23:15:26 -0700 Bug 2000941 - Vendor libwebrtc from 52cbf242e9 Upstream commit: https://webrtc.googlesource.com/src/+/52cbf242e96035991dedc8b4a17c7434a0a179fd Move frame_instrumentation_generator to api/video Moves the frame_instrumentation_generator files from video/corruption_detection to api/video/corruption_detection so that it can be used as a public dependency. Implementation moved inside the .cc in order to avoid DEPS problems. Bug: webrtc:358039777 Change-Id: I885d0fb965e4ceecf88d2c3067e1785351a9f1b3 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/410820 Commit-Queue: Erik Språng <sprang@webrtc.org> Reviewed-by: Per Kjellander <perkj@webrtc.org> Cr-Commit-Position: refs/heads/main@{#45710} Diffstat:
20 files changed, 1159 insertions(+), 1074 deletions(-)
diff --git a/third_party/libwebrtc/README.mozilla.last-vendor b/third_party/libwebrtc/README.mozilla.last-vendor @@ -1,4 +1,4 @@ # ./mach python dom/media/webrtc/third_party_build/vendor-libwebrtc.py --from-local /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc --commit mozpatches libwebrtc -libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-12-02T06:12:18.453329+00:00. +libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-12-02T06:15:02.689857+00:00. # base of lastest vendoring -4ed70dbed2 +52cbf242e9 diff --git a/third_party/libwebrtc/api/BUILD.gn b/third_party/libwebrtc/api/BUILD.gn @@ -1735,6 +1735,7 @@ if (rtc_include_tests) { "video/corruption_detection:frame_instrumentation_data_reader_unittest", "video/corruption_detection:frame_instrumentation_data_unittest", "video/corruption_detection:frame_instrumentation_evaluation_unittest", + "video/corruption_detection:frame_instrumentation_generator_unittest", "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/strings:string_view", diff --git a/third_party/libwebrtc/api/video/corruption_detection/BUILD.gn b/third_party/libwebrtc/api/video/corruption_detection/BUILD.gn @@ -61,6 +61,32 @@ rtc_library("frame_instrumentation_evaluation") { ] } +rtc_library("frame_instrumentation_generator") { + visibility = [ "*" ] + sources = [ + "frame_instrumentation_generator.cc", + "frame_instrumentation_generator.h", + ] + deps = [ + ":filter_settings", + ":frame_instrumentation_data", + "../:encoded_image", + "../:video_frame", + "../:video_frame_type", + "../../../modules:module_api_public", + "../../../modules/video_coding:video_coding_utility", + "../../../rtc_base:checks", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base/synchronization:mutex", + "../../../video/corruption_detection:generic_mapping_functions", + "../../../video/corruption_detection:halton_frame_sampler", + "../../../video/corruption_detection:utils", + "../../video_codecs:video_codecs_api", + "//third_party/abseil-cpp/absl/algorithm:container", + ] +} + if (rtc_include_tests) { rtc_library("frame_instrumentation_data_unittest") { testonly = true @@ -96,4 +122,22 @@ if (rtc_include_tests) { "../../../test:test_support", ] } + + rtc_library("frame_instrumentation_generator_unittest") { + testonly = true + sources = [ "frame_instrumentation_generator_unittest.cc" ] + deps = [ + ":filter_settings", + ":frame_instrumentation_data", + ":frame_instrumentation_generator", + "../:encoded_image", + "../:video_frame", + "../:video_frame_type", + "../../../api:make_ref_counted", + "../../../api:scoped_refptr", + "../../../rtc_base:refcount", + "../../../test:test_support", + "../../../video/corruption_detection:utils", + ] + } } diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_generator.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_generator.cc @@ -0,0 +1,261 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_generator.h" + +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <map> +#include <memory> +#include <optional> +#include <queue> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "api/video/corruption_detection/corruption_detection_filter_settings.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/encoded_image.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "api/video_codecs/video_codec.h" +#include "modules/include/module_common_types_public.h" +#include "modules/video_coding/utility/qp_parser.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "video/corruption_detection/generic_mapping_functions.h" +#include "video/corruption_detection/halton_frame_sampler.h" +#include "video/corruption_detection/utils.h" + +namespace webrtc { +namespace { + +// Avoid holding on to frames that might have been dropped by encoder, as that +// can lead to frame buffer pools draining. +constexpr size_t kMaxPendingFrames = 3; + +std::optional<CorruptionDetectionFilterSettings> GetCorruptionFilterSettings( + const EncodedImage& encoded_image, + VideoCodecType video_codec_type, + int layer_id) { + std::optional<CorruptionDetectionFilterSettings> filter_settings = + encoded_image.corruption_detection_filter_settings(); + + if (!filter_settings.has_value()) { + // No implementation specific filter settings available, using a generic + // QP-based settings instead. + int qp = encoded_image.qp_; + if (qp == -1) { + std::optional<uint32_t> parsed_qp = + QpParser().Parse(video_codec_type, layer_id, encoded_image.data(), + encoded_image.size()); + if (!parsed_qp.has_value()) { + RTC_LOG(LS_VERBOSE) + << "Missing QP for " << CodecTypeToPayloadString(video_codec_type) + << " layer " << layer_id << "."; + return std::nullopt; + } + qp = *parsed_qp; + } + + filter_settings = GetCorruptionFilterSettings(qp, video_codec_type); + } + return filter_settings; +} + +} // namespace + +class FrameInstrumentationGeneratorImpl : public FrameInstrumentationGenerator { + public: + explicit FrameInstrumentationGeneratorImpl(VideoCodecType video_codec_type); + + FrameInstrumentationGeneratorImpl(const FrameInstrumentationGeneratorImpl&) = + delete; + FrameInstrumentationGeneratorImpl& operator=( + const FrameInstrumentationGeneratorImpl&) = delete; + + ~FrameInstrumentationGeneratorImpl() override = default; + + void OnCapturedFrame(VideoFrame frame) override RTC_LOCKS_EXCLUDED(mutex_); + std::optional<FrameInstrumentationData> OnEncodedImage( + const EncodedImage& encoded_image) override RTC_LOCKS_EXCLUDED(mutex_); + + std::optional<int> GetHaltonSequenceIndex(int layer_id) const override + RTC_LOCKS_EXCLUDED(mutex_); + void SetHaltonSequenceIndex(int index, int layer_id) override + RTC_LOCKS_EXCLUDED(mutex_); + + private: + struct Context { + HaltonFrameSampler frame_sampler; + uint32_t rtp_timestamp_of_last_key_frame = 0; + }; + + // Incoming video frames in capture order. + std::queue<VideoFrame> captured_frames_ RTC_GUARDED_BY(mutex_); + // Map from spatial or simulcast index to sampling context. + std::map<int, Context> contexts_ RTC_GUARDED_BY(mutex_); + const VideoCodecType video_codec_type_; + mutable Mutex mutex_; +}; + +FrameInstrumentationGeneratorImpl::FrameInstrumentationGeneratorImpl( + VideoCodecType video_codec_type) + : video_codec_type_(video_codec_type) {} + +void FrameInstrumentationGeneratorImpl::OnCapturedFrame(VideoFrame frame) { + MutexLock lock(&mutex_); + while (captured_frames_.size() >= kMaxPendingFrames) { + captured_frames_.pop(); + } + captured_frames_.push(frame); +} + +std::optional<FrameInstrumentationData> +FrameInstrumentationGeneratorImpl::OnEncodedImage( + const EncodedImage& encoded_image) { + uint32_t rtp_timestamp_encoded_image = encoded_image.RtpTimestamp(); + std::optional<VideoFrame> captured_frame; + int layer_id; + FrameInstrumentationData data; + std::vector<HaltonFrameSampler::Coordinates> sample_coordinates; + { + MutexLock lock(&mutex_); + while (!captured_frames_.empty() && + IsNewerTimestamp(rtp_timestamp_encoded_image, + captured_frames_.front().rtp_timestamp())) { + captured_frames_.pop(); + } + if (captured_frames_.empty() || captured_frames_.front().rtp_timestamp() != + rtp_timestamp_encoded_image) { + RTC_LOG(LS_VERBOSE) << "No captured frames for RTC timestamp " + << rtp_timestamp_encoded_image << "."; + return std::nullopt; + } + captured_frame = captured_frames_.front(); + + layer_id = GetSpatialLayerId(encoded_image); + + bool is_key_frame = + encoded_image.FrameType() == VideoFrameType::kVideoFrameKey; + if (!is_key_frame) { + for (const auto& [unused, context] : contexts_) { + if (context.rtp_timestamp_of_last_key_frame == + rtp_timestamp_encoded_image) { + // Upper layer of an SVC key frame. + is_key_frame = true; + break; + } + } + } + if (is_key_frame) { + contexts_[layer_id].rtp_timestamp_of_last_key_frame = + encoded_image.RtpTimestamp(); + } else if (contexts_.find(layer_id) == contexts_.end()) { + // TODO: bugs.webrtc.org/358039777 - Update this if statement such that + // LxTy scalability modes work properly. It is not a problem for LxTy_KEY + // scalability. + // + // For LxTy, it sometimes hinders calculating corruption score on the + // higher spatial layers. Because e.g. in L3T1 the first frame might not + // create 3 spatial layers but, only 2. Then, we end up not creating this + // in the map and will therefore not get any corruption score until a new + // key frame is sent. + RTC_LOG(LS_INFO) << "The first frame of a spatial or simulcast layer is " + "not a key frame."; + return std::nullopt; + } + + int sequence_index = contexts_[layer_id].frame_sampler.GetCurrentIndex(); + if (is_key_frame && ((sequence_index & 0b0111'1111) != 0)) { + // Increase until all the last 7 bits are zeroes. + sequence_index >>= 7; + sequence_index += 1; + sequence_index <<= 7; + contexts_[layer_id].frame_sampler.SetCurrentIndex(sequence_index); + } + + if (sequence_index >= (1 << 14)) { + // Overflow of 14 bit counter, reset to 0. + sequence_index = 0; + contexts_[layer_id].frame_sampler.SetCurrentIndex(sequence_index); + } + + RTC_CHECK(data.SetSequenceIndex(sequence_index)); + + // TODO: bugs.webrtc.org/358039777 - Maybe allow other sample sizes as well + sample_coordinates = + contexts_[layer_id] + .frame_sampler.GetSampleCoordinatesForFrameIfFrameShouldBeSampled( + is_key_frame, captured_frame->rtp_timestamp(), + /*num_samples=*/13); + if (sample_coordinates.empty()) { + if (!is_key_frame) { + return std::nullopt; + } + return data; + } + } + RTC_DCHECK(captured_frame.has_value()); + RTC_DCHECK(!sample_coordinates.empty()); + + std::optional<CorruptionDetectionFilterSettings> filter_settings = + GetCorruptionFilterSettings(encoded_image, video_codec_type_, layer_id); + if (!filter_settings.has_value()) { + return std::nullopt; + } + + RTC_CHECK(data.SetStdDev(filter_settings->std_dev)); + RTC_CHECK(data.SetLumaErrorThreshold(filter_settings->luma_error_threshold)); + RTC_CHECK( + data.SetChromaErrorThreshold(filter_settings->chroma_error_threshold)); + + std::vector<double> plain_values; + std::vector<FilteredSample> samples = GetSampleValuesForFrame( + *captured_frame, sample_coordinates, encoded_image._encodedWidth, + encoded_image._encodedHeight, filter_settings->std_dev); + plain_values.reserve(samples.size()); + absl::c_transform(samples, std::back_inserter(plain_values), + [](const FilteredSample& sample) { return sample.value; }); + + RTC_CHECK(data.SetSampleValues(std::move(plain_values))); + + return data; +} + +std::optional<int> FrameInstrumentationGeneratorImpl::GetHaltonSequenceIndex( + int layer_id) const { + MutexLock lock(&mutex_); + auto it = contexts_.find(layer_id); + if (it == contexts_.end()) { + return std::nullopt; + } + return it->second.frame_sampler.GetCurrentIndex(); +} + +void FrameInstrumentationGeneratorImpl::SetHaltonSequenceIndex(int index, + int layer_id) { + MutexLock lock(&mutex_); + if (index <= 0x3FFF) { + contexts_[layer_id].frame_sampler.SetCurrentIndex(index); + } + RTC_DCHECK_LE(index, 0x3FFF) << "Index must not be larger than 0x3FFF"; +} + +std::unique_ptr<FrameInstrumentationGenerator> +FrameInstrumentationGenerator::Create(VideoCodecType video_codec_type) { + return std::make_unique<FrameInstrumentationGeneratorImpl>(video_codec_type); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_generator.h b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_generator.h @@ -0,0 +1,53 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_GENERATOR_H_ +#define API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_GENERATOR_H_ + +#include <memory> +#include <optional> + +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/encoded_image.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" + +namespace webrtc { + +// Class that, given raw input frames via `OnCapturedFrame()` and corresponding +// encoded frames via `OnEncodedImage()` will generate FrameInstrumentationData +// for a subset of those frames. This data can be written to RTP packets as +// corruption detection header extensions, allowing the receiver on the other +// end to validate whether the media stream contains any video corruptions or +// not. +class FrameInstrumentationGenerator { + public: + static std::unique_ptr<FrameInstrumentationGenerator> Create( + VideoCodecType video_codec_type); + + virtual ~FrameInstrumentationGenerator() = default; + + virtual void OnCapturedFrame(VideoFrame frame) = 0; + virtual std::optional<FrameInstrumentationData> OnEncodedImage( + const EncodedImage& encoded_image) = 0; + + // Returns `std::nullopt` if there is no context for the given layer. + // The layer id is the simulcast id or SVC spatial layer id depending on + // which structure is used, or zero if no spatial scalability is used. + virtual std::optional<int> GetHaltonSequenceIndex(int layer_id) const = 0; + virtual void SetHaltonSequenceIndex(int index, int layer_id) = 0; + + protected: + FrameInstrumentationGenerator() = default; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_GENERATOR_H_ diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_generator_unittest.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_generator_unittest.cc @@ -0,0 +1,744 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_generator.h" + +#include <cstdint> +#include <memory> +#include <optional> +#include <vector> + +#include "api/make_ref_counted.h" +#include "api/scoped_refptr.h" +#include "api/video/corruption_detection/corruption_detection_filter_settings.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "rtc_base/ref_counted_object.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "video/corruption_detection/utils.h" + +namespace webrtc { +namespace { +using ::testing::DoubleEq; +using ::testing::ElementsAre; +using ::testing::Pointwise; + +constexpr int kDefaultScaledWidth = 4; +constexpr int kDefaultScaledHeight = 4; + +scoped_refptr<I420Buffer> MakeDefaultI420FrameBuffer() { + // Create an I420 frame of size 4x4. + const int kDefaultLumaWidth = 4; + const int kDefaultLumaHeight = 4; + const int kDefaultChromaWidth = 2; + const int kDefaultPixelValue = 30; + std::vector<uint8_t> kDefaultYContent(16, kDefaultPixelValue); + std::vector<uint8_t> kDefaultUContent(4, kDefaultPixelValue); + std::vector<uint8_t> kDefaultVContent(4, kDefaultPixelValue); + + return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight, + kDefaultYContent.data(), kDefaultLumaWidth, + kDefaultUContent.data(), kDefaultChromaWidth, + kDefaultVContent.data(), kDefaultChromaWidth); +} + +scoped_refptr<I420Buffer> MakeI420FrameBufferWithDifferentPixelValues() { + // Create an I420 frame of size 4x4. + const int kDefaultLumaWidth = 4; + const int kDefaultLumaHeight = 4; + const int kDefaultChromaWidth = 2; + std::vector<uint8_t> kDefaultYContent = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + std::vector<uint8_t> kDefaultUContent = {17, 18, 19, 20}; + std::vector<uint8_t> kDefaultVContent = {21, 22, 23, 24}; + + return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight, + kDefaultYContent.data(), kDefaultLumaWidth, + kDefaultUContent.data(), kDefaultChromaWidth, + kDefaultVContent.data(), kDefaultChromaWidth); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsNothingWhenNoFramesHaveBeenProvided) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecGeneric); + + EXPECT_FALSE(generator->OnEncodedImage(EncodedImage()).has_value()); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsNothingWhenNoFrameWithTheSameTimestampIsProvided) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecGeneric); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(2); + + generator->OnCapturedFrame(frame); + + EXPECT_FALSE(generator->OnEncodedImage(encoded_image).has_value()); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsNothingWhenTheFirstFrameOfASpatialOrSimulcastLayerIsNotAKeyFrame) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecGeneric); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + + // Delta frame with no preceding key frame. + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameDelta); + encoded_image.SetSpatialIndex(0); + encoded_image.SetSimulcastIndex(0); + + generator->OnCapturedFrame(frame); + + // The first frame of a spatial or simulcast layer is not a key frame. + EXPECT_FALSE(generator->OnEncodedImage(encoded_image).has_value()); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsNothingWhenQpIsUnsetAndNotParseable) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecGeneric); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + + // Frame where QP is unset and QP is not parseable from the encoded data. + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); + + generator->OnCapturedFrame(frame); + + EXPECT_FALSE(generator->OnEncodedImage(encoded_image).has_value()); +} + +#if GTEST_HAS_DEATH_TEST +TEST(FrameInstrumentationGeneratorTest, FailsWhenCodecIsUnsupported) { + // No available mapping from codec to filter parameters. + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecGeneric); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image.qp_ = 10; + + generator->OnCapturedFrame(frame); + + EXPECT_DEATH(generator->OnEncodedImage(encoded_image), + "Codec type Generic is not supported"); +} +#endif // GTEST_HAS_DEATH_TEST + +TEST(FrameInstrumentationGeneratorTest, + ReturnsInstrumentationDataForVP8KeyFrameWithQpSet) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + // VP8 key frame with QP set. + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image.qp_ = 10; + encoded_image._encodedWidth = kDefaultScaledWidth; + encoded_image._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator->OnEncodedImage(encoded_image); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); + EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); + EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); + EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); + EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsInstrumentationDataWhenQpIsParseable) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + + // VP8 key frame with parseable QP. + constexpr uint8_t kCodedFrameVp8Qp25[] = { + 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00, + 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c, + 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0}; + scoped_refptr<EncodedImageBuffer> encoded_image_buffer = + EncodedImageBuffer::Create(kCodedFrameVp8Qp25, + sizeof(kCodedFrameVp8Qp25)); + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image.SetEncodedData(encoded_image_buffer); + encoded_image._encodedWidth = kDefaultScaledWidth; + encoded_image._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator->OnEncodedImage(encoded_image); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); + EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); + EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); + EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); + EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsInstrumentationDataForUpperLayerOfAnSvcKeyFrame) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP9); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(1); + encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image1.SetSpatialIndex(0); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + + // Delta frame that is an upper layer of an SVC key frame. + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(1); + encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); + encoded_image2.SetSpatialIndex(1); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame); + generator->OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator->OnEncodedImage(encoded_image2); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); + EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); + EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); + EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); + EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsNothingWhenNotEnoughTimeHasPassedSinceLastSampledFrame) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + VideoFrame frame1 = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + VideoFrame frame2 = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(2) + .build(); + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(1); + encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image1.SetSpatialIndex(0); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + + // Delta frame that is too recent in comparison to the last sampled frame: + // passed time < 90'000. + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(2); + encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); + encoded_image2.SetSpatialIndex(0); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame1); + generator->OnCapturedFrame(frame2); + generator->OnEncodedImage(encoded_image1); + + ASSERT_FALSE(generator->OnEncodedImage(encoded_image2).has_value()); +} + +TEST(FrameInstrumentationGeneratorTest, + ReturnsInstrumentationDataForUpperLayerOfASecondSvcKeyFrame) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP9); + VideoFrame frame1 = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + VideoFrame frame2 = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(2) + .build(); + for (const VideoFrame& frame : {frame1, frame2}) { + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(frame.rtp_timestamp()); + encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image1.SetSpatialIndex(0); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(frame.rtp_timestamp()); + encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); + encoded_image2.SetSpatialIndex(1); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame); + + std::optional<FrameInstrumentationData> data1 = + generator->OnEncodedImage(encoded_image1); + + std::optional<FrameInstrumentationData> data2 = + generator->OnEncodedImage(encoded_image2); + + ASSERT_TRUE(data1.has_value()); + ASSERT_TRUE(data2.has_value()); + + EXPECT_TRUE(data1->holds_upper_bits()); + EXPECT_TRUE(data2->holds_upper_bits()); + } +} + +TEST(FrameInstrumentationGeneratorTest, + SvcLayersSequenceIndicesIncreaseIndependentOnEachother) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP9); + VideoFrame frame1 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(1) + .build(); + VideoFrame frame2 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(2) + .build(); + for (const VideoFrame& frame : {frame1, frame2}) { + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(frame.rtp_timestamp()); + encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image1.SetSpatialIndex(0); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(frame.rtp_timestamp()); + encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); + encoded_image2.SetSpatialIndex(1); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame); + + std::optional<FrameInstrumentationData> data1 = + generator->OnEncodedImage(encoded_image1); + + std::optional<FrameInstrumentationData> data2 = + generator->OnEncodedImage(encoded_image2); + + ASSERT_TRUE(data1.has_value()); + ASSERT_TRUE(data2.has_value()); + + EXPECT_TRUE(data1->holds_upper_bits()); + EXPECT_TRUE(data2->holds_upper_bits()); + + EXPECT_EQ(data1->sequence_index(), data2->sequence_index()); + + // In the test the frames have equal frame buffers so the sample values + // should be equal. + EXPECT_THAT(data1->sample_values(), + Pointwise(DoubleEq(), data2->sample_values())); + } +} + +TEST(FrameInstrumentationGeneratorTest, + OutputsDeltaFrameInstrumentationDataForSimulcast) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP9); + bool has_found_delta_frame = false; + // 34 frames is the minimum number of frames to be able to sample a delta + // frame. + for (int i = 0; i < 34; ++i) { + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(i) + .build(); + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(frame.rtp_timestamp()); + encoded_image1.SetFrameType(i == 0 ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta); + encoded_image1.SetSimulcastIndex(0); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(frame.rtp_timestamp()); + encoded_image2.SetFrameType(i == 0 ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta); + encoded_image2.SetSimulcastIndex(1); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame); + + std::optional<FrameInstrumentationData> data1 = + generator->OnEncodedImage(encoded_image1); + + std::optional<FrameInstrumentationData> data2 = + generator->OnEncodedImage(encoded_image2); + + if (i == 0) { + ASSERT_TRUE(data1.has_value()); + ASSERT_TRUE(data2.has_value()); + + EXPECT_TRUE(data1->holds_upper_bits()); + EXPECT_TRUE(data2->holds_upper_bits()); + } else if (data1.has_value() || data2.has_value()) { + if (data1.has_value()) { + EXPECT_FALSE(data1->holds_upper_bits()); + } + if (data2.has_value()) { + EXPECT_FALSE(data2->holds_upper_bits()); + } + has_found_delta_frame = true; + } + } + EXPECT_TRUE(has_found_delta_frame); +} + +TEST(FrameInstrumentationGeneratorTest, + SequenceIndexIncreasesCorrectlyAtNewKeyFrame) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + VideoFrame frame1 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(1) + .build(); + VideoFrame frame2 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(2) + .build(); + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(1); + encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + + // Delta frame that is an upper layer of an SVC key frame. + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(2); + encoded_image2.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame1); + generator->OnCapturedFrame(frame2); + + ASSERT_EQ(GetSpatialLayerId(encoded_image1), + GetSpatialLayerId(encoded_image2)); + generator->SetHaltonSequenceIndex(0b0010'1010, + GetSpatialLayerId(encoded_image1)); + + std::optional<FrameInstrumentationData> data1 = + generator->OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data2 = + generator->OnEncodedImage(encoded_image2); + + ASSERT_TRUE(data1.has_value()); + ASSERT_TRUE(data2.has_value()); + + EXPECT_EQ(data1->sequence_index(), 0b0000'1000'0000); + EXPECT_EQ(data2->sequence_index(), 0b0001'0000'0000); +} + +TEST(FrameInstrumentationGeneratorTest, + SequenceIndexThatWouldOverflowTo15BitsIncreasesCorrectlyAtNewKeyFrame) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + VideoFrame frame1 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(1) + .build(); + VideoFrame frame2 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(2) + .build(); + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(1); + encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + encoded_image1.SetSimulcastIndex(0); + + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(2); + encoded_image2.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + encoded_image2.SetSimulcastIndex(0); + + generator->OnCapturedFrame(frame1); + generator->OnCapturedFrame(frame2); + + ASSERT_EQ(GetSpatialLayerId(encoded_image1), + GetSpatialLayerId(encoded_image2)); + generator->SetHaltonSequenceIndex(0b11'1111'1111'1111, + GetSpatialLayerId(encoded_image1)); + std::optional<FrameInstrumentationData> data1 = + generator->OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data2 = + generator->OnEncodedImage(encoded_image2); + + ASSERT_TRUE(data1.has_value()); + ASSERT_TRUE(data2.has_value()); + + EXPECT_EQ(data1->sequence_index(), 0); + EXPECT_EQ(data2->sequence_index(), 0b1000'0000); +} + +TEST(FrameInstrumentationGeneratorTest, + SequenceIndexIncreasesCorrectlyAtNewKeyFrameAlreadyZeroes) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + VideoFrame frame1 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(1) + .build(); + VideoFrame frame2 = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(2) + .build(); + EncodedImage encoded_image1; + encoded_image1.SetRtpTimestamp(1); + encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image1.qp_ = 10; + encoded_image1._encodedWidth = kDefaultScaledWidth; + encoded_image1._encodedHeight = kDefaultScaledHeight; + + // Delta frame that is an upper layer of an SVC key frame. + EncodedImage encoded_image2; + encoded_image2.SetRtpTimestamp(2); + encoded_image2.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image2.qp_ = 10; + encoded_image2._encodedWidth = kDefaultScaledWidth; + encoded_image2._encodedHeight = kDefaultScaledHeight; + + generator->OnCapturedFrame(frame1); + generator->OnCapturedFrame(frame2); + + ASSERT_EQ(GetSpatialLayerId(encoded_image1), + GetSpatialLayerId(encoded_image2)); + generator->SetHaltonSequenceIndex(0b1000'0000, + GetSpatialLayerId(encoded_image1)); + + std::optional<FrameInstrumentationData> data1 = + generator->OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data2 = + generator->OnEncodedImage(encoded_image2); + + ASSERT_TRUE(data1.has_value()); + ASSERT_TRUE(data2.has_value()); + + EXPECT_EQ(data1->sequence_index(), 0b0000'1000'0000); + EXPECT_EQ(data2->sequence_index(), 0b0001'0000'0000); +} +TEST(FrameInstrumentationGeneratorTest, + SequenceIndexThatWouldOverflowTo15BitsIncreasesCorrectlyAtNewDeltaFrame) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + generator->OnCapturedFrame( + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(1) + .build()); + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameDelta); + encoded_image.qp_ = 10; + encoded_image._encodedWidth = kDefaultScaledWidth; + encoded_image._encodedHeight = kDefaultScaledHeight; + encoded_image.SetSimulcastIndex(0); + + constexpr int kMaxSequenceIndex = 0b11'1111'1111'1111; + + generator->SetHaltonSequenceIndex(kMaxSequenceIndex, + GetSpatialLayerId(encoded_image)); + std::optional<FrameInstrumentationData> data = + generator->OnEncodedImage(encoded_image); + + ASSERT_TRUE(data.has_value()); + EXPECT_EQ(data->sequence_index(), kMaxSequenceIndex); + + // Loop until we get a new delta frame. + bool has_found_delta_frame = false; + for (int i = 0; i < 34; ++i) { + generator->OnCapturedFrame( + VideoFrame::Builder() + .set_video_frame_buffer( + MakeI420FrameBufferWithDifferentPixelValues()) + .set_rtp_timestamp(i + 2) + .build()); + + encoded_image.SetRtpTimestamp(i + 2); + + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator->OnEncodedImage(encoded_image); + if (frame_instrumentation_data.has_value()) { + has_found_delta_frame = true; + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); + break; + } + } + EXPECT_TRUE(has_found_delta_frame); +} + +TEST(FrameInstrumentationGeneratorTest, GetterAndSetterOperatesAsExpected) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + // `std::nullopt` when uninitialized. + EXPECT_FALSE(generator->GetHaltonSequenceIndex(1).has_value()); + + // Zero is a valid index. + generator->SetHaltonSequenceIndex(0, 1); + std::optional<int> index = generator->GetHaltonSequenceIndex(1); + EXPECT_TRUE(index.has_value()); + EXPECT_EQ(*index, 0); + +#if GTEST_HAS_DEATH_TEST + // Negative values are not allowed to be set. + EXPECT_DEATH(generator->SetHaltonSequenceIndex(-2, 1), + "Index must be non-negative"); + index = generator->GetHaltonSequenceIndex(1); + EXPECT_TRUE(index.has_value()); + EXPECT_EQ(*index, 0); + + // Values requiring more than 15 bits are not allowed. + EXPECT_DEATH(generator->SetHaltonSequenceIndex(0x4000, 1), + "Index must not be larger than 0x3FFF"); + index = generator->GetHaltonSequenceIndex(1); + EXPECT_TRUE(index.has_value()); + EXPECT_EQ(*index, 0); +#endif // GTEST_HAS_DEATH_TEST +} + +TEST(FrameInstrumentationGeneratorTest, QueuesAtMostThreeInputFrames) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + + bool frames_destroyed[4] = {}; + class TestBuffer : public I420Buffer { + public: + TestBuffer(int width, int height, bool* frame_destroyed_indicator) + : I420Buffer(width, height), + frame_destroyed_indicator_(frame_destroyed_indicator) {} + + private: + friend class RefCountedObject<TestBuffer>; + ~TestBuffer() override { *frame_destroyed_indicator_ = true; } + + bool* frame_destroyed_indicator_; + }; + + // Insert four frames, the first one should expire and be released. + for (int i = 0; i < 4; ++i) { + generator->OnCapturedFrame( + VideoFrame::Builder() + .set_video_frame_buffer(make_ref_counted<TestBuffer>( + kDefaultScaledWidth, kDefaultScaledHeight, + &frames_destroyed[i])) + .set_rtp_timestamp(1 + (33 * i)) + .build()); + } + + EXPECT_THAT(frames_destroyed, ElementsAre(true, false, false, false)); + + generator.reset(); + EXPECT_THAT(frames_destroyed, ElementsAre(true, true, true, true)); +} + +TEST(FrameInstrumentationGeneratorTest, + UsesFilterSettingsFromFrameWhenAvailable) { + auto generator = + FrameInstrumentationGenerator::Create(VideoCodecType::kVideoCodecVP8); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + // No QP needed when frame provides filter settings. + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image._encodedWidth = kDefaultScaledWidth; + encoded_image._encodedHeight = kDefaultScaledHeight; + encoded_image.set_corruption_detection_filter_settings( + CorruptionDetectionFilterSettings{.std_dev = 1.0, + .luma_error_threshold = 2, + .chroma_error_threshold = 3}); + + generator->OnCapturedFrame(frame); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator->OnEncodedImage(encoded_image); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->std_dev(), 1.0); + EXPECT_EQ(frame_instrumentation_data->luma_error_threshold(), 2); + EXPECT_EQ(frame_instrumentation_data->chroma_error_threshold(), 3); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/moz-patch-stack/s0027.patch b/third_party/libwebrtc/moz-patch-stack/s0027.patch @@ -203,7 +203,7 @@ index 3efce2dd19..cbfc05f243 100644 } diff --git a/api/BUILD.gn b/api/BUILD.gn -index 02ca836077..5a072619ee 100644 +index def130e348..66941a5b2b 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -44,6 +44,9 @@ rtc_library("enable_media") { @@ -1258,7 +1258,7 @@ index 0c7ce99ac0..1873e67f52 100644 rtc_library("call_config_utils") { testonly = true diff --git a/video/BUILD.gn b/video/BUILD.gn -index 033331442f..ed00c2cbef 100644 +index 79b0e8e040..2e408d56c0 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -17,7 +17,7 @@ rtc_library("video_stream_encoder_interface") { diff --git a/third_party/libwebrtc/moz-patch-stack/s0068.patch b/third_party/libwebrtc/moz-patch-stack/s0068.patch @@ -10,10 +10,10 @@ Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/a7179d8d75313b6c9 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc -index 86a547f611..dc6f8a7dac 100644 +index a3a20911ad..d437837911 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc -@@ -1484,7 +1484,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { +@@ -1483,7 +1483,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { bool is_svc = false; bool single_stream_or_non_first_inactive = true; diff --git a/third_party/libwebrtc/moz-patch-stack/s0102.patch b/third_party/libwebrtc/moz-patch-stack/s0102.patch @@ -126,7 +126,7 @@ index 771e0b196a..7e1e8353ab 100644 "Generated during 'gn gen' by //BUILD.gn.", "", diff --git a/api/BUILD.gn b/api/BUILD.gn -index 5a072619ee..84569eb8c6 100644 +index 66941a5b2b..19cb4d850e 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -8,8 +8,8 @@ diff --git a/third_party/libwebrtc/moz-patch-stack/s0110.patch b/third_party/libwebrtc/moz-patch-stack/s0110.patch @@ -36,7 +36,7 @@ index e647cd6beb..d1cc8c05cc 100644 ~VideoStreamFactoryInterface() override {} }; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc -index dc6f8a7dac..f4ae2b0d46 100644 +index d437837911..90c9f141aa 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1096,6 +1096,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { diff --git a/third_party/libwebrtc/video/BUILD.gn b/third_party/libwebrtc/video/BUILD.gn @@ -466,6 +466,7 @@ rtc_library("video_stream_encoder_impl") { "../api/video:video_rtp_headers", "../api/video:video_stream_encoder", "../api/video/corruption_detection:frame_instrumentation_data", + "../api/video/corruption_detection:frame_instrumentation_generator", "../api/video_codecs:scalability_mode", "../api/video_codecs:video_codecs_api", "../call/adaptation:resource_adaptation", @@ -509,7 +510,6 @@ rtc_library("video_stream_encoder_impl") { "adaptation:video_adaptation", "config:encoder_config", "config:streams_config", - "corruption_detection:frame_instrumentation_generator", "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/cleanup", "//third_party/abseil-cpp/absl/container:inlined_vector", @@ -972,7 +972,6 @@ if (rtc_include_tests) { "../modules/pacing", "../modules/rtp_rtcp", "../modules/rtp_rtcp:rtp_rtcp_format", - "../modules/rtp_rtcp:rtp_rtcp_format", "../modules/rtp_rtcp:rtp_video_header", "../modules/video_coding", "../modules/video_coding:codec_globals_headers", diff --git a/third_party/libwebrtc/video/corruption_detection/BUILD.gn b/third_party/libwebrtc/video/corruption_detection/BUILD.gn @@ -21,31 +21,6 @@ rtc_library("corruption_classifier") { ] } -rtc_library("frame_instrumentation_generator") { - sources = [ - "frame_instrumentation_generator.cc", - "frame_instrumentation_generator.h", - ] - deps = [ - ":generic_mapping_functions", - ":halton_frame_sampler", - "../../api:scoped_refptr", - "../../api/video:encoded_image", - "../../api/video:video_frame", - "../../api/video:video_frame_type", - "../../api/video/corruption_detection:filter_settings", - "../../api/video/corruption_detection:frame_instrumentation_data", - "../../api/video_codecs:video_codecs_api", - "../../modules:module_api_public", - "../../modules/video_coding:video_coding_utility", - "../../rtc_base:checks", - "../../rtc_base:logging", - "../../rtc_base:macromagic", - "../../rtc_base/synchronization:mutex", - "//third_party/abseil-cpp/absl/algorithm:container", - ] -} - rtc_library("frame_pair_corruption_score") { sources = [ "frame_pair_corruption_score.cc", @@ -120,6 +95,7 @@ rtc_library("utils") { ] deps = [ "../../api:scoped_refptr", + "../../api/video:encoded_image", "../../api/video:video_frame", "../../rtc_base:checks", "//third_party/abseil-cpp/absl/strings", @@ -137,24 +113,6 @@ if (rtc_include_tests) { "../../test:test_support", ] } - - rtc_library("frame_instrumentation_generator_unittest") { - testonly = true - sources = [ "frame_instrumentation_generator_unittest.cc" ] - deps = [ - ":frame_instrumentation_generator", - "../../api:make_ref_counted", - "../../api:scoped_refptr", - "../../api/video:encoded_image", - "../../api/video:video_frame", - "../../api/video:video_frame_type", - "../../api/video/corruption_detection:filter_settings", - "../../api/video/corruption_detection:frame_instrumentation_data", - "../../rtc_base:refcount", - "../../test:test_support", - ] - } - rtc_library("frame_pair_corruption_score_unittest") { testonly = true sources = [ "frame_pair_corruption_score_unittest.cc" ] @@ -218,6 +176,7 @@ if (rtc_include_tests) { sources = [ "utils_unittest.cc" ] deps = [ ":utils", + "../../api/video:encoded_image", "../../api/video:video_frame", "../../test:test_support", ] @@ -228,7 +187,6 @@ if (rtc_include_tests) { sources = [] deps = [ ":corruption_classifier_unittest", - ":frame_instrumentation_generator_unittest", ":frame_pair_corruption_score_unittest", ":generic_mapping_functions_unittest", ":halton_frame_sampler_unittest", diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.cc @@ -1,223 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/corruption_detection/frame_instrumentation_generator.h" - -#include <algorithm> -#include <cstddef> -#include <cstdint> -#include <iterator> -#include <optional> -#include <utility> -#include <vector> - -#include "absl/algorithm/container.h" -#include "api/video/corruption_detection/corruption_detection_filter_settings.h" -#include "api/video/corruption_detection/frame_instrumentation_data.h" -#include "api/video/encoded_image.h" -#include "api/video/video_codec_type.h" -#include "api/video/video_frame.h" -#include "api/video/video_frame_type.h" -#include "api/video_codecs/video_codec.h" -#include "modules/include/module_common_types_public.h" -#include "modules/video_coding/utility/qp_parser.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/synchronization/mutex.h" -#include "video/corruption_detection/generic_mapping_functions.h" -#include "video/corruption_detection/halton_frame_sampler.h" - -namespace webrtc { -namespace { - -// Avoid holding on to frames that might have been dropped by encoder, as that -// can lead to frame buffer pools draining. -constexpr size_t kMaxPendingFrames = 3; - -std::optional<CorruptionDetectionFilterSettings> GetCorruptionFilterSettings( - const EncodedImage& encoded_image, - VideoCodecType video_codec_type, - int layer_id) { - std::optional<CorruptionDetectionFilterSettings> filter_settings = - encoded_image.corruption_detection_filter_settings(); - - if (!filter_settings.has_value()) { - // No implementation specific filter settings available, using a generic - // QP-based settings instead. - int qp = encoded_image.qp_; - if (qp == -1) { - std::optional<uint32_t> parsed_qp = - QpParser().Parse(video_codec_type, layer_id, encoded_image.data(), - encoded_image.size()); - if (!parsed_qp.has_value()) { - RTC_LOG(LS_VERBOSE) - << "Missing QP for " << CodecTypeToPayloadString(video_codec_type) - << " layer " << layer_id << "."; - return std::nullopt; - } - qp = *parsed_qp; - } - - filter_settings = GetCorruptionFilterSettings(qp, video_codec_type); - } - return filter_settings; -} - -} // namespace - -FrameInstrumentationGenerator::FrameInstrumentationGenerator( - VideoCodecType video_codec_type) - : video_codec_type_(video_codec_type) {} - -void FrameInstrumentationGenerator::OnCapturedFrame(VideoFrame frame) { - MutexLock lock(&mutex_); - while (captured_frames_.size() >= kMaxPendingFrames) { - captured_frames_.pop(); - } - captured_frames_.push(frame); -} - -std::optional<FrameInstrumentationData> -FrameInstrumentationGenerator::OnEncodedImage( - const EncodedImage& encoded_image) { - uint32_t rtp_timestamp_encoded_image = encoded_image.RtpTimestamp(); - std::optional<VideoFrame> captured_frame; - int layer_id; - FrameInstrumentationData data; - std::vector<HaltonFrameSampler::Coordinates> sample_coordinates; - { - MutexLock lock(&mutex_); - while (!captured_frames_.empty() && - IsNewerTimestamp(rtp_timestamp_encoded_image, - captured_frames_.front().rtp_timestamp())) { - captured_frames_.pop(); - } - if (captured_frames_.empty() || captured_frames_.front().rtp_timestamp() != - rtp_timestamp_encoded_image) { - RTC_LOG(LS_VERBOSE) << "No captured frames for RTC timestamp " - << rtp_timestamp_encoded_image << "."; - return std::nullopt; - } - captured_frame = captured_frames_.front(); - - layer_id = GetLayerId(encoded_image); - - bool is_key_frame = - encoded_image.FrameType() == VideoFrameType::kVideoFrameKey; - if (!is_key_frame) { - for (const auto& [unused, context] : contexts_) { - if (context.rtp_timestamp_of_last_key_frame == - rtp_timestamp_encoded_image) { - // Upper layer of an SVC key frame. - is_key_frame = true; - break; - } - } - } - if (is_key_frame) { - contexts_[layer_id].rtp_timestamp_of_last_key_frame = - encoded_image.RtpTimestamp(); - } else if (contexts_.find(layer_id) == contexts_.end()) { - // TODO: bugs.webrtc.org/358039777 - Update this if statement such that - // LxTy scalability modes work properly. It is not a problem for LxTy_KEY - // scalability. - // - // For LxTy, it sometimes hinders calculating corruption score on the - // higher spatial layers. Because e.g. in L3T1 the first frame might not - // create 3 spatial layers but, only 2. Then, we end up not creating this - // in the map and will therefore not get any corruption score until a new - // key frame is sent. - RTC_LOG(LS_INFO) << "The first frame of a spatial or simulcast layer is " - "not a key frame."; - return std::nullopt; - } - - int sequence_index = contexts_[layer_id].frame_sampler.GetCurrentIndex(); - if (is_key_frame && ((sequence_index & 0b0111'1111) != 0)) { - // Increase until all the last 7 bits are zeroes. - sequence_index >>= 7; - sequence_index += 1; - sequence_index <<= 7; - contexts_[layer_id].frame_sampler.SetCurrentIndex(sequence_index); - } - - if (sequence_index >= (1 << 14)) { - // Overflow of 14 bit counter, reset to 0. - sequence_index = 0; - contexts_[layer_id].frame_sampler.SetCurrentIndex(sequence_index); - } - - RTC_CHECK(data.SetSequenceIndex(sequence_index)); - - // TODO: bugs.webrtc.org/358039777 - Maybe allow other sample sizes as well - sample_coordinates = - contexts_[layer_id] - .frame_sampler.GetSampleCoordinatesForFrameIfFrameShouldBeSampled( - is_key_frame, captured_frame->rtp_timestamp(), - /*num_samples=*/13); - if (sample_coordinates.empty()) { - if (!is_key_frame) { - return std::nullopt; - } - return data; - } - } - RTC_DCHECK(captured_frame.has_value()); - RTC_DCHECK(!sample_coordinates.empty()); - - std::optional<CorruptionDetectionFilterSettings> filter_settings = - GetCorruptionFilterSettings(encoded_image, video_codec_type_, layer_id); - if (!filter_settings.has_value()) { - return std::nullopt; - } - - RTC_CHECK(data.SetStdDev(filter_settings->std_dev)); - RTC_CHECK(data.SetLumaErrorThreshold(filter_settings->luma_error_threshold)); - RTC_CHECK( - data.SetChromaErrorThreshold(filter_settings->chroma_error_threshold)); - - std::vector<double> plain_values; - std::vector<FilteredSample> samples = GetSampleValuesForFrame( - *captured_frame, sample_coordinates, encoded_image._encodedWidth, - encoded_image._encodedHeight, filter_settings->std_dev); - plain_values.reserve(samples.size()); - absl::c_transform(samples, std::back_inserter(plain_values), - [](const FilteredSample& sample) { return sample.value; }); - - RTC_CHECK(data.SetSampleValues(std::move(plain_values))); - - return data; -} - -std::optional<int> FrameInstrumentationGenerator::GetHaltonSequenceIndex( - int layer_id) const { - MutexLock lock(&mutex_); - auto it = contexts_.find(layer_id); - if (it == contexts_.end()) { - return std::nullopt; - } - return it->second.frame_sampler.GetCurrentIndex(); -} - -void FrameInstrumentationGenerator::SetHaltonSequenceIndex(int index, - int layer_id) { - MutexLock lock(&mutex_); - if (index <= 0x3FFF) { - contexts_[layer_id].frame_sampler.SetCurrentIndex(index); - } - RTC_DCHECK_LE(index, 0x3FFF) << "Index must not be larger than 0x3FFF"; -} - -int FrameInstrumentationGenerator::GetLayerId( - const EncodedImage& encoded_image) const { - return std::max(encoded_image.SpatialIndex().value_or(0), - encoded_image.SimulcastIndex().value_or(0)); -} -} // namespace webrtc diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.h b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.h @@ -1,68 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_GENERATOR_H_ -#define VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_GENERATOR_H_ - -#include <cstdint> -#include <map> -#include <optional> -#include <queue> - -#include "api/video/corruption_detection/frame_instrumentation_data.h" -#include "api/video/encoded_image.h" -#include "api/video/video_codec_type.h" -#include "api/video/video_frame.h" -#include "rtc_base/synchronization/mutex.h" -#include "rtc_base/thread_annotations.h" -#include "video/corruption_detection/halton_frame_sampler.h" - -namespace webrtc { - -class FrameInstrumentationGenerator { - public: - FrameInstrumentationGenerator() = delete; - explicit FrameInstrumentationGenerator(VideoCodecType video_codec_type); - - FrameInstrumentationGenerator(const FrameInstrumentationGenerator&) = delete; - FrameInstrumentationGenerator& operator=( - const FrameInstrumentationGenerator&) = delete; - - ~FrameInstrumentationGenerator() = default; - - void OnCapturedFrame(VideoFrame frame) RTC_LOCKS_EXCLUDED(mutex_); - std::optional<FrameInstrumentationData> OnEncodedImage( - const EncodedImage& encoded_image) RTC_LOCKS_EXCLUDED(mutex_); - - // Returns `std::nullopt` if there is no context for the given layer. - std::optional<int> GetHaltonSequenceIndex(int layer_id) const - RTC_LOCKS_EXCLUDED(mutex_); - void SetHaltonSequenceIndex(int index, int layer_id) - RTC_LOCKS_EXCLUDED(mutex_); - - int GetLayerId(const EncodedImage& encoded_image) const; - - private: - struct Context { - HaltonFrameSampler frame_sampler; - uint32_t rtp_timestamp_of_last_key_frame = 0; - }; - - // Incoming video frames in capture order. - std::queue<VideoFrame> captured_frames_ RTC_GUARDED_BY(mutex_); - // Map from spatial or simulcast index to sampling context. - std::map<int, Context> contexts_ RTC_GUARDED_BY(mutex_); - const VideoCodecType video_codec_type_; - mutable Mutex mutex_; -}; - -} // namespace webrtc - -#endif // VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_GENERATOR_H_ diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator_unittest.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator_unittest.cc @@ -1,725 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/corruption_detection/frame_instrumentation_generator.h" - -#include <cstdint> -#include <memory> -#include <optional> -#include <vector> - -#include "api/make_ref_counted.h" -#include "api/scoped_refptr.h" -#include "api/video/corruption_detection/corruption_detection_filter_settings.h" -#include "api/video/corruption_detection/frame_instrumentation_data.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_codec_type.h" -#include "api/video/video_frame.h" -#include "api/video/video_frame_type.h" -#include "rtc_base/ref_counted_object.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace { -using ::testing::DoubleEq; -using ::testing::ElementsAre; -using ::testing::Pointwise; - -constexpr int kDefaultScaledWidth = 4; -constexpr int kDefaultScaledHeight = 4; - -scoped_refptr<I420Buffer> MakeDefaultI420FrameBuffer() { - // Create an I420 frame of size 4x4. - const int kDefaultLumaWidth = 4; - const int kDefaultLumaHeight = 4; - const int kDefaultChromaWidth = 2; - const int kDefaultPixelValue = 30; - std::vector<uint8_t> kDefaultYContent(16, kDefaultPixelValue); - std::vector<uint8_t> kDefaultUContent(4, kDefaultPixelValue); - std::vector<uint8_t> kDefaultVContent(4, kDefaultPixelValue); - - return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight, - kDefaultYContent.data(), kDefaultLumaWidth, - kDefaultUContent.data(), kDefaultChromaWidth, - kDefaultVContent.data(), kDefaultChromaWidth); -} - -scoped_refptr<I420Buffer> MakeI420FrameBufferWithDifferentPixelValues() { - // Create an I420 frame of size 4x4. - const int kDefaultLumaWidth = 4; - const int kDefaultLumaHeight = 4; - const int kDefaultChromaWidth = 2; - std::vector<uint8_t> kDefaultYContent = {1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16}; - std::vector<uint8_t> kDefaultUContent = {17, 18, 19, 20}; - std::vector<uint8_t> kDefaultVContent = {21, 22, 23, 24}; - - return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight, - kDefaultYContent.data(), kDefaultLumaWidth, - kDefaultUContent.data(), kDefaultChromaWidth, - kDefaultVContent.data(), kDefaultChromaWidth); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsNothingWhenNoFramesHaveBeenProvided) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecGeneric); - - EXPECT_FALSE(generator.OnEncodedImage(EncodedImage()).has_value()); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsNothingWhenNoFrameWithTheSameTimestampIsProvided) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecGeneric); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(2); - - generator.OnCapturedFrame(frame); - - EXPECT_FALSE(generator.OnEncodedImage(encoded_image).has_value()); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsNothingWhenTheFirstFrameOfASpatialOrSimulcastLayerIsNotAKeyFrame) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecGeneric); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - - // Delta frame with no preceding key frame. - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(1); - encoded_image.SetFrameType(VideoFrameType::kVideoFrameDelta); - encoded_image.SetSpatialIndex(0); - encoded_image.SetSimulcastIndex(0); - - generator.OnCapturedFrame(frame); - - // The first frame of a spatial or simulcast layer is not a key frame. - EXPECT_FALSE(generator.OnEncodedImage(encoded_image).has_value()); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsNothingWhenQpIsUnsetAndNotParseable) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecGeneric); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - - // Frame where QP is unset and QP is not parseable from the encoded data. - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(1); - encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); - - generator.OnCapturedFrame(frame); - - EXPECT_FALSE(generator.OnEncodedImage(encoded_image).has_value()); -} - -#if GTEST_HAS_DEATH_TEST -TEST(FrameInstrumentationGeneratorTest, FailsWhenCodecIsUnsupported) { - // No available mapping from codec to filter parameters. - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecGeneric); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(1); - encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image.qp_ = 10; - - generator.OnCapturedFrame(frame); - - EXPECT_DEATH(generator.OnEncodedImage(encoded_image), - "Codec type Generic is not supported"); -} -#endif // GTEST_HAS_DEATH_TEST - -TEST(FrameInstrumentationGeneratorTest, - ReturnsInstrumentationDataForVP8KeyFrameWithQpSet) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - // VP8 key frame with QP set. - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(1); - encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image.qp_ = 10; - encoded_image._encodedWidth = kDefaultScaledWidth; - encoded_image._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame); - std::optional<FrameInstrumentationData> frame_instrumentation_data = - generator.OnEncodedImage(encoded_image); - - ASSERT_TRUE(frame_instrumentation_data.has_value()); - EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); - EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); - EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); - EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); - EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsInstrumentationDataWhenQpIsParseable) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - - // VP8 key frame with parseable QP. - constexpr uint8_t kCodedFrameVp8Qp25[] = { - 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00, - 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c, - 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0}; - scoped_refptr<EncodedImageBuffer> encoded_image_buffer = - EncodedImageBuffer::Create(kCodedFrameVp8Qp25, - sizeof(kCodedFrameVp8Qp25)); - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(1); - encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image.SetEncodedData(encoded_image_buffer); - encoded_image._encodedWidth = kDefaultScaledWidth; - encoded_image._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame); - std::optional<FrameInstrumentationData> frame_instrumentation_data = - generator.OnEncodedImage(encoded_image); - - ASSERT_TRUE(frame_instrumentation_data.has_value()); - EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); - EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); - EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); - EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); - EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsInstrumentationDataForUpperLayerOfAnSvcKeyFrame) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP9); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(1); - encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image1.SetSpatialIndex(0); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - - // Delta frame that is an upper layer of an SVC key frame. - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(1); - encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); - encoded_image2.SetSpatialIndex(1); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame); - generator.OnEncodedImage(encoded_image1); - std::optional<FrameInstrumentationData> frame_instrumentation_data = - generator.OnEncodedImage(encoded_image2); - - ASSERT_TRUE(frame_instrumentation_data.has_value()); - EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); - EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); - EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); - EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); - EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsNothingWhenNotEnoughTimeHasPassedSinceLastSampledFrame) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - VideoFrame frame1 = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - VideoFrame frame2 = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(2) - .build(); - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(1); - encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image1.SetSpatialIndex(0); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - - // Delta frame that is too recent in comparison to the last sampled frame: - // passed time < 90'000. - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(2); - encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); - encoded_image2.SetSpatialIndex(0); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame1); - generator.OnCapturedFrame(frame2); - generator.OnEncodedImage(encoded_image1); - - ASSERT_FALSE(generator.OnEncodedImage(encoded_image2).has_value()); -} - -TEST(FrameInstrumentationGeneratorTest, - ReturnsInstrumentationDataForUpperLayerOfASecondSvcKeyFrame) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP9); - VideoFrame frame1 = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - VideoFrame frame2 = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(2) - .build(); - for (const VideoFrame& frame : {frame1, frame2}) { - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(frame.rtp_timestamp()); - encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image1.SetSpatialIndex(0); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(frame.rtp_timestamp()); - encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); - encoded_image2.SetSpatialIndex(1); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame); - - std::optional<FrameInstrumentationData> data1 = - generator.OnEncodedImage(encoded_image1); - - std::optional<FrameInstrumentationData> data2 = - generator.OnEncodedImage(encoded_image2); - - ASSERT_TRUE(data1.has_value()); - ASSERT_TRUE(data2.has_value()); - - EXPECT_TRUE(data1->holds_upper_bits()); - EXPECT_TRUE(data2->holds_upper_bits()); - } -} - -TEST(FrameInstrumentationGeneratorTest, - SvcLayersSequenceIndicesIncreaseIndependentOnEachother) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP9); - VideoFrame frame1 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(1) - .build(); - VideoFrame frame2 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(2) - .build(); - for (const VideoFrame& frame : {frame1, frame2}) { - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(frame.rtp_timestamp()); - encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image1.SetSpatialIndex(0); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(frame.rtp_timestamp()); - encoded_image2.SetFrameType(VideoFrameType::kVideoFrameDelta); - encoded_image2.SetSpatialIndex(1); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame); - - std::optional<FrameInstrumentationData> data1 = - generator.OnEncodedImage(encoded_image1); - - std::optional<FrameInstrumentationData> data2 = - generator.OnEncodedImage(encoded_image2); - - ASSERT_TRUE(data1.has_value()); - ASSERT_TRUE(data2.has_value()); - - EXPECT_TRUE(data1->holds_upper_bits()); - EXPECT_TRUE(data2->holds_upper_bits()); - - EXPECT_EQ(data1->sequence_index(), data2->sequence_index()); - - // In the test the frames have equal frame buffers so the sample values - // should be equal. - EXPECT_THAT(data1->sample_values(), - Pointwise(DoubleEq(), data2->sample_values())); - } -} - -TEST(FrameInstrumentationGeneratorTest, - OutputsDeltaFrameInstrumentationDataForSimulcast) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP9); - bool has_found_delta_frame = false; - // 34 frames is the minimum number of frames to be able to sample a delta - // frame. - for (int i = 0; i < 34; ++i) { - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(i) - .build(); - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(frame.rtp_timestamp()); - encoded_image1.SetFrameType(i == 0 ? VideoFrameType::kVideoFrameKey - : VideoFrameType::kVideoFrameDelta); - encoded_image1.SetSimulcastIndex(0); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(frame.rtp_timestamp()); - encoded_image2.SetFrameType(i == 0 ? VideoFrameType::kVideoFrameKey - : VideoFrameType::kVideoFrameDelta); - encoded_image2.SetSimulcastIndex(1); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame); - - std::optional<FrameInstrumentationData> data1 = - generator.OnEncodedImage(encoded_image1); - - std::optional<FrameInstrumentationData> data2 = - generator.OnEncodedImage(encoded_image2); - - if (i == 0) { - ASSERT_TRUE(data1.has_value()); - ASSERT_TRUE(data2.has_value()); - - EXPECT_TRUE(data1->holds_upper_bits()); - EXPECT_TRUE(data2->holds_upper_bits()); - } else if (data1.has_value() || data2.has_value()) { - if (data1.has_value()) { - EXPECT_FALSE(data1->holds_upper_bits()); - } - if (data2.has_value()) { - EXPECT_FALSE(data2->holds_upper_bits()); - } - has_found_delta_frame = true; - } - } - EXPECT_TRUE(has_found_delta_frame); -} - -TEST(FrameInstrumentationGeneratorTest, - SequenceIndexIncreasesCorrectlyAtNewKeyFrame) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - VideoFrame frame1 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(1) - .build(); - VideoFrame frame2 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(2) - .build(); - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(1); - encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - - // Delta frame that is an upper layer of an SVC key frame. - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(2); - encoded_image2.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame1); - generator.OnCapturedFrame(frame2); - - ASSERT_EQ(generator.GetLayerId(encoded_image1), - generator.GetLayerId(encoded_image2)); - generator.SetHaltonSequenceIndex(0b0010'1010, - generator.GetLayerId(encoded_image1)); - - std::optional<FrameInstrumentationData> data1 = - generator.OnEncodedImage(encoded_image1); - std::optional<FrameInstrumentationData> data2 = - generator.OnEncodedImage(encoded_image2); - - ASSERT_TRUE(data1.has_value()); - ASSERT_TRUE(data2.has_value()); - - EXPECT_EQ(data1->sequence_index(), 0b0000'1000'0000); - EXPECT_EQ(data2->sequence_index(), 0b0001'0000'0000); -} - -TEST(FrameInstrumentationGeneratorTest, - SequenceIndexThatWouldOverflowTo15BitsIncreasesCorrectlyAtNewKeyFrame) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - VideoFrame frame1 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(1) - .build(); - VideoFrame frame2 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(2) - .build(); - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(1); - encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - encoded_image1.SetSimulcastIndex(0); - - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(2); - encoded_image2.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - encoded_image2.SetSimulcastIndex(0); - - generator.OnCapturedFrame(frame1); - generator.OnCapturedFrame(frame2); - - ASSERT_EQ(generator.GetLayerId(encoded_image1), - generator.GetLayerId(encoded_image2)); - generator.SetHaltonSequenceIndex(0b11'1111'1111'1111, - generator.GetLayerId(encoded_image1)); - std::optional<FrameInstrumentationData> data1 = - generator.OnEncodedImage(encoded_image1); - std::optional<FrameInstrumentationData> data2 = - generator.OnEncodedImage(encoded_image2); - - ASSERT_TRUE(data1.has_value()); - ASSERT_TRUE(data2.has_value()); - - EXPECT_EQ(data1->sequence_index(), 0); - EXPECT_EQ(data2->sequence_index(), 0b1000'0000); -} - -TEST(FrameInstrumentationGeneratorTest, - SequenceIndexIncreasesCorrectlyAtNewKeyFrameAlreadyZeroes) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - VideoFrame frame1 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(1) - .build(); - VideoFrame frame2 = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(2) - .build(); - EncodedImage encoded_image1; - encoded_image1.SetRtpTimestamp(1); - encoded_image1.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image1.qp_ = 10; - encoded_image1._encodedWidth = kDefaultScaledWidth; - encoded_image1._encodedHeight = kDefaultScaledHeight; - - // Delta frame that is an upper layer of an SVC key frame. - EncodedImage encoded_image2; - encoded_image2.SetRtpTimestamp(2); - encoded_image2.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image2.qp_ = 10; - encoded_image2._encodedWidth = kDefaultScaledWidth; - encoded_image2._encodedHeight = kDefaultScaledHeight; - - generator.OnCapturedFrame(frame1); - generator.OnCapturedFrame(frame2); - - ASSERT_EQ(generator.GetLayerId(encoded_image1), - generator.GetLayerId(encoded_image2)); - generator.SetHaltonSequenceIndex(0b1000'0000, - generator.GetLayerId(encoded_image1)); - - std::optional<FrameInstrumentationData> data1 = - generator.OnEncodedImage(encoded_image1); - std::optional<FrameInstrumentationData> data2 = - generator.OnEncodedImage(encoded_image2); - - ASSERT_TRUE(data1.has_value()); - ASSERT_TRUE(data2.has_value()); - - EXPECT_EQ(data1->sequence_index(), 0b0000'1000'0000); - EXPECT_EQ(data2->sequence_index(), 0b0001'0000'0000); -} -TEST(FrameInstrumentationGeneratorTest, - SequenceIndexThatWouldOverflowTo15BitsIncreasesCorrectlyAtNewDeltaFrame) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - generator.OnCapturedFrame( - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(1) - .build()); - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(1); - encoded_image.SetFrameType(VideoFrameType::kVideoFrameDelta); - encoded_image.qp_ = 10; - encoded_image._encodedWidth = kDefaultScaledWidth; - encoded_image._encodedHeight = kDefaultScaledHeight; - encoded_image.SetSimulcastIndex(0); - - constexpr int kMaxSequenceIndex = 0b11'1111'1111'1111; - - generator.SetHaltonSequenceIndex(kMaxSequenceIndex, - generator.GetLayerId(encoded_image)); - std::optional<FrameInstrumentationData> data = - generator.OnEncodedImage(encoded_image); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index(), kMaxSequenceIndex); - - // Loop until we get a new delta frame. - bool has_found_delta_frame = false; - for (int i = 0; i < 34; ++i) { - generator.OnCapturedFrame( - VideoFrame::Builder() - .set_video_frame_buffer( - MakeI420FrameBufferWithDifferentPixelValues()) - .set_rtp_timestamp(i + 2) - .build()); - - encoded_image.SetRtpTimestamp(i + 2); - - std::optional<FrameInstrumentationData> frame_instrumentation_data = - generator.OnEncodedImage(encoded_image); - if (frame_instrumentation_data.has_value()) { - has_found_delta_frame = true; - EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); - break; - } - } - EXPECT_TRUE(has_found_delta_frame); -} - -TEST(FrameInstrumentationGeneratorTest, GetterAndSetterOperatesAsExpected) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - // `std::nullopt` when uninitialized. - EXPECT_FALSE(generator.GetHaltonSequenceIndex(1).has_value()); - - // Zero is a valid index. - generator.SetHaltonSequenceIndex(0, 1); - std::optional<int> index = generator.GetHaltonSequenceIndex(1); - EXPECT_TRUE(index.has_value()); - EXPECT_EQ(*index, 0); - -#if GTEST_HAS_DEATH_TEST - // Negative values are not allowed to be set. - EXPECT_DEATH(generator.SetHaltonSequenceIndex(-2, 1), - "Index must be non-negative"); - index = generator.GetHaltonSequenceIndex(1); - EXPECT_TRUE(index.has_value()); - EXPECT_EQ(*index, 0); - - // Values requiring more than 15 bits are not allowed. - EXPECT_DEATH(generator.SetHaltonSequenceIndex(0x4000, 1), - "Index must not be larger than 0x3FFF"); - index = generator.GetHaltonSequenceIndex(1); - EXPECT_TRUE(index.has_value()); - EXPECT_EQ(*index, 0); -#endif // GTEST_HAS_DEATH_TEST -} - -TEST(FrameInstrumentationGeneratorTest, QueuesAtMostThreeInputFrames) { - auto generator = std::make_unique<FrameInstrumentationGenerator>( - VideoCodecType::kVideoCodecVP8); - - bool frames_destroyed[4] = {}; - class TestBuffer : public I420Buffer { - public: - TestBuffer(int width, int height, bool* frame_destroyed_indicator) - : I420Buffer(width, height), - frame_destroyed_indicator_(frame_destroyed_indicator) {} - - private: - friend class RefCountedObject<TestBuffer>; - ~TestBuffer() override { *frame_destroyed_indicator_ = true; } - - bool* frame_destroyed_indicator_; - }; - - // Insert four frames, the first one should expire and be released. - for (int i = 0; i < 4; ++i) { - generator->OnCapturedFrame( - VideoFrame::Builder() - .set_video_frame_buffer(make_ref_counted<TestBuffer>( - kDefaultScaledWidth, kDefaultScaledHeight, - &frames_destroyed[i])) - .set_rtp_timestamp(1 + (33 * i)) - .build()); - } - - EXPECT_THAT(frames_destroyed, ElementsAre(true, false, false, false)); - - generator.reset(); - EXPECT_THAT(frames_destroyed, ElementsAre(true, true, true, true)); -} - -TEST(FrameInstrumentationGeneratorTest, - UsesFilterSettingsFromFrameWhenAvailable) { - FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); - VideoFrame frame = VideoFrame::Builder() - .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) - .set_rtp_timestamp(1) - .build(); - // No QP needed when frame provides filter settings. - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(1); - encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); - encoded_image._encodedWidth = kDefaultScaledWidth; - encoded_image._encodedHeight = kDefaultScaledHeight; - encoded_image.set_corruption_detection_filter_settings( - CorruptionDetectionFilterSettings{.std_dev = 1.0, - .luma_error_threshold = 2, - .chroma_error_threshold = 3}); - - generator.OnCapturedFrame(frame); - std::optional<FrameInstrumentationData> frame_instrumentation_data = - generator.OnEncodedImage(encoded_image); - - ASSERT_TRUE(frame_instrumentation_data.has_value()); - EXPECT_EQ(frame_instrumentation_data->std_dev(), 1.0); - EXPECT_EQ(frame_instrumentation_data->luma_error_threshold(), 2); - EXPECT_EQ(frame_instrumentation_data->chroma_error_threshold(), 3); -} - -} // namespace -} // namespace webrtc diff --git a/third_party/libwebrtc/video/corruption_detection/utils.cc b/third_party/libwebrtc/video/corruption_detection/utils.cc @@ -10,9 +10,12 @@ #include "video/corruption_detection/utils.h" +#include <algorithm> + #include "absl/strings/match.h" #include "absl/strings/string_view.h" #include "api/scoped_refptr.h" +#include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame_buffer.h" @@ -69,4 +72,9 @@ scoped_refptr<I420Buffer> GetAsI420Buffer( return frame_as_i420_buffer; } +int GetSpatialLayerId(const EncodedImage& encoded_image) { + return std::max(encoded_image.SpatialIndex().value_or(0), + encoded_image.SimulcastIndex().value_or(0)); +} + } // namespace webrtc diff --git a/third_party/libwebrtc/video/corruption_detection/utils.h b/third_party/libwebrtc/video/corruption_detection/utils.h @@ -13,6 +13,7 @@ #include "absl/strings/string_view.h" #include "api/scoped_refptr.h" +#include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame_buffer.h" @@ -24,6 +25,10 @@ VideoCodecType GetVideoCodecType(absl::string_view codec_name); scoped_refptr<I420Buffer> GetAsI420Buffer( scoped_refptr<I420BufferInterface> i420_buffer_interface); +// Returns effective spatial layer id. Since spatial layers can be implemented +// either using simulcast or using SVC, this helper picks the one in use. +int GetSpatialLayerId(const EncodedImage& encoded_image); + } // namespace webrtc #endif // VIDEO_CORRUPTION_DETECTION_UTILS_H_ diff --git a/third_party/libwebrtc/video/corruption_detection/utils_unittest.cc b/third_party/libwebrtc/video/corruption_detection/utils_unittest.cc @@ -10,6 +10,7 @@ #include "video/corruption_detection/utils.h" +#include "api/video/encoded_image.h" #include "api/video/video_codec_type.h" #include "test/gmock.h" #include "test/gtest.h" @@ -34,5 +35,33 @@ TEST(UtilsTest, IfCodecDoesNotExistRaiseError) { } #endif // GTEST_HAS_DEATH_TEST +TEST(UtilsTest, GetLayerIdReturnsZeroForEmptyIndices) { + EncodedImage encoded_image; + EXPECT_EQ(GetSpatialLayerId(encoded_image), 0); +} + +TEST(UtilsTest, GetLayerIdReturnsSpatialIndexWhenSet) { + EncodedImage encoded_image; + encoded_image.SetSpatialIndex(2); + EXPECT_EQ(GetSpatialLayerId(encoded_image), 2); +} + +TEST(UtilsTest, GetLayerIdReturnsSimulcastIndexWhenSet) { + EncodedImage encoded_image; + encoded_image.SetSimulcastIndex(1); + EXPECT_EQ(GetSpatialLayerId(encoded_image), 1); +} + +TEST(UtilsTest, GetLayerIdReturnsMaxOfSpatialAndSimulcastIndex) { + EncodedImage encoded_image; + encoded_image.SetSpatialIndex(1); + encoded_image.SetSimulcastIndex(2); + EXPECT_EQ(GetSpatialLayerId(encoded_image), 2); + + encoded_image.SetSpatialIndex(3); + encoded_image.SetSimulcastIndex(1); + EXPECT_EQ(GetSpatialLayerId(encoded_image), 3); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/video/video_stream_encoder.cc b/third_party/libwebrtc/video/video_stream_encoder.cc @@ -39,6 +39,7 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/corruption_detection/frame_instrumentation_generator.h" #include "api/video/encoded_image.h" #include "api/video/render_resolution.h" #include "api/video/video_adaptation_counters.h" @@ -88,7 +89,6 @@ #include "video/alignment_adjuster.h" #include "video/config/encoder_stream_factory.h" #include "video/config/video_encoder_config.h" -#include "video/corruption_detection/frame_instrumentation_generator.h" #include "video/encoder_bitrate_adjuster.h" #include "video/frame_cadence_adapter.h" #include "video/frame_dumping_encoder.h" @@ -1401,8 +1401,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { VideoFrameType::kVideoFrameKey); if (settings_.enable_frame_instrumentation_generator) { frame_instrumentation_generator_ = - std::make_unique<FrameInstrumentationGenerator>( - encoder_config_.codec_type); + FrameInstrumentationGenerator::Create(encoder_config_.codec_type); } } diff --git a/third_party/libwebrtc/video/video_stream_encoder.h b/third_party/libwebrtc/video/video_stream_encoder.h @@ -32,6 +32,7 @@ #include "api/units/data_rate.h" #include "api/units/data_size.h" #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_generator.h" #include "api/video/encoded_image.h" #include "api/video/video_adaptation_counters.h" #include "api/video/video_adaptation_reason.h" @@ -58,7 +59,6 @@ #include "video/adaptation/overuse_frame_detector.h" #include "video/adaptation/video_stream_encoder_resource_manager.h" #include "video/config/video_encoder_config.h" -#include "video/corruption_detection/frame_instrumentation_generator.h" #include "video/encoder_bitrate_adjuster.h" #include "video/frame_cadence_adapter.h" #include "video/frame_encode_metadata_writer.h"