commit ae58dbd5cb9fb194e150b6fa830bb9488067917b parent 04c32349af4b3a19ad32f179c480282cf4f554d4 Author: Dan Baker <dbaker@mozilla.com> Date: Fri, 24 Oct 2025 14:09:36 -0600 Bug 1995393 - Vendor libwebrtc from 81d959c673 Upstream commit: https://webrtc.googlesource.com/src/+/81d959c673a8abc0c63353ecf0be47b1fdb6d9be Move FrameInstrumentationEvaluation to api/video/corruption_detection. This exposes the api surface for external usage. Bug: webrtc:358039777 Change-Id: I19734e0d6ee2a99a9823a5aac4fbedfd70cc15a0 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/405440 Auto-Submit: Erik Språng <sprang@webrtc.org> Commit-Queue: Erik Språng <sprang@webrtc.org> Reviewed-by: Per Kjellander <perkj@webrtc.org> Cr-Commit-Position: refs/heads/main@{#45405} Diffstat:
21 files changed, 374 insertions(+), 343 deletions(-)
diff --git a/third_party/libwebrtc/README.mozilla.last-vendor b/third_party/libwebrtc/README.mozilla.last-vendor @@ -1,4 +1,4 @@ # ./mach python dom/media/webrtc/third_party_build/vendor-libwebrtc.py --from-local /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc --commit mozpatches libwebrtc -libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-10-24T20:07:02.954184+00:00. +libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-10-24T20:09:10.842564+00:00. # base of lastest vendoring -33f3b4a356 +81d959c673 diff --git a/third_party/libwebrtc/api/BUILD.gn b/third_party/libwebrtc/api/BUILD.gn @@ -1690,6 +1690,7 @@ if (rtc_include_tests) { "video:video_frame_metadata_unittest", "video/corruption_detection:frame_instrumentation_data_reader_unittest", "video/corruption_detection:frame_instrumentation_data_unittest", + "video/corruption_detection:frame_instrumentation_evaluation_unittest", "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/strings:string_view", diff --git a/third_party/libwebrtc/api/video/corruption_detection/BUILD.gn b/third_party/libwebrtc/api/video/corruption_detection/BUILD.gn @@ -42,24 +42,57 @@ rtc_library("frame_instrumentation_data_reader") { ] } -rtc_library("frame_instrumentation_data_unittest") { - testonly = true - sources = [ "frame_instrumentation_data_unittest.cc" ] - +rtc_library("frame_instrumentation_evaluation") { + sources = [ + "frame_instrumentation_evaluation.cc", + "frame_instrumentation_evaluation.h", + ] deps = [ ":frame_instrumentation_data", - "../../../test:test_support", + "../:video_frame", + "../:video_rtp_headers", + "../../:array_view", + "../../:scoped_refptr", + "../../../rtc_base:checks", + "../../../rtc_base:logging", + "../../../video/corruption_detection:corruption_classifier", + "../../../video/corruption_detection:halton_frame_sampler", ] } -rtc_library("frame_instrumentation_data_reader_unittest") { - testonly = true - sources = [ "frame_instrumentation_data_reader_unittest.cc" ] +if (rtc_include_tests) { + rtc_library("frame_instrumentation_data_unittest") { + testonly = true + sources = [ "frame_instrumentation_data_unittest.cc" ] - deps = [ - ":frame_instrumentation_data", - ":frame_instrumentation_data_reader", - "../../../test:test_support", - "../../transport/rtp:corruption_detection_message", - ] + deps = [ + ":frame_instrumentation_data", + "../../../test:test_support", + ] + } + + rtc_library("frame_instrumentation_data_reader_unittest") { + testonly = true + sources = [ "frame_instrumentation_data_reader_unittest.cc" ] + + deps = [ + ":frame_instrumentation_data", + ":frame_instrumentation_data_reader", + "../../../test:test_support", + "../../transport/rtp:corruption_detection_message", + ] + } + + rtc_library("frame_instrumentation_evaluation_unittest") { + testonly = true + sources = [ "frame_instrumentation_evaluation_unittest.cc" ] + deps = [ + ":frame_instrumentation_data", + ":frame_instrumentation_evaluation", + "../:video_frame", + "../:video_rtp_headers", + "../../:scoped_refptr", + "../../../test:test_support", + ] + } } diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_evaluation.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_evaluation.cc @@ -0,0 +1,111 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_evaluation.h" + +#include <cstddef> +#include <memory> +#include <vector> + +#include "api/array_view.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/video_content_type.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "video/corruption_detection/corruption_classifier.h" +#include "video/corruption_detection/halton_frame_sampler.h" + +namespace webrtc { + +namespace { + +std::vector<FilteredSample> ConvertSampleValuesToFilteredSamples( + ArrayView<const double> values, + ArrayView<const FilteredSample> samples) { + RTC_CHECK_EQ(values.size(), samples.size()) + << "values and samples must have the same size"; + std::vector<FilteredSample> filtered_samples; + filtered_samples.reserve(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + filtered_samples.push_back({.value = values[i], .plane = samples[i].plane}); + } + return filtered_samples; +} + +} // namespace + +class FrameInstrumentationEvaluationImpl + : public FrameInstrumentationEvaluation { + public: + explicit FrameInstrumentationEvaluationImpl(CorruptionScoreObserver* observer) + : observer_(observer), classifier_(/*scale_factor=*/3) { + RTC_CHECK(observer); + } + ~FrameInstrumentationEvaluationImpl() override = default; + + void OnInstrumentedFrame(const FrameInstrumentationData& data, + const VideoFrame& frame, + VideoContentType frame_type) override { + if (data.sample_values().empty()) { + // Likely a sync message. Silently ignore. + return; + } + + frame_sampler_.SetCurrentIndex(data.sequence_index()); + std::vector<HaltonFrameSampler::Coordinates> sample_coordinates = + frame_sampler_.GetSampleCoordinatesForFrame( + data.sample_values().size()); + if (sample_coordinates.empty()) { + RTC_LOG(LS_ERROR) << "Failed to get sample coordinates for frame."; + return; + } + + std::vector<FilteredSample> samples = + GetSampleValuesForFrame(frame, sample_coordinates, frame.width(), + frame.height(), data.std_dev()); + if (samples.empty()) { + RTC_LOG(LS_ERROR) << "Failed to get sample values for frame"; + return; + } + + std::vector<FilteredSample> data_samples = + ConvertSampleValuesToFilteredSamples(data.sample_values(), samples); + if (data_samples.empty()) { + RTC_LOG(LS_ERROR) + << "Failed to convert sample values to filtered samples"; + return; + } + + double score = classifier_.CalculateCorruptionProbability( + data_samples, samples, data.luma_error_threshold(), + data.chroma_error_threshold()); + + observer_->OnCorruptionScore(score, frame_type); + } + + private: + CorruptionScoreObserver* const observer_; + + HaltonFrameSampler frame_sampler_; + CorruptionClassifier classifier_; +}; + +std::unique_ptr<FrameInstrumentationEvaluation> +FrameInstrumentationEvaluation::Create(CorruptionScoreObserver* observer) { + if (!observer) { + RTC_LOG(LS_WARNING) + << "Cannot create FrameInstrumentationEvaluation with no observer."; + return nullptr; + } + return std::make_unique<FrameInstrumentationEvaluationImpl>(observer); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_evaluation.h b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_evaluation.h @@ -0,0 +1,49 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ +#define API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ + +#include <memory> + +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/video_content_type.h" +#include "api/video/video_frame.h" + +namespace webrtc { + +class CorruptionScoreObserver { + public: + CorruptionScoreObserver() = default; + virtual ~CorruptionScoreObserver() = default; + + // Results of corruption detection for a single frame, with a likelihood score + // in the range [0.0, 1.0]. + virtual void OnCorruptionScore(double corruption_score, + VideoContentType content_type) = 0; +}; + +class FrameInstrumentationEvaluation { + public: + static std::unique_ptr<FrameInstrumentationEvaluation> Create( + CorruptionScoreObserver* observer); + + virtual ~FrameInstrumentationEvaluation() = default; + virtual void OnInstrumentedFrame(const FrameInstrumentationData& data, + const VideoFrame& frame, + VideoContentType frame_type) = 0; + + protected: + FrameInstrumentationEvaluation() = default; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc @@ -0,0 +1,146 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_evaluation.h" + +#include <cstdint> +#include <memory> +#include <vector> + +#include "api/scoped_refptr.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_content_type.h" +#include "api/video/video_frame.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::_; +using ::testing::AllOf; +using ::testing::Ge; +using ::testing::Le; + +class MockCorruptionScoreObserver : public CorruptionScoreObserver { + public: + MOCK_METHOD(void, OnCorruptionScore, (double, VideoContentType), (override)); +}; + +scoped_refptr<I420Buffer> MakeI420FrameBufferWithDifferentPixelValues() { + // Create an I420 frame of size 4x4. + const int kDefaultLumaWidth = 4; + const int kDefaultLumaHeight = 4; + const int kDefaultChromaWidth = 2; + std::vector<uint8_t> kDefaultYContent = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + std::vector<uint8_t> kDefaultUContent = {17, 18, 19, 20}; + std::vector<uint8_t> kDefaultVContent = {21, 22, 23, 24}; + + return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight, + kDefaultYContent.data(), kDefaultLumaWidth, + kDefaultUContent.data(), kDefaultChromaWidth, + kDefaultVContent.data(), kDefaultChromaWidth); +} + +TEST(FrameInstrumentationEvaluationTest, + HaveNoCorruptionScoreWhenNoSampleValuesAreProvided) { + FrameInstrumentationData data; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + MockCorruptionScoreObserver observer; + std::unique_ptr<FrameInstrumentationEvaluation> evaluator = + FrameInstrumentationEvaluation::Create(&observer); + EXPECT_CALL(observer, OnCorruptionScore).Times(0); + evaluator->OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); +} + +TEST(FrameInstrumentationEvaluationTest, + HaveACorruptionScoreWhenSampleValuesAreProvided) { + FrameInstrumentationData data; + data.SetStdDev(1.0); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + MockCorruptionScoreObserver observer; + std::unique_ptr<FrameInstrumentationEvaluation> evaluator = + FrameInstrumentationEvaluation::Create(&observer); + EXPECT_CALL(observer, OnCorruptionScore(1.0, VideoContentType::SCREENSHARE)); + evaluator->OnInstrumentedFrame(data, frame, VideoContentType::SCREENSHARE); +} + +TEST(FrameInstrumentationEvaluationTest, + ApplyThresholdsWhenNonNegativeThresholdsAreProvided) { + FrameInstrumentationData data; + data.SetStdDev(1.0); + data.SetLumaErrorThreshold(8); + data.SetChromaErrorThreshold(8); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + MockCorruptionScoreObserver observer; + std::unique_ptr<FrameInstrumentationEvaluation> evaluator = + FrameInstrumentationEvaluation::Create(&observer); + EXPECT_CALL(observer, OnCorruptionScore(AllOf(Ge(0.0), Le(1.0)), _)); + evaluator->OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); +} + +TEST(FrameInstrumentationEvaluationTest, + ApplyStdDevWhenNonNegativeStdDevIsProvided) { + FrameInstrumentationData data; + data.SetStdDev(0.6); + data.SetLumaErrorThreshold(8); + data.SetChromaErrorThreshold(8); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); + + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + MockCorruptionScoreObserver observer; + std::unique_ptr<FrameInstrumentationEvaluation> evaluator = + FrameInstrumentationEvaluation::Create(&observer); + EXPECT_CALL(observer, OnCorruptionScore(AllOf(Ge(0.0), Le(1.0)), _)); + evaluator->OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); +} + +TEST(FrameInstrumentationEvaluationTest, ApplySequenceIndexWhenProvided) { + FrameInstrumentationData data; + data.SetSequenceIndex(1); + data.SetStdDev(0.6); + data.SetLumaErrorThreshold(8); + data.SetChromaErrorThreshold(8); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); + + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + MockCorruptionScoreObserver observer; + std::unique_ptr<FrameInstrumentationEvaluation> evaluator = + FrameInstrumentationEvaluation::Create(&observer); + EXPECT_CALL(observer, OnCorruptionScore(AllOf(Ge(0.0), Le(1.0)), _)); + evaluator->OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/BUILD.gn b/third_party/libwebrtc/modules/video_coding/BUILD.gn @@ -243,6 +243,7 @@ rtc_library("video_coding") { "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", "../../api/video/corruption_detection:frame_instrumentation_data", + "../../api/video/corruption_detection:frame_instrumentation_evaluation", "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../common_video", @@ -272,7 +273,6 @@ rtc_library("video_coding") { "../../system_wrappers", "../../system_wrappers:metrics", "../../video/config:encoder_config", - "../../video/corruption_detection:frame_instrumentation_evaluation", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", "../rtp_rtcp:rtp_video_header", diff --git a/third_party/libwebrtc/moz-patch-stack/s0027.patch b/third_party/libwebrtc/moz-patch-stack/s0027.patch @@ -203,7 +203,7 @@ index ef07265bf3..348be89bb9 100644 } diff --git a/api/BUILD.gn b/api/BUILD.gn -index ea3a8a4acd..3603d4f4b0 100644 +index 5b25a93d90..a9adaab17d 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -44,6 +44,9 @@ rtc_library("enable_media") { @@ -1258,7 +1258,7 @@ index 2e8752ab0c..f9bcdf7a0d 100644 rtc_library("call_config_utils") { testonly = true diff --git a/video/BUILD.gn b/video/BUILD.gn -index 0e43130593..eeeddba86f 100644 +index 033331442f..ed00c2cbef 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -17,7 +17,7 @@ rtc_library("video_stream_encoder_interface") { diff --git a/third_party/libwebrtc/moz-patch-stack/s0037.patch b/third_party/libwebrtc/moz-patch-stack/s0037.patch @@ -53,7 +53,7 @@ index 2ca52615a7..52f998ddc2 100644 // Implements RtpVideoFrameReceiver. void ManageFrame(std::unique_ptr<RtpFrameObject> frame) override; diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc -index ca0271cc97..f3170157c0 100644 +index 5238e8d3a6..4877b8421b 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -626,6 +626,14 @@ VideoReceiveStreamInterface::Stats VideoReceiveStream2::GetStats() const { diff --git a/third_party/libwebrtc/moz-patch-stack/s0042.patch b/third_party/libwebrtc/moz-patch-stack/s0042.patch @@ -153,7 +153,7 @@ index 52f998ddc2..1acb0ac70e 100644 private: // Implements RtpVideoFrameReceiver. diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc -index f3170157c0..e7a224e8c3 100644 +index 4877b8421b..bc997550df 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -632,7 +632,8 @@ VideoReceiveStreamInterface::Stats VideoReceiveStream2::GetStats() const { diff --git a/third_party/libwebrtc/moz-patch-stack/s0057.patch b/third_party/libwebrtc/moz-patch-stack/s0057.patch @@ -202,7 +202,7 @@ index 84b00b711e..c8058ea132 100644 RTC_DCHECK_RUN_ON(&main_thread_); last_codec_type_ = codec_type; diff --git a/video/receive_statistics_proxy.h b/video/receive_statistics_proxy.h -index 111b7f3c45..fb411d4bbf 100644 +index e3cd922647..9e58f41521 100644 --- a/video/receive_statistics_proxy.h +++ b/video/receive_statistics_proxy.h @@ -106,6 +106,7 @@ class ReceiveStatisticsProxy : public VideoStreamBufferControllerStatsObserver, @@ -272,7 +272,7 @@ index 1acb0ac70e..272f83ccba 100644 RTC_GUARDED_BY(packet_sequence_checker_); // h26x_packet_buffer_ is applicable to H.264 and H.265. For H.265 it is diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc -index e7a224e8c3..7fc4e69446 100644 +index bc997550df..59b5282229 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -256,6 +256,7 @@ VideoReceiveStream2::VideoReceiveStream2( diff --git a/third_party/libwebrtc/moz-patch-stack/s0100.patch b/third_party/libwebrtc/moz-patch-stack/s0100.patch @@ -39,7 +39,7 @@ index 0f2341a462..51d055bcf0 100644 group("media") { diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn -index 32fd20badc..8416dec319 100644 +index 3bd0bfb79f..b7561e53b6 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -7,7 +7,7 @@ diff --git a/third_party/libwebrtc/moz-patch-stack/s0102.patch b/third_party/libwebrtc/moz-patch-stack/s0102.patch @@ -126,7 +126,7 @@ index b602b8d2ce..4603ed3cfd 100644 "Generated during 'gn gen' by //BUILD.gn.", "", diff --git a/api/BUILD.gn b/api/BUILD.gn -index 3603d4f4b0..77cb83a9c4 100644 +index a9adaab17d..9544232d23 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -8,8 +8,8 @@ @@ -588,7 +588,7 @@ index a693604389..72972d18e1 100644 import("../../webrtc.gni") diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn -index 8416dec319..44328eade0 100644 +index b7561e53b6..fe7eb57423 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -6,7 +6,7 @@ diff --git a/third_party/libwebrtc/video/BUILD.gn b/third_party/libwebrtc/video/BUILD.gn @@ -123,6 +123,7 @@ rtc_library("video") { "../api/video:video_stream_encoder", "../api/video/corruption_detection:frame_instrumentation_data", "../api/video/corruption_detection:frame_instrumentation_data_reader", + "../api/video/corruption_detection:frame_instrumentation_evaluation", "../api/video_codecs:video_codecs_api", "../call:bitrate_allocator", "../call:call_interfaces", @@ -185,7 +186,6 @@ rtc_library("video") { "../system_wrappers", "../system_wrappers:metrics", "../video/config:encoder_config", - "../video/corruption_detection:frame_instrumentation_evaluation", "adaptation:video_adaptation", "render:incoming_video_stream", "//third_party/abseil-cpp/absl/algorithm:container", diff --git a/third_party/libwebrtc/video/corruption_detection/BUILD.gn b/third_party/libwebrtc/video/corruption_detection/BUILD.gn @@ -21,24 +21,6 @@ rtc_library("corruption_classifier") { ] } -rtc_library("frame_instrumentation_evaluation") { - sources = [ - "frame_instrumentation_evaluation.cc", - "frame_instrumentation_evaluation.h", - ] - deps = [ - ":corruption_classifier", - ":halton_frame_sampler", - "../../api:array_view", - "../../api:scoped_refptr", - "../../api/video:video_frame", - "../../api/video:video_rtp_headers", - "../../api/video/corruption_detection:frame_instrumentation_data", - "../../rtc_base:checks", - "../../rtc_base:logging", - ] -} - rtc_library("frame_instrumentation_generator") { sources = [ "frame_instrumentation_generator.cc", @@ -156,19 +138,6 @@ if (rtc_include_tests) { ] } - rtc_library("frame_instrumentation_evaluation_unittest") { - testonly = true - sources = [ "frame_instrumentation_evaluation_unittest.cc" ] - deps = [ - ":frame_instrumentation_evaluation", - "../../api:scoped_refptr", - "../../api/video:video_frame", - "../../api/video:video_rtp_headers", - "../../api/video/corruption_detection:frame_instrumentation_data", - "../../test:test_support", - ] - } - rtc_library("frame_instrumentation_generator_unittest") { testonly = true sources = [ "frame_instrumentation_generator_unittest.cc" ] @@ -259,7 +228,6 @@ if (rtc_include_tests) { sources = [] deps = [ ":corruption_classifier_unittest", - ":frame_instrumentation_evaluation_unittest", ":frame_instrumentation_generator_unittest", ":frame_pair_corruption_score_unittest", ":generic_mapping_functions_unittest", diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.cc @@ -1,87 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/corruption_detection/frame_instrumentation_evaluation.h" - -#include <cstddef> -#include <vector> - -#include "api/array_view.h" -#include "api/video/corruption_detection/frame_instrumentation_data.h" -#include "api/video/video_content_type.h" -#include "api/video/video_frame.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "video/corruption_detection/halton_frame_sampler.h" - -namespace webrtc { - -namespace { - -std::vector<FilteredSample> ConvertSampleValuesToFilteredSamples( - ArrayView<const double> values, - ArrayView<const FilteredSample> samples) { - RTC_CHECK_EQ(values.size(), samples.size()) - << "values and samples must have the same size"; - std::vector<FilteredSample> filtered_samples; - filtered_samples.reserve(values.size()); - for (size_t i = 0; i < values.size(); ++i) { - filtered_samples.push_back({.value = values[i], .plane = samples[i].plane}); - } - return filtered_samples; -} - -} // namespace - -FrameInstrumentationEvaluation::FrameInstrumentationEvaluation( - CorruptionScoreObserver* observer) - : observer_(observer), classifier_(/*scale_factor=*/3) { - RTC_CHECK(observer); -} - -void FrameInstrumentationEvaluation::OnInstrumentedFrame( - const FrameInstrumentationData& data, - const VideoFrame& frame, - VideoContentType content_type) { - if (data.sample_values().empty()) { - // Likely a sync message. Silently ignore. - return; - } - - frame_sampler_.SetCurrentIndex(data.sequence_index()); - std::vector<HaltonFrameSampler::Coordinates> sample_coordinates = - frame_sampler_.GetSampleCoordinatesForFrame(data.sample_values().size()); - if (sample_coordinates.empty()) { - RTC_LOG(LS_ERROR) << "Failed to get sample coordinates for frame."; - return; - } - - std::vector<FilteredSample> samples = GetSampleValuesForFrame( - frame, sample_coordinates, frame.width(), frame.height(), data.std_dev()); - if (samples.empty()) { - RTC_LOG(LS_ERROR) << "Failed to get sample values for frame"; - return; - } - - std::vector<FilteredSample> data_samples = - ConvertSampleValuesToFilteredSamples(data.sample_values(), samples); - if (data_samples.empty()) { - RTC_LOG(LS_ERROR) << "Failed to convert sample values to filtered samples"; - return; - } - - double score = classifier_.CalculateCorruptionProbability( - data_samples, samples, data.luma_error_threshold(), - data.chroma_error_threshold()); - - observer_->OnCorruptionScore(score, content_type); -} - -} // namespace webrtc diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.h b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.h @@ -1,50 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ -#define VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ - -#include "api/video/corruption_detection/frame_instrumentation_data.h" -#include "api/video/video_content_type.h" -#include "api/video/video_frame.h" -#include "video/corruption_detection/corruption_classifier.h" -#include "video/corruption_detection/halton_frame_sampler.h" - -namespace webrtc { - -class CorruptionScoreObserver { - public: - CorruptionScoreObserver() = default; - virtual ~CorruptionScoreObserver() = default; - - // Results of corruption detection for a single frame, with a likelihood score - // in the range [0.0, 1.0]. - virtual void OnCorruptionScore(double corruption_score, - VideoContentType content_type) = 0; -}; - -class FrameInstrumentationEvaluation { - public: - explicit FrameInstrumentationEvaluation(CorruptionScoreObserver* observer); - - void OnInstrumentedFrame(const FrameInstrumentationData& data, - const VideoFrame& frame, - VideoContentType frame_type); - - private: - CorruptionScoreObserver* const observer_; - - HaltonFrameSampler frame_sampler_; - CorruptionClassifier classifier_; -}; - -} // namespace webrtc - -#endif // VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc @@ -1,140 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/corruption_detection/frame_instrumentation_evaluation.h" - -#include <cstdint> -#include <vector> - -#include "api/scoped_refptr.h" -#include "api/video/corruption_detection/frame_instrumentation_data.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_content_type.h" -#include "api/video/video_frame.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace { - -using ::testing::_; -using ::testing::AllOf; -using ::testing::Ge; -using ::testing::Le; - -class MockCorruptionScoreObserver : public CorruptionScoreObserver { - public: - MOCK_METHOD(void, OnCorruptionScore, (double, VideoContentType), (override)); -}; - -scoped_refptr<I420Buffer> MakeI420FrameBufferWithDifferentPixelValues() { - // Create an I420 frame of size 4x4. - const int kDefaultLumaWidth = 4; - const int kDefaultLumaHeight = 4; - const int kDefaultChromaWidth = 2; - std::vector<uint8_t> kDefaultYContent = {1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16}; - std::vector<uint8_t> kDefaultUContent = {17, 18, 19, 20}; - std::vector<uint8_t> kDefaultVContent = {21, 22, 23, 24}; - - return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight, - kDefaultYContent.data(), kDefaultLumaWidth, - kDefaultUContent.data(), kDefaultChromaWidth, - kDefaultVContent.data(), kDefaultChromaWidth); -} - -TEST(FrameInstrumentationEvaluationTest, - HaveNoCorruptionScoreWhenNoSampleValuesAreProvided) { - FrameInstrumentationData data; - VideoFrame frame = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .build(); - - MockCorruptionScoreObserver observer; - FrameInstrumentationEvaluation evaluator(&observer); - EXPECT_CALL(observer, OnCorruptionScore).Times(0); - evaluator.OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); -} - -TEST(FrameInstrumentationEvaluationTest, - HaveACorruptionScoreWhenSampleValuesAreProvided) { - FrameInstrumentationData data; - data.SetStdDev(1.0); - data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); - VideoFrame frame = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .build(); - - MockCorruptionScoreObserver observer; - FrameInstrumentationEvaluation evaluator(&observer); - EXPECT_CALL(observer, OnCorruptionScore(1.0, VideoContentType::SCREENSHARE)); - evaluator.OnInstrumentedFrame(data, frame, VideoContentType::SCREENSHARE); -} - -TEST(FrameInstrumentationEvaluationTest, - ApplyThresholdsWhenNonNegativeThresholdsAreProvided) { - FrameInstrumentationData data; - data.SetStdDev(1.0); - data.SetLumaErrorThreshold(8); - data.SetChromaErrorThreshold(8); - data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); - VideoFrame frame = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .build(); - - MockCorruptionScoreObserver observer; - FrameInstrumentationEvaluation evaluator(&observer); - EXPECT_CALL(observer, OnCorruptionScore(AllOf(Ge(0.0), Le(1.0)), _)); - evaluator.OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); -} - -TEST(FrameInstrumentationEvaluationTest, - ApplyStdDevWhenNonNegativeStdDevIsProvided) { - FrameInstrumentationData data; - data.SetStdDev(0.6); - data.SetLumaErrorThreshold(8); - data.SetChromaErrorThreshold(8); - data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); - - VideoFrame frame = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .build(); - - MockCorruptionScoreObserver observer; - FrameInstrumentationEvaluation evaluator(&observer); - EXPECT_CALL(observer, OnCorruptionScore(AllOf(Ge(0.0), Le(1.0)), _)); - evaluator.OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); -} - -TEST(FrameInstrumentationEvaluationTest, ApplySequenceIndexWhenProvided) { - FrameInstrumentationData data; - data.SetSequenceIndex(1); - data.SetStdDev(0.6); - data.SetLumaErrorThreshold(8); - data.SetChromaErrorThreshold(8); - data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); - - VideoFrame frame = - VideoFrame::Builder() - .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) - .build(); - - MockCorruptionScoreObserver observer; - FrameInstrumentationEvaluation evaluator(&observer); - EXPECT_CALL(observer, OnCorruptionScore(AllOf(Ge(0.0), Le(1.0)), _)); - evaluator.OnInstrumentedFrame(data, frame, VideoContentType::UNSPECIFIED); -} - -} // namespace -} // namespace webrtc diff --git a/third_party/libwebrtc/video/receive_statistics_proxy.h b/third_party/libwebrtc/video/receive_statistics_proxy.h @@ -23,6 +23,7 @@ #include "api/task_queue/task_queue_base.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_evaluation.h" #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" @@ -41,7 +42,6 @@ #include "rtc_base/rate_tracker.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/thread_annotations.h" -#include "video/corruption_detection/frame_instrumentation_evaluation.h" #include "video/stats_counter.h" #include "video/video_quality_observer2.h" #include "video/video_stream_buffer_controller.h" diff --git a/third_party/libwebrtc/video/video_receive_stream2.cc b/third_party/libwebrtc/video/video_receive_stream2.cc @@ -43,6 +43,7 @@ #include "api/units/timestamp.h" #include "api/video/color_space.h" #include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/corruption_detection/frame_instrumentation_evaluation.h" #include "api/video/encoded_frame.h" #include "api/video/encoded_image.h" #include "api/video/recordable_encoded_frame.h" @@ -85,7 +86,6 @@ #include "system_wrappers/include/clock.h" #include "system_wrappers/include/ntp_time.h" #include "video/call_stats2.h" -#include "video/corruption_detection/frame_instrumentation_evaluation.h" #include "video/decode_synchronizer.h" #include "video/frame_decode_scheduler.h" #include "video/frame_dumping_decoder.h" @@ -267,7 +267,7 @@ VideoReceiveStream2::VideoReceiveStream2( max_wait_for_frame_(DetermineMaxWaitForFrame( TimeDelta::Millis(config_.rtp.nack.rtp_history_ms), false)), - frame_evaluator_(&stats_proxy_), + frame_evaluator_(FrameInstrumentationEvaluation::Create(&stats_proxy_)), decode_queue_(env_.task_queue_factory().CreateTaskQueue( "DecodingQueue", TaskQueueFactory::Priority::HIGH)) { @@ -667,8 +667,8 @@ void VideoReceiveStream2::CalculateCorruptionScore( const FrameInstrumentationData& frame_instrumentation_data, VideoContentType content_type) { RTC_DCHECK_RUN_ON(&decode_sequence_checker_); - frame_evaluator_.OnInstrumentedFrame(frame_instrumentation_data, frame, - content_type); + frame_evaluator_->OnInstrumentedFrame(frame_instrumentation_data, frame, + content_type); } bool VideoReceiveStream2::SetBaseMinimumPlayoutDelayMs(int delay_ms) { diff --git a/third_party/libwebrtc/video/video_receive_stream2.h b/third_party/libwebrtc/video/video_receive_stream2.h @@ -33,6 +33,7 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "api/video/corruption_detection/frame_instrumentation_evaluation.h" #include "api/video/encoded_frame.h" #include "api/video/recordable_encoded_frame.h" #include "api/video/video_content_type.h" @@ -51,7 +52,6 @@ #include "rtc_base/system/no_unique_address.h" #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/ntp_time.h" -#include "video/corruption_detection/frame_instrumentation_evaluation.h" #include "video/decode_synchronizer.h" #include "video/receive_statistics_proxy.h" #include "video/rtp_streams_synchronizer2.h" @@ -362,7 +362,7 @@ class VideoReceiveStream2 std::vector<std::unique_ptr<EncodedFrame>> buffered_encoded_frames_ RTC_GUARDED_BY(decode_sequence_checker_); - FrameInstrumentationEvaluation frame_evaluator_ + std::unique_ptr<FrameInstrumentationEvaluation> frame_evaluator_ RTC_GUARDED_BY(decode_sequence_checker_); // Used to signal destruction to potentially pending tasks.