commit d9f1d2a993d706d2dd6f49009ef5d5a95c287877 parent 453b72faea86ab41fc4c8f3c9557de9a8987b53e Author: Dan Baker <dbaker@mozilla.com> Date: Fri, 24 Oct 2025 13:32:37 -0600 Bug 1995393 - Vendor libwebrtc from d79dd43c9b Upstream commit: https://webrtc.googlesource.com/src/+/d79dd43c9b16776ff21d2a96b2ad3526b6831f8c Refactor FrameInstrumentationData and expose it in api/ This CL introduces several changes: * Makes FrameInstrumentationData a class with accesors * Removes FrameInstrumentationSyncData as moves that functionalitt into FrameInstrumentationData, removing the need for absl::variant. * FrameInstrumentationData.sequence_index() is now intended to be the full unwrapped sequence index, and the "communicate upper bits" part is only done in CorruptionDetectionMessage. * Updated the converter functions: - FrameInstrumentationData::AsCorruptionDetectionMessage() - New class FrameInstrumentationDataReader to convert from message to instrumentatio data and handle sequence number unwrapping. * Exposes FrameInstrumentationData and FrameInstrumentationDataReader in the api/ folder for public access. Tested: Automatic tests plus manual test with this patched in chromium. Bug: webrtc:358039777 Change-Id: Ic5395d3548c677868166a6697877cce66aa2d210 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/404220 Reviewed-by: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Per Kjellander <perkj@webrtc.org> Commit-Queue: Per Kjellander <perkj@webrtc.org> Auto-Submit: Erik Språng <sprang@webrtc.org> Cr-Commit-Position: refs/heads/main@{#45395} Diffstat:
64 files changed, 1270 insertions(+), 1229 deletions(-)
diff --git a/third_party/libwebrtc/README.mozilla.last-vendor b/third_party/libwebrtc/README.mozilla.last-vendor @@ -1,4 +1,4 @@ # ./mach python dom/media/webrtc/third_party_build/vendor-libwebrtc.py --from-local /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc --commit mozpatches libwebrtc -libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-10-24T19:00:10.783793+00:00. +libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-10-24T19:32:14.488814+00:00. # base of lastest vendoring -ddfbd2782c +d79dd43c9b diff --git a/third_party/libwebrtc/api/BUILD.gn b/third_party/libwebrtc/api/BUILD.gn @@ -1688,6 +1688,8 @@ if (rtc_include_tests) { "video:rtp_video_frame_assembler_unittests", "video:video_frame", "video:video_frame_metadata_unittest", + "video/corruption_detection:frame_instrumentation_data_reader_unittest", + "video/corruption_detection:frame_instrumentation_data_unittest", "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/strings:string_view", diff --git a/third_party/libwebrtc/api/transport/rtp/BUILD.gn b/third_party/libwebrtc/api/transport/rtp/BUILD.gn @@ -34,11 +34,16 @@ rtc_library("dependency_descriptor") { ] } -rtc_source_set("corruption_detection_message") { +rtc_library("corruption_detection_message") { visibility = [ "*" ] - sources = [ "corruption_detection_message.h" ] + sources = [ + "corruption_detection_message.cc", + "corruption_detection_message.h", + ] deps = [ - "../../../api:array_view", + "../../:array_view", + "../../../rtc_base:checks", + "../../video/corruption_detection:frame_instrumentation_data", "//third_party/abseil-cpp/absl/container:inlined_vector", ] } @@ -50,6 +55,7 @@ if (rtc_include_tests && !build_with_chromium) { deps = [ ":corruption_detection_message", "../../../test:test_support", + "../../video/corruption_detection:frame_instrumentation_data", ] } } diff --git a/third_party/libwebrtc/api/transport/rtp/corruption_detection_message.cc b/third_party/libwebrtc/api/transport/rtp/corruption_detection_message.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2025 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/transport/rtp/corruption_detection_message.h" + +#include <optional> +#include <utility> + +#include "api/array_view.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { + +constexpr double kMaxStdDev = 40.0; +constexpr int kMaxErrorThreshold = 15; + +} // namespace + +CorruptionDetectionMessage +CorruptionDetectionMessage::FromFrameInstrumentationData( + const FrameInstrumentationData& frame_instrumentation) { + int transmitted_sequence_index = + frame_instrumentation.holds_upper_bits() + ? frame_instrumentation.sequence_index() >> 7 + : (frame_instrumentation.sequence_index() & 0b0111'1111); + Builder builder; + builder.WithSequenceIndex(transmitted_sequence_index) + .WithInterpretSequenceIndexAsMostSignificantBits( + frame_instrumentation.holds_upper_bits()); + if (!frame_instrumentation.is_sync_only()) { + builder.WithStdDev(frame_instrumentation.std_dev()) + .WithLumaErrorThreshold(frame_instrumentation.luma_error_threshold()) + .WithChromaErrorThreshold( + frame_instrumentation.chroma_error_threshold()) + .WithSampleValues(frame_instrumentation.sample_values()); + } + + // The setter values of `FrameInstrumentationData` has already validated the + // values, so this should never return nullopt; + std::optional<CorruptionDetectionMessage> message = builder.Build(); + RTC_DCHECK(message.has_value()); + + return std::move(*message); +} + +std::optional<CorruptionDetectionMessage> +CorruptionDetectionMessage::Builder::Build() { + if (message_.sequence_index_ < 0 || message_.sequence_index_ > 0b0111'1111) { + return std::nullopt; + } + if (message_.std_dev_ < 0.0 || message_.std_dev_ > kMaxStdDev) { + return std::nullopt; + } + if (message_.luma_error_threshold_ < 0 || + message_.luma_error_threshold_ > kMaxErrorThreshold) { + return std::nullopt; + } + if (message_.chroma_error_threshold_ < 0 || + message_.chroma_error_threshold_ > kMaxErrorThreshold) { + return std::nullopt; + } + if (message_.sample_values_.size() > kMaxSampleSize) { + return std::nullopt; + } + for (double sample_value : message_.sample_values_) { + if (sample_value < 0.0 || sample_value > 255.0) { + return std::nullopt; + } + } + return message_; +} + +CorruptionDetectionMessage::Builder& +CorruptionDetectionMessage::Builder::WithSequenceIndex(int sequence_index) { + message_.sequence_index_ = sequence_index; + return *this; +} + +CorruptionDetectionMessage::Builder& CorruptionDetectionMessage::Builder:: + WithInterpretSequenceIndexAsMostSignificantBits( + bool interpret_sequence_index_as_most_significant_bits) { + message_.interpret_sequence_index_as_most_significant_bits_ = + interpret_sequence_index_as_most_significant_bits; + return *this; +} + +CorruptionDetectionMessage::Builder& +CorruptionDetectionMessage::Builder::WithStdDev(double std_dev) { + message_.std_dev_ = std_dev; + return *this; +} + +CorruptionDetectionMessage::Builder& +CorruptionDetectionMessage::Builder::WithLumaErrorThreshold( + int luma_error_threshold) { + message_.luma_error_threshold_ = luma_error_threshold; + return *this; +} + +CorruptionDetectionMessage::Builder& +CorruptionDetectionMessage::Builder::WithChromaErrorThreshold( + int chroma_error_threshold) { + message_.chroma_error_threshold_ = chroma_error_threshold; + return *this; +} + +CorruptionDetectionMessage::Builder& +CorruptionDetectionMessage::Builder::WithSampleValues( + const ArrayView<const double>& sample_values) { + message_.sample_values_.assign(sample_values.cbegin(), sample_values.cend()); + return *this; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/transport/rtp/corruption_detection_message.h b/third_party/libwebrtc/api/transport/rtp/corruption_detection_message.h @@ -16,6 +16,7 @@ #include "absl/container/inlined_vector.h" #include "api/array_view.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" namespace webrtc { @@ -42,6 +43,9 @@ class CorruptionDetectionMessage { return MakeArrayView(sample_values_.data(), sample_values_.size()); } + static CorruptionDetectionMessage FromFrameInstrumentationData( + const FrameInstrumentationData& frame_instrumentation); + private: friend class CorruptionDetectionExtension; @@ -78,71 +82,20 @@ class CorruptionDetectionMessage { class CorruptionDetectionMessage::Builder { public: Builder() = default; - Builder(const Builder&) = default; Builder& operator=(const Builder&) = default; ~Builder() = default; - std::optional<CorruptionDetectionMessage> Build() { - if (message_.sequence_index_ < 0 || - message_.sequence_index_ > 0b0111'1111) { - return std::nullopt; - } - if (message_.std_dev_ < 0.0 || message_.std_dev_ > 40.0) { - return std::nullopt; - } - if (message_.luma_error_threshold_ < 0 || - message_.luma_error_threshold_ > 15) { - return std::nullopt; - } - if (message_.chroma_error_threshold_ < 0 || - message_.chroma_error_threshold_ > 15) { - return std::nullopt; - } - if (message_.sample_values_.size() > kMaxSampleSize) { - return std::nullopt; - } - for (double sample_value : message_.sample_values_) { - if (sample_value < 0.0 || sample_value > 255.0) { - return std::nullopt; - } - } - return message_; - } - - Builder& WithSequenceIndex(int sequence_index) { - message_.sequence_index_ = sequence_index; - return *this; - } + std::optional<CorruptionDetectionMessage> Build(); + Builder& WithSequenceIndex(int sequence_index); Builder& WithInterpretSequenceIndexAsMostSignificantBits( - bool interpret_sequence_index_as_most_significant_bits) { - message_.interpret_sequence_index_as_most_significant_bits_ = - interpret_sequence_index_as_most_significant_bits; - return *this; - } - - Builder& WithStdDev(double std_dev) { - message_.std_dev_ = std_dev; - return *this; - } - - Builder& WithLumaErrorThreshold(int luma_error_threshold) { - message_.luma_error_threshold_ = luma_error_threshold; - return *this; - } - - Builder& WithChromaErrorThreshold(int chroma_error_threshold) { - message_.chroma_error_threshold_ = chroma_error_threshold; - return *this; - } - - Builder& WithSampleValues(const ArrayView<const double>& sample_values) { - message_.sample_values_.assign(sample_values.cbegin(), - sample_values.cend()); - return *this; - } + bool interpret_sequence_index_as_most_significant_bits); + Builder& WithStdDev(double std_dev); + Builder& WithLumaErrorThreshold(int luma_error_threshold); + Builder& WithChromaErrorThreshold(int chroma_error_threshold); + Builder& WithSampleValues(const ArrayView<const double>& sample_values); private: CorruptionDetectionMessage message_; diff --git a/third_party/libwebrtc/api/transport/rtp/corruption_detection_message_unittest.cc b/third_party/libwebrtc/api/transport/rtp/corruption_detection_message_unittest.cc @@ -13,8 +13,13 @@ #include <optional> #include <vector> +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "test/gmock.h" #include "test/gtest.h" +using ::testing::ElementsAre; +using ::testing::IsEmpty; + namespace webrtc { namespace { @@ -120,5 +125,120 @@ TEST(CorruptionDetectionMessageTest, CreatesWhenValidParametersAreSpecified) { std::nullopt); } +TEST(CorruptionDetectionMessageFromFrameInstrumentationDataTest, + ConvertsValidData) { + FrameInstrumentationData data; + EXPECT_TRUE(data.SetSequenceIndex(1)); + EXPECT_TRUE(data.SetStdDev(1)); + EXPECT_TRUE(data.SetLumaErrorThreshold(5)); + EXPECT_TRUE(data.SetChromaErrorThreshold(5)); + EXPECT_TRUE(data.SetSampleValues({1.0, 2.0, 3.0, 4.0, 5.0})); + + CorruptionDetectionMessage message = + CorruptionDetectionMessage::FromFrameInstrumentationData(data); + EXPECT_EQ(message.sequence_index(), 1); + EXPECT_FALSE(message.interpret_sequence_index_as_most_significant_bits()); + EXPECT_EQ(message.std_dev(), 1.0); + EXPECT_EQ(message.luma_error_threshold(), 5); + EXPECT_EQ(message.chroma_error_threshold(), 5); + EXPECT_THAT(message.sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); +} + +TEST(CorruptionDetectionMessageFromFrameInstrumentationDataTest, + ConvertsSequenceIndexWhenSetToUseUpperBits) { + FrameInstrumentationData data; + EXPECT_TRUE(data.SetSequenceIndex(117 << 7)); + EXPECT_TRUE(data.SetStdDev(1)); + EXPECT_TRUE(data.SetLumaErrorThreshold(5)); + EXPECT_TRUE(data.SetChromaErrorThreshold(5)); + EXPECT_TRUE(data.SetSampleValues({1.0, 2.0, 3.0, 4.0, 5.0})); + + CorruptionDetectionMessage message = + CorruptionDetectionMessage::FromFrameInstrumentationData(data); + EXPECT_EQ(message.sequence_index(), 117); + EXPECT_TRUE(message.interpret_sequence_index_as_most_significant_bits()); + EXPECT_EQ(message.std_dev(), 1.0); + EXPECT_EQ(message.luma_error_threshold(), 5); + EXPECT_EQ(message.chroma_error_threshold(), 5); + EXPECT_THAT(message.sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); +} + +TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, + ConvertsSequenceIndexWhenSetToUseLowerBits) { + FrameInstrumentationData data; + EXPECT_TRUE(data.SetSequenceIndex((117 << 7) + 118)); + EXPECT_TRUE(data.SetStdDev(1)); + EXPECT_TRUE(data.SetLumaErrorThreshold(5)); + EXPECT_TRUE(data.SetChromaErrorThreshold(5)); + EXPECT_TRUE(data.SetSampleValues({1.0, 2.0, 3.0, 4.0, 5.0})); + + CorruptionDetectionMessage message = + CorruptionDetectionMessage::FromFrameInstrumentationData(data); + EXPECT_EQ(message.sequence_index(), 118); + EXPECT_FALSE(message.interpret_sequence_index_as_most_significant_bits()); + EXPECT_EQ(message.std_dev(), 1.0); + EXPECT_EQ(message.luma_error_threshold(), 5); + EXPECT_EQ(message.chroma_error_threshold(), 5); + EXPECT_THAT(message.sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); +} + +TEST(CorruptionDetectionMessageFromFrameInstrumentationDataTest, + SendsLowerSequenceIndexWhenDroppable) { + FrameInstrumentationData data; + EXPECT_TRUE(data.SetSequenceIndex(117 << 7)); + data.set_droppable(true); + EXPECT_TRUE(data.SetStdDev(1)); + EXPECT_TRUE(data.SetLumaErrorThreshold(5)); + EXPECT_TRUE(data.SetChromaErrorThreshold(5)); + EXPECT_TRUE(data.SetSampleValues({1.0, 2.0, 3.0, 4.0, 5.0})); + + CorruptionDetectionMessage message = + CorruptionDetectionMessage::FromFrameInstrumentationData(data); + EXPECT_EQ(message.sequence_index(), 0); + EXPECT_FALSE(message.interpret_sequence_index_as_most_significant_bits()); + EXPECT_EQ(message.std_dev(), 1.0); + EXPECT_EQ(message.luma_error_threshold(), 5); + EXPECT_EQ(message.chroma_error_threshold(), 5); + EXPECT_THAT(message.sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); +} + +TEST(CorruptionDetectionMessageFromFrameInstrumentationDataTest, + CanCreateLsbSyncMessage) { + FrameInstrumentationData data; + EXPECT_TRUE(data.SetSequenceIndex((117 << 7) + 118)); + EXPECT_TRUE(data.SetStdDev(1)); + EXPECT_TRUE(data.SetLumaErrorThreshold(5)); + EXPECT_TRUE(data.SetChromaErrorThreshold(5)); + EXPECT_TRUE(data.is_sync_only()); + + CorruptionDetectionMessage message = + CorruptionDetectionMessage::FromFrameInstrumentationData(data); + EXPECT_EQ(message.sequence_index(), 118); + EXPECT_FALSE(message.interpret_sequence_index_as_most_significant_bits()); + EXPECT_EQ(message.std_dev(), 0.0); + EXPECT_EQ(message.luma_error_threshold(), 0); + EXPECT_EQ(message.chroma_error_threshold(), 0); + EXPECT_THAT(message.sample_values(), IsEmpty()); +} + +TEST(CorruptionDetectionMessageFromFrameInstrumentationDataTest, + CanCreateMsbSyncMessage) { + FrameInstrumentationData data; + EXPECT_TRUE(data.SetSequenceIndex(117 << 7)); + EXPECT_TRUE(data.SetStdDev(1)); + EXPECT_TRUE(data.SetLumaErrorThreshold(5)); + EXPECT_TRUE(data.SetChromaErrorThreshold(5)); + EXPECT_TRUE(data.is_sync_only()); + + CorruptionDetectionMessage message = + CorruptionDetectionMessage::FromFrameInstrumentationData(data); + EXPECT_EQ(message.sequence_index(), 117); + EXPECT_TRUE(message.interpret_sequence_index_as_most_significant_bits()); + EXPECT_EQ(message.std_dev(), 0.0); + EXPECT_EQ(message.luma_error_threshold(), 0); + EXPECT_EQ(message.chroma_error_threshold(), 0); + EXPECT_THAT(message.sample_values(), IsEmpty()); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/api/video/BUILD.gn b/third_party/libwebrtc/api/video/BUILD.gn @@ -186,11 +186,11 @@ rtc_library("encoded_frame") { deps = [ ":encoded_image", ":video_frame", - "../../common_video:frame_instrumentation_data", "../../modules/rtp_rtcp:rtp_video_header", "../../modules/video_coding:codec_globals_headers", "../../modules/video_coding:video_codec_interface", "../units:timestamp", + "corruption_detection:frame_instrumentation_data", ] } diff --git a/third_party/libwebrtc/api/video/corruption_detection/BUILD.gn b/third_party/libwebrtc/api/video/corruption_detection/BUILD.gn @@ -12,3 +12,54 @@ rtc_source_set("filter_settings") { visibility = [ "*" ] sources = [ "corruption_detection_filter_settings.h" ] } + +rtc_library("frame_instrumentation_data") { + visibility = [ "*" ] + sources = [ + "frame_instrumentation_data.cc", + "frame_instrumentation_data.h", + ] + + deps = [ + "../../:array_view", + "../../../rtc_base:checks", + ] +} + +rtc_library("frame_instrumentation_data_reader") { + visibility = [ "*" ] + sources = [ + "frame_instrumentation_data_reader.cc", + "frame_instrumentation_data_reader.h", + ] + + deps = [ + ":frame_instrumentation_data", + "../../:array_view", + "../../../rtc_base:checks", + "../../../rtc_base:logging", + "../../transport/rtp:corruption_detection_message", + ] +} + +rtc_library("frame_instrumentation_data_unittest") { + testonly = true + sources = [ "frame_instrumentation_data_unittest.cc" ] + + deps = [ + ":frame_instrumentation_data", + "../../../test:test_support", + ] +} + +rtc_library("frame_instrumentation_data_reader_unittest") { + testonly = true + sources = [ "frame_instrumentation_data_reader_unittest.cc" ] + + deps = [ + ":frame_instrumentation_data", + ":frame_instrumentation_data_reader", + "../../../test:test_support", + "../../transport/rtp:corruption_detection_message", + ] +} diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data.cc @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2025 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_data.h" + +#include <utility> +#include <vector> + +#include "api/array_view.h" + +namespace webrtc { + +constexpr int kMaxSequenceIndex = (1 << 14) - 1; +constexpr double kMaxStdDev = 40.0; +constexpr int kMaxErrorThreshold = 15; + +FrameInstrumentationData::FrameInstrumentationData() + : sequence_index_(0), + droppable_(false), + std_dev_(0), + luma_error_threshold_(0), + chroma_error_threshold_(0) {} + +bool FrameInstrumentationData::SetSequenceIndex(int index) { + if (index < 0 || index > kMaxSequenceIndex) + return false; + + sequence_index_ = index; + return true; +} + +bool FrameInstrumentationData::SetStdDev(double std_dev) { + if (std_dev < 0.0 || std_dev > kMaxStdDev) + return false; + + std_dev_ = std_dev; + return true; +} + +bool FrameInstrumentationData::SetLumaErrorThreshold(int threshold) { + if (threshold < 0 || threshold > kMaxErrorThreshold) + return false; + + luma_error_threshold_ = threshold; + return true; +} + +bool FrameInstrumentationData::SetChromaErrorThreshold(int threshold) { + if (threshold < 0 || threshold > kMaxErrorThreshold) + return false; + + chroma_error_threshold_ = threshold; + return true; +} + +bool FrameInstrumentationData::SetSampleValues( + webrtc::ArrayView<const double> samples) { + for (double sample_value : samples) { + if (sample_value < 0.0 || sample_value > 255.0) { + return false; + } + } + sample_values_.assign(samples.begin(), samples.end()); + return true; +} + +bool FrameInstrumentationData::SetSampleValues(std::vector<double>&& samples) { + for (double sample_value : samples) { + if (sample_value < 0.0 || sample_value > 255.0) { + return false; + } + } + sample_values_ = std::move(samples); + return true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data.h b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data.h @@ -0,0 +1,56 @@ +/* + * Copyright 2025 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_DATA_H_ +#define API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_DATA_H_ + +#include <vector> + +#include "api/array_view.h" + +namespace webrtc { + +class FrameInstrumentationData { + public: + FrameInstrumentationData(); + + int sequence_index() const { return sequence_index_; } + bool is_droppable() const { return droppable_; } + double std_dev() const { return std_dev_; } + int luma_error_threshold() const { return luma_error_threshold_; } + int chroma_error_threshold() const { return chroma_error_threshold_; } + ArrayView<const double> sample_values() const { return sample_values_; } + + bool SetSequenceIndex(int index); + void set_droppable(bool droppable) { droppable_ = droppable; } + bool SetStdDev(double std_dev); + bool SetLumaErrorThreshold(int threshold); + bool SetChromaErrorThreshold(int threshold); + bool SetSampleValues(ArrayView<const double> samples); + bool SetSampleValues(std::vector<double>&& samples); + + // Convenience methods.. + bool holds_upper_bits() const { + return !droppable_ && (sequence_index_ & 0b0111'1111) == 0; + } + bool is_sync_only() const { return sample_values_.empty(); } + + private: + int sequence_index_; + bool droppable_; + double std_dev_; + int luma_error_threshold_; + int chroma_error_threshold_; + std::vector<double> sample_values_; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_DATA_H_ diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_reader.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_reader.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2025 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_data_reader.h" + +#include <optional> + +#include "api/array_view.h" +#include "api/transport/rtp/corruption_detection_message.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +std::optional<FrameInstrumentationData> +FrameInstrumentationDataReader::ParseMessage( + const CorruptionDetectionMessage& message) { + FrameInstrumentationData data; + if (message.interpret_sequence_index_as_most_significant_bits()) { + data.SetSequenceIndex(message.sequence_index() << 7); + } else { + if (!last_seen_sequence_index_.has_value()) { + RTC_LOG(LS_WARNING) + << "Got Corruption Detection Message with relative sequence index " + "where no earlier sequence index is know. Ignoring."; + return std::nullopt; + } + + int upper_bits = *last_seen_sequence_index_ & 0b0011'1111'1000'0000; + if (message.sequence_index() < (*last_seen_sequence_index_ & 0b0111'1111)) { + // Assume one and only one wraparound has happened. + upper_bits += 0b1000'0000; + } + // Replace the lowest bits with the bits from the update. + data.SetSequenceIndex(upper_bits + message.sequence_index()); + } + + // The sequence index field of the message refers to the halton sequence index + // for the first sample in the message. In order to figure out the next + // expected sequence index we must increment it by the number of samples. + ArrayView<const double> sample_values = message.sample_values(); + last_seen_sequence_index_ = data.sequence_index() + sample_values.size(); + + if (!sample_values.empty()) { + data.SetStdDev(message.std_dev()); + data.SetLumaErrorThreshold(message.luma_error_threshold()); + data.SetChromaErrorThreshold(message.chroma_error_threshold()); + data.SetSampleValues(sample_values); + } + + return data; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_reader.h b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_reader.h @@ -0,0 +1,34 @@ +/* + * Copyright 2025 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_DATA_READER_H_ +#define API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_DATA_READER_H_ + +#include <optional> + +#include "api/transport/rtp/corruption_detection_message.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" + +namespace webrtc { + +class FrameInstrumentationDataReader { + public: + FrameInstrumentationDataReader() = default; + + std::optional<FrameInstrumentationData> ParseMessage( + const CorruptionDetectionMessage& message); + + private: + std::optional<int> last_seen_sequence_index_; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_DATA_READER_H_ diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_reader_unittest.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_reader_unittest.cc @@ -0,0 +1,263 @@ +/* + * Copyright 2025 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_data_reader.h" + +#include <optional> +#include <vector> + +#include "api/transport/rtp/corruption_detection_message.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::ElementsAre; +using ::testing::Eq; + +namespace webrtc { + +TEST(FrameInstrumentationDataReaderTest, AcceptsMsbFromStart) { + FrameInstrumentationDataReader reader; + std::optional<FrameInstrumentationData> data = reader.ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(1) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()); + + EXPECT_TRUE(data.has_value()); + EXPECT_THAT(data->sequence_index(), Eq(1 << 7)); +} + +TEST(FrameInstrumentationDataReaderTest, RejectsLsbFromStart) { + FrameInstrumentationDataReader reader; + std::optional<FrameInstrumentationData> data = reader.ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(1) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()); + + EXPECT_FALSE(data.has_value()); +} + +TEST(FrameInstrumentationDataReaderTest, + IgnorePreviousSequenceIndexWhenSetToUpdateTheMostSignificantBits) { + FrameInstrumentationDataReader reader; + + // Prime with sequence index 11 << 7. + EXPECT_TRUE( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(11) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()) + .has_value()); + + // New absolute value 12 << 7 take precedence. + std::optional<FrameInstrumentationData> data = reader.ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(12) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()); + + ASSERT_TRUE(data.has_value()); + EXPECT_THAT(data->sequence_index(), Eq(12 << 7)); +} + +TEST(FrameInstrumentationDataReaderTest, + UseMessageSequenceIndexWhenHigherThanPrevious) { + FrameInstrumentationDataReader reader; + + // Prime with sequence index 11 << 7. + EXPECT_TRUE( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(11) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()) + .has_value()); + + std::optional<FrameInstrumentationData> data = reader.ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(12) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()); + + ASSERT_TRUE(data.has_value()); + EXPECT_EQ(data->sequence_index(), (11 << 7) + 12); +} + +TEST(FrameInstrumentationDataReaderTest, HandlesMsbRollOver) { + FrameInstrumentationDataReader reader; + + // Prime with sequence index 11 << 7. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(11) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()) + ->sequence_index(), + Eq(11 << 7)); + + // Bump index by 100. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(100) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()) + ->sequence_index(), + Eq((11 << 7) + 100)); + + // Bumping it again so that LSB = 1, MSB should increment to 12. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(1) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()) + ->sequence_index(), + Eq((12 << 7) + 1)); +} + +TEST(FrameInstrumentationDataReaderTest, + IgnoreIndexUpdateWhenTheLowerBitsSuppliedAreTheSameAsInThePreviousIndex) { + FrameInstrumentationDataReader reader; + + // Prime with sequence index 11 << 7. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(11) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()) + ->sequence_index(), + Eq(11 << 7)); + + // LSB = 0, meaning it's the same sequence again - no increment. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(0) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()) + ->sequence_index(), + Eq(11 << 7)); +} + +TEST(FrameInstrumentationDataReaderTest, MaximumRollover) { + FrameInstrumentationDataReader reader; + + // Prime with sequence index 11 << 7. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(11) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()) + ->sequence_index(), + Eq(11 << 7)); + + // Bump index by 1. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(1) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()) + ->sequence_index(), + Eq((11 << 7) + 1)); + + // Setting the LSB to one lower than current => maximum +127 jump. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(0) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()) + ->sequence_index(), + Eq(12 << 7)); +} + +TEST(FrameInstrumentationDataReaderTest, RolloverWithSamples) { + FrameInstrumentationDataReader reader; + + // Prime with sequence index 11 << 7. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(11) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .Build()) + ->sequence_index(), + Eq(11 << 7)); + + // Bump index by one, but include 4 samples values, which count as index too. + std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0}; + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(1) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .WithSampleValues(sample_values) + .Build()) + ->sequence_index(), + Eq((11 << 7) + 1)); + + // Set the LSB to 4, which is one less than the previous seen sequence, this + // counts as a wraparound. + EXPECT_THAT( + reader + .ParseMessage( + *CorruptionDetectionMessage::Builder() + .WithSequenceIndex(4) + .WithInterpretSequenceIndexAsMostSignificantBits(false) + .Build()) + ->sequence_index(), + Eq((12 << 7) + 4)); +} + +TEST(FrameInstrumentationDataReaderTest, ConvertAllFields) { + std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; + std::optional<CorruptionDetectionMessage> message = + CorruptionDetectionMessage::Builder() + .WithSequenceIndex(11) + .WithInterpretSequenceIndexAsMostSignificantBits(true) + .WithStdDev(1.2) + .WithLumaErrorThreshold(10) + .WithChromaErrorThreshold(10) + .WithSampleValues(sample_values) + .Build(); + ASSERT_TRUE(message.has_value()); + + FrameInstrumentationDataReader reader; + std::optional<FrameInstrumentationData> data = reader.ParseMessage(*message); + + ASSERT_TRUE(data.has_value()); + EXPECT_EQ(data->sequence_index(), 11 << 7); + EXPECT_NEAR(data->std_dev(), 1.2, 0.024); // ~2% + EXPECT_EQ(data->luma_error_threshold(), 10); + EXPECT_EQ(data->chroma_error_threshold(), 10); + EXPECT_THAT(data->sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_unittest.cc b/third_party/libwebrtc/api/video/corruption_detection/frame_instrumentation_data_unittest.cc @@ -0,0 +1,119 @@ +/* + * Copyright 2025 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/corruption_detection/frame_instrumentation_data.h" + +#include <cmath> +#include <limits> +#include <utility> +#include <vector> + +#include "test/gtest.h" + +namespace webrtc { + +TEST(FrameInstrumentationDataTest, RespectsSequenceIndexRange) { + FrameInstrumentationData data; + EXPECT_FALSE(data.SetSequenceIndex(-1)); + EXPECT_TRUE(data.SetSequenceIndex(0)); + EXPECT_TRUE(data.SetSequenceIndex((1 << 14) - 1)); + EXPECT_FALSE(data.SetSequenceIndex(1 << 14)); +} + +TEST(FrameInstrumentationDataTest, RespectsStdDevRange) { + FrameInstrumentationData data; + EXPECT_FALSE(data.SetStdDev( + std::nextafter(0.0, std::numeric_limits<double>::lowest()))); + EXPECT_TRUE(data.SetStdDev(0.0)); + EXPECT_TRUE(data.SetStdDev(40.0)); + EXPECT_FALSE( + data.SetStdDev(std::nextafter(40.0, std::numeric_limits<double>::max()))); +} + +TEST(FrameInstrumentationDataTest, RespectsLumaRange) { + FrameInstrumentationData data; + EXPECT_FALSE(data.SetLumaErrorThreshold(-1)); + EXPECT_TRUE(data.SetLumaErrorThreshold(0)); + EXPECT_TRUE(data.SetLumaErrorThreshold(15)); + EXPECT_FALSE(data.SetLumaErrorThreshold(16)); +} + +TEST(FrameInstrumentationDataTest, RespectsChromaRange) { + FrameInstrumentationData data; + EXPECT_FALSE(data.SetChromaErrorThreshold(-1)); + EXPECT_TRUE(data.SetChromaErrorThreshold(0)); + EXPECT_TRUE(data.SetChromaErrorThreshold(15)); + EXPECT_FALSE(data.SetChromaErrorThreshold(16)); +} + +TEST(FrameInstrumentationDataTest, RejectsLowSampleValues) { + FrameInstrumentationData data; + + const double kLowValue[] = { + std::nextafter(0.0, std::numeric_limits<double>::lowest())}; + EXPECT_FALSE(data.SetSampleValues(kLowValue)); + + std::vector vec = {kLowValue[0]}; + EXPECT_FALSE(data.SetSampleValues(std::move(vec))); +} + +TEST(FrameInstrumentationDataTest, AcceptsValidSampleValues) { + FrameInstrumentationData data; + + const double kValues[] = {0.0, 255.0}; + EXPECT_TRUE(data.SetSampleValues(kValues)); + + std::vector vec(std::begin(kValues), std::end(kValues)); + EXPECT_TRUE(data.SetSampleValues(std::move(vec))); +} + +TEST(FrameInstrumentationDataTest, RejectsHighSampleValues) { + FrameInstrumentationData data; + + const double kHighValue[] = { + std::nextafter(255.0, std::numeric_limits<double>::max())}; + EXPECT_FALSE(data.SetSampleValues(kHighValue)); + + std::vector vec = {kHighValue[0]}; + EXPECT_FALSE(data.SetSampleValues(std::move(vec))); +} + +TEST(FrameInstrumentationDataTest, ReportsUpperBits) { + FrameInstrumentationData data; + + EXPECT_TRUE(data.SetSequenceIndex(0b0111'1111)); + EXPECT_FALSE(data.holds_upper_bits()); + + EXPECT_TRUE(data.SetSequenceIndex(0b1111'1111)); + EXPECT_FALSE(data.holds_upper_bits()); + + EXPECT_TRUE(data.SetSequenceIndex(0b1000'0000)); + EXPECT_TRUE(data.holds_upper_bits()); +} + +TEST(FrameInstrumentationDataTest, NoUpperBitsWhenDroppable) { + FrameInstrumentationData data; + + EXPECT_TRUE(data.SetSequenceIndex(0b1000'0000)); + EXPECT_TRUE(data.holds_upper_bits()); + + data.set_droppable(true); + EXPECT_FALSE(data.holds_upper_bits()); +} + +TEST(FrameInstrumentationDataTest, ReportsSyncOnly) { + FrameInstrumentationData data; + + EXPECT_TRUE(data.is_sync_only()); + + EXPECT_TRUE(data.SetSampleValues({0.0})); + EXPECT_FALSE(data.is_sync_only()); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video/encoded_frame.h b/third_party/libwebrtc/api/video/encoded_frame.h @@ -15,12 +15,11 @@ #include <stdint.h> #include <optional> -#include <variant> #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/video_codec_type.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -77,9 +76,7 @@ class EncodedFrame : public EncodedImage { _codecSpecificInfo = *codec_specific; } void SetFrameInstrumentationData( - const std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - frame_instrumentation) { + const std::optional<FrameInstrumentationData> frame_instrumentation) { _codecSpecificInfo.frame_instrumentation_data = frame_instrumentation; } diff --git a/third_party/libwebrtc/common_video/BUILD.gn b/third_party/libwebrtc/common_video/BUILD.gn @@ -8,28 +8,12 @@ import("../webrtc.gni") -rtc_library("corruption_detection_converters") { - sources = [ - "corruption_detection_converters.cc", - "corruption_detection_converters.h", - ] - deps = [ - ":frame_instrumentation_data", - "../api/transport/rtp:corruption_detection_message", - "../rtc_base:checks", - ] -} - -rtc_library("frame_instrumentation_data") { - sources = [ "frame_instrumentation_data.h" ] -} - rtc_library("corruption_score_calculator") { sources = [ "include/corruption_score_calculator.h" ] deps = [ - ":frame_instrumentation_data", "../api/video:video_frame", "../api/video:video_rtp_headers", + "../api/video/corruption_detection:frame_instrumentation_data", ] } @@ -138,18 +122,6 @@ if (rtc_include_tests && !build_with_chromium) { } } - rtc_library("corruption_detection_converters_unittest") { - testonly = true - sources = [ "corruption_detection_converters_unittest.cc" ] - deps = [ - ":corruption_detection_converters", - ":frame_instrumentation_data", - "../api/transport/rtp:corruption_detection_message", - "../rtc_base:checks", - "../test:test_support", - ] - } - rtc_test("common_video_unittests") { testonly = true @@ -177,7 +149,6 @@ if (rtc_include_tests && !build_with_chromium) { deps = [ ":common_video", - ":corruption_detection_converters_unittest", "../api:array_view", "../api:scoped_refptr", "../api/units:time_delta", diff --git a/third_party/libwebrtc/common_video/corruption_detection_converters.cc b/third_party/libwebrtc/common_video/corruption_detection_converters.cc @@ -1,139 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "common_video/corruption_detection_converters.h" - -#include <optional> -#include <vector> - -#include "api/transport/rtp/corruption_detection_message.h" -#include "common_video/frame_instrumentation_data.h" -#include "rtc_base/checks.h" - -namespace webrtc { - -namespace { - -int GetFullSequenceIndex(int previous_sequence_index, - int sequence_index_update, - bool update_the_most_significant_bits) { - RTC_CHECK_GE(previous_sequence_index, 0) - << "previous_sequence_index must not be negative"; - RTC_CHECK_LE(previous_sequence_index, 0x7FFF) - << "previous_sequence_index must be at most 15 bits"; - RTC_CHECK_GE(sequence_index_update, 0) - << "sequence_index_update must not be negative"; - RTC_CHECK_LE(sequence_index_update, 0b0111'1111) - << "sequence_index_update must be at most 7 bits"; - if (update_the_most_significant_bits) { - // Reset LSB. - return sequence_index_update << 7; - } - int upper_bits = previous_sequence_index & 0b0011'1111'1000'0000; - if (sequence_index_update < (previous_sequence_index & 0b0111'1111)) { - // Assume one and only one wraparound has happened. - upper_bits += 0b1000'0000; - } - // Replace the lowest bits with the bits from the update. - return upper_bits + sequence_index_update; -} - -int GetSequenceIndexForMessage(int sequence_index, - bool communicate_upper_bits) { - return communicate_upper_bits ? (sequence_index >> 7) - : (sequence_index & 0b0111'1111); -} - -} // namespace - -std::optional<FrameInstrumentationData> -ConvertCorruptionDetectionMessageToFrameInstrumentationData( - const CorruptionDetectionMessage& message, - int previous_sequence_index) { - if (previous_sequence_index < 0) { - return std::nullopt; - } - if (message.sample_values().empty()) { - return std::nullopt; - } - int full_sequence_index = GetFullSequenceIndex( - previous_sequence_index, message.sequence_index(), - message.interpret_sequence_index_as_most_significant_bits()); - std::vector<double> sample_values(message.sample_values().cbegin(), - message.sample_values().cend()); - return FrameInstrumentationData{ - .sequence_index = full_sequence_index, - .communicate_upper_bits = - message.interpret_sequence_index_as_most_significant_bits(), - .std_dev = message.std_dev(), - .luma_error_threshold = message.luma_error_threshold(), - .chroma_error_threshold = message.chroma_error_threshold(), - .sample_values = sample_values}; -} - -std::optional<FrameInstrumentationSyncData> -ConvertCorruptionDetectionMessageToFrameInstrumentationSyncData( - const CorruptionDetectionMessage& message, - int previous_sequence_index) { - if (previous_sequence_index < 0) { - return std::nullopt; - } - if (!message.sample_values().empty()) { - return std::nullopt; - } - if (!message.interpret_sequence_index_as_most_significant_bits()) { - return std::nullopt; - } - return FrameInstrumentationSyncData{ - .sequence_index = GetFullSequenceIndex( - previous_sequence_index, message.sequence_index(), - /*update_the_most_significant_bits=*/true), - .communicate_upper_bits = true}; -} - -std::optional<CorruptionDetectionMessage> -ConvertFrameInstrumentationDataToCorruptionDetectionMessage( - const FrameInstrumentationData& data) { - if (data.sequence_index < 0 || data.sequence_index > 0b0011'1111'1111'1111) { - return std::nullopt; - } - // Frame instrumentation data must have sample values. - if (data.sample_values.empty()) { - return std::nullopt; - } - return CorruptionDetectionMessage::Builder() - .WithSequenceIndex(GetSequenceIndexForMessage( - data.sequence_index, data.communicate_upper_bits)) - .WithInterpretSequenceIndexAsMostSignificantBits( - data.communicate_upper_bits) - .WithStdDev(data.std_dev) - .WithLumaErrorThreshold(data.luma_error_threshold) - .WithChromaErrorThreshold(data.chroma_error_threshold) - .WithSampleValues(data.sample_values) - .Build(); -} - -std::optional<CorruptionDetectionMessage> -ConvertFrameInstrumentationSyncDataToCorruptionDetectionMessage( - const FrameInstrumentationSyncData& data) { - RTC_DCHECK(data.communicate_upper_bits) - << "FrameInstrumentationSyncData data must always send the upper bits."; - - if (data.sequence_index < 0 || data.sequence_index > 0b0011'1111'1111'1111) { - return std::nullopt; - } - return CorruptionDetectionMessage::Builder() - .WithSequenceIndex(GetSequenceIndexForMessage( - data.sequence_index, data.communicate_upper_bits)) - .WithInterpretSequenceIndexAsMostSignificantBits(true) - .Build(); -} - -} // namespace webrtc diff --git a/third_party/libwebrtc/common_video/corruption_detection_converters.h b/third_party/libwebrtc/common_video/corruption_detection_converters.h @@ -1,37 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef COMMON_VIDEO_CORRUPTION_DETECTION_CONVERTERS_H_ -#define COMMON_VIDEO_CORRUPTION_DETECTION_CONVERTERS_H_ - -#include <optional> - -#include "api/transport/rtp/corruption_detection_message.h" -#include "common_video/frame_instrumentation_data.h" - -namespace webrtc { - -std::optional<FrameInstrumentationData> -ConvertCorruptionDetectionMessageToFrameInstrumentationData( - const CorruptionDetectionMessage& message, - int previous_sequence_index); -std::optional<FrameInstrumentationSyncData> -ConvertCorruptionDetectionMessageToFrameInstrumentationSyncData( - const CorruptionDetectionMessage& message, - int previous_sequence_index); -std::optional<CorruptionDetectionMessage> -ConvertFrameInstrumentationDataToCorruptionDetectionMessage( - const FrameInstrumentationData& frame_instrumentation_data); -std::optional<CorruptionDetectionMessage> -ConvertFrameInstrumentationSyncDataToCorruptionDetectionMessage( - const FrameInstrumentationSyncData& frame_instrumentation_sync_data); -} // namespace webrtc - -#endif // COMMON_VIDEO_CORRUPTION_DETECTION_CONVERTERS_H_ diff --git a/third_party/libwebrtc/common_video/corruption_detection_converters_unittest.cc b/third_party/libwebrtc/common_video/corruption_detection_converters_unittest.cc @@ -1,391 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "common_video/corruption_detection_converters.h" - -#include <optional> -#include <vector> - -#include "api/transport/rtp/corruption_detection_message.h" -#include "common_video/frame_instrumentation_data.h" -#include "rtc_base/checks.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace { - -using ::testing::_; -using ::testing::ElementsAre; - -TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, - ConvertsValidData) { - FrameInstrumentationData data = {.sequence_index = 1, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 5, - .chroma_error_threshold = 5, - .sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(data); - ASSERT_TRUE(message.has_value()); - EXPECT_EQ(message->sequence_index(), 1); - EXPECT_FALSE(message->interpret_sequence_index_as_most_significant_bits()); - EXPECT_EQ(message->std_dev(), 1.0); - EXPECT_EQ(message->luma_error_threshold(), 5); - EXPECT_EQ(message->chroma_error_threshold(), 5); - EXPECT_THAT(message->sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); -} - -TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, - ReturnsNulloptWhenSequenceIndexIsNegative) { - FrameInstrumentationData data = {.sequence_index = -1, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 5, - .chroma_error_threshold = 5, - .sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(data); - ASSERT_FALSE(message.has_value()); -} - -TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, - ReturnsNulloptWhenSequenceIndexIsTooLarge) { - // Sequence index must be at max 14 bits. - FrameInstrumentationData data = {.sequence_index = 0x4000, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 5, - .chroma_error_threshold = 5, - .sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(data); - ASSERT_FALSE(message.has_value()); -} - -TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, - ReturnsNulloptWhenThereAreNoSampleValues) { - // FrameInstrumentationData must by definition have at least one sample value. - FrameInstrumentationData data = {.sequence_index = 1, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 5, - .chroma_error_threshold = 5, - .sample_values = {}}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(data); - ASSERT_FALSE(message.has_value()); -} - -TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, - ReturnsNulloptWhenNotSpecifyingSampleValues) { - FrameInstrumentationData data = {.sequence_index = 1, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 5, - .chroma_error_threshold = 5}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(data); - ASSERT_FALSE(message.has_value()); -} - -TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, - ConvertsSequenceIndexWhenSetToUseUpperBits) { - FrameInstrumentationData data = {.sequence_index = 0b0000'0110'0000'0101, - .communicate_upper_bits = true, - .std_dev = 1.0, - .luma_error_threshold = 5, - .chroma_error_threshold = 5, - .sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(data); - ASSERT_TRUE(message.has_value()); - EXPECT_EQ(message->sequence_index(), 0b0000'1100); - EXPECT_TRUE(message->interpret_sequence_index_as_most_significant_bits()); - EXPECT_EQ(message->std_dev(), 1.0); - EXPECT_EQ(message->luma_error_threshold(), 5); - EXPECT_EQ(message->chroma_error_threshold(), 5); - EXPECT_THAT(message->sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); -} - -TEST(FrameInstrumentationDataToCorruptionDetectionMessageTest, - ConvertsSequenceIndexWhenSetToUseLowerBits) { - FrameInstrumentationData data = {.sequence_index = 0b0000'0110'0000'0101, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 5, - .chroma_error_threshold = 5, - .sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(data); - ASSERT_TRUE(message.has_value()); - EXPECT_EQ(message->sequence_index(), 0b0000'0101); - EXPECT_FALSE(message->interpret_sequence_index_as_most_significant_bits()); - EXPECT_EQ(message->std_dev(), 1.0); - EXPECT_EQ(message->luma_error_threshold(), 5); - EXPECT_EQ(message->chroma_error_threshold(), 5); - EXPECT_THAT(message->sample_values(), ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); -} - -TEST(FrameInstrumentationSyncDataToCorruptionDetectionMessageTest, - ConvertsValidSyncData) { - FrameInstrumentationSyncData data = {.sequence_index = 1, - .communicate_upper_bits = true}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationSyncDataToCorruptionDetectionMessage(data); - ASSERT_TRUE(message.has_value()); - EXPECT_EQ(message->sequence_index(), 0); - EXPECT_TRUE(message->interpret_sequence_index_as_most_significant_bits()); -} - -#if GTEST_HAS_DEATH_TEST && RTC_DCHECK_IS_ON -TEST(FrameInstrumentationSyncDataToCorruptionDetectionMessageTest, - FailsWhenSetToNotCommunicateUpperBits) { - FrameInstrumentationSyncData data = {.sequence_index = 1, - .communicate_upper_bits = false}; - - EXPECT_DEATH( - ConvertFrameInstrumentationSyncDataToCorruptionDetectionMessage(data), _); -} -#endif // GTEST_HAS_DEATH_TEST - -TEST(FrameInstrumentationSyncDataToCorruptionDetectionMessageTest, - ReturnsNulloptWhenSyncSequenceIndexIsNegative) { - FrameInstrumentationSyncData data = {.sequence_index = -1, - .communicate_upper_bits = true}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationSyncDataToCorruptionDetectionMessage(data); - ASSERT_FALSE(message.has_value()); -} - -TEST(FrameInstrumentationSyncDataToCorruptionDetectionMessageTest, - ReturnsNulloptWhenSyncSequenceIndexIsTooLarge) { - FrameInstrumentationSyncData data = {.sequence_index = 0x4000, - .communicate_upper_bits = true}; - - std::optional<CorruptionDetectionMessage> message = - ConvertFrameInstrumentationSyncDataToCorruptionDetectionMessage(data); - ASSERT_FALSE(message.has_value()); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationData, - FailWhenPreviousSequenceIndexIsNegative) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - EXPECT_FALSE( - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, -1) - .has_value()); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationData, - FailWhenNoSampleValuesAreProvided) { - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder().Build(); - ASSERT_TRUE(message.has_value()); - - EXPECT_FALSE( - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, 0) - .has_value()); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationData, - IgnorePreviousSequenceIndexWhenSetToUpdateTheMostSignificantBits) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSequenceIndex(11) - .WithInterpretSequenceIndexAsMostSignificantBits(true) - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - std::optional<FrameInstrumentationData> data = - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, 12); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index, 0b0101'1000'0000); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationData, - UseMessageSequenceIndexWhenHigherThanPrevious) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSequenceIndex(11) - .WithInterpretSequenceIndexAsMostSignificantBits(false) - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - std::optional<FrameInstrumentationData> data = - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, 0); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index, 11); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationData, - IncreaseThePreviousIdxUntilLsbsAreEqualToTheUpdateWhenTheUpdateIsLsbs) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSequenceIndex(11) - .WithInterpretSequenceIndexAsMostSignificantBits(false) - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - std::optional<FrameInstrumentationData> data = - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, - 1 + 128); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index, 11 + 128); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationData, - IgnoreIndexUpdateWhenTheLowerBitsSuppliedAreTheSameAsInThePreviousIndex) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSequenceIndex(11) - .WithInterpretSequenceIndexAsMostSignificantBits(false) - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - std::optional<FrameInstrumentationData> data = - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, - 11 + 128); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index, 11 + 128); -} - -TEST( - CorruptionDetectionMessageToFrameInstrumentationData, - IncreaseTheMsbsByOneAndSetTheMessagesLsbWhenMessageLsbIsLowerThanPrevious) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSequenceIndex(11) - .WithInterpretSequenceIndexAsMostSignificantBits(false) - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - std::optional<FrameInstrumentationData> data = - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, 12); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index, 11 + 128); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationData, ConvertAllFields) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSequenceIndex(11) - .WithInterpretSequenceIndexAsMostSignificantBits(false) - .WithStdDev(1.2) - .WithLumaErrorThreshold(10) - .WithChromaErrorThreshold(10) - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - std::optional<FrameInstrumentationData> data = - ConvertCorruptionDetectionMessageToFrameInstrumentationData(*message, 0); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index, 11); - EXPECT_FALSE(data->communicate_upper_bits); - EXPECT_NEAR(data->std_dev, 1.2, 0.024); // ~2% - EXPECT_EQ(data->luma_error_threshold, 10); - EXPECT_EQ(data->chroma_error_threshold, 10); - EXPECT_THAT(data->sample_values, ElementsAre(1.0, 2.0, 3.0, 4.0, 5.0)); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationSyncData, - FailWhenPreviousSequenceIndexIsNegative) { - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithInterpretSequenceIndexAsMostSignificantBits(true) - .Build(); - ASSERT_TRUE(message.has_value()); - - EXPECT_FALSE(ConvertCorruptionDetectionMessageToFrameInstrumentationSyncData( - *message, -1) - .has_value()); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationSyncData, - FailWhenSampleValuesArePresent) { - std::vector<double> sample_values = {1.0, 2.0, 3.0, 4.0, 5.0}; - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithInterpretSequenceIndexAsMostSignificantBits(true) - .WithSampleValues(sample_values) - .Build(); - ASSERT_TRUE(message.has_value()); - - EXPECT_FALSE(ConvertCorruptionDetectionMessageToFrameInstrumentationSyncData( - *message, 0) - .has_value()); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationSyncData, - FailWhenSetToUpdateTheLowerBits) { - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithInterpretSequenceIndexAsMostSignificantBits(false) - .Build(); - ASSERT_TRUE(message.has_value()); - - EXPECT_FALSE(ConvertCorruptionDetectionMessageToFrameInstrumentationSyncData( - *message, 0) - .has_value()); -} - -TEST(CorruptionDetectionMessageToFrameInstrumentationSyncData, - IgnorePreviousSequenceIndex) { - std::optional<CorruptionDetectionMessage> message = - CorruptionDetectionMessage::Builder() - .WithSequenceIndex(11) - .WithInterpretSequenceIndexAsMostSignificantBits(true) - .Build(); - ASSERT_TRUE(message.has_value()); - - std::optional<FrameInstrumentationSyncData> data = - ConvertCorruptionDetectionMessageToFrameInstrumentationSyncData(*message, - 12); - - ASSERT_TRUE(data.has_value()); - EXPECT_EQ(data->sequence_index, 0b0101'1000'0000); - EXPECT_TRUE(data->communicate_upper_bits); -} - -} // namespace -} // namespace webrtc diff --git a/third_party/libwebrtc/common_video/frame_instrumentation_data.h b/third_party/libwebrtc/common_video/frame_instrumentation_data.h @@ -1,35 +0,0 @@ -/* - * Copyright 2024 The WebRTC project authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef COMMON_VIDEO_FRAME_INSTRUMENTATION_DATA_H_ -#define COMMON_VIDEO_FRAME_INSTRUMENTATION_DATA_H_ - -#include <vector> - -namespace webrtc { - -// TODO: bugs.webrtc.org/358039777 - Error handling: negative values etc. -struct FrameInstrumentationSyncData { - int sequence_index; - bool communicate_upper_bits; -}; - -struct FrameInstrumentationData { - int sequence_index; - bool communicate_upper_bits; - double std_dev; - int luma_error_threshold; - int chroma_error_threshold; - std::vector<double> sample_values; -}; - -} // namespace webrtc - -#endif // COMMON_VIDEO_FRAME_INSTRUMENTATION_DATA_H_ diff --git a/third_party/libwebrtc/common_video/include/corruption_score_calculator.h b/third_party/libwebrtc/common_video/include/corruption_score_calculator.h @@ -11,9 +11,9 @@ #ifndef COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_ #define COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_ +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" -#include "common_video/frame_instrumentation_data.h" namespace webrtc { diff --git a/third_party/libwebrtc/modules/rtp_rtcp/BUILD.gn b/third_party/libwebrtc/modules/rtp_rtcp/BUILD.gn @@ -316,11 +316,10 @@ rtc_library("rtp_rtcp") { "../../api/video:video_frame_type", "../../api/video:video_layers_allocation", "../../api/video:video_rtp_headers", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../api/video_codecs:video_codecs_api", "../../call:rtp_interfaces", "../../common_video", - "../../common_video:corruption_detection_converters", - "../../common_video:frame_instrumentation_data", "../../logging:rtc_event_rtp_rtcp", "../../rtc_base:bit_buffer", "../../rtc_base:bitrate_tracker", @@ -430,7 +429,7 @@ rtc_library("rtp_video_header") { "../../api/video:video_frame_metadata", "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", - "../../common_video:frame_instrumentation_data", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../modules/video_coding:codec_globals_headers", "//third_party/abseil-cpp/absl/container:inlined_vector", ] @@ -689,12 +688,12 @@ if (rtc_include_tests) { "../../api/video:video_frame_type", "../../api/video:video_layers_allocation", "../../api/video:video_rtp_headers", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../api/video_codecs:video_codecs_api", "../../call:rtp_interfaces", "../../call:rtp_receiver", "../../call:video_receive_stream_api", "../../common_video", - "../../common_video:frame_instrumentation_data", "../../common_video/generic_frame_descriptor", "../../common_video/test:utilities", "../../logging:mocks", diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/frame_object.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/frame_object.cc @@ -13,11 +13,11 @@ #include <cstdint> #include <optional> #include <utility> -#include <variant> #include "api/rtp_packet_infos.h" #include "api/scoped_refptr.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" @@ -25,7 +25,6 @@ #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" namespace webrtc { @@ -45,9 +44,7 @@ RtpFrameObject::RtpFrameObject( VideoContentType content_type, const RTPVideoHeader& video_header, const std::optional<class ColorSpace>& color_space, - const std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>& - frame_instrumentation_data, + const std::optional<FrameInstrumentationData>& frame_instrumentation_data, RtpPacketInfos packet_infos, scoped_refptr<EncodedImageBuffer> image_buffer) : image_buffer_(image_buffer), diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/frame_object.h b/third_party/libwebrtc/modules/rtp_rtcp/source/frame_object.h @@ -13,12 +13,12 @@ #include <cstdint> #include <optional> -#include <variant> #include <vector> #include "api/rtp_packet_infos.h" #include "api/scoped_refptr.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/encoded_image.h" #include "api/video/video_codec_type.h" @@ -27,33 +27,31 @@ #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" namespace webrtc { class RtpFrameObject : public EncodedFrame { public: - RtpFrameObject(uint16_t first_seq_num, - uint16_t last_seq_num, - bool markerBit, - int times_nacked, - int64_t first_packet_received_time, - int64_t last_packet_received_time, - uint32_t rtp_timestamp, - int64_t ntp_time_ms, - const VideoSendTiming& timing, - uint8_t payload_type, - VideoCodecType codec, - VideoRotation rotation, - VideoContentType content_type, - const RTPVideoHeader& video_header, - const std::optional<class ColorSpace>& color_space, - const std::optional<std::variant<FrameInstrumentationSyncData, - FrameInstrumentationData>>& - frame_instrumentation_data, - RtpPacketInfos packet_infos, - scoped_refptr<EncodedImageBuffer> image_buffer); + RtpFrameObject( + uint16_t first_seq_num, + uint16_t last_seq_num, + bool markerBit, + int times_nacked, + int64_t first_packet_received_time, + int64_t last_packet_received_time, + uint32_t rtp_timestamp, + int64_t ntp_time_ms, + const VideoSendTiming& timing, + uint8_t payload_type, + VideoCodecType codec, + VideoRotation rotation, + VideoContentType content_type, + const RTPVideoHeader& video_header, + const std::optional<class ColorSpace>& color_space, + const std::optional<FrameInstrumentationData>& frame_instrumentation_data, + RtpPacketInfos packet_infos, + scoped_refptr<EncodedImageBuffer> image_buffer); ~RtpFrameObject() override; uint16_t first_seq_num() const; diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -39,8 +39,6 @@ #include "api/video/video_layers_allocation.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" -#include "common_video/corruption_detection_converters.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" #include "modules/rtp_rtcp/source/corruption_detection_extension.h" @@ -484,26 +482,9 @@ void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header, } if (last_packet && video_header.frame_instrumentation_data) { - std::optional<CorruptionDetectionMessage> message; - if (const auto* data = std::get_if<FrameInstrumentationData>( - &(*video_header.frame_instrumentation_data))) { - message = - ConvertFrameInstrumentationDataToCorruptionDetectionMessage(*data); - } else if (const auto* sync_data = - std::get_if<FrameInstrumentationSyncData>( - &(*video_header.frame_instrumentation_data))) { - message = ConvertFrameInstrumentationSyncDataToCorruptionDetectionMessage( - *sync_data); - } else { - RTC_DCHECK_NOTREACHED(); - } - - if (message.has_value()) { - packet->SetExtension<CorruptionDetectionExtension>(*message); - } else { - RTC_LOG(LS_WARNING) << "Failed to convert frame instrumentation data to " - "corruption detection message."; - } + packet->SetExtension<CorruptionDetectionExtension>( + CorruptionDetectionMessage::FromFrameInstrumentationData( + *video_header.frame_instrumentation_data)); } } diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -38,6 +38,7 @@ #include "api/units/data_rate.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/video_codec_constants.h" #include "api/video/video_codec_type.h" @@ -45,7 +46,6 @@ #include "api/video/video_layers_allocation.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/rtp_rtcp/include/rtp_cvo.h" #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" @@ -272,13 +272,13 @@ TEST_F(RtpSenderVideoTest, kCorruptionDetectionExtensionId); RTPVideoHeader hdr; hdr.frame_type = VideoFrameType::kVideoFrameKey; - hdr.frame_instrumentation_data = FrameInstrumentationData{ - .sequence_index = 130, // 128 + 2 - .communicate_upper_bits = false, - .std_dev = 2.0, - .luma_error_threshold = 3, - .chroma_error_threshold = 2, - .sample_values = {12.0, 12.0, 12.0, 12.0, 12.0, 12.0, 12.0, 12.0}}; + FrameInstrumentationData data; + data.SetSequenceIndex(130); // 128 + 2 + data.SetStdDev(2.0); + data.SetLumaErrorThreshold(3); + data.SetChromaErrorThreshold(2); + data.SetSampleValues({12.0, 12.0, 12.0, 12.0, 12.0, 12.0, 12.0, 12.0}); + hdr.frame_instrumentation_data = data; CorruptionDetectionMessage message; rtp_sender_video_->SendVideo( @@ -310,10 +310,9 @@ TEST_F(RtpSenderVideoTest, kCorruptionDetectionExtensionId); RTPVideoHeader hdr; hdr.frame_type = VideoFrameType::kVideoFrameKey; - hdr.frame_instrumentation_data = FrameInstrumentationSyncData{ - .sequence_index = 130, // 128 + 2 - .communicate_upper_bits = true, - }; + // Send data with sequence index divisible by 2^7 and no sample values in + // order create a sync message with upper bits set. + hdr.frame_instrumentation_data.emplace().SetSequenceIndex(128); CorruptionDetectionMessage message; rtp_sender_video_->SendVideo( diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_header.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_header.h @@ -19,13 +19,13 @@ #include "api/rtp_headers.h" #include "api/transport/rtp/dependency_descriptor.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_metadata.h" #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -97,9 +97,7 @@ struct RTPVideoHeader { std::optional<AbsoluteCaptureTime> absolute_capture_time; // Required for automatic corruption detection. - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - frame_instrumentation_data; + std::optional<FrameInstrumentationData> frame_instrumentation_data; }; } // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/BUILD.gn b/third_party/libwebrtc/modules/video_coding/BUILD.gn @@ -242,11 +242,11 @@ rtc_library("video_coding") { "../../api/video:video_frame", "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../common_video", "../../common_video:corruption_score_calculator", - "../../common_video:frame_instrumentation_data", "../../rtc_base:base64", "../../rtc_base:buffer", "../../rtc_base:byte_buffer", @@ -305,10 +305,10 @@ rtc_library("video_codec_interface") { "../../api/video:video_frame", "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../common_video", - "../../common_video:frame_instrumentation_data", "../../common_video/generic_frame_descriptor", "../../rtc_base:checks", "../../rtc_base/system:rtc_export", @@ -1309,12 +1309,12 @@ if (rtc_include_tests) { "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", "../../api/video/corruption_detection:filter_settings", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../api/video_codecs:vp8_temporal_layers_factory", "../../common_video", "../../common_video:corruption_score_calculator", - "../../common_video:frame_instrumentation_data", "../../common_video/generic_frame_descriptor", "../../common_video/test:utilities", "../../media:media_constants", diff --git a/third_party/libwebrtc/modules/video_coding/frame_helpers_unittest.cc b/third_party/libwebrtc/modules/video_coding/frame_helpers_unittest.cc @@ -15,14 +15,13 @@ #include <memory> #include <optional> #include <utility> -#include <variant> #include "absl/container/inlined_vector.h" #include "api/scoped_refptr.h" #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/encoded_image.h" -#include "common_video/frame_instrumentation_data.h" #include "test/gmock.h" #include "test/gtest.h" @@ -68,48 +67,41 @@ TEST(FrameInstrumentationDataTest, CombinedFrameHasSameDataAsHighestSpatialLayer) { // Assume L2T1 scalability mode. EncodedFrame spatial_layer_1 = CreateEncodedImageOfSizeN(/*n=*/10, /*x=*/1); - const FrameInstrumentationData frame_ins_data_1 = { - .sequence_index = 100, - .communicate_upper_bits = false, - .std_dev = 0.5, - .luma_error_threshold = 5, - .chroma_error_threshold = 4, - .sample_values = {0.2, 0.7, 1.9}}; + FrameInstrumentationData frame_ins_data_1; + frame_ins_data_1.SetSequenceIndex(100); + frame_ins_data_1.SetStdDev(0.5); + frame_ins_data_1.SetLumaErrorThreshold(5); + frame_ins_data_1.SetChromaErrorThreshold(5); + frame_ins_data_1.SetSampleValues({0.2, 0.7, 1.9}); spatial_layer_1.SetFrameInstrumentationData(frame_ins_data_1); EncodedFrame spatial_layer_2 = CreateEncodedImageOfSizeN(/*n=*/10, /*x=*/11); - FrameInstrumentationData frame_ins_data_2 = { - .sequence_index = 10, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 3, - .chroma_error_threshold = 4, - .sample_values = {0.1, 0.3, 2.1}}; + FrameInstrumentationData frame_ins_data_2; + frame_ins_data_2.SetSequenceIndex(10); + frame_ins_data_2.SetStdDev(1.0); + frame_ins_data_2.SetLumaErrorThreshold(3); + frame_ins_data_2.SetChromaErrorThreshold(4); + frame_ins_data_2.SetSampleValues({0.1, 0.3, 2.1}); spatial_layer_2.SetFrameInstrumentationData(frame_ins_data_2); absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> frames; frames.push_back(std::make_unique<EncodedFrame>(spatial_layer_1)); frames.push_back(std::make_unique<EncodedFrame>(spatial_layer_2)); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data = CombineAndDeleteFrames(std::move(frames)) - ->CodecSpecific() - ->frame_instrumentation_data; + std::optional<FrameInstrumentationData> frame_instrumentation_data = + CombineAndDeleteFrames(std::move(frames)) + ->CodecSpecific() + ->frame_instrumentation_data; - ASSERT_TRUE(data.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data)); - FrameInstrumentationData frame_instrumentation_data = - std::get<FrameInstrumentationData>(*data); + ASSERT_TRUE(frame_instrumentation_data.has_value()); // Expect to have the same frame_instrumentation_data as the highest spatial // layer. - EXPECT_EQ(frame_instrumentation_data.sequence_index, 10); - EXPECT_FALSE(frame_instrumentation_data.communicate_upper_bits); - EXPECT_EQ(frame_instrumentation_data.std_dev, 1.0); - EXPECT_EQ(frame_instrumentation_data.luma_error_threshold, 3); - EXPECT_EQ(frame_instrumentation_data.chroma_error_threshold, 4); - EXPECT_THAT(frame_instrumentation_data.sample_values, + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 10); + EXPECT_EQ(frame_instrumentation_data->std_dev(), 1.0); + EXPECT_EQ(frame_instrumentation_data->luma_error_threshold(), 3); + EXPECT_EQ(frame_instrumentation_data->chroma_error_threshold(), 4); + EXPECT_THAT(frame_instrumentation_data->sample_values(), ElementsAre(0.1, 0.3, 2.1)); } diff --git a/third_party/libwebrtc/modules/video_coding/generic_decoder.cc b/third_party/libwebrtc/modules/video_coding/generic_decoder.cc @@ -17,13 +17,13 @@ #include <optional> #include <tuple> #include <utility> -#include <variant> #include "absl/algorithm/container.h" #include "api/field_trials_view.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/encoded_image.h" #include "api/video/video_content_type.h" @@ -31,7 +31,6 @@ #include "api/video/video_frame_type.h" #include "api/video/video_timing.h" #include "api/video_codecs/video_decoder.h" -#include "common_video/frame_instrumentation_data.h" #include "common_video/include/corruption_score_calculator.h" #include "modules/include/module_common_types_public.h" #include "modules/video_coding/encoded_frame.h" @@ -247,12 +246,9 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, if (corruption_score_calculator_ && frame_info->frame_instrumentation_data.has_value()) { - if (const FrameInstrumentationData* data = - std::get_if<FrameInstrumentationData>( - &*frame_info->frame_instrumentation_data)) { - corruption_score_calculator_->CalculateCorruptionScore( - decodedImage, *data, frame_info->content_type); - } + corruption_score_calculator_->CalculateCorruptionScore( + decodedImage, *frame_info->frame_instrumentation_data, + frame_info->content_type); } } @@ -327,9 +323,7 @@ int32_t VCMGenericDecoder::Decode( const EncodedImage& frame, Timestamp now, int64_t render_time_ms, - const std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>& - frame_instrumentation_data) { + const std::optional<FrameInstrumentationData>& frame_instrumentation_data) { TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", frame.RtpTimestamp()); FrameInfo frame_info; diff --git a/third_party/libwebrtc/modules/video_coding/generic_decoder.h b/third_party/libwebrtc/modules/video_coding/generic_decoder.h @@ -16,13 +16,13 @@ #include <deque> #include <optional> #include <utility> -#include <variant> #include "api/field_trials_view.h" #include "api/rtp_packet_infos.h" #include "api/sequence_checker.h" #include "api/units/timestamp.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/encoded_image.h" #include "api/video/video_content_type.h" @@ -30,7 +30,6 @@ #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video_codecs/video_decoder.h" -#include "common_video/frame_instrumentation_data.h" #include "common_video/include/corruption_score_calculator.h" #include "modules/video_coding/encoded_frame.h" #include "modules/video_coding/timing/timing.h" @@ -62,9 +61,7 @@ struct FrameInfo { RtpPacketInfos packet_infos; // ColorSpace is not stored here, as it might be modified by decoders. VideoFrameType frame_type; - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - frame_instrumentation_data; + std::optional<FrameInstrumentationData> frame_instrumentation_data; std::optional<ColorSpace> color_space; }; @@ -142,8 +139,7 @@ class VCMGenericDecoder { int32_t Decode(const EncodedImage& frame, Timestamp now, int64_t render_time_ms, - const std::optional<std::variant<FrameInstrumentationSyncData, - FrameInstrumentationData>>& + const std::optional<FrameInstrumentationData>& frame_instrumentation_data); VCMDecodedFrameCallback* _callback = nullptr; VideoDecoder* const decoder_; diff --git a/third_party/libwebrtc/modules/video_coding/generic_decoder_unittest.cc b/third_party/libwebrtc/modules/video_coding/generic_decoder_unittest.cc @@ -21,6 +21,7 @@ #include "api/scoped_refptr.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/i420_buffer.h" #include "api/video/video_codec_type.h" @@ -29,7 +30,6 @@ #include "api/video/video_frame_type.h" #include "api/video/video_timing.h" #include "api/video_codecs/video_decoder.h" -#include "common_video/frame_instrumentation_data.h" #include "common_video/include/corruption_score_calculator.h" #include "common_video/test/utilities.h" #include "modules/video_coding/include/video_coding_defines.h" @@ -220,8 +220,8 @@ TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) { TEST_F(GenericDecoderTest, CallCalculateCorruptionScoreInDecoded) { constexpr uint32_t kRtpTimestamp = 1; FrameInfo frame_info; - frame_info.frame_instrumentation_data = - FrameInstrumentationData{.sequence_index = 1}; + frame_info.frame_instrumentation_data.emplace(); + frame_info.frame_instrumentation_data->SetSequenceIndex(1); frame_info.rtp_timestamp = kRtpTimestamp; frame_info.decode_start = Timestamp::Zero(); frame_info.content_type = VideoContentType::SCREENSHARE; @@ -235,7 +235,7 @@ TEST_F(GenericDecoderTest, CallCalculateCorruptionScoreInDecoded) { EXPECT_CALL(corruption_score_calculator_, CalculateCorruptionScore( Property(&VideoFrame::rtp_timestamp, Eq(kRtpTimestamp)), - Field(&FrameInstrumentationData::sequence_index, Eq(1)), + Property(&FrameInstrumentationData::sequence_index, Eq(1)), VideoContentType::SCREENSHARE)); vcm_callback_.Decoded(video_frame); } diff --git a/third_party/libwebrtc/modules/video_coding/include/video_codec_interface.h b/third_party/libwebrtc/modules/video_coding/include/video_codec_interface.h @@ -15,13 +15,12 @@ #include <cstdint> #include <optional> #include <type_traits> -#include <variant> #include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/video_codec_type.h" #include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_encoder.h" -#include "common_video/frame_instrumentation_data.h" #include "common_video/generic_frame_descriptor/generic_frame_info.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -124,9 +123,7 @@ struct RTC_EXPORT CodecSpecificInfo { std::optional<ScalabilityMode> scalability_mode; // Required for automatic corruption detection. - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - frame_instrumentation_data; + std::optional<FrameInstrumentationData> frame_instrumentation_data; }; } // namespace webrtc diff --git a/third_party/libwebrtc/moz-patch-stack/s0027.patch b/third_party/libwebrtc/moz-patch-stack/s0027.patch @@ -203,7 +203,7 @@ index ef07265bf3..348be89bb9 100644 } diff --git a/api/BUILD.gn b/api/BUILD.gn -index 036a23ca9b..ac2234a065 100644 +index ea3a8a4acd..3603d4f4b0 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -44,6 +44,9 @@ rtc_library("enable_media") { @@ -1258,7 +1258,7 @@ index 2e8752ab0c..f9bcdf7a0d 100644 rtc_library("call_config_utils") { testonly = true diff --git a/video/BUILD.gn b/video/BUILD.gn -index e3122cbc18..b58bef6ad6 100644 +index 0e43130593..eeeddba86f 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -17,7 +17,7 @@ rtc_library("video_stream_encoder_interface") { diff --git a/third_party/libwebrtc/moz-patch-stack/s0030.patch b/third_party/libwebrtc/moz-patch-stack/s0030.patch @@ -25,10 +25,10 @@ index ae9951e6f9..c69ec1a674 100644 // See LntfConfig for description. diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index 77153416ed..ea6325087d 100644 +index 054b9c1126..30e97de170 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -1412,7 +1412,7 @@ void RtpVideoStreamReceiver2::StartReceive() { +@@ -1366,7 +1366,7 @@ void RtpVideoStreamReceiver2::StartReceive() { // Change REMB candidate egibility. packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), diff --git a/third_party/libwebrtc/moz-patch-stack/s0035.patch b/third_party/libwebrtc/moz-patch-stack/s0035.patch @@ -261,10 +261,10 @@ index 7a178ea664..41c3d608ad 100644 VideoBitrateAllocationObserver* bitrate_allocation_observer = nullptr; diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index ea6325087d..30a0d37d5a 100644 +index 30e97de170..07c84e5ada 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -130,7 +130,8 @@ std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpRtcpModule( +@@ -127,7 +127,8 @@ std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpRtcpModule( RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, RtcpCnameCallback* rtcp_cname_callback, bool non_sender_rtt_measurement, @@ -274,7 +274,7 @@ index ea6325087d..30a0d37d5a 100644 RtpRtcpInterface::Configuration configuration; configuration.audio = false; configuration.receiver_only = true; -@@ -141,6 +142,7 @@ std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpRtcpModule( +@@ -138,6 +139,7 @@ std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpRtcpModule( rtcp_packet_type_counter_observer; configuration.rtcp_cname_callback = rtcp_cname_callback; configuration.local_media_ssrc = local_ssrc; @@ -282,7 +282,7 @@ index ea6325087d..30a0d37d5a 100644 configuration.non_sender_rtt_measurement = non_sender_rtt_measurement; auto rtp_rtcp = std::make_unique<ModuleRtpRtcpImpl2>(env, configuration); -@@ -314,7 +316,8 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( +@@ -311,7 +313,8 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( rtcp_packet_type_counter_observer, rtcp_cname_callback, config_.rtp.rtcp_xr.receiver_reference_time_report, diff --git a/third_party/libwebrtc/moz-patch-stack/s0037.patch b/third_party/libwebrtc/moz-patch-stack/s0037.patch @@ -15,10 +15,10 @@ Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/edac9d01a9ac7594f 3 files changed, 24 insertions(+) diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index 30a0d37d5a..3c2119cd2d 100644 +index 07c84e5ada..216b5823b2 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -1178,6 +1178,16 @@ std::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedKeyframePacketMs() +@@ -1132,6 +1132,16 @@ std::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedKeyframePacketMs() return std::nullopt; } @@ -36,10 +36,10 @@ index 30a0d37d5a..3c2119cd2d 100644 RtpVideoStreamReceiver2::GetSenderReportStats() const { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h -index 18ee7ffa46..7e358c3c13 100644 +index 2ca52615a7..52f998ddc2 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h -@@ -229,6 +229,12 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, +@@ -228,6 +228,12 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, std::optional<RtpRtcpInterface::SenderReportStats> GetSenderReportStats() const; @@ -53,7 +53,7 @@ index 18ee7ffa46..7e358c3c13 100644 // Implements RtpVideoFrameReceiver. void ManageFrame(std::unique_ptr<RtpFrameObject> frame) override; diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc -index eccec79bb3..ce63bc1fc8 100644 +index ca0271cc97..f3170157c0 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -626,6 +626,14 @@ VideoReceiveStreamInterface::Stats VideoReceiveStream2::GetStats() const { diff --git a/third_party/libwebrtc/moz-patch-stack/s0042.patch b/third_party/libwebrtc/moz-patch-stack/s0042.patch @@ -122,10 +122,10 @@ index 41c3d608ad..5c6a9fcb85 100644 // Within this list, the sender-source SSRC pair is unique and per-pair the // ReportBlockData represents the latest Report Block that was received for diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index 3c2119cd2d..cf492a3a41 100644 +index 216b5823b2..022099a780 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -1183,9 +1183,10 @@ std::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedKeyframePacketMs() +@@ -1137,9 +1137,10 @@ std::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedKeyframePacketMs() // seem to be any support for these stats right now. So, we hack this in. void RtpVideoStreamReceiver2::RemoteRTCPSenderInfo( uint32_t* packet_count, uint32_t* octet_count, @@ -139,10 +139,10 @@ index 3c2119cd2d..cf492a3a41 100644 std::optional<RtpRtcpInterface::SenderReportStats> diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h -index 7e358c3c13..05498bd640 100644 +index 52f998ddc2..1acb0ac70e 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h -@@ -233,7 +233,8 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, +@@ -232,7 +232,8 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, // stats at all, and even on the most recent libwebrtc code there does not // seem to be any support for these stats right now. So, we hack this in. void RemoteRTCPSenderInfo(uint32_t* packet_count, uint32_t* octet_count, @@ -153,7 +153,7 @@ index 7e358c3c13..05498bd640 100644 private: // Implements RtpVideoFrameReceiver. diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc -index ce63bc1fc8..78f5a62bbb 100644 +index f3170157c0..e7a224e8c3 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -632,7 +632,8 @@ VideoReceiveStreamInterface::Stats VideoReceiveStream2::GetStats() const { diff --git a/third_party/libwebrtc/moz-patch-stack/s0057.patch b/third_party/libwebrtc/moz-patch-stack/s0057.patch @@ -214,10 +214,10 @@ index 111b7f3c45..fb411d4bbf 100644 int current_delay_ms, int target_delay_ms, diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index cf492a3a41..0ed27ba85d 100644 +index 022099a780..7f2537ac18 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -287,6 +287,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( +@@ -284,6 +284,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, RtcpCnameCallback* rtcp_cname_callback, NackPeriodicProcessor* nack_periodic_processor, @@ -225,7 +225,7 @@ index cf492a3a41..0ed27ba85d 100644 OnCompleteFrameCallback* complete_frame_callback, scoped_refptr<FrameDecryptorInterface> frame_decryptor, scoped_refptr<FrameTransformerInterface> frame_transformer) -@@ -330,6 +331,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( +@@ -327,6 +328,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( config_.rtp.nack, &rtcp_feedback_buffer_, &rtcp_feedback_buffer_)), @@ -233,7 +233,7 @@ index cf492a3a41..0ed27ba85d 100644 packet_buffer_(kPacketBufferStartSize, PacketBufferMaxSize(env_.field_trials())), reference_finder_(std::make_unique<RtpFrameReferenceFinder>()), -@@ -1403,7 +1405,8 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { +@@ -1357,7 +1359,8 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { int64_t unwrapped_rtp_seq_num = rtp_seq_num_unwrapper_.Unwrap(seq_num); packet_infos_.erase(packet_infos_.begin(), packet_infos_.upper_bound(unwrapped_rtp_seq_num)); @@ -244,10 +244,10 @@ index cf492a3a41..0ed27ba85d 100644 } } diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h -index 05498bd640..e35711dc8b 100644 +index 1acb0ac70e..272f83ccba 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h -@@ -69,6 +69,7 @@ +@@ -68,6 +68,7 @@ #include "rtc_base/thread_annotations.h" #include "video/buffered_frame_decryptor.h" #include "video/unique_timestamp_counter.h" @@ -255,7 +255,7 @@ index 05498bd640..e35711dc8b 100644 namespace webrtc { -@@ -111,6 +112,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, +@@ -110,6 +111,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, RtcpCnameCallback* rtcp_cname_callback, NackPeriodicProcessor* nack_periodic_processor, @@ -263,7 +263,7 @@ index 05498bd640..e35711dc8b 100644 // The KeyFrameRequestSender is optional; if not provided, key frame // requests are sent via the internal RtpRtcp module. OnCompleteFrameCallback* complete_frame_callback, -@@ -395,6 +397,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, +@@ -390,6 +392,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, std::unique_ptr<LossNotificationController> loss_notification_controller_ RTC_GUARDED_BY(packet_sequence_checker_); @@ -272,7 +272,7 @@ index 05498bd640..e35711dc8b 100644 RTC_GUARDED_BY(packet_sequence_checker_); // h26x_packet_buffer_ is applicable to H.264 and H.265. For H.265 it is diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc -index 78f5a62bbb..ad2ecf5da9 100644 +index e7a224e8c3..7fc4e69446 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -256,6 +256,7 @@ VideoReceiveStream2::VideoReceiveStream2( diff --git a/third_party/libwebrtc/moz-patch-stack/s0058.patch b/third_party/libwebrtc/moz-patch-stack/s0058.patch @@ -30,10 +30,10 @@ index c8058ea132..568fd441d2 100644 void ReceiveStatisticsProxy::OnPreDecode(VideoCodecType codec_type, int qp) { diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index 0ed27ba85d..a488da81e2 100644 +index 7f2537ac18..4573171a5f 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -1406,7 +1406,9 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { +@@ -1360,7 +1360,9 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { packet_infos_.erase(packet_infos_.begin(), packet_infos_.upper_bound(unwrapped_rtp_seq_num)); uint32_t num_packets_cleared = packet_buffer_.ClearTo(seq_num); diff --git a/third_party/libwebrtc/moz-patch-stack/s0059.patch b/third_party/libwebrtc/moz-patch-stack/s0059.patch @@ -166,10 +166,10 @@ index 568fd441d2..30eec9f090 100644 } diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index a488da81e2..e0bd1549ce 100644 +index 4573171a5f..a5b1a12fd0 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -92,6 +92,7 @@ +@@ -89,6 +89,7 @@ #include "rtc_base/numerics/sequence_number_util.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/thread.h" @@ -177,7 +177,7 @@ index a488da81e2..e0bd1549ce 100644 #include "system_wrappers/include/ntp_time.h" #include "video/buffered_frame_decryptor.h" -@@ -1407,6 +1408,9 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { +@@ -1361,6 +1362,9 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { packet_infos_.upper_bound(unwrapped_rtp_seq_num)); uint32_t num_packets_cleared = packet_buffer_.ClearTo(seq_num); if (num_packets_cleared > 0) { diff --git a/third_party/libwebrtc/moz-patch-stack/s0060.patch b/third_party/libwebrtc/moz-patch-stack/s0060.patch @@ -9,10 +9,10 @@ Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/5b2a7894ef1cf096d 1 file changed, 6 insertions(+) diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index e0bd1549ce..4a3d672892 100644 +index a5b1a12fd0..cb7070ecda 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -830,6 +830,12 @@ void RtpVideoStreamReceiver2::OnRtpPacket(const RtpPacketReceived& packet) { +@@ -784,6 +784,12 @@ void RtpVideoStreamReceiver2::OnRtpPacket(const RtpPacketReceived& packet) { void RtpVideoStreamReceiver2::RequestKeyFrame() { RTC_DCHECK_RUN_ON(&worker_task_checker_); diff --git a/third_party/libwebrtc/moz-patch-stack/s0068.patch b/third_party/libwebrtc/moz-patch-stack/s0068.patch @@ -10,10 +10,10 @@ Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/a7179d8d75313b6c9 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc -index 9464b2b892..f20f81d449 100644 +index 2e402e0896..0b5ec1050f 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc -@@ -1443,7 +1443,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { +@@ -1442,7 +1442,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { bool is_svc = false; bool single_stream_or_non_first_inactive = true; diff --git a/third_party/libwebrtc/moz-patch-stack/s0071.patch b/third_party/libwebrtc/moz-patch-stack/s0071.patch @@ -50,10 +50,10 @@ index 1e6678b4ee..1ed48c08d7 100644 Clock* const clock_; bool short_circuit_ RTC_GUARDED_BY(network_sequence_checker_) = false; diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc -index 4a3d672892..44a2ae0709 100644 +index cb7070ecda..5249f6aed3 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc -@@ -384,7 +384,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( +@@ -381,7 +381,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( frame_transformer_delegate_ = make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( this, &env_.clock(), std::move(frame_transformer), diff --git a/third_party/libwebrtc/moz-patch-stack/s0072.patch b/third_party/libwebrtc/moz-patch-stack/s0072.patch @@ -45,10 +45,10 @@ index 80c1c84043..2c900dde28 100644 return flexfec_ssrc_; } diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc -index 82e8b16791..8014841d32 100644 +index ee7414b582..d8c46a5819 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc -@@ -186,6 +186,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) +@@ -184,6 +184,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) this, config.frame_transformer, rtp_sender_->SSRC(), diff --git a/third_party/libwebrtc/moz-patch-stack/s0093.patch b/third_party/libwebrtc/moz-patch-stack/s0093.patch @@ -30,10 +30,10 @@ index 82aede5e28..b602b8d2ce 100644 if (rtc_use_perfetto) { rtc_library("webrtc_libperfetto") { diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc -index 88c93170c6..b7abdd14c4 100644 +index 7dc827a0d2..8ab283fa95 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc -@@ -121,9 +121,9 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, +@@ -120,9 +120,9 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, std::optional<int32_t> decode_time_ms, std::optional<uint8_t> qp) { RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point"; @@ -46,10 +46,10 @@ index 88c93170c6..b7abdd14c4 100644 // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). std::optional<FrameInfo> frame_info; -@@ -330,8 +330,8 @@ int32_t VCMGenericDecoder::Decode( - const std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>& - frame_instrumentation_data) { +@@ -324,8 +324,8 @@ int32_t VCMGenericDecoder::Decode( + Timestamp now, + int64_t render_time_ms, + const std::optional<FrameInstrumentationData>& frame_instrumentation_data) { - TRACE_EVENT("webrtc", "VCMGenericDecoder::Decode", - perfetto::Flow::ProcessScoped(frame.RtpTimestamp())); + TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", diff --git a/third_party/libwebrtc/moz-patch-stack/s0100.patch b/third_party/libwebrtc/moz-patch-stack/s0100.patch @@ -39,7 +39,7 @@ index 0f2341a462..51d055bcf0 100644 group("media") { diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn -index 1cc12a4636..33c4316577 100644 +index 32fd20badc..8416dec319 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -7,7 +7,7 @@ diff --git a/third_party/libwebrtc/moz-patch-stack/s0102.patch b/third_party/libwebrtc/moz-patch-stack/s0102.patch @@ -126,7 +126,7 @@ index b602b8d2ce..4603ed3cfd 100644 "Generated during 'gn gen' by //BUILD.gn.", "", diff --git a/api/BUILD.gn b/api/BUILD.gn -index ac2234a065..2aee479f4c 100644 +index 3603d4f4b0..77cb83a9c4 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -8,8 +8,8 @@ @@ -246,7 +246,7 @@ index 40714b196e..2c6b39f59e 100644 if (rtc_include_tests) { diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn -index cd74281a22..16af840261 100644 +index 14cf3604b4..b162e7da27 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -8,8 +8,8 @@ @@ -588,7 +588,7 @@ index a693604389..72972d18e1 100644 import("../../webrtc.gni") diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn -index 33c4316577..034a8c2cf1 100644 +index 8416dec319..44328eade0 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -6,7 +6,7 @@ diff --git a/third_party/libwebrtc/moz-patch-stack/s0110.patch b/third_party/libwebrtc/moz-patch-stack/s0110.patch @@ -36,10 +36,10 @@ index 7fda03ffa8..8ca8462dd0 100644 ~VideoStreamFactoryInterface() override {} }; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc -index f20f81d449..3a01bce061 100644 +index 0b5ec1050f..8d1cf10e65 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc -@@ -1055,6 +1055,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { +@@ -1054,6 +1054,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::vector<VideoStream> streams; if (encoder_config_.video_stream_factory) { // Note: only tests set their own EncoderStreamFactory... diff --git a/third_party/libwebrtc/video/BUILD.gn b/third_party/libwebrtc/video/BUILD.gn @@ -121,6 +121,8 @@ rtc_library("video") { "../api/video:video_layers_allocation", "../api/video:video_rtp_headers", "../api/video:video_stream_encoder", + "../api/video/corruption_detection:frame_instrumentation_data", + "../api/video/corruption_detection:frame_instrumentation_data_reader", "../api/video_codecs:video_codecs_api", "../call:bitrate_allocator", "../call:call_interfaces", @@ -130,10 +132,8 @@ rtc_library("video") { "../call:video_receive_stream_api", "../call:video_send_stream_api", "../common_video", - "../common_video:corruption_detection_converters", "../common_video:corruption_score_calculator", "../common_video:frame_counts", - "../common_video:frame_instrumentation_data", "../media:media_constants", "../media:rtc_sdp_video_format_utils", "../modules:module_api", @@ -465,11 +465,11 @@ rtc_library("video_stream_encoder_impl") { "../api/video:video_layers_allocation", "../api/video:video_rtp_headers", "../api/video:video_stream_encoder", + "../api/video/corruption_detection:frame_instrumentation_data", "../api/video_codecs:scalability_mode", "../api/video_codecs:video_codecs_api", "../call/adaptation:resource_adaptation", "../common_video", - "../common_video:frame_instrumentation_data", "../media:media_channel", "../modules:module_api_public", "../modules/video_coding", @@ -940,6 +940,7 @@ if (rtc_include_tests) { "../api/video:video_layers_allocation", "../api/video:video_rtp_headers", "../api/video:video_stream_encoder", + "../api/video/corruption_detection:frame_instrumentation_data", "../api/video/test:video_frame_matchers", "../api/video_codecs:scalability_mode", "../api/video_codecs:video_codecs_api", @@ -959,7 +960,6 @@ if (rtc_include_tests) { "../call/adaptation:resource_adaptation_test_utilities", "../common_video", "../common_video:frame_counts", - "../common_video:frame_instrumentation_data", "../common_video/test:utilities", "../media:codec", "../media:media_constants", diff --git a/third_party/libwebrtc/video/corruption_detection/BUILD.gn b/third_party/libwebrtc/video/corruption_detection/BUILD.gn @@ -33,7 +33,7 @@ rtc_library("frame_instrumentation_evaluation") { "../../api:scoped_refptr", "../../api/video:video_frame", "../../api/video:video_rtp_headers", - "../../common_video:frame_instrumentation_data", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../rtc_base:checks", "../../rtc_base:logging", ] @@ -52,8 +52,8 @@ rtc_library("frame_instrumentation_generator") { "../../api/video:video_frame", "../../api/video:video_frame_type", "../../api/video/corruption_detection:filter_settings", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../api/video_codecs:video_codecs_api", - "../../common_video:frame_instrumentation_data", "../../modules:module_api_public", "../../modules/video_coding:video_coding_utility", "../../rtc_base:checks", @@ -164,7 +164,7 @@ if (rtc_include_tests) { "../../api:scoped_refptr", "../../api/video:video_frame", "../../api/video:video_rtp_headers", - "../../common_video:frame_instrumentation_data", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../test:test_support", ] } @@ -180,7 +180,7 @@ if (rtc_include_tests) { "../../api/video:video_frame", "../../api/video:video_frame_type", "../../api/video/corruption_detection:filter_settings", - "../../common_video:frame_instrumentation_data", + "../../api/video/corruption_detection:frame_instrumentation_data", "../../rtc_base:refcount", "../../test:test_support", ] diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.cc @@ -14,9 +14,9 @@ #include <vector> #include "api/array_view.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" -#include "common_video/frame_instrumentation_data.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "video/corruption_detection/halton_frame_sampler.h" @@ -50,37 +50,36 @@ void FrameInstrumentationEvaluation::OnInstrumentedFrame( const FrameInstrumentationData& data, const VideoFrame& frame, VideoContentType content_type) { - if (data.sample_values.empty()) { - RTC_LOG(LS_WARNING) - << "Samples are needed to calculate a corruption score."; + if (data.sample_values().empty()) { + // Likely a sync message. Silently ignore. return; } - frame_sampler_.SetCurrentIndex(data.sequence_index); + frame_sampler_.SetCurrentIndex(data.sequence_index()); std::vector<HaltonFrameSampler::Coordinates> sample_coordinates = - frame_sampler_.GetSampleCoordinatesForFrame(data.sample_values.size()); + frame_sampler_.GetSampleCoordinatesForFrame(data.sample_values().size()); if (sample_coordinates.empty()) { RTC_LOG(LS_ERROR) << "Failed to get sample coordinates for frame."; return; } std::vector<FilteredSample> samples = GetSampleValuesForFrame( - frame, sample_coordinates, frame.width(), frame.height(), data.std_dev); + frame, sample_coordinates, frame.width(), frame.height(), data.std_dev()); if (samples.empty()) { RTC_LOG(LS_ERROR) << "Failed to get sample values for frame"; return; } std::vector<FilteredSample> data_samples = - ConvertSampleValuesToFilteredSamples(data.sample_values, samples); + ConvertSampleValuesToFilteredSamples(data.sample_values(), samples); if (data_samples.empty()) { RTC_LOG(LS_ERROR) << "Failed to convert sample values to filtered samples"; return; } double score = classifier_.CalculateCorruptionProbability( - data_samples, samples, data.luma_error_threshold, - data.chroma_error_threshold); + data_samples, samples, data.luma_error_threshold(), + data.chroma_error_threshold()); observer_->OnCorruptionScore(score, content_type); } diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.h b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation.h @@ -11,9 +11,9 @@ #ifndef VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ #define VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" -#include "common_video/frame_instrumentation_data.h" #include "video/corruption_detection/corruption_classifier.h" #include "video/corruption_detection/halton_frame_sampler.h" diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc @@ -14,10 +14,10 @@ #include <vector> #include "api/scoped_refptr.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/i420_buffer.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" -#include "common_video/frame_instrumentation_data.h" #include "test/gmock.h" #include "test/gtest.h" @@ -52,12 +52,7 @@ scoped_refptr<I420Buffer> MakeI420FrameBufferWithDifferentPixelValues() { TEST(FrameInstrumentationEvaluationTest, HaveNoCorruptionScoreWhenNoSampleValuesAreProvided) { - FrameInstrumentationData data = {.sequence_index = 0, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 0, - .chroma_error_threshold = 0, - .sample_values = {}}; + FrameInstrumentationData data; VideoFrame frame = VideoFrame::Builder() .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) @@ -71,13 +66,9 @@ TEST(FrameInstrumentationEvaluationTest, TEST(FrameInstrumentationEvaluationTest, HaveACorruptionScoreWhenSampleValuesAreProvided) { - FrameInstrumentationData data = { - .sequence_index = 0, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 0, - .chroma_error_threshold = 0, - .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; + FrameInstrumentationData data; + data.SetStdDev(1.0); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); VideoFrame frame = VideoFrame::Builder() .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) @@ -91,13 +82,11 @@ TEST(FrameInstrumentationEvaluationTest, TEST(FrameInstrumentationEvaluationTest, ApplyThresholdsWhenNonNegativeThresholdsAreProvided) { - FrameInstrumentationData data = { - .sequence_index = 0, - .communicate_upper_bits = false, - .std_dev = 1.0, - .luma_error_threshold = 8, - .chroma_error_threshold = 8, - .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; + FrameInstrumentationData data; + data.SetStdDev(1.0); + data.SetLumaErrorThreshold(8); + data.SetChromaErrorThreshold(8); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); VideoFrame frame = VideoFrame::Builder() .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) @@ -111,15 +100,12 @@ TEST(FrameInstrumentationEvaluationTest, TEST(FrameInstrumentationEvaluationTest, ApplyStdDevWhenNonNegativeStdDevIsProvided) { - FrameInstrumentationData data = { - .sequence_index = 0, - .communicate_upper_bits = false, - .std_dev = 0.6, - .luma_error_threshold = 8, - .chroma_error_threshold = 8, - .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; - - std::vector<double> sample_values = {12, 12, 12, 12, 12, 12, 12, 12}; + FrameInstrumentationData data; + data.SetStdDev(0.6); + data.SetLumaErrorThreshold(8); + data.SetChromaErrorThreshold(8); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); + VideoFrame frame = VideoFrame::Builder() .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) @@ -132,15 +118,13 @@ TEST(FrameInstrumentationEvaluationTest, } TEST(FrameInstrumentationEvaluationTest, ApplySequenceIndexWhenProvided) { - FrameInstrumentationData data = { - .sequence_index = 1, - .communicate_upper_bits = false, - .std_dev = 0.6, - .luma_error_threshold = 8, - .chroma_error_threshold = 8, - .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; - - std::vector<double> sample_values = {12, 12, 12, 12, 12, 12, 12, 12}; + FrameInstrumentationData data; + data.SetSequenceIndex(1); + data.SetStdDev(0.6); + data.SetLumaErrorThreshold(8); + data.SetChromaErrorThreshold(8); + data.SetSampleValues({12, 12, 12, 12, 12, 12, 12, 12}); + VideoFrame frame = VideoFrame::Builder() .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.cc @@ -15,17 +15,17 @@ #include <cstdint> #include <iterator> #include <optional> -#include <variant> +#include <utility> #include <vector> #include "absl/algorithm/container.h" #include "api/video/corruption_detection/corruption_detection_filter_settings.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame.h" #include "api/video/video_frame_type.h" #include "api/video_codecs/video_codec.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/include/module_common_types_public.h" #include "modules/video_coding/utility/qp_parser.h" #include "rtc_base/checks.h" @@ -84,15 +84,13 @@ void FrameInstrumentationGenerator::OnCapturedFrame(VideoFrame frame) { captured_frames_.push(frame); } -std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> +std::optional<FrameInstrumentationData> FrameInstrumentationGenerator::OnEncodedImage( const EncodedImage& encoded_image) { uint32_t rtp_timestamp_encoded_image = encoded_image.RtpTimestamp(); std::optional<VideoFrame> captured_frame; int layer_id; - int sequence_index; - bool communicate_upper_bits; + FrameInstrumentationData data; std::vector<HaltonFrameSampler::Coordinates> sample_coordinates; { MutexLock lock(&mutex_); @@ -141,12 +139,9 @@ FrameInstrumentationGenerator::OnEncodedImage( return std::nullopt; } - sequence_index = contexts_[layer_id].frame_sampler.GetCurrentIndex(); - communicate_upper_bits = false; + int sequence_index = contexts_[layer_id].frame_sampler.GetCurrentIndex(); if (is_key_frame) { - communicate_upper_bits = true; // Increase until all the last 7 bits are zeroes. - // If this would overflow to 15 bits, reset to 0. if (sequence_index > 0b0011'1111'1000'0000) { sequence_index = 0; @@ -159,6 +154,8 @@ FrameInstrumentationGenerator::OnEncodedImage( contexts_[layer_id].frame_sampler.SetCurrentIndex(sequence_index); } + RTC_CHECK(data.SetSequenceIndex(sequence_index)); + // TODO: bugs.webrtc.org/358039777 - Maybe allow other sample sizes as well sample_coordinates = contexts_[layer_id] @@ -169,8 +166,7 @@ FrameInstrumentationGenerator::OnEncodedImage( if (!is_key_frame) { return std::nullopt; } - return FrameInstrumentationSyncData{.sequence_index = sequence_index, - .communicate_upper_bits = true}; + return data; } } RTC_DCHECK(captured_frame.has_value()); @@ -182,18 +178,21 @@ FrameInstrumentationGenerator::OnEncodedImage( return std::nullopt; } - FrameInstrumentationData data = { - .sequence_index = sequence_index, - .communicate_upper_bits = communicate_upper_bits, - .std_dev = filter_settings->std_dev, - .luma_error_threshold = filter_settings->luma_error_threshold, - .chroma_error_threshold = filter_settings->chroma_error_threshold}; + RTC_CHECK(data.SetStdDev(filter_settings->std_dev)); + RTC_CHECK(data.SetLumaErrorThreshold(filter_settings->luma_error_threshold)); + RTC_CHECK( + data.SetChromaErrorThreshold(filter_settings->chroma_error_threshold)); + + std::vector<double> plain_values; std::vector<FilteredSample> samples = GetSampleValuesForFrame( *captured_frame, sample_coordinates, encoded_image._encodedWidth, encoded_image._encodedHeight, filter_settings->std_dev); - data.sample_values.reserve(samples.size()); - absl::c_transform(samples, std::back_inserter(data.sample_values), + plain_values.reserve(samples.size()); + absl::c_transform(samples, std::back_inserter(plain_values), [](const FilteredSample& sample) { return sample.value; }); + + RTC_CHECK(data.SetSampleValues(std::move(plain_values))); + return data; } diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.h b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator.h @@ -15,12 +15,11 @@ #include <map> #include <optional> #include <queue> -#include <variant> +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame.h" -#include "common_video/frame_instrumentation_data.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" #include "video/corruption_detection/halton_frame_sampler.h" @@ -39,9 +38,8 @@ class FrameInstrumentationGenerator { ~FrameInstrumentationGenerator() = default; void OnCapturedFrame(VideoFrame frame) RTC_LOCKS_EXCLUDED(mutex_); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - OnEncodedImage(const EncodedImage& encoded_image) RTC_LOCKS_EXCLUDED(mutex_); + std::optional<FrameInstrumentationData> OnEncodedImage( + const EncodedImage& encoded_image) RTC_LOCKS_EXCLUDED(mutex_); // Returns `std::nullopt` if there is no context for the given layer. std::optional<int> GetHaltonSequenceIndex(int layer_id) const diff --git a/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator_unittest.cc b/third_party/libwebrtc/video/corruption_detection/frame_instrumentation_generator_unittest.cc @@ -13,25 +13,26 @@ #include <cstdint> #include <memory> #include <optional> -#include <variant> #include <vector> #include "api/make_ref_counted.h" #include "api/scoped_refptr.h" #include "api/video/corruption_detection/corruption_detection_filter_settings.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame.h" #include "api/video/video_frame_type.h" -#include "common_video/frame_instrumentation_data.h" #include "rtc_base/ref_counted_object.h" #include "test/gmock.h" #include "test/gtest.h" namespace webrtc { namespace { +using ::testing::DoubleEq; using ::testing::ElementsAre; +using ::testing::Pointwise; constexpr int kDefaultScaledWidth = 4; constexpr int kDefaultScaledHeight = 4; @@ -165,20 +166,15 @@ TEST(FrameInstrumentationGeneratorTest, encoded_image._encodedHeight = kDefaultScaledHeight; generator.OnCapturedFrame(frame); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data = generator.OnEncodedImage(encoded_image); - - ASSERT_TRUE(data.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data)); - FrameInstrumentationData frame_instrumentation_data = - std::get<FrameInstrumentationData>(*data); - EXPECT_EQ(frame_instrumentation_data.sequence_index, 0); - EXPECT_TRUE(frame_instrumentation_data.communicate_upper_bits); - EXPECT_NE(frame_instrumentation_data.std_dev, 0.0); - EXPECT_NE(frame_instrumentation_data.luma_error_threshold, 0); - EXPECT_NE(frame_instrumentation_data.chroma_error_threshold, 0); - EXPECT_FALSE(frame_instrumentation_data.sample_values.empty()); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator.OnEncodedImage(encoded_image); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); + EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); + EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); + EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); + EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); } TEST(FrameInstrumentationGeneratorTest, @@ -205,20 +201,15 @@ TEST(FrameInstrumentationGeneratorTest, encoded_image._encodedHeight = kDefaultScaledHeight; generator.OnCapturedFrame(frame); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data = generator.OnEncodedImage(encoded_image); - - ASSERT_TRUE(data.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data)); - FrameInstrumentationData frame_instrumentation_data = - std::get<FrameInstrumentationData>(*data); - EXPECT_EQ(frame_instrumentation_data.sequence_index, 0); - EXPECT_TRUE(frame_instrumentation_data.communicate_upper_bits); - EXPECT_NE(frame_instrumentation_data.std_dev, 0.0); - EXPECT_NE(frame_instrumentation_data.luma_error_threshold, 0); - EXPECT_NE(frame_instrumentation_data.chroma_error_threshold, 0); - EXPECT_FALSE(frame_instrumentation_data.sample_values.empty()); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator.OnEncodedImage(encoded_image); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); + EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); + EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); + EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); + EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); } TEST(FrameInstrumentationGeneratorTest, @@ -247,20 +238,15 @@ TEST(FrameInstrumentationGeneratorTest, generator.OnCapturedFrame(frame); generator.OnEncodedImage(encoded_image1); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data = generator.OnEncodedImage(encoded_image2); - - ASSERT_TRUE(data.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data)); - FrameInstrumentationData frame_instrumentation_data = - std::get<FrameInstrumentationData>(*data); - EXPECT_EQ(frame_instrumentation_data.sequence_index, 0); - EXPECT_TRUE(frame_instrumentation_data.communicate_upper_bits); - EXPECT_NE(frame_instrumentation_data.std_dev, 0.0); - EXPECT_NE(frame_instrumentation_data.luma_error_threshold, 0); - EXPECT_NE(frame_instrumentation_data.chroma_error_threshold, 0); - EXPECT_FALSE(frame_instrumentation_data.sample_values.empty()); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator.OnEncodedImage(encoded_image2); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->sequence_index(), 0); + EXPECT_NE(frame_instrumentation_data->std_dev(), 0.0); + EXPECT_NE(frame_instrumentation_data->luma_error_threshold(), 0); + EXPECT_NE(frame_instrumentation_data->chroma_error_threshold(), 0); + EXPECT_FALSE(frame_instrumentation_data->sample_values().empty()); } TEST(FrameInstrumentationGeneratorTest, @@ -329,24 +315,17 @@ TEST(FrameInstrumentationGeneratorTest, generator.OnCapturedFrame(frame); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data1 = generator.OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data1 = + generator.OnEncodedImage(encoded_image1); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data2 = generator.OnEncodedImage(encoded_image2); + std::optional<FrameInstrumentationData> data2 = + generator.OnEncodedImage(encoded_image2); ASSERT_TRUE(data1.has_value()); ASSERT_TRUE(data2.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data1)); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data2)); - - EXPECT_TRUE( - std::get<FrameInstrumentationData>(*data1).communicate_upper_bits); - EXPECT_TRUE( - std::get<FrameInstrumentationData>(*data2).communicate_upper_bits); + EXPECT_TRUE(data1->holds_upper_bits()); + EXPECT_TRUE(data2->holds_upper_bits()); } } @@ -382,35 +361,24 @@ TEST(FrameInstrumentationGeneratorTest, generator.OnCapturedFrame(frame); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data1 = generator.OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data1 = + generator.OnEncodedImage(encoded_image1); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data2 = generator.OnEncodedImage(encoded_image2); + std::optional<FrameInstrumentationData> data2 = + generator.OnEncodedImage(encoded_image2); ASSERT_TRUE(data1.has_value()); ASSERT_TRUE(data2.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data1)); - - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data2)); - FrameInstrumentationData frame_instrumentation_data1 = - std::get<FrameInstrumentationData>(*data1); - FrameInstrumentationData frame_instrumentation_data2 = - std::get<FrameInstrumentationData>(*data2); + EXPECT_TRUE(data1->holds_upper_bits()); + EXPECT_TRUE(data2->holds_upper_bits()); - EXPECT_TRUE(frame_instrumentation_data1.communicate_upper_bits); - EXPECT_TRUE(frame_instrumentation_data2.communicate_upper_bits); - - EXPECT_EQ(frame_instrumentation_data1.sequence_index, - frame_instrumentation_data2.sequence_index); + EXPECT_EQ(data1->sequence_index(), data2->sequence_index()); // In the test the frames have equal frame buffers so the sample values // should be equal. - EXPECT_THAT(frame_instrumentation_data1.sample_values, - frame_instrumentation_data2.sample_values); + EXPECT_THAT(data1->sample_values(), + Pointwise(DoubleEq(), data2->sample_values())); } } @@ -445,35 +413,24 @@ TEST(FrameInstrumentationGeneratorTest, generator.OnCapturedFrame(frame); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data1 = generator.OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data1 = + generator.OnEncodedImage(encoded_image1); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data2 = generator.OnEncodedImage(encoded_image2); + std::optional<FrameInstrumentationData> data2 = + generator.OnEncodedImage(encoded_image2); if (i == 0) { ASSERT_TRUE(data1.has_value()); ASSERT_TRUE(data2.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data1)); - - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data2)); - EXPECT_TRUE( - std::get<FrameInstrumentationData>(*data1).communicate_upper_bits); - EXPECT_TRUE( - std::get<FrameInstrumentationData>(*data2).communicate_upper_bits); + EXPECT_TRUE(data1->holds_upper_bits()); + EXPECT_TRUE(data2->holds_upper_bits()); } else if (data1.has_value() || data2.has_value()) { if (data1.has_value()) { - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data1)); - EXPECT_FALSE( - std::get<FrameInstrumentationData>(*data1).communicate_upper_bits); + EXPECT_FALSE(data1->holds_upper_bits()); } if (data2.has_value()) { - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data2)); - EXPECT_FALSE( - std::get<FrameInstrumentationData>(*data2).communicate_upper_bits); + EXPECT_FALSE(data2->holds_upper_bits()); } has_found_delta_frame = true; } @@ -517,25 +474,16 @@ TEST(FrameInstrumentationGeneratorTest, generator.SetHaltonSequenceIndex(0b0010'1010, generator.GetLayerId(encoded_image1)); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data1 = generator.OnEncodedImage(encoded_image1); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data2 = generator.OnEncodedImage(encoded_image2); + std::optional<FrameInstrumentationData> data1 = + generator.OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data2 = + generator.OnEncodedImage(encoded_image2); ASSERT_TRUE(data1.has_value()); ASSERT_TRUE(data2.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data1)); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data2)); - FrameInstrumentationData frame_instrumentation_data1 = - std::get<FrameInstrumentationData>(*data1); - FrameInstrumentationData frame_instrumentation_data2 = - std::get<FrameInstrumentationData>(*data2); - - EXPECT_EQ(frame_instrumentation_data1.sequence_index, 0b0000'1000'0000); - EXPECT_EQ(frame_instrumentation_data2.sequence_index, 0b0001'0000'0000); + EXPECT_EQ(data1->sequence_index(), 0b0000'1000'0000); + EXPECT_EQ(data2->sequence_index(), 0b0001'0000'0000); } TEST(FrameInstrumentationGeneratorTest, @@ -574,25 +522,16 @@ TEST(FrameInstrumentationGeneratorTest, generator.GetLayerId(encoded_image2)); generator.SetHaltonSequenceIndex(0b11'1111'1111'1111, generator.GetLayerId(encoded_image1)); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data1 = generator.OnEncodedImage(encoded_image1); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data2 = generator.OnEncodedImage(encoded_image2); + std::optional<FrameInstrumentationData> data1 = + generator.OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data2 = + generator.OnEncodedImage(encoded_image2); ASSERT_TRUE(data1.has_value()); ASSERT_TRUE(data2.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data1)); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data2)); - - FrameInstrumentationData frame_instrumentation_data1 = - std::get<FrameInstrumentationData>(*data1); - FrameInstrumentationData frame_instrumentation_data2 = - std::get<FrameInstrumentationData>(*data2); - EXPECT_EQ(frame_instrumentation_data1.sequence_index, 0); - EXPECT_EQ(frame_instrumentation_data2.sequence_index, 0b1000'0000); + EXPECT_EQ(data1->sequence_index(), 0); + EXPECT_EQ(data2->sequence_index(), 0b1000'0000); } TEST(FrameInstrumentationGeneratorTest, @@ -631,25 +570,16 @@ TEST(FrameInstrumentationGeneratorTest, generator.SetHaltonSequenceIndex(0b1000'0000, generator.GetLayerId(encoded_image1)); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data1 = generator.OnEncodedImage(encoded_image1); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data2 = generator.OnEncodedImage(encoded_image2); + std::optional<FrameInstrumentationData> data1 = + generator.OnEncodedImage(encoded_image1); + std::optional<FrameInstrumentationData> data2 = + generator.OnEncodedImage(encoded_image2); ASSERT_TRUE(data1.has_value()); ASSERT_TRUE(data2.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data1)); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data2)); - FrameInstrumentationData frame_instrumentation_data1 = - std::get<FrameInstrumentationData>(*data1); - FrameInstrumentationData frame_instrumentation_data2 = - std::get<FrameInstrumentationData>(*data2); - - EXPECT_EQ(frame_instrumentation_data1.sequence_index, 0b0000'1000'0000); - EXPECT_EQ(frame_instrumentation_data2.sequence_index, 0b0001'0000'0000); + EXPECT_EQ(data1->sequence_index(), 0b0000'1000'0000); + EXPECT_EQ(data2->sequence_index(), 0b0001'0000'0000); } TEST(FrameInstrumentationGeneratorTest, GetterAndSetterOperatesAsExpected) { @@ -734,17 +664,13 @@ TEST(FrameInstrumentationGeneratorTest, .chroma_error_threshold = 3}); generator.OnCapturedFrame(frame); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data = generator.OnEncodedImage(encoded_image); - - ASSERT_TRUE(data.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data)); - FrameInstrumentationData frame_instrumentation_data = - std::get<FrameInstrumentationData>(*data); - EXPECT_EQ(frame_instrumentation_data.std_dev, 1.0); - EXPECT_EQ(frame_instrumentation_data.luma_error_threshold, 2); - EXPECT_EQ(frame_instrumentation_data.chroma_error_threshold, 3); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + generator.OnEncodedImage(encoded_image); + + ASSERT_TRUE(frame_instrumentation_data.has_value()); + EXPECT_EQ(frame_instrumentation_data->std_dev(), 1.0); + EXPECT_EQ(frame_instrumentation_data->luma_error_threshold(), 2); + EXPECT_EQ(frame_instrumentation_data->chroma_error_threshold(), 3); } } // namespace diff --git a/third_party/libwebrtc/video/rtp_video_stream_receiver2.cc b/third_party/libwebrtc/video/rtp_video_stream_receiver2.cc @@ -19,7 +19,6 @@ #include <optional> #include <string> #include <utility> -#include <variant> #include <vector> #include "absl/algorithm/container.h" @@ -51,8 +50,6 @@ #include "call/rtp_packet_sink_interface.h" #include "call/syncable.h" #include "call/video_receive_stream.h" -#include "common_video/corruption_detection_converters.h" -#include "common_video/frame_instrumentation_data.h" #include "media/base/media_constants.h" #include "modules/include/module_common_types.h" #include "modules/pacing/packet_router.h" @@ -552,25 +549,6 @@ RtpVideoStreamReceiver2::ParseGenericDependenciesExtension( return kHasGenericDescriptor; } -void RtpVideoStreamReceiver2::SetLastCorruptionDetectionIndex( - const std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>& - frame_instrumentation_data, - int spatial_idx) { - RTC_CHECK_GE(spatial_idx, 0); - RTC_CHECK_LT(spatial_idx, kMaxSpatialLayers); - if (const auto* sync_data = std::get_if<FrameInstrumentationSyncData>( - &frame_instrumentation_data)) { - last_corruption_detection_state_by_layer_[spatial_idx].sequence_index = - sync_data->sequence_index; - } else if (const auto* data = std::get_if<FrameInstrumentationData>( - &frame_instrumentation_data)) { - last_corruption_detection_state_by_layer_[spatial_idx].sequence_index = - data->sequence_index + data->sample_values.size(); - } else { - RTC_DCHECK_NOTREACHED(); - } -} - bool RtpVideoStreamReceiver2::OnReceivedPayloadData( CopyOnWriteBuffer codec_payload, const RtpPacketReceived& rtp_packet, @@ -675,7 +653,7 @@ bool RtpVideoStreamReceiver2::OnReceivedPayloadData( spatial_id = video_header.generic->spatial_index; if (spatial_id >= kMaxSpatialLayers) { RTC_LOG(LS_WARNING) << "Invalid spatial id: " << *spatial_id - << ". Ignoring corruption detection mesaage."; + << " in generic descriptor."; spatial_id.reset(); } } else { @@ -686,33 +664,9 @@ bool RtpVideoStreamReceiver2::OnReceivedPayloadData( rtp_packet.GetExtension<CorruptionDetectionExtension>(); if (message.has_value() && spatial_id.has_value()) { RTC_CHECK_GE(*spatial_id, 0); - if (message->sample_values().empty()) { - video_header.frame_instrumentation_data = - ConvertCorruptionDetectionMessageToFrameInstrumentationSyncData( - *message, last_corruption_detection_state_by_layer_[*spatial_id] - .sequence_index); - } else { - // `OnReceivedPayloadData` might be called several times, however, we - // don't want to increase the sequence index each time. - if (!last_corruption_detection_state_by_layer_[*spatial_id] - .timestamp.has_value() || - rtp_packet.Timestamp() != - last_corruption_detection_state_by_layer_[*spatial_id] - .timestamp) { - video_header.frame_instrumentation_data = - ConvertCorruptionDetectionMessageToFrameInstrumentationData( - *message, - last_corruption_detection_state_by_layer_[*spatial_id] - .sequence_index); - last_corruption_detection_state_by_layer_[*spatial_id].timestamp = - rtp_packet.Timestamp(); - } - } - - if (video_header.frame_instrumentation_data.has_value()) { - SetLastCorruptionDetectionIndex( - *video_header.frame_instrumentation_data, *spatial_id); - } + video_header.frame_instrumentation_data = + last_corruption_detection_state_by_layer_[*spatial_id].ParseMessage( + *message); } } video_header.video_frame_tracking_id = diff --git a/third_party/libwebrtc/video/rtp_video_stream_receiver2.h b/third_party/libwebrtc/video/rtp_video_stream_receiver2.h @@ -17,7 +17,6 @@ #include <map> #include <memory> #include <optional> -#include <variant> #include <vector> #include "api/array_view.h" @@ -34,13 +33,13 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data_reader.h" #include "api/video/encoded_frame.h" #include "api/video/video_codec_constants.h" #include "api/video/video_codec_type.h" #include "call/rtp_packet_sink_interface.h" #include "call/syncable.h" #include "call/video_receive_stream.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/include/module_common_types.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/recovered_packet_receiver.h" @@ -341,10 +340,6 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, void UpdatePacketReceiveTimestamps(const RtpPacketReceived& packet, bool is_keyframe) RTC_RUN_ON(packet_sequence_checker_); - void SetLastCorruptionDetectionIndex( - const std::variant<FrameInstrumentationSyncData, - FrameInstrumentationData>& frame_instrumentation_data, - int spatial_idx); std::optional<VideoCodecType> GetCodecFromPayloadType( uint8_t payload_type) const RTC_RUN_ON(packet_sequence_checker_); @@ -490,11 +485,8 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, Timestamp::MinusInfinity(); bool sps_pps_idr_is_h264_keyframe_ = false; - struct CorruptionDetectionLayerState { - int sequence_index = 0; - std::optional<uint32_t> timestamp; - }; - std::array<CorruptionDetectionLayerState, kMaxSpatialLayers> + // TODO: bugs.webrtc.org/358039777 - Move this to after the frame assembler. + std::array<FrameInstrumentationDataReader, kMaxSpatialLayers> last_corruption_detection_state_by_layer_; }; diff --git a/third_party/libwebrtc/video/rtp_video_stream_receiver2_unittest.cc b/third_party/libwebrtc/video/rtp_video_stream_receiver2_unittest.cc @@ -16,7 +16,6 @@ #include <memory> #include <optional> #include <string> -#include <variant> #include <vector> #include "api/array_view.h" @@ -35,6 +34,7 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame_type.h" @@ -42,7 +42,6 @@ #include "call/rtp_packet_sink_interface.h" #include "call/test/mock_rtp_packet_sink_interface.h" #include "call/video_receive_stream.h" -#include "common_video/frame_instrumentation_data.h" #include "common_video/h264/h264_common.h" #include "media/base/media_constants.h" #include "modules/include/module_common_types.h" @@ -493,12 +492,20 @@ TEST_F(RtpVideoStreamReceiver2Test, // Have corruption header on the key frame. RtpPacketReceived key_frame_packet = received_packet_generator.NextPacket(/*include_corruption_header=*/true); + // Generate delta frame packet. received_packet_generator.SetPayload(kDeltaFramePayload, VideoFrameType::kVideoFrameDelta); - // Don't have corruption header on the delta frame (is not a general rule). + + // Second message sets the LSB instead of MSB, increasing the counter by the + // number of samples in the first message. + corruption_detection_msg = GetCorruptionDetectionMessage( + /*sequence_idx=*/kNumSamples, /*interpret_as_MSB*/ false); + ASSERT_TRUE(corruption_detection_msg.has_value()); + received_packet_generator.SetCorruptionDetectionHeader( + *corruption_detection_msg); RtpPacketReceived delta_frame_packet = - received_packet_generator.NextPacket(/*include_corruption_header=*/false); + received_packet_generator.NextPacket(/*include_corruption_header=*/true); rtp_video_stream_receiver_->StartReceive(); mock_on_complete_frame_callback_.AppendExpectedBitstream( @@ -512,26 +519,20 @@ TEST_F(RtpVideoStreamReceiver2Test, }); rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet); ASSERT_TRUE(key_encoded_frame != nullptr); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data_key_frame = - key_encoded_frame->CodecSpecific()->frame_instrumentation_data; - ASSERT_TRUE(data_key_frame.has_value()); - ASSERT_TRUE( - std::holds_alternative<FrameInstrumentationData>(*data_key_frame)); - FrameInstrumentationData frame_inst_data_key_frame = - std::get<FrameInstrumentationData>(*data_key_frame); - EXPECT_EQ(frame_inst_data_key_frame.sequence_index, 0); - EXPECT_TRUE(frame_inst_data_key_frame.communicate_upper_bits); - EXPECT_THAT(frame_inst_data_key_frame.std_dev, DoubleNear(kStd, 0.1)); - EXPECT_EQ(frame_inst_data_key_frame.luma_error_threshold, kLumaThreshold); - EXPECT_EQ(frame_inst_data_key_frame.chroma_error_threshold, kChormaThreshold); + std::optional<FrameInstrumentationData> frame_inst_data_key_frame = + key_encoded_frame->CodecSpecific()->frame_instrumentation_data; + ASSERT_TRUE(frame_inst_data_key_frame.has_value()); + EXPECT_EQ(frame_inst_data_key_frame->sequence_index(), 0); + EXPECT_THAT(frame_inst_data_key_frame->std_dev(), DoubleNear(kStd, 0.1)); + EXPECT_EQ(frame_inst_data_key_frame->luma_error_threshold(), kLumaThreshold); + EXPECT_EQ(frame_inst_data_key_frame->chroma_error_threshold(), + kChormaThreshold); mock_on_complete_frame_callback_.ClearExpectedBitstream(); mock_on_complete_frame_callback_.AppendExpectedBitstream( kDeltaFramePayload.data(), kDeltaFramePayload.size()); - EXPECT_FALSE(delta_frame_packet.GetExtension<CorruptionDetectionExtension>()); + EXPECT_TRUE(delta_frame_packet.GetExtension<CorruptionDetectionExtension>()); std::unique_ptr<EncodedFrame> delta_encoded_frame; EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) .WillOnce([&](EncodedFrame* encoded_frame) { @@ -539,10 +540,18 @@ TEST_F(RtpVideoStreamReceiver2Test, }); rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet); ASSERT_TRUE(delta_encoded_frame != nullptr); - // Not delta frame specific but as this test is designed, second frame - // shouldnt have corruption header. - EXPECT_FALSE(delta_encoded_frame->CodecSpecific() - ->frame_instrumentation_data.has_value()); + ASSERT_TRUE(delta_encoded_frame->CodecSpecific() + ->frame_instrumentation_data.has_value()); + + std::optional<FrameInstrumentationData> frame_inst_data_delta_frame = + delta_encoded_frame->CodecSpecific()->frame_instrumentation_data; + ASSERT_TRUE(frame_inst_data_delta_frame.has_value()); + EXPECT_EQ(frame_inst_data_delta_frame->sequence_index(), kNumSamples); + EXPECT_THAT(frame_inst_data_delta_frame->std_dev(), DoubleNear(kStd, 0.1)); + EXPECT_EQ(frame_inst_data_delta_frame->luma_error_threshold(), + kLumaThreshold); + EXPECT_EQ(frame_inst_data_delta_frame->chroma_error_threshold(), + kChormaThreshold); } TEST_F(RtpVideoStreamReceiver2Test, @@ -608,17 +617,13 @@ TEST_F(RtpVideoStreamReceiver2Test, }); rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet); ASSERT_TRUE(delta_encoded_frame != nullptr); - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - data = delta_encoded_frame->CodecSpecific()->frame_instrumentation_data; - ASSERT_TRUE(data.has_value()); - ASSERT_TRUE(std::holds_alternative<FrameInstrumentationData>(*data)); - FrameInstrumentationData frame_inst_data = - std::get<FrameInstrumentationData>(*data); - if (frame_inst_data.sequence_index < (kMaxSequenceIdx + 1)) { - EXPECT_EQ(frame_inst_data.sequence_index, sequence_idx); + std::optional<FrameInstrumentationData> frame_inst_data = + delta_encoded_frame->CodecSpecific()->frame_instrumentation_data; + ASSERT_TRUE(frame_inst_data.has_value()); + if (frame_inst_data->sequence_index() < (kMaxSequenceIdx + 1)) { + EXPECT_EQ(frame_inst_data->sequence_index(), sequence_idx); } else { - EXPECT_EQ(frame_inst_data.sequence_index, + EXPECT_EQ(frame_inst_data->sequence_index(), sequence_idx + kMaxSequenceIdx + 1); } } diff --git a/third_party/libwebrtc/video/video_receive_stream2.cc b/third_party/libwebrtc/video/video_receive_stream2.cc @@ -42,6 +42,7 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/encoded_image.h" #include "api/video/recordable_encoded_frame.h" @@ -64,7 +65,6 @@ #include "call/rtx_receive_stream.h" #include "call/syncable.h" #include "call/video_receive_stream.h" -#include "common_video/frame_instrumentation_data.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" diff --git a/third_party/libwebrtc/video/video_receive_stream2.h b/third_party/libwebrtc/video/video_receive_stream2.h @@ -32,6 +32,7 @@ #include "api/transport/rtp/rtp_source.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_frame.h" #include "api/video/recordable_encoded_frame.h" #include "api/video/video_content_type.h" @@ -41,7 +42,6 @@ #include "call/rtp_packet_sink_interface.h" #include "call/syncable.h" #include "call/video_receive_stream.h" -#include "common_video/frame_instrumentation_data.h" #include "common_video/include/corruption_score_calculator.h" #include "modules/include/module_common_types.h" #include "modules/rtp_rtcp/source/source_tracker.h" diff --git a/third_party/libwebrtc/video/video_stream_encoder.cc b/third_party/libwebrtc/video/video_stream_encoder.cc @@ -18,7 +18,6 @@ #include <string> #include <tuple> #include <utility> -#include <variant> #include <vector> #include "absl/algorithm/container.h" @@ -39,6 +38,7 @@ #include "api/units/data_size.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/render_resolution.h" #include "api/video/video_adaptation_counters.h" @@ -65,7 +65,6 @@ #include "call/adaptation/resource_adaptation_processor.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_adapter.h" -#include "common_video/frame_instrumentation_data.h" #include "media/base/media_channel.h" #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/include/video_codec_initializer.h" @@ -2254,10 +2253,8 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( std::unique_ptr<CodecSpecificInfo> codec_specific_info_copy; if (codec_specific_info && frame_instrumentation_generator_) { - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - frame_instrumentation_data = - frame_instrumentation_generator_->OnEncodedImage(image_copy); + std::optional<FrameInstrumentationData> frame_instrumentation_data = + frame_instrumentation_generator_->OnEncodedImage(image_copy); RTC_CHECK(!codec_specific_info->frame_instrumentation_data.has_value()) << "CodecSpecificInfo must not have frame_instrumentation_data set."; if (frame_instrumentation_data.has_value()) { diff --git a/third_party/libwebrtc/video/video_stream_encoder_unittest.cc b/third_party/libwebrtc/video/video_stream_encoder_unittest.cc @@ -18,7 +18,6 @@ #include <optional> #include <string> #include <utility> -#include <variant> #include <vector> #include "absl/algorithm/container.h" @@ -46,6 +45,7 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/corruption_detection/frame_instrumentation_data.h" #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/nv12_buffer.h" @@ -78,7 +78,6 @@ #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_adapter.h" #include "call/video_send_stream.h" -#include "common_video/frame_instrumentation_data.h" #include "common_video/h264/h264_common.h" #include "media/base/video_adapter.h" #include "media/engine/webrtc_video_engine.h" @@ -1553,9 +1552,8 @@ class VideoStreamEncoderTest : public ::testing::Test { return number_of_layers_allocations_; } - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - GetLastFrameInstrumentationData() const { + std::optional<FrameInstrumentationData> GetLastFrameInstrumentationData() + const { MutexLock lock(&mutex_); return last_frame_instrumentation_data_; } @@ -1654,9 +1652,8 @@ class VideoStreamEncoderTest : public ::testing::Test { int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0; VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_); int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0; - std::optional< - std::variant<FrameInstrumentationSyncData, FrameInstrumentationData>> - last_frame_instrumentation_data_ RTC_GUARDED_BY(&mutex_); + std::optional<FrameInstrumentationData> last_frame_instrumentation_data_ + RTC_GUARDED_BY(&mutex_); }; class VideoBitrateAllocatorProxyFactory