commit 86e89286a5a64899e5c7207dc3642a50b9b3d94b parent 365cd48b044b0921e15fc554ba55bbbec5882971 Author: Dan Baker <dbaker@mozilla.com> Date: Mon, 1 Dec 2025 22:31:34 -0700 Bug 2000941 - Vendor libwebrtc from d3fcfe01c9 Upstream commit: https://webrtc.googlesource.com/src/+/d3fcfe01c9b583fc7b7a9a403b91dcdca0f8c309 Support mixed-codec simulcast in SimulcastEncoderAdapter SimulcastEncoderAdapter now allows specifying different codecs for each stream. To support this, video format has been added to both SimulcastStream and VideoStream. Bug: webrtc:362277533 Change-Id: I1700022021c704c608384c257a071ab6346c9bbe Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/387320 Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Commit-Queue: Henrik Boström <hbos@webrtc.org> Reviewed-by: Sergey Silkin <ssilkin@webrtc.org> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Henrik Boström <hbos@webrtc.org> Cr-Commit-Position: refs/heads/main@{#45694} Diffstat:
19 files changed, 515 insertions(+), 34 deletions(-)
diff --git a/third_party/libwebrtc/README.mozilla.last-vendor b/third_party/libwebrtc/README.mozilla.last-vendor @@ -1,4 +1,4 @@ # ./mach python dom/media/webrtc/third_party_build/vendor-libwebrtc.py --from-local /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc --commit mozpatches libwebrtc -libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-12-02T05:28:57.082727+00:00. +libwebrtc updated from /Users/danielbaker/elm/.moz-fast-forward/moz-libwebrtc commit mozpatches on 2025-12-02T05:31:18.899684+00:00. # base of lastest vendoring -824fad7309 +d3fcfe01c9 diff --git a/third_party/libwebrtc/api/video_codecs/simulcast_stream.h b/third_party/libwebrtc/api/video_codecs/simulcast_stream.h @@ -14,6 +14,7 @@ #include <optional> #include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/sdp_video_format.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -41,6 +42,10 @@ struct RTC_EXPORT SimulcastStream { unsigned int minBitrate = 0; // kilobits/sec. unsigned int qpMax = 0; // minimum quality bool active = false; // encoded and sent. + // The video format for this stream. + // This should be set for mixed-codec simulcast, while for other cases, + // it is optional and can be unset. + std::optional<SdpVideoFormat> format; }; } // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/test/BUILD.gn b/third_party/libwebrtc/api/video_codecs/test/BUILD.gn @@ -15,6 +15,7 @@ if (rtc_include_tests) { "builtin_video_encoder_factory_unittest.cc", "h264_profile_level_id_unittest.cc", "sdp_video_format_unittest.cc", + "video_codec_unittest.cc", "video_decoder_software_fallback_wrapper_unittest.cc", "video_encoder_software_fallback_wrapper_unittest.cc", ] diff --git a/third_party/libwebrtc/api/video_codecs/test/video_codec_unittest.cc b/third_party/libwebrtc/api/video_codecs/test/video_codec_unittest.cc @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2025 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_codec.h" + +#include <cstddef> +#include <optional> +#include <vector> + +#include "api/video/video_codec_type.h" +#include "api/video_codecs/sdp_video_format.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +VideoCodec CreateVideoCodecForMixedCodec( + std::optional<VideoCodecType> codec_type, + std::vector<bool> active_streams, + std::vector<std::optional<SdpVideoFormat>> formats) { + RTC_DCHECK(active_streams.size() == formats.size()); + + VideoCodec codec; + if (codec_type) { + codec.codecType = *codec_type; + } + codec.numberOfSimulcastStreams = static_cast<unsigned char>(formats.size()); + for (size_t i = 0; i < formats.size(); ++i) { + codec.simulcastStream[i].active = active_streams[i]; + codec.simulcastStream[i].format = formats[i]; + } + return codec; +} + +} // namespace + +TEST(VideoCodecTest, TestIsMixedCodec) { + VideoCodec codec; + + // Non mixed-codec cases + codec = CreateVideoCodecForMixedCodec(std::nullopt, {}, {}); + EXPECT_FALSE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec(std::nullopt, {true}, + {SdpVideoFormat::VP8()}); + EXPECT_FALSE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec( + std::nullopt, {true, true}, + {SdpVideoFormat::VP8(), SdpVideoFormat::VP8()}); + EXPECT_FALSE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec( + std::nullopt, {true, true, true}, + {SdpVideoFormat::VP8(), SdpVideoFormat::VP8(), SdpVideoFormat::VP8()}); + EXPECT_FALSE(codec.IsMixedCodec()); + + // Mixed-codec cases + codec = CreateVideoCodecForMixedCodec( + std::nullopt, {true, true}, + {SdpVideoFormat::VP8(), SdpVideoFormat::VP9Profile0()}); + EXPECT_TRUE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec( + std::nullopt, {true, true}, + {SdpVideoFormat::VP9Profile0(), SdpVideoFormat::VP9Profile1()}); + EXPECT_TRUE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec( + std::nullopt, {true, true, true}, + {SdpVideoFormat::VP9Profile0(), SdpVideoFormat::VP9Profile1(), + SdpVideoFormat::VP9Profile0()}); + EXPECT_TRUE(codec.IsMixedCodec()); + + // If formats are only partially set, it will never be a mixed-codec + codec = CreateVideoCodecForMixedCodec(kVideoCodecVP8, {true, true}, + {std::nullopt, std::nullopt}); + EXPECT_FALSE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec( + kVideoCodecVP8, {true, true, true}, + {SdpVideoFormat::VP8(), std::nullopt, SdpVideoFormat::VP9Profile0()}); + EXPECT_FALSE(codec.IsMixedCodec()); + + // The format of non-active streams are ignored + codec = CreateVideoCodecForMixedCodec( + kVideoCodecVP8, {false, true, true}, + {std::nullopt, SdpVideoFormat::VP8(), SdpVideoFormat::VP9Profile0()}); + EXPECT_TRUE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec( + kVideoCodecVP9, {true, false, true}, + {SdpVideoFormat::VP8(), std::nullopt, SdpVideoFormat::VP9Profile0()}); + EXPECT_TRUE(codec.IsMixedCodec()); + + codec = CreateVideoCodecForMixedCodec( + kVideoCodecVP8, {true, true, false}, + {SdpVideoFormat::VP8(), SdpVideoFormat::VP8(), + SdpVideoFormat::VP9Profile0()}); + EXPECT_FALSE(codec.IsMixedCodec()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_codec.cc b/third_party/libwebrtc/api/video_codecs/video_codec.cc @@ -11,11 +11,13 @@ #include "api/video_codecs/video_codec.h" #include <cstring> +#include <optional> #include <string> #include "absl/strings/match.h" #include "api/video/video_codec_type.h" #include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/simulcast_stream.h" #include "rtc_base/checks.h" #include "rtc_base/strings/string_builder.h" @@ -191,4 +193,22 @@ void VideoCodec::SetFrameDropEnabled(bool enabled) { frame_drop_enabled_ = enabled; } +bool VideoCodec::IsMixedCodec() const { + std::optional<SdpVideoFormat> first_format; + for (size_t i = 0; i < numberOfSimulcastStreams; ++i) { + if (!simulcastStream[i].active) { + continue; + } + if (!simulcastStream[i].format.has_value()) { + return false; // Format is always set for active layers in mixed-codec. + } + if (!first_format.has_value()) { + first_format = simulcastStream[i].format; // First active layer's format. + } else if (!first_format->IsSameCodec(*simulcastStream[i].format)) { + return true; + } + } + return false; +} + } // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_codec.h b/third_party/libwebrtc/api/video_codecs/video_codec.h @@ -143,6 +143,8 @@ class RTC_EXPORT VideoCodec { bool IsSinglecast() const { return numberOfSimulcastStreams <= 1; } bool IsSimulcast() const { return !IsSinglecast(); } + // Returns true if the codec is a mixed-codec simulcast. + bool IsMixedCodec() const; // Public variables. TODO(hta): Make them private with accessors. VideoCodecType codecType; diff --git a/third_party/libwebrtc/media/engine/simulcast_encoder_adapter.cc b/third_party/libwebrtc/media/engine/simulcast_encoder_adapter.cc @@ -161,11 +161,13 @@ SimulcastEncoderAdapter::EncoderContext::EncoderContext( std::unique_ptr<VideoEncoder> encoder, bool prefer_temporal_support, VideoEncoder::EncoderInfo primary_info, - VideoEncoder::EncoderInfo fallback_info) + VideoEncoder::EncoderInfo fallback_info, + SdpVideoFormat video_format) : encoder_(std::move(encoder)), prefer_temporal_support_(prefer_temporal_support), primary_info_(std::move(primary_info)), - fallback_info_(std::move(fallback_info)) {} + fallback_info_(std::move(fallback_info)), + video_format_(std::move(video_format)) {} void SimulcastEncoderAdapter::EncoderContext::Release() { if (encoder_) { @@ -343,11 +345,16 @@ int SimulcastEncoderAdapter::InitEncode( std::unique_ptr<EncoderContext> encoder_context = FetchOrCreateEncoderContext( /*is_lowest_quality_stream=*/( is_legacy_singlecast || - codec_.simulcastStream[lowest_quality_stream_idx].active)); + codec_.simulcastStream[lowest_quality_stream_idx].active), + /*stream_idx=*/is_legacy_singlecast + ? std::nullopt + : std::make_optional(lowest_quality_stream_idx)); if (encoder_context == nullptr) { return WEBRTC_VIDEO_CODEC_MEMORY; } + bool is_mixed_codec = codec_.IsMixedCodec(); + // Two distinct scenarios: // * Singlecast (total_streams_count == 1) or simulcast with simulcast-capable // underlaying encoder implementation if active_streams_count > 1. SEA @@ -365,6 +372,7 @@ int SimulcastEncoderAdapter::InitEncode( // forces the use of SEA with separate encoders to support per-layer // handling of PLIs. bool separate_encoders_needed = + is_mixed_codec || !encoder_context->encoder().GetEncoderInfo().supports_simulcast || active_streams_count == 1 || per_layer_pli_; RTC_LOG(LS_INFO) << "[SEA] InitEncode: total_streams_count: " @@ -410,9 +418,10 @@ int SimulcastEncoderAdapter::InitEncode( continue; } - if (encoder_context == nullptr) { + if (encoder_context == nullptr || is_mixed_codec) { encoder_context = FetchOrCreateEncoderContext( - /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx); + /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx, + stream_idx); } if (encoder_context == nullptr) { Release(); @@ -713,6 +722,10 @@ EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage( stream_image.SetSimulcastIndex(stream_idx); + if (codec_.IsMixedCodec()) { + stream_image.SetSpatialIndex(std::nullopt); + } + return encoded_complete_callback_->OnEncodedImage(stream_image, &stream_codec_specific); } @@ -734,20 +747,30 @@ void SimulcastEncoderAdapter::DestroyStoredEncoders() { std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> SimulcastEncoderAdapter::FetchOrCreateEncoderContext( - bool is_lowest_quality_stream) const { + bool is_lowest_quality_stream, + std::optional<int> stream_idx) const { RTC_DCHECK_RUN_ON(&encoder_queue_); + if (stream_idx) { + RTC_CHECK_LT(*stream_idx, codec_.numberOfSimulcastStreams); + } + SdpVideoFormat video_format = + stream_idx + ? codec_.simulcastStream[*stream_idx].format.value_or(video_format_) + : video_format_; bool prefer_temporal_support = fallback_encoder_factory_ != nullptr && is_lowest_quality_stream && prefer_temporal_support_on_base_layer_; // Toggling of `prefer_temporal_support` requires encoder recreation. Find - // and reuse encoder with desired `prefer_temporal_support`. Otherwise, if - // there is no such encoder in the cache, create a new instance. + // and reuse encoder with desired `prefer_temporal_support` and + // `video_format`. Otherwise, if there is no such encoder in the cache, create + // a new instance. auto encoder_context_iter = std::find_if(cached_encoder_contexts_.begin(), cached_encoder_contexts_.end(), [&](auto& encoder_context) { return encoder_context->prefer_temporal_support() == - prefer_temporal_support; + prefer_temporal_support && + encoder_context->video_format() == video_format; }); std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context; @@ -756,11 +779,11 @@ SimulcastEncoderAdapter::FetchOrCreateEncoderContext( cached_encoder_contexts_.erase(encoder_context_iter); } else { std::unique_ptr<VideoEncoder> primary_encoder = - primary_encoder_factory_->Create(env_, video_format_); + primary_encoder_factory_->Create(env_, video_format); std::unique_ptr<VideoEncoder> fallback_encoder; if (fallback_encoder_factory_ != nullptr) { - fallback_encoder = fallback_encoder_factory_->Create(env_, video_format_); + fallback_encoder = fallback_encoder_factory_->Create(env_, video_format); } std::unique_ptr<VideoEncoder> encoder; @@ -779,20 +802,20 @@ SimulcastEncoderAdapter::FetchOrCreateEncoderContext( prefer_temporal_support); } } else if (fallback_encoder != nullptr) { - RTC_LOG(LS_WARNING) << "Failed to create primary " << video_format_.name + RTC_LOG(LS_WARNING) << "Failed to create primary " << video_format.name << " encoder. Use fallback encoder."; fallback_info = fallback_encoder->GetEncoderInfo(); primary_info = fallback_info; encoder = std::move(fallback_encoder); } else { RTC_LOG(LS_ERROR) << "Failed to create primary and fallback " - << video_format_.name << " encoders."; + << video_format.name << " encoders."; return nullptr; } encoder_context = std::make_unique<SimulcastEncoderAdapter::EncoderContext>( std::move(encoder), prefer_temporal_support, primary_info, - fallback_info); + fallback_info, std::move(video_format)); } encoder_context->encoder().RegisterEncodeCompleteCallback( @@ -808,7 +831,12 @@ VideoCodec SimulcastEncoderAdapter::MakeStreamCodec( bool is_highest_quality_stream) { VideoCodec codec_params = codec; const SimulcastStream& stream_params = codec.simulcastStream[stream_idx]; + webrtc::VideoCodecType codec_type = + stream_params.format + ? PayloadStringToCodecType(stream_params.format->name) + : codec.codecType; + codec_params.codecType = codec_type; codec_params.numberOfSimulcastStreams = 0; codec_params.width = stream_params.width; codec_params.height = stream_params.height; @@ -846,7 +874,29 @@ VideoCodec SimulcastEncoderAdapter::MakeStreamCodec( codec_params.qpMax = kLowestResMaxQp; } } - if (codec.codecType == kVideoCodecVP8) { + + // Ensure default codec specifics matches the correct codec type for this + // stream. This can only differ in mixed-codec simulcast. + if (codec_type != codec.codecType) { + switch (codec_type) { + case kVideoCodecVP8: + *codec_params.VP8() = VideoEncoder::GetDefaultVp8Settings(); + break; + case kVideoCodecVP9: + *codec_params.VP9() = VideoEncoder::GetDefaultVp9Settings(); + break; + case kVideoCodecH264: + *codec_params.H264() = VideoEncoder::GetDefaultH264Settings(); + break; + case kVideoCodecAV1: + memset(codec_params.AV1(), 0, sizeof(VideoCodecAV1)); + break; + default: + break; + } + } + + if (codec_type == kVideoCodecVP8) { codec_params.VP8()->numberOfTemporalLayers = stream_params.numberOfTemporalLayers; if (!is_highest_quality_stream) { @@ -860,11 +910,11 @@ VideoCodec SimulcastEncoderAdapter::MakeStreamCodec( // Turn off denoising for all streams but the highest resolution. codec_params.VP8()->denoisingOn = false; } - } else if (codec.codecType == kVideoCodecH264) { + } else if (codec_type == kVideoCodecH264) { codec_params.H264()->numberOfTemporalLayers = stream_params.numberOfTemporalLayers; - } else if (codec.codecType == kVideoCodecVP9 && - scalability_mode.has_value() && !only_active_stream) { + } else if (codec_type == kVideoCodecVP9 && scalability_mode.has_value() && + !only_active_stream) { // If VP9 simulcast then explicitly set a single spatial layer for each // simulcast stream. codec_params.VP9()->numberOfSpatialLayers = 1; @@ -926,7 +976,8 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { // Create one encoder and query it. std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context = - FetchOrCreateEncoderContext(/*is_lowest_quality_stream=*/true); + FetchOrCreateEncoderContext(/*is_lowest_quality_stream=*/true, + std::nullopt); if (encoder_context == nullptr) { return encoder_info; } diff --git a/third_party/libwebrtc/media/engine/simulcast_encoder_adapter.h b/third_party/libwebrtc/media/engine/simulcast_encoder_adapter.h @@ -81,7 +81,8 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { EncoderContext(std::unique_ptr<VideoEncoder> encoder, bool prefer_temporal_support, VideoEncoder::EncoderInfo primary_info, - VideoEncoder::EncoderInfo fallback_info); + VideoEncoder::EncoderInfo fallback_info, + SdpVideoFormat video_format); EncoderContext& operator=(EncoderContext&&) = delete; VideoEncoder& encoder() { return *encoder_; } @@ -92,11 +93,14 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; } + const SdpVideoFormat& video_format() { return video_format_; } + private: std::unique_ptr<VideoEncoder> encoder_; bool prefer_temporal_support_; const VideoEncoder::EncoderInfo primary_info_; const VideoEncoder::EncoderInfo fallback_info_; + const SdpVideoFormat video_format_; }; class StreamContext : public EncodedImageCallback { @@ -156,7 +160,8 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { // `cached_encoder_contexts_`. It's const because it's used from // const GetEncoderInfo(). std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext( - bool is_lowest_quality_stream) const; + bool is_lowest_quality_stream, + std::optional<int> stream_idx) const; VideoCodec MakeStreamCodec(const VideoCodec& codec, int stream_idx, diff --git a/third_party/libwebrtc/media/engine/simulcast_encoder_adapter_unittest.cc b/third_party/libwebrtc/media/engine/simulcast_encoder_adapter_unittest.cc @@ -67,6 +67,7 @@ using FramerateFractions = absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>; namespace webrtc { + namespace test { namespace { @@ -95,6 +96,7 @@ std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture( std::move(decoder_factory), SdpVideoFormat::VP8()); } + } // namespace TEST(SimulcastEncoderAdapterSimulcastTest, TestKeyFrameRequestsOnAllStreams) { @@ -548,6 +550,34 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, adapter_->RegisterEncodeCompleteCallback(this); } + struct StreamDescription { + bool active; + SdpVideoFormat format; + }; + + void SetupMixedCodec(std::vector<StreamDescription> stream_descriptions) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + ASSERT_LE(stream_descriptions.size(), codec_.numberOfSimulcastStreams); + codec_.numberOfSimulcastStreams = stream_descriptions.size(); + for (size_t stream_idx = 0; stream_idx < kMaxSimulcastStreams; + ++stream_idx) { + if (stream_idx >= codec_.numberOfSimulcastStreams) { + // Reset parameters of unspecified stream. + codec_.simulcastStream[stream_idx] = {0}; + } else { + codec_.simulcastStream[stream_idx].active = + stream_descriptions[stream_idx].active; + codec_.simulcastStream[stream_idx].format = + stream_descriptions[stream_idx].format; + } + } + rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + } + void SetupCodecWithEarlyEncodeCompleteCallback( std::vector<bool> active_streams) { SimulcastTestFixtureImpl::DefaultSettings( @@ -2118,6 +2148,91 @@ TEST_F(TestSimulcastEncoderAdapterFake, PopulatesScalabilityModeOfSubcodecs) { ScalabilityMode::kL1T3); } +// In the case of mixed-codec simulcast, verify whether each encoder is created +// with the specified video format. +TEST_F(TestSimulcastEncoderAdapterFake, InitEncodeForMixedCodec) { + std::vector<SdpVideoFormat> codecs = {SdpVideoFormat::VP8(), + SdpVideoFormat::VP9Profile0(), + SdpVideoFormat::VP9Profile1()}; + SetupMixedCodec({{true, SdpVideoFormat::VP8()}, + {true, SdpVideoFormat::VP9Profile0()}, + {true, SdpVideoFormat::VP9Profile1()}}); + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + EXPECT_EQ(encoders[0]->video_format(), SdpVideoFormat::VP8()); + EXPECT_EQ(encoders[1]->video_format(), SdpVideoFormat::VP9Profile0()); + EXPECT_EQ(encoders[2]->video_format(), SdpVideoFormat::VP9Profile1()); + EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP8); + EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); + EXPECT_EQ(encoders[2]->codec().codecType, webrtc::kVideoCodecVP9); + + SetupMixedCodec({{false, SdpVideoFormat::VP8()}, + {true, SdpVideoFormat::VP9Profile0()}, + {true, SdpVideoFormat::VP9Profile1()}}); + encoders = helper_->factory()->encoders(); + ASSERT_EQ(2u, helper_->factory()->encoders().size()); + EXPECT_EQ(encoders[0]->video_format(), SdpVideoFormat::VP9Profile0()); + EXPECT_EQ(encoders[1]->video_format(), SdpVideoFormat::VP9Profile1()); + EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP9); + EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); +} + +TEST_F(TestSimulcastEncoderAdapterFake, + CodecSpecificSettingsIsInitializedDefaultValueForMixedCodec) { + std::vector<SdpVideoFormat> codecs = {SdpVideoFormat::VP8(), + SdpVideoFormat::VP9Profile0(), + SdpVideoFormat::VP9Profile1()}; + SetupMixedCodec({{true, SdpVideoFormat::VP8()}, + {true, SdpVideoFormat::VP9Profile0()}, + {true, SdpVideoFormat::VP9Profile1()}}); + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP8); + EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); + EXPECT_EQ(encoders[2]->codec().codecType, webrtc::kVideoCodecVP9); + + // Fields in the codec specific settings that are not set in + // SimulcastEncoderAdapter should be initialized with default values. + auto vp8_defaults = VideoEncoder::GetDefaultVp8Settings(); + auto vp9_defaults = VideoEncoder::GetDefaultVp9Settings(); + EXPECT_EQ(encoders[0]->codec().VP8().automaticResizeOn, + vp8_defaults.automaticResizeOn); + EXPECT_EQ(encoders[0]->codec().VP8().keyFrameInterval, + vp8_defaults.keyFrameInterval); + for (int i = 1; i <= 2; i++) { + EXPECT_EQ(encoders[i]->codec().VP9().denoisingOn, vp9_defaults.denoisingOn); + EXPECT_EQ(encoders[i]->codec().VP9().keyFrameInterval, + vp9_defaults.keyFrameInterval); + EXPECT_EQ(encoders[i]->codec().VP9().adaptiveQpMode, + vp9_defaults.adaptiveQpMode); + EXPECT_EQ(encoders[i]->codec().VP9().automaticResizeOn, + vp9_defaults.automaticResizeOn); + EXPECT_EQ(encoders[i]->codec().VP9().flexibleMode, + vp9_defaults.flexibleMode); + } +} + +// In the case of mixed-codec simulcast, multiple encoders are used even if +// supports_simulcast() == true. +TEST_F(TestSimulcastEncoderAdapterFake, + CreateMultipleEncodersEvenIfSimulcastIsSupportedForMixedCodec) { + std::vector<SdpVideoFormat> codecs = {SdpVideoFormat::VP8(), + SdpVideoFormat::VP9Profile0(), + SdpVideoFormat::VP9Profile1()}; + helper_->factory()->set_supports_simulcast(true); + SetupMixedCodec({{true, SdpVideoFormat::VP8()}, + {true, SdpVideoFormat::VP9Profile0()}, + {true, SdpVideoFormat::VP9Profile1()}}); + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + EXPECT_EQ(encoders[0]->video_format(), SdpVideoFormat::VP8()); + EXPECT_EQ(encoders[1]->video_format(), SdpVideoFormat::VP9Profile0()); + EXPECT_EQ(encoders[2]->video_format(), SdpVideoFormat::VP9Profile1()); + EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP8); + EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); + EXPECT_EQ(encoders[2]->codec().codecType, webrtc::kVideoCodecVP9); +} + TEST_F(TestSimulcastEncoderAdapterFake, EncodeDropsFrameIfResolutionIsNotAlignedByDefault) { field_trials_.Set("WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride", diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.cc b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -23,6 +23,7 @@ #include "api/video/video_bitrate_allocator.h" #include "api/video/video_codec_constants.h" #include "api/video/video_codec_type.h" +#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/simulcast_stream.h" #include "api/video_codecs/video_codec.h" #include "rtc_base/checks.h" @@ -330,11 +331,14 @@ const VideoCodec& SimulcastRateAllocator::GetCodec() const { } int SimulcastRateAllocator::NumTemporalStreams(size_t simulcast_id) const { + bool is_vp8 = codec_.simulcastStream[simulcast_id].format + ? codec_.simulcastStream[simulcast_id].format->IsSameCodec( + webrtc::SdpVideoFormat::VP8()) + : codec_.codecType == kVideoCodecVP8; return std::max<uint8_t>( - 1, - codec_.codecType == kVideoCodecVP8 && codec_.numberOfSimulcastStreams == 0 - ? codec_.VP8().numberOfTemporalLayers - : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers); + 1, is_vp8 && codec_.numberOfSimulcastStreams == 0 + ? codec_.VP8().numberOfTemporalLayers + : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers); } void SimulcastRateAllocator::SetLegacyConferenceMode(bool enabled) { diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -14,6 +14,7 @@ #include <cstdint> #include <limits> #include <memory> +#include <optional> #include <vector> #include "api/environment/environment.h" @@ -23,6 +24,7 @@ #include "api/video/video_bitrate_allocation.h" #include "api/video/video_bitrate_allocator.h" #include "api/video/video_codec_type.h" +#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/vp8_frame_buffer_controller.h" #include "api/video_codecs/vp8_frame_config.h" @@ -589,6 +591,47 @@ TEST_F(SimulcastRateAllocatorTest, NonConferenceModeScreenshare) { EXPECT_EQ(alloc.GetTemporalLayerAllocation(2).size(), 3u); } +TEST_F(SimulcastRateAllocatorTest, TemporalLayerForMixedCodec) { + // Singlecast stream + { + SetupCodec3TL(); + codec_.numberOfSimulcastStreams = 0; + CreateAllocator(); + + const uint32_t bitrate = codec_.simulcastStream[0].maxBitrate + + codec_.simulcastStream[1].maxBitrate + + codec_.simulcastStream[2].maxBitrate; + const VideoBitrateAllocation alloc = GetAllocation(bitrate); + + // If the single stream, use default temporal layer count. + EXPECT_EQ(alloc.GetTemporalLayerAllocation(0).size(), 3u); + } + + // Simulcast stream + { + SetupCodec3SL3TL({true, true, true}); + codec_.numberOfSimulcastStreams = 3; + codec_.simulcastStream[0].format = std::nullopt; + codec_.simulcastStream[0].numberOfTemporalLayers = 1; + codec_.simulcastStream[1].format = SdpVideoFormat::VP8(); + codec_.simulcastStream[1].numberOfTemporalLayers = 2; + codec_.simulcastStream[2].format = SdpVideoFormat::VP9Profile0(); + codec_.simulcastStream[2].numberOfTemporalLayers = 0; + CreateAllocator(); + + const uint32_t bitrate = codec_.simulcastStream[0].maxBitrate + + codec_.simulcastStream[1].maxBitrate + + codec_.simulcastStream[2].maxBitrate; + const VideoBitrateAllocation alloc = GetAllocation(bitrate); + + // If the simulcast stream, use the layer specific temporal layer count + // (minimum is 1). + EXPECT_EQ(alloc.GetTemporalLayerAllocation(0).size(), 1u); + EXPECT_EQ(alloc.GetTemporalLayerAllocation(1).size(), 2u); + EXPECT_EQ(alloc.GetTemporalLayerAllocation(2).size(), 1u); + } +} + class ScreenshareRateAllocationTest : public SimulcastRateAllocatorTest { public: void SetupConferenceScreenshare(bool use_simulcast, bool active = true) { diff --git a/third_party/libwebrtc/modules/video_coding/video_codec_initializer.cc b/third_party/libwebrtc/modules/video_coding/video_codec_initializer.cc @@ -121,6 +121,7 @@ VideoCodec VideoCodecInitializer::SetupCodec( sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; sim_stream->qpMax = streams[i].max_qp; + sim_stream->format = config.GetSimulcastVideoFormat(i); int num_temporal_layers = streams[i].scalability_mode.has_value() diff --git a/third_party/libwebrtc/moz-patch-stack/s0068.patch b/third_party/libwebrtc/moz-patch-stack/s0068.patch @@ -10,10 +10,10 @@ Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/a7179d8d75313b6c9 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc -index 876c5cbb9f..fef402d236 100644 +index 86a547f611..dc6f8a7dac 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc -@@ -1468,7 +1468,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { +@@ -1484,7 +1484,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { bool is_svc = false; bool single_stream_or_non_first_inactive = true; diff --git a/third_party/libwebrtc/moz-patch-stack/s0110.patch b/third_party/libwebrtc/moz-patch-stack/s0110.patch @@ -14,7 +14,7 @@ Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/8b5653392e88ccd0c 2 files changed, 5 insertions(+) diff --git a/video/config/video_encoder_config.h b/video/config/video_encoder_config.h -index 7fda03ffa8..8ca8462dd0 100644 +index e647cd6beb..d1cc8c05cc 100644 --- a/video/config/video_encoder_config.h +++ b/video/config/video_encoder_config.h @@ -26,6 +26,7 @@ @@ -25,7 +25,7 @@ index 7fda03ffa8..8ca8462dd0 100644 namespace webrtc { -@@ -153,6 +154,9 @@ class VideoEncoderConfig { +@@ -158,6 +159,9 @@ class VideoEncoderConfig { int frame_height, const VideoEncoderConfig& encoder_config) = 0; @@ -36,10 +36,10 @@ index 7fda03ffa8..8ca8462dd0 100644 ~VideoStreamFactoryInterface() override {} }; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc -index fef402d236..67ff17db0c 100644 +index dc6f8a7dac..f4ae2b0d46 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc -@@ -1080,6 +1080,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { +@@ -1096,6 +1096,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::vector<VideoStream> streams; if (encoder_config_.video_stream_factory) { // Note: only tests set their own EncoderStreamFactory... diff --git a/third_party/libwebrtc/video/config/video_encoder_config.cc b/third_party/libwebrtc/video/config/video_encoder_config.cc @@ -14,6 +14,7 @@ #include <string> #include "api/video/video_codec_type.h" +#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_codec.h" #include "rtc_base/checks.h" #include "rtc_base/strings/string_builder.h" @@ -109,6 +110,14 @@ bool VideoEncoderConfig::HasScaleResolutionDownTo() const { VideoEncoderConfig::VideoEncoderConfig(const VideoEncoderConfig&) = default; +SdpVideoFormat VideoEncoderConfig::GetSimulcastVideoFormat( + size_t stream_index) const { + if (stream_index >= simulcast_layers.size()) { + return video_format; + } + return simulcast_layers[stream_index].video_format.value_or(video_format); +} + void VideoEncoderConfig::EncoderSpecificSettings::FillEncoderSpecificSettings( VideoCodec* codec) const { if (codec->codecType == kVideoCodecVP8) { diff --git a/third_party/libwebrtc/video/config/video_encoder_config.h b/third_party/libwebrtc/video/config/video_encoder_config.h @@ -86,6 +86,11 @@ struct VideoStream { // e.g. if source only provides lower resolution or // if resource adaptation is active. std::optional<Resolution> scale_resolution_down_to; + + // The video format for the stream + // This should be set for mixed-codec simulcast, while for other cases, + // it is optional and can be unset. + std::optional<SdpVideoFormat> video_format; }; class VideoEncoderConfig { @@ -174,6 +179,8 @@ class VideoEncoderConfig { bool HasScaleResolutionDownTo() const; + SdpVideoFormat GetSimulcastVideoFormat(size_t stream_index) const; + // TODO(bugs.webrtc.org/6883): Consolidate on one of these. VideoCodecType codec_type; SdpVideoFormat video_format; diff --git a/third_party/libwebrtc/video/picture_id_tests.cc b/third_party/libwebrtc/video/picture_id_tests.cc @@ -286,6 +286,7 @@ void PictureIdTest::SetupEncoder(VideoEncoderFactory* encoder_factory, GetVideoSendConfig()->rtp.payload_name = payload_name; GetVideoEncoderConfig()->codec_type = PayloadStringToCodecType(payload_name); + GetVideoEncoderConfig()->video_format = SdpVideoFormat(payload_name); SetVideoEncoderConfig(/* number_of_streams */ 1); }); } diff --git a/third_party/libwebrtc/video/video_stream_encoder.cc b/third_party/libwebrtc/video/video_stream_encoder.cc @@ -1013,8 +1013,24 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, frame_cadence_adapter_->SetZeroHertzModeEnabled(std::nullopt); } + bool video_format_changed = + encoder_config_.video_format != config.video_format; + if (!video_format_changed && encoder_config_.simulcast_layers.size() != + config.simulcast_layers.size()) { + video_format_changed = true; + } + if (!video_format_changed) { + for (size_t i = 0; i < encoder_config_.simulcast_layers.size(); i++) { + if (!encoder_config_.GetSimulcastVideoFormat(i).IsSameCodec( + config.GetSimulcastVideoFormat(i))) { + video_format_changed = true; + break; + } + } + } + pending_encoder_creation_ = - (!encoder_ || encoder_config_.video_format != config.video_format || + (!encoder_ || video_format_changed || max_data_payload_length_ != max_data_payload_length); encoder_config_ = std::move(config); max_data_payload_length_ = max_data_payload_length; diff --git a/third_party/libwebrtc/video/video_stream_encoder_unittest.cc b/third_party/libwebrtc/video/video_stream_encoder_unittest.cc @@ -8690,6 +8690,95 @@ TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToCodecChange) { + const float downscale_factors[] = {4.0, 2.0, 1.0}; + const int number_layers = + sizeof(downscale_factors) / sizeof(downscale_factors[0]); + VideoEncoderConfig config; + test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config); + for (int i = 0; i < number_layers; ++i) { + config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i]; + config.simulcast_layers[i].active = true; + } + config.video_stream_factory = nullptr; + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kSimulcastTargetBitrate, kSimulcastTargetBitrate, 0, 0, 0); + + // First initialization. + // Encoder should be initialized. Next frame should be key frame. + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + sink_.SetNumExpectedLayers(number_layers); + int64_t timestamp_ms = kFrameIntervalMs; + video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(timestamp_ms); + EXPECT_EQ(1, fake_encoder_.GetNumInitializations()); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey})); + + // Set VP8 format + // Encoder should be re-initialized. Next frame should be key frame. + config.video_format = SdpVideoFormat::VP8(); + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + sink_.SetNumExpectedLayers(number_layers); + timestamp_ms += kFrameIntervalMs; + video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(timestamp_ms); + EXPECT_EQ(2, fake_encoder_.GetNumInitializations()); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey})); + + // No changes format + // Encoder shouldn't be re-initialized. Next frame should be delta frame. + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + sink_.SetNumExpectedLayers(number_layers); + timestamp_ms += kFrameIntervalMs; + video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(timestamp_ms); + EXPECT_EQ(2, fake_encoder_.GetNumInitializations()); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta, + VideoFrameType::kVideoFrameDelta, + VideoFrameType::kVideoFrameDelta})); + + // Set VP8 and VP9 format for layers + // Encoder should be re-initialized. Next frame should be key frame. + for (int i = 0; i < number_layers - 1; i++) { + config.simulcast_layers[i].video_format = SdpVideoFormat::VP8(); + } + config.simulcast_layers[number_layers - 1].video_format = + SdpVideoFormat::VP9Profile0(); + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + sink_.SetNumExpectedLayers(number_layers); + timestamp_ms += kFrameIntervalMs; + video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(timestamp_ms); + EXPECT_EQ(3, fake_encoder_.GetNumInitializations()); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey})); + + // Set VP9 format for layer 0 + // Encoder should be re-initialized. Next frame should be key frame. + config.simulcast_layers[0].video_format = SdpVideoFormat::VP9Profile0(); + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + sink_.SetNumExpectedLayers(number_layers); + timestamp_ms += kFrameIntervalMs; + video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(timestamp_ms); + EXPECT_EQ(4, fake_encoder_.GetNumInitializations()); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey, + VideoFrameType::kVideoFrameKey})); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) { const int kFrameWidth = 1280; const int kFrameHeight = 720;