simulcast_encoder_adapter_unittest.cc (95853B)
1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "media/engine/simulcast_encoder_adapter.h" 12 13 #include <array> 14 #include <cstddef> 15 #include <cstdint> 16 #include <memory> 17 #include <optional> 18 #include <string> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/container/inlined_vector.h" 23 #include "api/environment/environment.h" 24 #include "api/environment/environment_factory.h" 25 #include "api/fec_controller_override.h" 26 #include "api/field_trials.h" 27 #include "api/make_ref_counted.h" 28 #include "api/rtp_parameters.h" 29 #include "api/scoped_refptr.h" 30 #include "api/test/create_simulcast_test_fixture.h" 31 #include "api/test/mock_video_decoder.h" 32 #include "api/test/simulcast_test_fixture.h" 33 #include "api/test/video/function_video_decoder_factory.h" 34 #include "api/test/video/function_video_encoder_factory.h" 35 #include "api/units/data_rate.h" 36 #include "api/video/encoded_image.h" 37 #include "api/video/i420_buffer.h" 38 #include "api/video/video_bitrate_allocation.h" 39 #include "api/video/video_bitrate_allocator.h" 40 #include "api/video/video_codec_constants.h" 41 #include "api/video/video_codec_type.h" 42 #include "api/video/video_frame.h" 43 #include "api/video/video_frame_buffer.h" 44 #include "api/video/video_frame_type.h" 45 #include "api/video/video_rotation.h" 46 #include "api/video_codecs/scalability_mode.h" 47 #include "api/video_codecs/sdp_video_format.h" 48 #include "api/video_codecs/video_codec.h" 49 #include "api/video_codecs/video_decoder_factory.h" 50 #include "api/video_codecs/video_encoder.h" 51 #include "api/video_codecs/video_encoder_factory.h" 52 #include "media/engine/internal_encoder_factory.h" 53 #include "modules/video_coding/codecs/vp8/include/vp8.h" 54 #include "modules/video_coding/include/video_codec_interface.h" 55 #include "modules/video_coding/include/video_error_codes.h" 56 #include "modules/video_coding/utility/simulcast_rate_allocator.h" 57 #include "modules/video_coding/utility/simulcast_test_fixture_impl.h" 58 #include "rtc_base/checks.h" 59 #include "test/create_test_field_trials.h" 60 #include "test/gmock.h" 61 #include "test/gtest.h" 62 63 using ::testing::_; 64 using ::testing::Return; 65 using EncoderInfo = webrtc::VideoEncoder::EncoderInfo; 66 using FramerateFractions = 67 absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>; 68 69 namespace webrtc { 70 71 namespace test { 72 73 namespace { 74 75 constexpr int kDefaultWidth = 1280; 76 constexpr int kDefaultHeight = 720; 77 78 const VideoEncoder::Capabilities kCapabilities(false); 79 const VideoEncoder::Settings kSettings(kCapabilities, 1, 1200); 80 81 std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture( 82 VideoEncoderFactory* internal_encoder_factory) { 83 std::unique_ptr<VideoEncoderFactory> encoder_factory = 84 std::make_unique<FunctionVideoEncoderFactory>( 85 [internal_encoder_factory](const Environment& env, 86 const SdpVideoFormat& /* format */) { 87 return std::make_unique<SimulcastEncoderAdapter>( 88 env, internal_encoder_factory, nullptr, SdpVideoFormat::VP8()); 89 }); 90 std::unique_ptr<VideoDecoderFactory> decoder_factory = 91 std::make_unique<FunctionVideoDecoderFactory>( 92 [](const Environment& env, const SdpVideoFormat& /* format */) { 93 return CreateVp8Decoder(env); 94 }); 95 return CreateSimulcastTestFixture(std::move(encoder_factory), 96 std::move(decoder_factory), 97 SdpVideoFormat::VP8()); 98 } 99 100 } // namespace 101 102 TEST(SimulcastEncoderAdapterSimulcastTest, TestKeyFrameRequestsOnAllStreams) { 103 InternalEncoderFactory internal_encoder_factory; 104 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 105 fixture->TestKeyFrameRequestsOnAllStreams(); 106 } 107 108 TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingAllStreams) { 109 InternalEncoderFactory internal_encoder_factory; 110 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 111 fixture->TestPaddingAllStreams(); 112 } 113 114 TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingTwoStreams) { 115 InternalEncoderFactory internal_encoder_factory; 116 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 117 fixture->TestPaddingTwoStreams(); 118 } 119 120 TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingTwoStreamsOneMaxedOut) { 121 InternalEncoderFactory internal_encoder_factory; 122 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 123 fixture->TestPaddingTwoStreamsOneMaxedOut(); 124 } 125 126 TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingOneStream) { 127 InternalEncoderFactory internal_encoder_factory; 128 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 129 fixture->TestPaddingOneStream(); 130 } 131 132 TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingOneStreamTwoMaxedOut) { 133 InternalEncoderFactory internal_encoder_factory; 134 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 135 fixture->TestPaddingOneStreamTwoMaxedOut(); 136 } 137 138 TEST(SimulcastEncoderAdapterSimulcastTest, TestSendAllStreams) { 139 InternalEncoderFactory internal_encoder_factory; 140 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 141 fixture->TestSendAllStreams(); 142 } 143 144 TEST(SimulcastEncoderAdapterSimulcastTest, TestDisablingStreams) { 145 InternalEncoderFactory internal_encoder_factory; 146 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 147 fixture->TestDisablingStreams(); 148 } 149 150 TEST(SimulcastEncoderAdapterSimulcastTest, TestActiveStreams) { 151 InternalEncoderFactory internal_encoder_factory; 152 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 153 fixture->TestActiveStreams(); 154 } 155 156 TEST(SimulcastEncoderAdapterSimulcastTest, TestSwitchingToOneStream) { 157 InternalEncoderFactory internal_encoder_factory; 158 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 159 fixture->TestSwitchingToOneStream(); 160 } 161 162 TEST(SimulcastEncoderAdapterSimulcastTest, TestSwitchingToOneOddStream) { 163 InternalEncoderFactory internal_encoder_factory; 164 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 165 fixture->TestSwitchingToOneOddStream(); 166 } 167 168 TEST(SimulcastEncoderAdapterSimulcastTest, TestStrideEncodeDecode) { 169 InternalEncoderFactory internal_encoder_factory; 170 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 171 fixture->TestStrideEncodeDecode(); 172 } 173 174 TEST(SimulcastEncoderAdapterSimulcastTest, 175 TestSpatioTemporalLayers333PatternEncoder) { 176 InternalEncoderFactory internal_encoder_factory; 177 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 178 fixture->TestSpatioTemporalLayers333PatternEncoder(); 179 } 180 181 TEST(SimulcastEncoderAdapterSimulcastTest, 182 TestSpatioTemporalLayers321PatternEncoder) { 183 InternalEncoderFactory internal_encoder_factory; 184 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 185 fixture->TestSpatioTemporalLayers321PatternEncoder(); 186 } 187 188 TEST(SimulcastEncoderAdapterSimulcastTest, TestDecodeWidthHeightSet) { 189 InternalEncoderFactory internal_encoder_factory; 190 auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); 191 fixture->TestDecodeWidthHeightSet(); 192 } 193 194 class MockVideoEncoder; 195 196 class MockVideoEncoderFactory : public VideoEncoderFactory { 197 public: 198 std::vector<SdpVideoFormat> GetSupportedFormats() const override; 199 200 std::unique_ptr<VideoEncoder> Create(const Environment& env, 201 const SdpVideoFormat& format) override; 202 203 const std::vector<MockVideoEncoder*>& encoders() const; 204 void SetEncoderNames(const std::vector<const char*>& encoder_names); 205 void set_create_video_encode_return_nullptr(bool return_nullptr) { 206 create_video_encoder_return_nullptr_ = return_nullptr; 207 } 208 void set_init_encode_return_value(int32_t value); 209 void set_requested_resolution_alignments( 210 std::vector<uint32_t> requested_resolution_alignments) { 211 requested_resolution_alignments_ = requested_resolution_alignments; 212 } 213 void set_supports_simulcast(bool supports_simulcast) { 214 supports_simulcast_ = supports_simulcast; 215 } 216 void set_resolution_bitrate_limits( 217 std::vector<VideoEncoder::ResolutionBitrateLimits> limits) { 218 resolution_bitrate_limits_ = limits; 219 } 220 void set_fallback_from_simulcast(std::optional<int32_t> return_value) { 221 fallback_from_simulcast_ = return_value; 222 } 223 224 void DestroyVideoEncoder(VideoEncoder* encoder); 225 226 private: 227 bool create_video_encoder_return_nullptr_ = false; 228 int32_t init_encode_return_value_ = 0; 229 std::optional<int32_t> fallback_from_simulcast_; 230 std::vector<MockVideoEncoder*> encoders_; 231 std::vector<const char*> encoder_names_; 232 // Keep number of entries in sync with `kMaxSimulcastStreams`. 233 std::vector<uint32_t> requested_resolution_alignments_ = {1, 1, 1}; 234 bool supports_simulcast_ = false; 235 std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits_; 236 }; 237 238 class MockVideoEncoder : public VideoEncoder { 239 public: 240 explicit MockVideoEncoder(MockVideoEncoderFactory* factory) 241 : factory_(factory), 242 scaling_settings_(VideoEncoder::ScalingSettings::kOff), 243 video_format_("unknown"), 244 callback_(nullptr) {} 245 246 MOCK_METHOD(void, 247 SetFecControllerOverride, 248 (FecControllerOverride * fec_controller_override), 249 (override)); 250 251 int32_t InitEncode(const VideoCodec* codecSettings, 252 const VideoEncoder::Settings& /* settings */) override { 253 codec_ = *codecSettings; 254 if (codec_.numberOfSimulcastStreams > 1 && fallback_from_simulcast_) { 255 return *fallback_from_simulcast_; 256 } 257 return init_encode_return_value_; 258 } 259 260 MOCK_METHOD(int32_t, 261 Encode, 262 (const VideoFrame& inputImage, 263 const std::vector<VideoFrameType>* frame_types), 264 (override)); 265 266 int32_t RegisterEncodeCompleteCallback( 267 EncodedImageCallback* callback) override { 268 callback_ = callback; 269 return 0; 270 } 271 272 MOCK_METHOD(int32_t, Release, (), (override)); 273 274 void SetRates(const RateControlParameters& parameters) override { 275 last_set_rates_ = parameters; 276 } 277 278 EncoderInfo GetEncoderInfo() const override { 279 EncoderInfo info; 280 info.supports_native_handle = supports_native_handle_; 281 info.implementation_name = implementation_name_; 282 info.scaling_settings = scaling_settings_; 283 info.requested_resolution_alignment = requested_resolution_alignment_; 284 info.apply_alignment_to_all_simulcast_layers = 285 apply_alignment_to_all_simulcast_layers_; 286 info.has_trusted_rate_controller = has_trusted_rate_controller_; 287 info.is_hardware_accelerated = is_hardware_accelerated_; 288 info.fps_allocation[0] = fps_allocation_; 289 info.supports_simulcast = supports_simulcast_; 290 info.is_qp_trusted = is_qp_trusted_; 291 info.resolution_bitrate_limits = resolution_bitrate_limits; 292 return info; 293 } 294 295 ~MockVideoEncoder() override { factory_->DestroyVideoEncoder(this); } 296 297 const VideoCodec& codec() const { return codec_; } 298 299 EncodedImageCallback* callback() const { return callback_; } 300 301 void SendEncodedImage(int width, int height) { 302 // Sends a fake image of the given width/height. 303 EncodedImage image; 304 image._encodedWidth = width; 305 image._encodedHeight = height; 306 CodecSpecificInfo codec_specific_info; 307 codec_specific_info.codecType = kVideoCodecVP8; 308 callback_->OnEncodedImage(image, &codec_specific_info); 309 } 310 311 void set_supports_native_handle(bool enabled) { 312 supports_native_handle_ = enabled; 313 } 314 315 void set_implementation_name(const std::string& name) { 316 implementation_name_ = name; 317 } 318 319 void set_init_encode_return_value(int32_t value) { 320 init_encode_return_value_ = value; 321 } 322 323 void set_fallback_from_simulcast(std::optional<int32_t> value) { 324 fallback_from_simulcast_ = value; 325 } 326 327 void set_scaling_settings(const VideoEncoder::ScalingSettings& settings) { 328 scaling_settings_ = settings; 329 } 330 331 void set_requested_resolution_alignment( 332 uint32_t requested_resolution_alignment) { 333 requested_resolution_alignment_ = requested_resolution_alignment; 334 } 335 336 void set_apply_alignment_to_all_simulcast_layers(bool apply) { 337 apply_alignment_to_all_simulcast_layers_ = apply; 338 } 339 340 void set_has_trusted_rate_controller(bool trusted) { 341 has_trusted_rate_controller_ = trusted; 342 } 343 344 void set_is_hardware_accelerated(bool is_hardware_accelerated) { 345 is_hardware_accelerated_ = is_hardware_accelerated; 346 } 347 348 void set_fps_allocation(const FramerateFractions& fps_allocation) { 349 fps_allocation_ = fps_allocation; 350 } 351 352 RateControlParameters last_set_rates() const { return last_set_rates_; } 353 354 void set_supports_simulcast(bool supports_simulcast) { 355 supports_simulcast_ = supports_simulcast; 356 } 357 358 void set_video_format(const SdpVideoFormat& video_format) { 359 video_format_ = video_format; 360 } 361 362 void set_is_qp_trusted(std::optional<bool> is_qp_trusted) { 363 is_qp_trusted_ = is_qp_trusted; 364 } 365 366 void set_resolution_bitrate_limits( 367 std::vector<VideoEncoder::ResolutionBitrateLimits> limits) { 368 resolution_bitrate_limits = limits; 369 } 370 371 bool supports_simulcast() const { return supports_simulcast_; } 372 373 SdpVideoFormat video_format() const { return video_format_; } 374 375 private: 376 MockVideoEncoderFactory* const factory_; 377 bool supports_native_handle_ = false; 378 std::string implementation_name_ = "unknown"; 379 VideoEncoder::ScalingSettings scaling_settings_; 380 uint32_t requested_resolution_alignment_ = 1; 381 bool apply_alignment_to_all_simulcast_layers_ = false; 382 bool has_trusted_rate_controller_ = false; 383 bool is_hardware_accelerated_ = false; 384 int32_t init_encode_return_value_ = 0; 385 std::optional<int32_t> fallback_from_simulcast_; 386 VideoEncoder::RateControlParameters last_set_rates_; 387 FramerateFractions fps_allocation_; 388 bool supports_simulcast_ = false; 389 std::optional<bool> is_qp_trusted_; 390 SdpVideoFormat video_format_; 391 std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits; 392 393 VideoCodec codec_; 394 EncodedImageCallback* callback_; 395 }; 396 397 std::vector<SdpVideoFormat> MockVideoEncoderFactory::GetSupportedFormats() 398 const { 399 return {SdpVideoFormat::VP8()}; 400 } 401 402 std::unique_ptr<VideoEncoder> MockVideoEncoderFactory::Create( 403 const Environment& /* env */, 404 const SdpVideoFormat& format) { 405 if (create_video_encoder_return_nullptr_) { 406 return nullptr; 407 } 408 409 auto encoder = std::make_unique<::testing::NiceMock<MockVideoEncoder>>(this); 410 encoder->set_init_encode_return_value(init_encode_return_value_); 411 encoder->set_fallback_from_simulcast(fallback_from_simulcast_); 412 const char* encoder_name = encoder_names_.empty() 413 ? "codec_implementation_name" 414 : encoder_names_[encoders_.size()]; 415 encoder->set_implementation_name(encoder_name); 416 RTC_CHECK_LT(encoders_.size(), requested_resolution_alignments_.size()); 417 encoder->set_requested_resolution_alignment( 418 requested_resolution_alignments_[encoders_.size()]); 419 encoder->set_supports_simulcast(supports_simulcast_); 420 encoder->set_video_format(format); 421 encoder->set_resolution_bitrate_limits(resolution_bitrate_limits_); 422 encoders_.push_back(encoder.get()); 423 return encoder; 424 } 425 426 void MockVideoEncoderFactory::DestroyVideoEncoder(VideoEncoder* encoder) { 427 for (size_t i = 0; i < encoders_.size(); ++i) { 428 if (encoders_[i] == encoder) { 429 encoders_.erase(encoders_.begin() + i); 430 break; 431 } 432 } 433 } 434 435 const std::vector<MockVideoEncoder*>& MockVideoEncoderFactory::encoders() 436 const { 437 return encoders_; 438 } 439 void MockVideoEncoderFactory::SetEncoderNames( 440 const std::vector<const char*>& encoder_names) { 441 encoder_names_ = encoder_names; 442 } 443 void MockVideoEncoderFactory::set_init_encode_return_value(int32_t value) { 444 init_encode_return_value_ = value; 445 } 446 447 class TestSimulcastEncoderAdapterFakeHelper { 448 public: 449 explicit TestSimulcastEncoderAdapterFakeHelper( 450 const Environment& env, 451 bool use_fallback_factory, 452 const SdpVideoFormat& video_format) 453 : env_(env), 454 fallback_factory_(use_fallback_factory 455 ? std::make_unique<MockVideoEncoderFactory>() 456 : nullptr), 457 video_format_(video_format) {} 458 459 std::unique_ptr<VideoEncoder> CreateMockEncoderAdapter() { 460 return std::make_unique<SimulcastEncoderAdapter>( 461 env_, &primary_factory_, fallback_factory_.get(), video_format_); 462 } 463 464 MockVideoEncoderFactory* factory() { return &primary_factory_; } 465 MockVideoEncoderFactory* fallback_factory() { 466 return fallback_factory_.get(); 467 } 468 469 private: 470 const Environment env_; 471 MockVideoEncoderFactory primary_factory_; 472 std::unique_ptr<MockVideoEncoderFactory> fallback_factory_; 473 SdpVideoFormat video_format_; 474 }; 475 476 static const int kTestTemporalLayerProfile[3] = {3, 2, 1}; 477 478 class TestSimulcastEncoderAdapterFake : public ::testing::Test, 479 public EncodedImageCallback { 480 public: 481 TestSimulcastEncoderAdapterFake() : use_fallback_factory_(false) {} 482 483 ~TestSimulcastEncoderAdapterFake() override { 484 if (adapter_) { 485 adapter_->Release(); 486 } 487 } 488 489 void SetUp() override { 490 env_ = CreateEnvironment(field_trials_.CreateCopy()); 491 helper_ = std::make_unique<TestSimulcastEncoderAdapterFakeHelper>( 492 env_, use_fallback_factory_, 493 SdpVideoFormat("VP8", sdp_video_parameters_)); 494 adapter_ = helper_->CreateMockEncoderAdapter(); 495 last_encoded_image_width_ = std::nullopt; 496 last_encoded_image_height_ = std::nullopt; 497 last_encoded_image_simulcast_index_ = std::nullopt; 498 } 499 500 void ReSetUp() { 501 if (adapter_) { 502 adapter_->Release(); 503 // `helper_` owns factories which `adapter_` needs to destroy encoders. 504 // Release `adapter_` before `helper_` (released in SetUp()). 505 adapter_.reset(); 506 } 507 SetUp(); 508 } 509 510 Result OnEncodedImage( 511 const EncodedImage& encoded_image, 512 const CodecSpecificInfo* /* codec_specific_info */) override { 513 last_encoded_image_width_ = encoded_image._encodedWidth; 514 last_encoded_image_height_ = encoded_image._encodedHeight; 515 last_encoded_image_simulcast_index_ = encoded_image.SimulcastIndex(); 516 517 return Result(Result::OK, encoded_image.RtpTimestamp()); 518 } 519 520 bool GetLastEncodedImageInfo(std::optional<int>* out_width, 521 std::optional<int>* out_height, 522 std::optional<int>* out_simulcast_index) { 523 if (!last_encoded_image_width_.has_value()) { 524 return false; 525 } 526 *out_width = last_encoded_image_width_; 527 *out_height = last_encoded_image_height_; 528 *out_simulcast_index = last_encoded_image_simulcast_index_; 529 return true; 530 } 531 532 void SetupCodec() { SetupCodec(/*active_streams=*/{true, true, true}); } 533 534 void SetupCodec(std::vector<bool> active_streams) { 535 SimulcastTestFixtureImpl::DefaultSettings( 536 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 537 kVideoCodecVP8); 538 ASSERT_LE(active_streams.size(), codec_.numberOfSimulcastStreams); 539 codec_.numberOfSimulcastStreams = active_streams.size(); 540 for (size_t stream_idx = 0; stream_idx < kMaxSimulcastStreams; 541 ++stream_idx) { 542 if (stream_idx >= codec_.numberOfSimulcastStreams) { 543 // Reset parameters of unspecified stream. 544 codec_.simulcastStream[stream_idx] = {}; 545 } else { 546 codec_.simulcastStream[stream_idx].active = active_streams[stream_idx]; 547 } 548 } 549 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 550 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 551 adapter_->RegisterEncodeCompleteCallback(this); 552 } 553 554 struct StreamDescription { 555 bool active; 556 SdpVideoFormat format; 557 }; 558 559 void SetupMixedCodec(std::vector<StreamDescription> stream_descriptions) { 560 SimulcastTestFixtureImpl::DefaultSettings( 561 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 562 kVideoCodecVP8); 563 ASSERT_LE(stream_descriptions.size(), codec_.numberOfSimulcastStreams); 564 codec_.numberOfSimulcastStreams = stream_descriptions.size(); 565 for (size_t stream_idx = 0; stream_idx < kMaxSimulcastStreams; 566 ++stream_idx) { 567 if (stream_idx >= codec_.numberOfSimulcastStreams) { 568 // Reset parameters of unspecified stream. 569 codec_.simulcastStream[stream_idx] = {0}; 570 } else { 571 codec_.simulcastStream[stream_idx].active = 572 stream_descriptions[stream_idx].active; 573 codec_.simulcastStream[stream_idx].format = 574 stream_descriptions[stream_idx].format; 575 } 576 } 577 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 578 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 579 adapter_->RegisterEncodeCompleteCallback(this); 580 } 581 582 void SetupCodecWithEarlyEncodeCompleteCallback( 583 std::vector<bool> active_streams) { 584 SimulcastTestFixtureImpl::DefaultSettings( 585 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 586 kVideoCodecVP8); 587 ASSERT_LE(active_streams.size(), codec_.numberOfSimulcastStreams); 588 codec_.numberOfSimulcastStreams = active_streams.size(); 589 for (size_t stream_idx = 0; stream_idx < kMaxSimulcastStreams; 590 ++stream_idx) { 591 if (stream_idx >= codec_.numberOfSimulcastStreams) { 592 // Reset parameters of unspecified stream. 593 codec_.simulcastStream[stream_idx] = {}; 594 } else { 595 codec_.simulcastStream[stream_idx].active = active_streams[stream_idx]; 596 } 597 } 598 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 599 // Register the callback before the InitEncode(). 600 adapter_->RegisterEncodeCompleteCallback(this); 601 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 602 } 603 604 void VerifyCodec(const VideoCodec& ref, int stream_index) { 605 const VideoCodec& target = 606 helper_->factory()->encoders()[stream_index]->codec(); 607 EXPECT_EQ(ref.codecType, target.codecType); 608 EXPECT_EQ(ref.width, target.width); 609 EXPECT_EQ(ref.height, target.height); 610 EXPECT_EQ(ref.startBitrate, target.startBitrate); 611 EXPECT_EQ(ref.maxBitrate, target.maxBitrate); 612 EXPECT_EQ(ref.minBitrate, target.minBitrate); 613 EXPECT_EQ(ref.maxFramerate, target.maxFramerate); 614 EXPECT_EQ(ref.GetVideoEncoderComplexity(), 615 target.GetVideoEncoderComplexity()); 616 EXPECT_EQ(ref.VP8().numberOfTemporalLayers, 617 target.VP8().numberOfTemporalLayers); 618 EXPECT_EQ(ref.VP8().denoisingOn, target.VP8().denoisingOn); 619 EXPECT_EQ(ref.VP8().automaticResizeOn, target.VP8().automaticResizeOn); 620 EXPECT_EQ(ref.GetFrameDropEnabled(), target.GetFrameDropEnabled()); 621 EXPECT_EQ(ref.VP8().keyFrameInterval, target.VP8().keyFrameInterval); 622 EXPECT_EQ(ref.qpMax, target.qpMax); 623 EXPECT_EQ(0, target.numberOfSimulcastStreams); 624 EXPECT_EQ(ref.mode, target.mode); 625 626 // No need to compare simulcastStream as numberOfSimulcastStreams should 627 // always be 0. 628 } 629 630 void InitRefCodec(int stream_index, 631 VideoCodec* ref_codec, 632 bool reverse_layer_order = false) { 633 *ref_codec = codec_; 634 ref_codec->VP8()->numberOfTemporalLayers = 635 kTestTemporalLayerProfile[reverse_layer_order ? 2 - stream_index 636 : stream_index]; 637 ref_codec->width = codec_.simulcastStream[stream_index].width; 638 ref_codec->height = codec_.simulcastStream[stream_index].height; 639 ref_codec->maxBitrate = codec_.simulcastStream[stream_index].maxBitrate; 640 ref_codec->minBitrate = codec_.simulcastStream[stream_index].minBitrate; 641 ref_codec->qpMax = codec_.simulcastStream[stream_index].qpMax; 642 } 643 644 void VerifyCodecSettings() { 645 EXPECT_EQ(3u, helper_->factory()->encoders().size()); 646 VideoCodec ref_codec; 647 648 // stream 0, the lowest resolution stream. 649 InitRefCodec(0, &ref_codec); 650 ref_codec.qpMax = 45; 651 ref_codec.SetVideoEncoderComplexity( 652 VideoCodecComplexity::kComplexityHigher); 653 ref_codec.VP8()->denoisingOn = false; 654 ref_codec.startBitrate = 100; // Should equal to the target bitrate. 655 VerifyCodec(ref_codec, 0); 656 657 // stream 1 658 InitRefCodec(1, &ref_codec); 659 ref_codec.VP8()->denoisingOn = false; 660 // The start bitrate (300kbit) minus what we have for the lower layers 661 // (100kbit). 662 ref_codec.startBitrate = 200; 663 VerifyCodec(ref_codec, 1); 664 665 // stream 2, the biggest resolution stream. 666 InitRefCodec(2, &ref_codec); 667 // We don't have enough bits to send this, so the adapter should have 668 // configured it to use the min bitrate for this layer (600kbit) but turn 669 // off sending. 670 ref_codec.startBitrate = 600; 671 VerifyCodec(ref_codec, 2); 672 } 673 674 protected: 675 FieldTrials field_trials_ = CreateTestFieldTrials(); 676 Environment env_ = CreateEnvironment(field_trials_.CreateCopy()); 677 std::unique_ptr<TestSimulcastEncoderAdapterFakeHelper> helper_; 678 std::unique_ptr<VideoEncoder> adapter_; 679 VideoCodec codec_; 680 std::optional<int> last_encoded_image_width_; 681 std::optional<int> last_encoded_image_height_; 682 std::optional<int> last_encoded_image_simulcast_index_; 683 std::unique_ptr<SimulcastRateAllocator> rate_allocator_; 684 bool use_fallback_factory_; 685 CodecParameterMap sdp_video_parameters_; 686 }; 687 688 TEST_F(TestSimulcastEncoderAdapterFake, InitEncode) { 689 SetupCodec(); 690 VerifyCodecSettings(); 691 } 692 693 TEST_F(TestSimulcastEncoderAdapterFake, EarlyCallbackSetupNotLost) { 694 helper_->factory()->set_supports_simulcast(true); 695 helper_->factory()->set_fallback_from_simulcast( 696 WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); 697 SetupCodecWithEarlyEncodeCompleteCallback( 698 /*active_streams=*/{true, true, true}); 699 for (size_t idx = 0; idx < 3; ++idx) { 700 auto callback = helper_->factory()->encoders()[idx]->callback(); 701 EXPECT_NE(callback, nullptr); 702 } 703 } 704 705 TEST_F(TestSimulcastEncoderAdapterFake, ReleaseWithoutInitEncode) { 706 EXPECT_EQ(0, adapter_->Release()); 707 } 708 709 TEST_F(TestSimulcastEncoderAdapterFake, Reinit) { 710 SetupCodec(); 711 EXPECT_EQ(0, adapter_->Release()); 712 713 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 714 } 715 716 TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) { 717 SetupCodec(); 718 719 // Set bitrates so that we send all layers. 720 adapter_->SetRates(VideoEncoder::RateControlParameters( 721 rate_allocator_->Allocate(VideoBitrateAllocationParameters(1200, 30)), 722 30.0)); 723 724 // At this point, the simulcast encoder adapter should have 3 streams: HD, 725 // quarter HD, and quarter quarter HD. We're going to mostly ignore the exact 726 // resolutions, to test that the adapter forwards on the correct resolution 727 // and simulcast index values, going only off the encoder that generates the 728 // image. 729 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 730 ASSERT_EQ(3u, encoders.size()); 731 encoders[0]->SendEncodedImage(1152, 704); 732 std::optional<int> width; 733 std::optional<int> height; 734 std::optional<int> simulcast_index; 735 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 736 ASSERT_TRUE(width.has_value()); 737 EXPECT_EQ(1152, width.value()); 738 ASSERT_TRUE(height.has_value()); 739 EXPECT_EQ(704, height.value()); 740 // SEA doesn't intercept frame encode complete callback for the lowest stream. 741 EXPECT_FALSE(simulcast_index.has_value()); 742 743 encoders[1]->SendEncodedImage(300, 620); 744 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 745 ASSERT_TRUE(width.has_value()); 746 EXPECT_EQ(300, width.value()); 747 ASSERT_TRUE(height.has_value()); 748 EXPECT_EQ(620, height.value()); 749 ASSERT_TRUE(simulcast_index.has_value()); 750 EXPECT_EQ(1, simulcast_index.value()); 751 752 encoders[2]->SendEncodedImage(120, 240); 753 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 754 ASSERT_TRUE(width.has_value()); 755 EXPECT_EQ(120, width.value()); 756 ASSERT_TRUE(height.has_value()); 757 EXPECT_EQ(240, height.value()); 758 ASSERT_TRUE(simulcast_index.has_value()); 759 EXPECT_EQ(2, simulcast_index.value()); 760 } 761 762 // This test verifies that the underlying encoders are reused, when the adapter 763 // is reinited with different number of simulcast streams. It further checks 764 // that the allocated encoders are reused in the same order as before, starting 765 // with the lowest stream. 766 TEST_F(TestSimulcastEncoderAdapterFake, ReusesEncodersInOrder) { 767 // Set up common settings for three streams. 768 SimulcastTestFixtureImpl::DefaultSettings( 769 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 770 kVideoCodecVP8); 771 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 772 adapter_->RegisterEncodeCompleteCallback(this); 773 const uint32_t target_bitrate = 774 1000 * (codec_.simulcastStream[0].targetBitrate + 775 codec_.simulcastStream[1].targetBitrate + 776 codec_.simulcastStream[2].minBitrate); 777 778 // Input data. 779 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 780 VideoFrame input_frame = VideoFrame::Builder() 781 .set_video_frame_buffer(buffer) 782 .set_rtp_timestamp(100) 783 .set_timestamp_ms(1000) 784 .set_rotation(kVideoRotation_180) 785 .build(); 786 std::vector<VideoFrameType> frame_types; 787 788 // Encode with three streams. 789 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 790 VerifyCodecSettings(); 791 adapter_->SetRates(VideoEncoder::RateControlParameters( 792 rate_allocator_->Allocate( 793 VideoBitrateAllocationParameters(target_bitrate, 30)), 794 30.0)); 795 796 std::vector<MockVideoEncoder*> original_encoders = 797 helper_->factory()->encoders(); 798 ASSERT_EQ(3u, original_encoders.size()); 799 EXPECT_CALL(*original_encoders[0], Encode(_, _)) 800 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 801 EXPECT_CALL(*original_encoders[1], Encode(_, _)) 802 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 803 EXPECT_CALL(*original_encoders[2], Encode(_, _)) 804 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 805 frame_types.resize(3, VideoFrameType::kVideoFrameKey); 806 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 807 EXPECT_CALL(*original_encoders[0], Release()) 808 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 809 EXPECT_CALL(*original_encoders[1], Release()) 810 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 811 EXPECT_CALL(*original_encoders[2], Release()) 812 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 813 EXPECT_EQ(0, adapter_->Release()); 814 815 // Encode with two streams. 816 codec_.width /= 2; 817 codec_.height /= 2; 818 codec_.numberOfSimulcastStreams = 2; 819 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 820 adapter_->SetRates(VideoEncoder::RateControlParameters( 821 rate_allocator_->Allocate( 822 VideoBitrateAllocationParameters(target_bitrate, 30)), 823 30.0)); 824 std::vector<MockVideoEncoder*> new_encoders = helper_->factory()->encoders(); 825 ASSERT_EQ(2u, new_encoders.size()); 826 ASSERT_EQ(original_encoders[0], new_encoders[0]); 827 EXPECT_CALL(*original_encoders[0], Encode(_, _)) 828 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 829 ASSERT_EQ(original_encoders[1], new_encoders[1]); 830 EXPECT_CALL(*original_encoders[1], Encode(_, _)) 831 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 832 frame_types.resize(2, VideoFrameType::kVideoFrameKey); 833 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 834 EXPECT_CALL(*original_encoders[0], Release()) 835 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 836 EXPECT_CALL(*original_encoders[1], Release()) 837 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 838 EXPECT_EQ(0, adapter_->Release()); 839 840 // Encode with single stream. 841 codec_.width /= 2; 842 codec_.height /= 2; 843 codec_.numberOfSimulcastStreams = 1; 844 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 845 adapter_->SetRates(VideoEncoder::RateControlParameters( 846 rate_allocator_->Allocate( 847 VideoBitrateAllocationParameters(target_bitrate, 30)), 848 30.0)); 849 new_encoders = helper_->factory()->encoders(); 850 ASSERT_EQ(1u, new_encoders.size()); 851 ASSERT_EQ(original_encoders[0], new_encoders[0]); 852 EXPECT_CALL(*original_encoders[0], Encode(_, _)) 853 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 854 frame_types.resize(1, VideoFrameType::kVideoFrameKey); 855 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 856 EXPECT_CALL(*original_encoders[0], Release()) 857 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 858 EXPECT_EQ(0, adapter_->Release()); 859 860 // Encode with three streams, again. 861 codec_.width *= 4; 862 codec_.height *= 4; 863 codec_.numberOfSimulcastStreams = 3; 864 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 865 adapter_->SetRates(VideoEncoder::RateControlParameters( 866 rate_allocator_->Allocate( 867 VideoBitrateAllocationParameters(target_bitrate, 30)), 868 30.0)); 869 new_encoders = helper_->factory()->encoders(); 870 ASSERT_EQ(3u, new_encoders.size()); 871 // The first encoder is reused. 872 ASSERT_EQ(original_encoders[0], new_encoders[0]); 873 EXPECT_CALL(*original_encoders[0], Encode(_, _)) 874 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 875 // The second and third encoders are new. 876 EXPECT_CALL(*new_encoders[1], Encode(_, _)) 877 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 878 EXPECT_CALL(*new_encoders[2], Encode(_, _)) 879 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 880 frame_types.resize(3, VideoFrameType::kVideoFrameKey); 881 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 882 EXPECT_CALL(*original_encoders[0], Release()) 883 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 884 EXPECT_CALL(*new_encoders[1], Release()) 885 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 886 EXPECT_CALL(*new_encoders[2], Release()) 887 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 888 EXPECT_EQ(0, adapter_->Release()); 889 } 890 891 TEST_F(TestSimulcastEncoderAdapterFake, DoesNotLeakEncoders) { 892 SetupCodec(); 893 VerifyCodecSettings(); 894 895 EXPECT_EQ(3u, helper_->factory()->encoders().size()); 896 897 // The adapter should destroy all encoders it has allocated. Since 898 // `helper_->factory()` is owned by `adapter_`, however, we need to rely on 899 // lsan to find leaks here. 900 EXPECT_EQ(0, adapter_->Release()); 901 adapter_.reset(); 902 } 903 904 // This test verifies that an adapter reinit with the same codec settings as 905 // before does not change the underlying encoder codec settings. 906 TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderEncoderSettings) { 907 SetupCodec(); 908 VerifyCodecSettings(); 909 910 // Capture current codec settings. 911 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 912 ASSERT_EQ(3u, encoders.size()); 913 std::array<VideoCodec, 3> codecs_before; 914 for (int i = 0; i < 3; ++i) { 915 codecs_before[i] = encoders[i]->codec(); 916 } 917 918 // Reinitialize and verify that the new codec settings are the same. 919 EXPECT_EQ(0, adapter_->Release()); 920 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 921 for (int i = 0; i < 3; ++i) { 922 const VideoCodec& codec_before = codecs_before[i]; 923 const VideoCodec& codec_after = encoders[i]->codec(); 924 925 // webrtc::VideoCodec does not implement operator==. 926 EXPECT_EQ(codec_before.codecType, codec_after.codecType); 927 EXPECT_EQ(codec_before.width, codec_after.width); 928 EXPECT_EQ(codec_before.height, codec_after.height); 929 EXPECT_EQ(codec_before.startBitrate, codec_after.startBitrate); 930 EXPECT_EQ(codec_before.maxBitrate, codec_after.maxBitrate); 931 EXPECT_EQ(codec_before.minBitrate, codec_after.minBitrate); 932 EXPECT_EQ(codec_before.maxFramerate, codec_after.maxFramerate); 933 EXPECT_EQ(codec_before.qpMax, codec_after.qpMax); 934 EXPECT_EQ(codec_before.numberOfSimulcastStreams, 935 codec_after.numberOfSimulcastStreams); 936 EXPECT_EQ(codec_before.mode, codec_after.mode); 937 EXPECT_EQ(codec_before.expect_encode_from_texture, 938 codec_after.expect_encode_from_texture); 939 } 940 } 941 942 // This test is similar to the one above, except that it tests the simulcastIdx 943 // from the CodecSpecificInfo that is connected to an encoded frame. The 944 // PayloadRouter demuxes the incoming encoded frames on different RTP modules 945 // using the simulcastIdx, so it's important that there is no corresponding 946 // encoder reordering in between adapter reinits as this would lead to PictureID 947 // discontinuities. 948 TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderFrameSimulcastIdx) { 949 SetupCodec(); 950 adapter_->SetRates(VideoEncoder::RateControlParameters( 951 rate_allocator_->Allocate(VideoBitrateAllocationParameters(1200, 30)), 952 30.0)); 953 VerifyCodecSettings(); 954 955 // Send frames on all streams. 956 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 957 ASSERT_EQ(3u, encoders.size()); 958 encoders[0]->SendEncodedImage(1152, 704); 959 std::optional<int> width; 960 std::optional<int> height; 961 std::optional<int> simulcast_index; 962 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 963 // SEA doesn't intercept frame encode complete callback for the lowest stream. 964 EXPECT_FALSE(simulcast_index.has_value()); 965 966 encoders[1]->SendEncodedImage(300, 620); 967 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 968 ASSERT_TRUE(simulcast_index.has_value()); 969 EXPECT_EQ(1, simulcast_index.value()); 970 971 encoders[2]->SendEncodedImage(120, 240); 972 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 973 ASSERT_TRUE(simulcast_index.has_value()); 974 EXPECT_EQ(2, simulcast_index.value()); 975 976 // Reinitialize. 977 EXPECT_EQ(0, adapter_->Release()); 978 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 979 adapter_->SetRates(VideoEncoder::RateControlParameters( 980 rate_allocator_->Allocate(VideoBitrateAllocationParameters(1200, 30)), 981 30.0)); 982 983 // Verify that the same encoder sends out frames on the same simulcast index. 984 encoders[0]->SendEncodedImage(1152, 704); 985 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 986 EXPECT_FALSE(simulcast_index.has_value()); 987 988 encoders[1]->SendEncodedImage(300, 620); 989 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 990 ASSERT_TRUE(simulcast_index.has_value()); 991 EXPECT_EQ(1, simulcast_index.value()); 992 993 encoders[2]->SendEncodedImage(120, 240); 994 EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); 995 ASSERT_TRUE(simulcast_index.has_value()); 996 EXPECT_EQ(2, simulcast_index.value()); 997 } 998 999 TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) { 1000 SimulcastTestFixtureImpl::DefaultSettings( 1001 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1002 kVideoCodecVP8); 1003 codec_.numberOfSimulcastStreams = 1; 1004 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1005 adapter_->RegisterEncodeCompleteCallback(this); 1006 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 1007 helper_->factory()->encoders()[0]->set_supports_native_handle(true); 1008 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1009 EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); 1010 helper_->factory()->encoders()[0]->set_supports_native_handle(false); 1011 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1012 EXPECT_FALSE(adapter_->GetEncoderInfo().supports_native_handle); 1013 } 1014 1015 TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) { 1016 SimulcastTestFixtureImpl::DefaultSettings( 1017 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1018 kVideoCodecVP8); 1019 codec_.minBitrate = 50; 1020 codec_.numberOfSimulcastStreams = 1; 1021 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1022 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1023 1024 // Above min should be respected. 1025 VideoBitrateAllocation target_bitrate = rate_allocator_->Allocate( 1026 VideoBitrateAllocationParameters(codec_.minBitrate * 1000, 30)); 1027 adapter_->SetRates(VideoEncoder::RateControlParameters(target_bitrate, 30.0)); 1028 EXPECT_EQ(target_bitrate, 1029 helper_->factory()->encoders()[0]->last_set_rates().bitrate); 1030 1031 // Below min but non-zero should be replaced with the min bitrate. 1032 VideoBitrateAllocation too_low_bitrate = rate_allocator_->Allocate( 1033 VideoBitrateAllocationParameters((codec_.minBitrate - 1) * 1000, 30)); 1034 adapter_->SetRates( 1035 VideoEncoder::RateControlParameters(too_low_bitrate, 30.0)); 1036 EXPECT_EQ(target_bitrate, 1037 helper_->factory()->encoders()[0]->last_set_rates().bitrate); 1038 1039 // Zero should be passed on as is, since it means "pause". 1040 adapter_->SetRates( 1041 VideoEncoder::RateControlParameters(VideoBitrateAllocation(), 30.0)); 1042 EXPECT_EQ(VideoBitrateAllocation(), 1043 helper_->factory()->encoders()[0]->last_set_rates().bitrate); 1044 } 1045 1046 TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { 1047 SimulcastTestFixtureImpl::DefaultSettings( 1048 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1049 kVideoCodecVP8); 1050 codec_.numberOfSimulcastStreams = 2; 1051 std::vector<const char*> encoder_names; 1052 encoder_names.push_back("codec1"); 1053 encoder_names.push_back("codec2"); 1054 helper_->factory()->SetEncoderNames(encoder_names); 1055 EXPECT_EQ("SimulcastEncoderAdapter", 1056 adapter_->GetEncoderInfo().implementation_name); 1057 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1058 EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2)", 1059 adapter_->GetEncoderInfo().implementation_name); 1060 1061 // Single streams should not expose "SimulcastEncoderAdapter" in name. 1062 EXPECT_EQ(0, adapter_->Release()); 1063 codec_.numberOfSimulcastStreams = 1; 1064 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1065 adapter_->RegisterEncodeCompleteCallback(this); 1066 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 1067 EXPECT_EQ("codec1", adapter_->GetEncoderInfo().implementation_name); 1068 } 1069 1070 TEST_F(TestSimulcastEncoderAdapterFake, RuntimeEncoderInfoUpdate) { 1071 SimulcastTestFixtureImpl::DefaultSettings( 1072 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1073 kVideoCodecVP8); 1074 std::vector<const char*> encoder_names; 1075 encoder_names.push_back("codec1"); 1076 encoder_names.push_back("codec2"); 1077 encoder_names.push_back("codec3"); 1078 helper_->factory()->SetEncoderNames(encoder_names); 1079 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1080 EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2)", 1081 adapter_->GetEncoderInfo().implementation_name); 1082 1083 // Change name of first encoder to indicate it has done a fallback to another 1084 // implementation. 1085 helper_->factory()->encoders().front()->set_implementation_name("fallback1"); 1086 EXPECT_EQ("SimulcastEncoderAdapter (fallback1, codec2)", 1087 adapter_->GetEncoderInfo().implementation_name); 1088 } 1089 1090 TEST_F(TestSimulcastEncoderAdapterFake, EncoderInfoDeactiveLayersUpdatesName) { 1091 SimulcastTestFixtureImpl::DefaultSettings( 1092 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1093 kVideoCodecVP8); 1094 const DataRate target_bitrate = 1095 DataRate::KilobitsPerSec(codec_.simulcastStream[0].targetBitrate + 1096 codec_.simulcastStream[1].targetBitrate + 1097 codec_.simulcastStream[2].targetBitrate); 1098 const DataRate bandwidth_allocation = 1099 target_bitrate + DataRate::KilobitsPerSec(600); 1100 const DataRate target_bitrate_without_layer3 = 1101 target_bitrate - 1102 DataRate::KilobitsPerSec(codec_.simulcastStream[2].targetBitrate); 1103 const DataRate bandwidth_allocation_without_layer3 = 1104 target_bitrate + DataRate::KilobitsPerSec(300); 1105 1106 std::vector<const char*> encoder_names = {"codec1", "codec2", "codec3"}; 1107 helper_->factory()->SetEncoderNames(encoder_names); 1108 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1109 1110 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1111 adapter_->SetRates(VideoEncoder::RateControlParameters( 1112 rate_allocator_->Allocate( 1113 VideoBitrateAllocationParameters(target_bitrate, 30)), 1114 30.0, bandwidth_allocation)); 1115 EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)", 1116 adapter_->GetEncoderInfo().implementation_name); 1117 1118 // Disable the third encoder using bitrate allocation. 1119 adapter_->SetRates(VideoEncoder::RateControlParameters( 1120 rate_allocator_->Allocate( 1121 VideoBitrateAllocationParameters(target_bitrate_without_layer3, 30)), 1122 30.0, bandwidth_allocation_without_layer3)); 1123 EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2)", 1124 adapter_->GetEncoderInfo().implementation_name); 1125 1126 // Enable the third encoder again using bitrate allocation. 1127 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1128 adapter_->SetRates(VideoEncoder::RateControlParameters( 1129 rate_allocator_->Allocate( 1130 VideoBitrateAllocationParameters(target_bitrate, 30)), 1131 30.0, bandwidth_allocation)); 1132 EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)", 1133 adapter_->GetEncoderInfo().implementation_name); 1134 } 1135 1136 TEST_F(TestSimulcastEncoderAdapterFake, 1137 SupportsNativeHandleForMultipleStreams) { 1138 SimulcastTestFixtureImpl::DefaultSettings( 1139 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1140 kVideoCodecVP8); 1141 codec_.numberOfSimulcastStreams = 3; 1142 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1143 adapter_->RegisterEncodeCompleteCallback(this); 1144 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1145 for (MockVideoEncoder* encoder : helper_->factory()->encoders()) 1146 encoder->set_supports_native_handle(true); 1147 // As long as one encoder supports native handle, it's enabled. 1148 helper_->factory()->encoders()[0]->set_supports_native_handle(false); 1149 EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); 1150 // Once none do, then the adapter claims no support. 1151 helper_->factory()->encoders()[1]->set_supports_native_handle(false); 1152 helper_->factory()->encoders()[2]->set_supports_native_handle(false); 1153 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1154 EXPECT_FALSE(adapter_->GetEncoderInfo().supports_native_handle); 1155 } 1156 1157 class FakeNativeBufferI420 : public VideoFrameBuffer { 1158 public: 1159 FakeNativeBufferI420(int width, int height, bool allow_to_i420) 1160 : width_(width), height_(height), allow_to_i420_(allow_to_i420) {} 1161 1162 Type type() const override { return Type::kNative; } 1163 int width() const override { return width_; } 1164 int height() const override { return height_; } 1165 1166 scoped_refptr<I420BufferInterface> ToI420() override { 1167 if (allow_to_i420_) { 1168 return I420Buffer::Create(width_, height_); 1169 } else { 1170 RTC_DCHECK_NOTREACHED(); 1171 } 1172 return nullptr; 1173 } 1174 1175 private: 1176 const int width_; 1177 const int height_; 1178 const bool allow_to_i420_; 1179 }; 1180 1181 TEST_F(TestSimulcastEncoderAdapterFake, 1182 NativeHandleForwardingForMultipleStreams) { 1183 SimulcastTestFixtureImpl::DefaultSettings( 1184 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1185 kVideoCodecVP8); 1186 codec_.numberOfSimulcastStreams = 3; 1187 // High start bitrate, so all streams are enabled. 1188 codec_.startBitrate = 3000; 1189 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1190 adapter_->RegisterEncodeCompleteCallback(this); 1191 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1192 for (MockVideoEncoder* encoder : helper_->factory()->encoders()) 1193 encoder->set_supports_native_handle(true); 1194 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1195 EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); 1196 1197 scoped_refptr<VideoFrameBuffer> buffer( 1198 make_ref_counted<FakeNativeBufferI420>(1280, 720, 1199 /*allow_to_i420=*/false)); 1200 VideoFrame input_frame = VideoFrame::Builder() 1201 .set_video_frame_buffer(buffer) 1202 .set_rtp_timestamp(100) 1203 .set_timestamp_ms(1000) 1204 .set_rotation(kVideoRotation_180) 1205 .build(); 1206 // Expect calls with the given video frame verbatim, since it's a texture 1207 // frame and can't otherwise be modified/resized. 1208 for (MockVideoEncoder* encoder : helper_->factory()->encoders()) 1209 EXPECT_CALL(*encoder, Encode(::testing::Ref(input_frame), _)).Times(1); 1210 std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); 1211 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1212 } 1213 1214 TEST_F(TestSimulcastEncoderAdapterFake, NativeHandleForwardingOnlyIfSupported) { 1215 SimulcastTestFixtureImpl::DefaultSettings( 1216 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1217 kVideoCodecVP8); 1218 codec_.numberOfSimulcastStreams = 3; 1219 // High start bitrate, so all streams are enabled. 1220 codec_.startBitrate = 3000; 1221 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1222 adapter_->RegisterEncodeCompleteCallback(this); 1223 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1224 1225 // QVGA encoders has fallen back to software. 1226 auto& encoders = helper_->factory()->encoders(); 1227 encoders[0]->set_supports_native_handle(false); 1228 encoders[1]->set_supports_native_handle(true); 1229 encoders[2]->set_supports_native_handle(true); 1230 1231 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1232 EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); 1233 1234 scoped_refptr<VideoFrameBuffer> buffer( 1235 make_ref_counted<FakeNativeBufferI420>(1280, 720, 1236 /*allow_to_i420=*/true)); 1237 VideoFrame input_frame = VideoFrame::Builder() 1238 .set_video_frame_buffer(buffer) 1239 .set_rtp_timestamp(100) 1240 .set_timestamp_ms(1000) 1241 .set_rotation(kVideoRotation_180) 1242 .build(); 1243 // Expect calls with the given video frame verbatim, since it's a texture 1244 // frame and can't otherwise be modified/resized, but only on the two 1245 // streams supporting it... 1246 EXPECT_CALL(*encoders[1], Encode(::testing::Ref(input_frame), _)).Times(1); 1247 EXPECT_CALL(*encoders[2], Encode(::testing::Ref(input_frame), _)).Times(1); 1248 // ...the lowest one gets a software buffer. 1249 EXPECT_CALL(*encoders[0], Encode) 1250 .WillOnce([&](const VideoFrame& frame, 1251 const std::vector<VideoFrameType>* /* frame_types */) { 1252 EXPECT_EQ(frame.video_frame_buffer()->type(), 1253 VideoFrameBuffer::Type::kI420); 1254 return 0; 1255 }); 1256 std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); 1257 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1258 } 1259 1260 TEST_F(TestSimulcastEncoderAdapterFake, GeneratesKeyFramesOnRequestedLayers) { 1261 // Set up common settings for three streams. 1262 SimulcastTestFixtureImpl::DefaultSettings( 1263 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1264 kVideoCodecVP8); 1265 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1266 adapter_->RegisterEncodeCompleteCallback(this); 1267 1268 // Input data. 1269 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 1270 1271 // Encode with three streams. 1272 codec_.startBitrate = 3000; 1273 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1274 1275 std::vector<VideoFrameType> frame_types; 1276 frame_types.resize(3, VideoFrameType::kVideoFrameKey); 1277 1278 std::vector<VideoFrameType> expected_keyframe(1, 1279 VideoFrameType::kVideoFrameKey); 1280 std::vector<VideoFrameType> expected_deltaframe( 1281 1, VideoFrameType::kVideoFrameDelta); 1282 1283 std::vector<MockVideoEncoder*> original_encoders = 1284 helper_->factory()->encoders(); 1285 ASSERT_EQ(3u, original_encoders.size()); 1286 EXPECT_CALL(*original_encoders[0], 1287 Encode(_, ::testing::Pointee(::testing::Eq(expected_keyframe)))) 1288 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1289 EXPECT_CALL(*original_encoders[1], 1290 Encode(_, ::testing::Pointee(::testing::Eq(expected_keyframe)))) 1291 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1292 EXPECT_CALL(*original_encoders[2], 1293 Encode(_, ::testing::Pointee(::testing::Eq(expected_keyframe)))) 1294 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1295 VideoFrame first_frame = VideoFrame::Builder() 1296 .set_video_frame_buffer(buffer) 1297 .set_rtp_timestamp(0) 1298 .set_timestamp_ms(0) 1299 .build(); 1300 EXPECT_EQ(0, adapter_->Encode(first_frame, &frame_types)); 1301 1302 // Request [key, delta, delta]. 1303 EXPECT_CALL(*original_encoders[0], 1304 Encode(_, ::testing::Pointee(::testing::Eq(expected_keyframe)))) 1305 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1306 EXPECT_CALL(*original_encoders[1], 1307 Encode(_, ::testing::Pointee(::testing::Eq(expected_deltaframe)))) 1308 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1309 EXPECT_CALL(*original_encoders[2], 1310 Encode(_, ::testing::Pointee(::testing::Eq(expected_deltaframe)))) 1311 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1312 frame_types[1] = VideoFrameType::kVideoFrameKey; 1313 frame_types[1] = VideoFrameType::kVideoFrameDelta; 1314 frame_types[2] = VideoFrameType::kVideoFrameDelta; 1315 VideoFrame second_frame = VideoFrame::Builder() 1316 .set_video_frame_buffer(buffer) 1317 .set_rtp_timestamp(10000) 1318 .set_timestamp_ms(100000) 1319 .build(); 1320 EXPECT_EQ(0, adapter_->Encode(second_frame, &frame_types)); 1321 1322 // Request [delta, key, delta]. 1323 EXPECT_CALL(*original_encoders[0], 1324 Encode(_, ::testing::Pointee(::testing::Eq(expected_deltaframe)))) 1325 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1326 EXPECT_CALL(*original_encoders[1], 1327 Encode(_, ::testing::Pointee(::testing::Eq(expected_keyframe)))) 1328 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1329 EXPECT_CALL(*original_encoders[2], 1330 Encode(_, ::testing::Pointee(::testing::Eq(expected_deltaframe)))) 1331 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1332 frame_types[0] = VideoFrameType::kVideoFrameDelta; 1333 frame_types[1] = VideoFrameType::kVideoFrameKey; 1334 frame_types[2] = VideoFrameType::kVideoFrameDelta; 1335 VideoFrame third_frame = VideoFrame::Builder() 1336 .set_video_frame_buffer(buffer) 1337 .set_rtp_timestamp(20000) 1338 .set_timestamp_ms(200000) 1339 .build(); 1340 EXPECT_EQ(0, adapter_->Encode(third_frame, &frame_types)); 1341 } 1342 1343 TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) { 1344 SimulcastTestFixtureImpl::DefaultSettings( 1345 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1346 kVideoCodecVP8); 1347 codec_.numberOfSimulcastStreams = 3; 1348 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1349 adapter_->RegisterEncodeCompleteCallback(this); 1350 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1351 // Tell the 2nd encoder to request software fallback. 1352 EXPECT_CALL(*helper_->factory()->encoders()[1], Encode(_, _)) 1353 .WillOnce(Return(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)); 1354 1355 // Send a fake frame and assert the return is software fallback. 1356 scoped_refptr<I420Buffer> input_buffer = 1357 I420Buffer::Create(kDefaultWidth, kDefaultHeight); 1358 input_buffer->InitializeData(); 1359 VideoFrame input_frame = VideoFrame::Builder() 1360 .set_video_frame_buffer(input_buffer) 1361 .set_rtp_timestamp(0) 1362 .set_timestamp_us(0) 1363 .set_rotation(kVideoRotation_0) 1364 .build(); 1365 std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); 1366 EXPECT_EQ(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, 1367 adapter_->Encode(input_frame, &frame_types)); 1368 } 1369 1370 TEST_F(TestSimulcastEncoderAdapterFake, TestInitFailureCleansUpEncoders) { 1371 SimulcastTestFixtureImpl::DefaultSettings( 1372 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1373 kVideoCodecVP8); 1374 codec_.numberOfSimulcastStreams = 3; 1375 helper_->factory()->set_init_encode_return_value( 1376 WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); 1377 EXPECT_EQ(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, 1378 adapter_->InitEncode(&codec_, kSettings)); 1379 EXPECT_TRUE(helper_->factory()->encoders().empty()); 1380 } 1381 1382 TEST_F(TestSimulcastEncoderAdapterFake, DoesNotAlterMaxQpForScreenshare) { 1383 const int kHighMaxQp = 56; 1384 const int kLowMaxQp = 46; 1385 1386 SimulcastTestFixtureImpl::DefaultSettings( 1387 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1388 kVideoCodecVP8); 1389 codec_.numberOfSimulcastStreams = 3; 1390 codec_.simulcastStream[0].qpMax = kHighMaxQp; 1391 codec_.mode = VideoCodecMode::kScreensharing; 1392 1393 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1394 EXPECT_EQ(3u, helper_->factory()->encoders().size()); 1395 1396 // Just check the lowest stream, which is the one that where the adapter 1397 // might alter the max qp setting. 1398 VideoCodec ref_codec; 1399 InitRefCodec(0, &ref_codec); 1400 ref_codec.qpMax = kHighMaxQp; 1401 ref_codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityHigher); 1402 ref_codec.VP8()->denoisingOn = false; 1403 ref_codec.startBitrate = 100; // Should equal to the target bitrate. 1404 VerifyCodec(ref_codec, 0); 1405 1406 // Change the max qp and try again. 1407 codec_.simulcastStream[0].qpMax = kLowMaxQp; 1408 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1409 EXPECT_EQ(3u, helper_->factory()->encoders().size()); 1410 ref_codec.qpMax = kLowMaxQp; 1411 VerifyCodec(ref_codec, 0); 1412 } 1413 1414 TEST_F(TestSimulcastEncoderAdapterFake, 1415 DoesNotAlterMaxQpForScreenshareReversedLayer) { 1416 const int kHighMaxQp = 56; 1417 const int kLowMaxQp = 46; 1418 1419 SimulcastTestFixtureImpl::DefaultSettings( 1420 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1421 kVideoCodecVP8, true /* reverse_layer_order */); 1422 codec_.numberOfSimulcastStreams = 3; 1423 codec_.simulcastStream[2].qpMax = kHighMaxQp; 1424 codec_.mode = VideoCodecMode::kScreensharing; 1425 1426 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1427 EXPECT_EQ(3u, helper_->factory()->encoders().size()); 1428 1429 // Just check the lowest stream, which is the one that where the adapter 1430 // might alter the max qp setting. 1431 VideoCodec ref_codec; 1432 InitRefCodec(2, &ref_codec, true /* reverse_layer_order */); 1433 ref_codec.qpMax = kHighMaxQp; 1434 ref_codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityHigher); 1435 ref_codec.VP8()->denoisingOn = false; 1436 ref_codec.startBitrate = 100; // Should equal to the target bitrate. 1437 VerifyCodec(ref_codec, 2); 1438 1439 // Change the max qp and try again. 1440 codec_.simulcastStream[2].qpMax = kLowMaxQp; 1441 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1442 EXPECT_EQ(3u, helper_->factory()->encoders().size()); 1443 ref_codec.qpMax = kLowMaxQp; 1444 VerifyCodec(ref_codec, 2); 1445 } 1446 1447 TEST_F(TestSimulcastEncoderAdapterFake, ActivatesCorrectStreamsInInitEncode) { 1448 // Set up common settings for three streams. 1449 SimulcastTestFixtureImpl::DefaultSettings( 1450 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1451 kVideoCodecVP8); 1452 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1453 adapter_->RegisterEncodeCompleteCallback(this); 1454 1455 // Only enough start bitrate for the lowest stream. 1456 ASSERT_EQ(3u, codec_.numberOfSimulcastStreams); 1457 codec_.startBitrate = codec_.simulcastStream[0].targetBitrate + 1458 codec_.simulcastStream[1].minBitrate - 1; 1459 1460 // Input data. 1461 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 1462 VideoFrame input_frame = VideoFrame::Builder() 1463 .set_video_frame_buffer(buffer) 1464 .set_rtp_timestamp(100) 1465 .set_timestamp_ms(1000) 1466 .set_rotation(kVideoRotation_180) 1467 .build(); 1468 1469 // Encode with three streams. 1470 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1471 std::vector<MockVideoEncoder*> original_encoders = 1472 helper_->factory()->encoders(); 1473 ASSERT_EQ(3u, original_encoders.size()); 1474 // Only first encoder will be active and called. 1475 EXPECT_CALL(*original_encoders[0], Encode(_, _)) 1476 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1477 EXPECT_CALL(*original_encoders[1], Encode(_, _)).Times(0); 1478 EXPECT_CALL(*original_encoders[2], Encode(_, _)).Times(0); 1479 1480 std::vector<VideoFrameType> frame_types; 1481 frame_types.resize(3, VideoFrameType::kVideoFrameKey); 1482 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1483 } 1484 1485 TEST_F(TestSimulcastEncoderAdapterFake, TrustedRateControl) { 1486 // Set up common settings for three streams. 1487 SimulcastTestFixtureImpl::DefaultSettings( 1488 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1489 kVideoCodecVP8); 1490 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1491 adapter_->RegisterEncodeCompleteCallback(this); 1492 1493 // Only enough start bitrate for the lowest stream. 1494 ASSERT_EQ(3u, codec_.numberOfSimulcastStreams); 1495 codec_.startBitrate = codec_.simulcastStream[0].targetBitrate + 1496 codec_.simulcastStream[1].minBitrate - 1; 1497 1498 // Input data. 1499 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 1500 VideoFrame input_frame = VideoFrame::Builder() 1501 .set_video_frame_buffer(buffer) 1502 .set_rtp_timestamp(100) 1503 .set_timestamp_ms(1000) 1504 .set_rotation(kVideoRotation_180) 1505 .build(); 1506 1507 // No encoder trusted, so simulcast adapter should not be either. 1508 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1509 EXPECT_FALSE(adapter_->GetEncoderInfo().has_trusted_rate_controller); 1510 1511 // Encode with three streams. 1512 std::vector<MockVideoEncoder*> original_encoders = 1513 helper_->factory()->encoders(); 1514 1515 // All encoders are trusted, so simulcast adapter should be too. 1516 original_encoders[0]->set_has_trusted_rate_controller(true); 1517 original_encoders[1]->set_has_trusted_rate_controller(true); 1518 original_encoders[2]->set_has_trusted_rate_controller(true); 1519 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1520 EXPECT_TRUE(adapter_->GetEncoderInfo().has_trusted_rate_controller); 1521 1522 // One encoder not trusted, so simulcast adapter should not be either. 1523 original_encoders[2]->set_has_trusted_rate_controller(false); 1524 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1525 EXPECT_FALSE(adapter_->GetEncoderInfo().has_trusted_rate_controller); 1526 1527 // No encoder trusted, so simulcast adapter should not be either. 1528 original_encoders[0]->set_has_trusted_rate_controller(false); 1529 original_encoders[1]->set_has_trusted_rate_controller(false); 1530 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1531 EXPECT_FALSE(adapter_->GetEncoderInfo().has_trusted_rate_controller); 1532 } 1533 1534 TEST_F(TestSimulcastEncoderAdapterFake, ReportsHardwareAccelerated) { 1535 SimulcastTestFixtureImpl::DefaultSettings( 1536 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1537 kVideoCodecVP8); 1538 codec_.numberOfSimulcastStreams = 3; 1539 adapter_->RegisterEncodeCompleteCallback(this); 1540 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1541 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1542 1543 // None of the encoders uses HW support, so simulcast adapter reports false. 1544 for (MockVideoEncoder* encoder : helper_->factory()->encoders()) { 1545 encoder->set_is_hardware_accelerated(false); 1546 } 1547 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1548 EXPECT_FALSE(adapter_->GetEncoderInfo().is_hardware_accelerated); 1549 1550 // One encoder uses HW support, so simulcast adapter reports true. 1551 helper_->factory()->encoders()[2]->set_is_hardware_accelerated(true); 1552 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1553 EXPECT_TRUE(adapter_->GetEncoderInfo().is_hardware_accelerated); 1554 } 1555 1556 TEST_F(TestSimulcastEncoderAdapterFake, 1557 ReportsLeastCommonMultipleOfRequestedResolutionAlignments) { 1558 SimulcastTestFixtureImpl::DefaultSettings( 1559 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1560 kVideoCodecVP8); 1561 codec_.numberOfSimulcastStreams = 3; 1562 helper_->factory()->set_requested_resolution_alignments({2, 4, 7}); 1563 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1564 1565 EXPECT_EQ(adapter_->GetEncoderInfo().requested_resolution_alignment, 28u); 1566 } 1567 1568 TEST_F(TestSimulcastEncoderAdapterFake, 1569 ReportsApplyAlignmentToSimulcastLayers) { 1570 SimulcastTestFixtureImpl::DefaultSettings( 1571 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1572 kVideoCodecVP8); 1573 codec_.numberOfSimulcastStreams = 3; 1574 1575 // No encoder has apply_alignment_to_all_simulcast_layers, report false. 1576 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1577 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1578 for (MockVideoEncoder* encoder : helper_->factory()->encoders()) { 1579 encoder->set_apply_alignment_to_all_simulcast_layers(false); 1580 } 1581 EXPECT_FALSE( 1582 adapter_->GetEncoderInfo().apply_alignment_to_all_simulcast_layers); 1583 1584 // One encoder has apply_alignment_to_all_simulcast_layers, report true. 1585 helper_->factory() 1586 ->encoders()[1] 1587 ->set_apply_alignment_to_all_simulcast_layers(true); 1588 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1589 EXPECT_TRUE( 1590 adapter_->GetEncoderInfo().apply_alignment_to_all_simulcast_layers); 1591 } 1592 1593 TEST_F( 1594 TestSimulcastEncoderAdapterFake, 1595 EncoderInfoFromFieldTrialDoesNotOverrideExistingBitrateLimitsInSinglecast) { 1596 field_trials_.Set("WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride", 1597 "frame_size_pixels:123|456|789," 1598 "min_start_bitrate_bps:11000|22000|33000," 1599 "min_bitrate_bps:44000|55000|66000," 1600 "max_bitrate_bps:77000|88000|99000"); 1601 SetUp(); 1602 1603 std::vector<VideoEncoder::ResolutionBitrateLimits> bitrate_limits; 1604 bitrate_limits.push_back( 1605 VideoEncoder::ResolutionBitrateLimits(111, 11100, 44400, 77700)); 1606 bitrate_limits.push_back( 1607 VideoEncoder::ResolutionBitrateLimits(444, 22200, 55500, 88700)); 1608 bitrate_limits.push_back( 1609 VideoEncoder::ResolutionBitrateLimits(777, 33300, 66600, 99900)); 1610 SetUp(); 1611 helper_->factory()->set_resolution_bitrate_limits(bitrate_limits); 1612 1613 SimulcastTestFixtureImpl::DefaultSettings( 1614 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1615 kVideoCodecVP8); 1616 codec_.numberOfSimulcastStreams = 1; 1617 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1618 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 1619 EXPECT_EQ(adapter_->GetEncoderInfo().resolution_bitrate_limits, 1620 bitrate_limits); 1621 } 1622 1623 TEST_F(TestSimulcastEncoderAdapterFake, EncoderInfoFromFieldTrial) { 1624 field_trials_.Set("WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride", 1625 "requested_resolution_alignment:8," 1626 "apply_alignment_to_all_simulcast_layers"); 1627 SetUp(); 1628 SimulcastTestFixtureImpl::DefaultSettings( 1629 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1630 kVideoCodecVP8); 1631 codec_.numberOfSimulcastStreams = 3; 1632 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1633 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1634 1635 EXPECT_EQ(8u, adapter_->GetEncoderInfo().requested_resolution_alignment); 1636 EXPECT_TRUE( 1637 adapter_->GetEncoderInfo().apply_alignment_to_all_simulcast_layers); 1638 EXPECT_TRUE(adapter_->GetEncoderInfo().resolution_bitrate_limits.empty()); 1639 } 1640 1641 TEST_F(TestSimulcastEncoderAdapterFake, 1642 EncoderInfoFromFieldTrialForSingleStream) { 1643 field_trials_.Set("WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride", 1644 "requested_resolution_alignment:9," 1645 "frame_size_pixels:123|456|789," 1646 "min_start_bitrate_bps:11000|22000|33000," 1647 "min_bitrate_bps:44000|55000|66000," 1648 "max_bitrate_bps:77000|88000|99000"); 1649 SetUp(); 1650 SimulcastTestFixtureImpl::DefaultSettings( 1651 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1652 kVideoCodecVP8); 1653 codec_.numberOfSimulcastStreams = 1; 1654 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1655 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 1656 1657 EXPECT_EQ(9u, adapter_->GetEncoderInfo().requested_resolution_alignment); 1658 EXPECT_FALSE( 1659 adapter_->GetEncoderInfo().apply_alignment_to_all_simulcast_layers); 1660 EXPECT_THAT( 1661 adapter_->GetEncoderInfo().resolution_bitrate_limits, 1662 ::testing::ElementsAre( 1663 VideoEncoder::ResolutionBitrateLimits{123, 11000, 44000, 77000}, 1664 VideoEncoder::ResolutionBitrateLimits{456, 22000, 55000, 88000}, 1665 VideoEncoder::ResolutionBitrateLimits{789, 33000, 66000, 99000})); 1666 } 1667 1668 TEST_F(TestSimulcastEncoderAdapterFake, ReportsIsQpTrusted) { 1669 SimulcastTestFixtureImpl::DefaultSettings( 1670 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1671 kVideoCodecVP8); 1672 codec_.numberOfSimulcastStreams = 3; 1673 adapter_->RegisterEncodeCompleteCallback(this); 1674 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1675 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1676 1677 // All encoders have internal source, simulcast adapter reports true. 1678 for (MockVideoEncoder* encoder : helper_->factory()->encoders()) { 1679 encoder->set_is_qp_trusted(true); 1680 } 1681 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1682 EXPECT_TRUE(adapter_->GetEncoderInfo().is_qp_trusted.value_or(false)); 1683 1684 // One encoder reports QP not trusted, simulcast adapter reports false. 1685 helper_->factory()->encoders()[2]->set_is_qp_trusted(false); 1686 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1687 EXPECT_FALSE(adapter_->GetEncoderInfo().is_qp_trusted.value_or(true)); 1688 } 1689 1690 TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) { 1691 SimulcastTestFixtureImpl::DefaultSettings( 1692 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1693 kVideoCodecVP8); 1694 codec_.numberOfSimulcastStreams = 3; 1695 adapter_->RegisterEncodeCompleteCallback(this); 1696 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1697 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 1698 1699 // Combination of three different supported mode: 1700 // Simulcast stream 0 has undefined fps behavior. 1701 // Simulcast stream 1 has three temporal layers. 1702 // Simulcast stream 2 has 1 temporal layer. 1703 FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; 1704 expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 4); 1705 expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2); 1706 expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction); 1707 expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction); 1708 1709 // All encoders have internal source, simulcast adapter reports true. 1710 for (size_t i = 0; i < codec_.numberOfSimulcastStreams; ++i) { 1711 MockVideoEncoder* encoder = helper_->factory()->encoders()[i]; 1712 encoder->set_fps_allocation(expected_fps_allocation[i]); 1713 } 1714 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1715 EXPECT_THAT(adapter_->GetEncoderInfo().fps_allocation, 1716 ::testing::ElementsAreArray(expected_fps_allocation)); 1717 } 1718 1719 TEST_F(TestSimulcastEncoderAdapterFake, SetRateDistributesBandwithAllocation) { 1720 SimulcastTestFixtureImpl::DefaultSettings( 1721 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1722 kVideoCodecVP8); 1723 codec_.numberOfSimulcastStreams = 3; 1724 const DataRate target_bitrate = 1725 DataRate::KilobitsPerSec(codec_.simulcastStream[0].targetBitrate + 1726 codec_.simulcastStream[1].targetBitrate + 1727 codec_.simulcastStream[2].minBitrate); 1728 const DataRate bandwidth_allocation = 1729 target_bitrate + DataRate::KilobitsPerSec(600); 1730 1731 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1732 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1733 adapter_->RegisterEncodeCompleteCallback(this); 1734 1735 // Set bitrates so that we send all layers. 1736 adapter_->SetRates(VideoEncoder::RateControlParameters( 1737 rate_allocator_->Allocate( 1738 VideoBitrateAllocationParameters(target_bitrate.bps(), 30)), 1739 30.0, bandwidth_allocation)); 1740 1741 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 1742 1743 ASSERT_EQ(3u, encoders.size()); 1744 1745 for (size_t i = 0; i < 3; ++i) { 1746 const uint32_t layer_bitrate_bps = 1747 (i < static_cast<size_t>(codec_.numberOfSimulcastStreams) - 1 1748 ? codec_.simulcastStream[i].targetBitrate 1749 : codec_.simulcastStream[i].minBitrate) * 1750 1000; 1751 EXPECT_EQ(layer_bitrate_bps, 1752 encoders[i]->last_set_rates().bitrate.get_sum_bps()) 1753 << i; 1754 EXPECT_EQ( 1755 (layer_bitrate_bps * bandwidth_allocation.bps()) / target_bitrate.bps(), 1756 encoders[i]->last_set_rates().bandwidth_allocation.bps()) 1757 << i; 1758 } 1759 } 1760 1761 TEST_F(TestSimulcastEncoderAdapterFake, CanSetZeroBitrateWithHeadroom) { 1762 SimulcastTestFixtureImpl::DefaultSettings( 1763 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1764 kVideoCodecVP8); 1765 codec_.numberOfSimulcastStreams = 3; 1766 1767 rate_allocator_ = std::make_unique<SimulcastRateAllocator>(env_, codec_); 1768 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1769 adapter_->RegisterEncodeCompleteCallback(this); 1770 1771 // Set allocated bitrate to 0, but keep (network) bandwidth allocation. 1772 VideoEncoder::RateControlParameters rate_params; 1773 rate_params.framerate_fps = 30; 1774 rate_params.bandwidth_allocation = DataRate::KilobitsPerSec(600); 1775 1776 adapter_->SetRates(rate_params); 1777 1778 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 1779 1780 ASSERT_EQ(3u, encoders.size()); 1781 for (size_t i = 0; i < 3; ++i) { 1782 EXPECT_EQ(0u, encoders[i]->last_set_rates().bitrate.get_sum_bps()); 1783 } 1784 } 1785 1786 TEST_F(TestSimulcastEncoderAdapterFake, SupportsSimulcast) { 1787 SimulcastTestFixtureImpl::DefaultSettings( 1788 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1789 kVideoCodecVP8); 1790 codec_.numberOfSimulcastStreams = 3; 1791 1792 // Indicate that mock encoders internally support simulcast. 1793 helper_->factory()->set_supports_simulcast(true); 1794 adapter_->RegisterEncodeCompleteCallback(this); 1795 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1796 1797 // Only one encoder should have been produced. 1798 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 1799 1800 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 1801 VideoFrame input_frame = VideoFrame::Builder() 1802 .set_video_frame_buffer(buffer) 1803 .set_rtp_timestamp(100) 1804 .set_timestamp_ms(1000) 1805 .set_rotation(kVideoRotation_180) 1806 .build(); 1807 EXPECT_CALL(*helper_->factory()->encoders()[0], Encode) 1808 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1809 std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); 1810 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1811 } 1812 1813 TEST_F(TestSimulcastEncoderAdapterFake, PassesSdpVideoFormatToEncoder) { 1814 sdp_video_parameters_ = {{"test_param", "test_value"}}; 1815 SetUp(); 1816 SetupCodec(); 1817 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 1818 ASSERT_GT(encoders.size(), 0u); 1819 EXPECT_EQ(encoders[0]->video_format(), 1820 SdpVideoFormat("VP8", sdp_video_parameters_)); 1821 } 1822 1823 TEST_F(TestSimulcastEncoderAdapterFake, SupportsFallback) { 1824 // Enable support for fallback encoder factory and re-setup. 1825 use_fallback_factory_ = true; 1826 SetUp(); 1827 1828 SetupCodec(); 1829 1830 // Make sure we have bitrate for all layers. 1831 DataRate max_bitrate = DataRate::Zero(); 1832 for (int i = 0; i < 3; ++i) { 1833 max_bitrate += 1834 DataRate::KilobitsPerSec(codec_.simulcastStream[i].maxBitrate); 1835 } 1836 const auto rate_settings = VideoEncoder::RateControlParameters( 1837 rate_allocator_->Allocate( 1838 VideoBitrateAllocationParameters(max_bitrate.bps(), 30)), 1839 30.0, max_bitrate); 1840 adapter_->SetRates(rate_settings); 1841 1842 std::vector<MockVideoEncoder*> primary_encoders = 1843 helper_->factory()->encoders(); 1844 std::vector<MockVideoEncoder*> fallback_encoders = 1845 helper_->fallback_factory()->encoders(); 1846 1847 ASSERT_EQ(3u, primary_encoders.size()); 1848 ASSERT_EQ(3u, fallback_encoders.size()); 1849 1850 // Create frame to test with. 1851 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 1852 VideoFrame input_frame = VideoFrame::Builder() 1853 .set_video_frame_buffer(buffer) 1854 .set_rtp_timestamp(100) 1855 .set_timestamp_ms(1000) 1856 .set_rotation(kVideoRotation_180) 1857 .build(); 1858 std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); 1859 1860 // All primary encoders used. 1861 for (auto codec : primary_encoders) { 1862 EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1863 } 1864 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1865 1866 // Trigger fallback on first encoder. 1867 primary_encoders[0]->set_init_encode_return_value( 1868 WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); 1869 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1870 adapter_->SetRates(rate_settings); 1871 EXPECT_CALL(*fallback_encoders[0], Encode) 1872 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1873 EXPECT_CALL(*primary_encoders[1], Encode) 1874 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1875 EXPECT_CALL(*primary_encoders[2], Encode) 1876 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1877 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1878 1879 // Trigger fallback on all encoder. 1880 primary_encoders[1]->set_init_encode_return_value( 1881 WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); 1882 primary_encoders[2]->set_init_encode_return_value( 1883 WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); 1884 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1885 adapter_->SetRates(rate_settings); 1886 EXPECT_CALL(*fallback_encoders[0], Encode) 1887 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1888 EXPECT_CALL(*fallback_encoders[1], Encode) 1889 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1890 EXPECT_CALL(*fallback_encoders[2], Encode) 1891 .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1892 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1893 1894 // Return to primary encoders on all streams. 1895 for (int i = 0; i < 3; ++i) { 1896 primary_encoders[i]->set_init_encode_return_value(WEBRTC_VIDEO_CODEC_OK); 1897 } 1898 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 1899 adapter_->SetRates(rate_settings); 1900 for (auto codec : primary_encoders) { 1901 EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1902 } 1903 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1904 } 1905 1906 TEST_F(TestSimulcastEncoderAdapterFake, 1907 SupportsHardwareSimulcastWithBadParametrs) { 1908 SimulcastTestFixtureImpl::DefaultSettings( 1909 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1910 kVideoCodecVP8); 1911 1912 // Enable support for fallback encoder factory and re-setup. 1913 use_fallback_factory_ = true; 1914 SetUp(); 1915 1916 helper_->factory()->set_supports_simulcast(true); 1917 // Make encoders reject the simulcast configuration despite supporting it 1918 // because parameters are not good for simulcast (e.g. different temporal 1919 // layers setting). 1920 helper_->factory()->set_fallback_from_simulcast( 1921 WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED); 1922 1923 SetupCodec(); 1924 1925 // Make sure we have bitrate for all layers. 1926 DataRate max_bitrate = DataRate::Zero(); 1927 for (int i = 0; i < 3; ++i) { 1928 max_bitrate += 1929 DataRate::KilobitsPerSec(codec_.simulcastStream[i].maxBitrate); 1930 } 1931 const auto rate_settings = VideoEncoder::RateControlParameters( 1932 rate_allocator_->Allocate( 1933 VideoBitrateAllocationParameters(max_bitrate.bps(), 30)), 1934 30.0, max_bitrate); 1935 adapter_->SetRates(rate_settings); 1936 1937 std::vector<MockVideoEncoder*> primary_encoders = 1938 helper_->factory()->encoders(); 1939 std::vector<MockVideoEncoder*> fallback_encoders = 1940 helper_->fallback_factory()->encoders(); 1941 1942 ASSERT_EQ(3u, primary_encoders.size()); 1943 ASSERT_EQ(3u, fallback_encoders.size()); 1944 1945 // Create frame to test with. 1946 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 1947 VideoFrame input_frame = VideoFrame::Builder() 1948 .set_video_frame_buffer(buffer) 1949 .set_rtp_timestamp(100) 1950 .set_timestamp_ms(1000) 1951 .set_rotation(kVideoRotation_180) 1952 .build(); 1953 std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); 1954 1955 // All primary encoders must be used. 1956 for (auto codec : primary_encoders) { 1957 EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 1958 } 1959 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 1960 } 1961 1962 TEST_F(TestSimulcastEncoderAdapterFake, SupportsHardwareSimulcast) { 1963 SimulcastTestFixtureImpl::DefaultSettings( 1964 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 1965 kVideoCodecVP8); 1966 1967 // Enable support for fallback encoder factory and re-setup. 1968 use_fallback_factory_ = true; 1969 SetUp(); 1970 1971 helper_->factory()->set_supports_simulcast(true); 1972 helper_->factory()->set_fallback_from_simulcast(std::nullopt); 1973 1974 SetupCodec(); 1975 1976 // Make sure we have bitrate for all layers. 1977 DataRate max_bitrate = DataRate::Zero(); 1978 for (int i = 0; i < 3; ++i) { 1979 max_bitrate += 1980 DataRate::KilobitsPerSec(codec_.simulcastStream[i].maxBitrate); 1981 } 1982 const auto rate_settings = VideoEncoder::RateControlParameters( 1983 rate_allocator_->Allocate( 1984 VideoBitrateAllocationParameters(max_bitrate.bps(), 30)), 1985 30.0, max_bitrate); 1986 adapter_->SetRates(rate_settings); 1987 1988 std::vector<MockVideoEncoder*> primary_encoders = 1989 helper_->factory()->encoders(); 1990 std::vector<MockVideoEncoder*> fallback_encoders = 1991 helper_->fallback_factory()->encoders(); 1992 1993 ASSERT_EQ(1u, primary_encoders.size()); 1994 ASSERT_EQ(1u, fallback_encoders.size()); 1995 1996 // Create frame to test with. 1997 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 1998 VideoFrame input_frame = VideoFrame::Builder() 1999 .set_video_frame_buffer(buffer) 2000 .set_rtp_timestamp(100) 2001 .set_timestamp_ms(1000) 2002 .set_rotation(kVideoRotation_180) 2003 .build(); 2004 std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); 2005 2006 // A primary encoders must be used. 2007 for (auto codec : primary_encoders) { 2008 EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); 2009 } 2010 EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); 2011 } 2012 2013 TEST_F(TestSimulcastEncoderAdapterFake, SupportsPerSimulcastLayerMaxFramerate) { 2014 SimulcastTestFixtureImpl::DefaultSettings( 2015 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 2016 kVideoCodecVP8); 2017 codec_.numberOfSimulcastStreams = 3; 2018 codec_.simulcastStream[0].maxFramerate = 60; 2019 codec_.simulcastStream[1].maxFramerate = 30; 2020 codec_.simulcastStream[2].maxFramerate = 10; 2021 2022 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 2023 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 2024 EXPECT_EQ(60u, helper_->factory()->encoders()[0]->codec().maxFramerate); 2025 EXPECT_EQ(30u, helper_->factory()->encoders()[1]->codec().maxFramerate); 2026 EXPECT_EQ(10u, helper_->factory()->encoders()[2]->codec().maxFramerate); 2027 } 2028 2029 TEST_F(TestSimulcastEncoderAdapterFake, CreatesEncoderOnlyIfStreamIsActive) { 2030 // Legacy singlecast 2031 SetupCodec(/*active_streams=*/{}); 2032 EXPECT_EQ(1u, helper_->factory()->encoders().size()); 2033 2034 // Simulcast-capable underlaying encoder 2035 ReSetUp(); 2036 helper_->factory()->set_supports_simulcast(true); 2037 SetupCodec(/*active_streams=*/{true, true}); 2038 EXPECT_EQ(1u, helper_->factory()->encoders().size()); 2039 2040 // Muti-encoder simulcast 2041 ReSetUp(); 2042 helper_->factory()->set_supports_simulcast(false); 2043 SetupCodec(/*active_streams=*/{true, true}); 2044 EXPECT_EQ(2u, helper_->factory()->encoders().size()); 2045 2046 // Singlecast via layers deactivation. Lowest layer is active. 2047 ReSetUp(); 2048 helper_->factory()->set_supports_simulcast(false); 2049 SetupCodec(/*active_streams=*/{true, false}); 2050 EXPECT_EQ(1u, helper_->factory()->encoders().size()); 2051 2052 // Singlecast via layers deactivation. Highest layer is active. 2053 ReSetUp(); 2054 helper_->factory()->set_supports_simulcast(false); 2055 SetupCodec(/*active_streams=*/{false, true}); 2056 EXPECT_EQ(1u, helper_->factory()->encoders().size()); 2057 } 2058 2059 TEST_F(TestSimulcastEncoderAdapterFake, 2060 RecreateEncoderIfPreferTemporalSupportIsEnabled) { 2061 // Normally SEA reuses encoders. But, when TL-based SW fallback is enabled, 2062 // the encoder which served the lowest stream should be recreated before it 2063 // can be used to process an upper layer and vice-versa. 2064 field_trials_.Set("WebRTC-Video-PreferTemporalSupportOnBaseLayer", "Enabled"); 2065 use_fallback_factory_ = true; 2066 ReSetUp(); 2067 2068 // Legacy singlecast 2069 SetupCodec(/*active_streams=*/{}); 2070 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 2071 2072 // Singlecast, the lowest stream is active. Encoder should be reused. 2073 MockVideoEncoder* prev_encoder = helper_->factory()->encoders()[0]; 2074 SetupCodec(/*active_streams=*/{true, false}); 2075 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 2076 EXPECT_EQ(helper_->factory()->encoders()[0], prev_encoder); 2077 2078 // Singlecast, an upper stream is active. Encoder should be recreated. 2079 EXPECT_CALL(*prev_encoder, Release()).Times(1); 2080 SetupCodec(/*active_streams=*/{false, true}); 2081 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 2082 EXPECT_NE(helper_->factory()->encoders()[0], prev_encoder); 2083 2084 // Singlecast, the lowest stream is active. Encoder should be recreated. 2085 prev_encoder = helper_->factory()->encoders()[0]; 2086 EXPECT_CALL(*prev_encoder, Release()).Times(1); 2087 SetupCodec(/*active_streams=*/{true, false}); 2088 ASSERT_EQ(1u, helper_->factory()->encoders().size()); 2089 EXPECT_NE(helper_->factory()->encoders()[0], prev_encoder); 2090 } 2091 2092 TEST_F(TestSimulcastEncoderAdapterFake, 2093 UseFallbackEncoderIfCreatePrimaryEncoderFailed) { 2094 // Enable support for fallback encoder factory and re-setup. 2095 use_fallback_factory_ = true; 2096 SetUp(); 2097 SimulcastTestFixtureImpl::DefaultSettings( 2098 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 2099 kVideoCodecVP8); 2100 codec_.numberOfSimulcastStreams = 1; 2101 helper_->factory()->SetEncoderNames({"primary"}); 2102 helper_->fallback_factory()->SetEncoderNames({"fallback"}); 2103 2104 // Emulate failure at creating of primary encoder and verify that SEA switches 2105 // to fallback encoder. 2106 helper_->factory()->set_create_video_encode_return_nullptr(true); 2107 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 2108 ASSERT_EQ(0u, helper_->factory()->encoders().size()); 2109 ASSERT_EQ(1u, helper_->fallback_factory()->encoders().size()); 2110 EXPECT_EQ("fallback", adapter_->GetEncoderInfo().implementation_name); 2111 } 2112 2113 TEST_F(TestSimulcastEncoderAdapterFake, 2114 InitEncodeReturnsErrorIfEncoderCannotBeCreated) { 2115 // Enable support for fallback encoder factory and re-setup. 2116 use_fallback_factory_ = true; 2117 SetUp(); 2118 SimulcastTestFixtureImpl::DefaultSettings( 2119 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 2120 kVideoCodecVP8); 2121 codec_.numberOfSimulcastStreams = 1; 2122 helper_->factory()->SetEncoderNames({"primary"}); 2123 helper_->fallback_factory()->SetEncoderNames({"fallback"}); 2124 2125 // Emulate failure at creating of primary and fallback encoders and verify 2126 // that `InitEncode` returns an error. 2127 helper_->factory()->set_create_video_encode_return_nullptr(true); 2128 helper_->fallback_factory()->set_create_video_encode_return_nullptr(true); 2129 EXPECT_EQ(WEBRTC_VIDEO_CODEC_MEMORY, 2130 adapter_->InitEncode(&codec_, kSettings)); 2131 } 2132 2133 TEST_F(TestSimulcastEncoderAdapterFake, PopulatesScalabilityModeOfSubcodecs) { 2134 SimulcastTestFixtureImpl::DefaultSettings( 2135 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 2136 kVideoCodecVP8); 2137 codec_.numberOfSimulcastStreams = 3; 2138 codec_.simulcastStream[0].numberOfTemporalLayers = 1; 2139 codec_.simulcastStream[1].numberOfTemporalLayers = 2; 2140 codec_.simulcastStream[2].numberOfTemporalLayers = 3; 2141 2142 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 2143 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 2144 EXPECT_EQ(helper_->factory()->encoders()[0]->codec().GetScalabilityMode(), 2145 ScalabilityMode::kL1T1); 2146 EXPECT_EQ(helper_->factory()->encoders()[1]->codec().GetScalabilityMode(), 2147 ScalabilityMode::kL1T2); 2148 EXPECT_EQ(helper_->factory()->encoders()[2]->codec().GetScalabilityMode(), 2149 ScalabilityMode::kL1T3); 2150 } 2151 2152 // In the case of mixed-codec simulcast, verify whether each encoder is created 2153 // with the specified video format. 2154 TEST_F(TestSimulcastEncoderAdapterFake, InitEncodeForMixedCodec) { 2155 std::vector<SdpVideoFormat> codecs = {SdpVideoFormat::VP8(), 2156 SdpVideoFormat::VP9Profile0(), 2157 SdpVideoFormat::VP9Profile1()}; 2158 SetupMixedCodec({{true, SdpVideoFormat::VP8()}, 2159 {true, SdpVideoFormat::VP9Profile0()}, 2160 {true, SdpVideoFormat::VP9Profile1()}}); 2161 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 2162 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 2163 EXPECT_EQ(encoders[0]->video_format(), SdpVideoFormat::VP8()); 2164 EXPECT_EQ(encoders[1]->video_format(), SdpVideoFormat::VP9Profile0()); 2165 EXPECT_EQ(encoders[2]->video_format(), SdpVideoFormat::VP9Profile1()); 2166 EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP8); 2167 EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); 2168 EXPECT_EQ(encoders[2]->codec().codecType, webrtc::kVideoCodecVP9); 2169 2170 SetupMixedCodec({{false, SdpVideoFormat::VP8()}, 2171 {true, SdpVideoFormat::VP9Profile0()}, 2172 {true, SdpVideoFormat::VP9Profile1()}}); 2173 encoders = helper_->factory()->encoders(); 2174 ASSERT_EQ(2u, helper_->factory()->encoders().size()); 2175 EXPECT_EQ(encoders[0]->video_format(), SdpVideoFormat::VP9Profile0()); 2176 EXPECT_EQ(encoders[1]->video_format(), SdpVideoFormat::VP9Profile1()); 2177 EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP9); 2178 EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); 2179 } 2180 2181 TEST_F(TestSimulcastEncoderAdapterFake, 2182 CodecSpecificSettingsIsInitializedDefaultValueForMixedCodec) { 2183 std::vector<SdpVideoFormat> codecs = {SdpVideoFormat::VP8(), 2184 SdpVideoFormat::VP9Profile0(), 2185 SdpVideoFormat::VP9Profile1()}; 2186 SetupMixedCodec({{true, SdpVideoFormat::VP8()}, 2187 {true, SdpVideoFormat::VP9Profile0()}, 2188 {true, SdpVideoFormat::VP9Profile1()}}); 2189 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 2190 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 2191 EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP8); 2192 EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); 2193 EXPECT_EQ(encoders[2]->codec().codecType, webrtc::kVideoCodecVP9); 2194 2195 // Fields in the codec specific settings that are not set in 2196 // SimulcastEncoderAdapter should be initialized with default values. 2197 auto vp8_defaults = VideoEncoder::GetDefaultVp8Settings(); 2198 auto vp9_defaults = VideoEncoder::GetDefaultVp9Settings(); 2199 EXPECT_EQ(encoders[0]->codec().VP8().automaticResizeOn, 2200 vp8_defaults.automaticResizeOn); 2201 EXPECT_EQ(encoders[0]->codec().VP8().keyFrameInterval, 2202 vp8_defaults.keyFrameInterval); 2203 for (int i = 1; i <= 2; i++) { 2204 EXPECT_EQ(encoders[i]->codec().VP9().denoisingOn, vp9_defaults.denoisingOn); 2205 EXPECT_EQ(encoders[i]->codec().VP9().keyFrameInterval, 2206 vp9_defaults.keyFrameInterval); 2207 EXPECT_EQ(encoders[i]->codec().VP9().adaptiveQpMode, 2208 vp9_defaults.adaptiveQpMode); 2209 EXPECT_EQ(encoders[i]->codec().VP9().automaticResizeOn, 2210 vp9_defaults.automaticResizeOn); 2211 EXPECT_EQ(encoders[i]->codec().VP9().flexibleMode, 2212 vp9_defaults.flexibleMode); 2213 } 2214 } 2215 2216 // In the case of mixed-codec simulcast, multiple encoders are used even if 2217 // supports_simulcast() == true. 2218 TEST_F(TestSimulcastEncoderAdapterFake, 2219 CreateMultipleEncodersEvenIfSimulcastIsSupportedForMixedCodec) { 2220 std::vector<SdpVideoFormat> codecs = {SdpVideoFormat::VP8(), 2221 SdpVideoFormat::VP9Profile0(), 2222 SdpVideoFormat::VP9Profile1()}; 2223 helper_->factory()->set_supports_simulcast(true); 2224 SetupMixedCodec({{true, SdpVideoFormat::VP8()}, 2225 {true, SdpVideoFormat::VP9Profile0()}, 2226 {true, SdpVideoFormat::VP9Profile1()}}); 2227 std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); 2228 ASSERT_EQ(3u, helper_->factory()->encoders().size()); 2229 EXPECT_EQ(encoders[0]->video_format(), SdpVideoFormat::VP8()); 2230 EXPECT_EQ(encoders[1]->video_format(), SdpVideoFormat::VP9Profile0()); 2231 EXPECT_EQ(encoders[2]->video_format(), SdpVideoFormat::VP9Profile1()); 2232 EXPECT_EQ(encoders[0]->codec().codecType, webrtc::kVideoCodecVP8); 2233 EXPECT_EQ(encoders[1]->codec().codecType, webrtc::kVideoCodecVP9); 2234 EXPECT_EQ(encoders[2]->codec().codecType, webrtc::kVideoCodecVP9); 2235 } 2236 2237 TEST_F(TestSimulcastEncoderAdapterFake, 2238 EncodeDropsFrameIfResolutionIsNotAlignedByDefault) { 2239 field_trials_.Set("WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride", 2240 "requested_resolution_alignment:8," 2241 "apply_alignment_to_all_simulcast_layers"); 2242 SetUp(); 2243 SimulcastTestFixtureImpl::DefaultSettings( 2244 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 2245 kVideoCodecVP8); 2246 SetupCodec(); 2247 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 2248 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 2249 VideoFrame input_frame = VideoFrame::Builder() 2250 .set_video_frame_buffer(buffer) 2251 .set_rtp_timestamp(0) 2252 .set_timestamp_ms(0) 2253 .build(); 2254 std::vector<VideoFrameType> frame_types; 2255 frame_types.resize(codec_.numberOfSimulcastStreams, 2256 VideoFrameType::kVideoFrameKey); 2257 EXPECT_EQ(WEBRTC_VIDEO_CODEC_NO_OUTPUT, 2258 adapter_->Encode(input_frame, &frame_types)); 2259 } 2260 2261 TEST_F(TestSimulcastEncoderAdapterFake, 2262 EncodeReturnsErrorIfResolutionIsNotAlignedAndDroppingIsDisabled) { 2263 field_trials_.Set("WebRTC-SimulcastEncoderAdapter-DropUnalignedResolution", 2264 "Disabled"); 2265 field_trials_.Set("WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride", 2266 "requested_resolution_alignment:8," 2267 "apply_alignment_to_all_simulcast_layers"); 2268 SetUp(); 2269 SimulcastTestFixtureImpl::DefaultSettings( 2270 &codec_, static_cast<const int*>(kTestTemporalLayerProfile), 2271 kVideoCodecVP8); 2272 SetupCodec(); 2273 EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); 2274 scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); 2275 VideoFrame input_frame = VideoFrame::Builder() 2276 .set_video_frame_buffer(buffer) 2277 .set_rtp_timestamp(0) 2278 .set_timestamp_ms(0) 2279 .build(); 2280 std::vector<VideoFrameType> frame_types; 2281 frame_types.resize(codec_.numberOfSimulcastStreams, 2282 VideoFrameType::kVideoFrameKey); 2283 EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERROR, 2284 adapter_->Encode(input_frame, &frame_types)); 2285 } 2286 2287 } // namespace test 2288 } // namespace webrtc