call_perf_tests.cc (44548B)
1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <algorithm> 12 #include <cstddef> 13 #include <cstdint> 14 #include <cstdlib> 15 #include <iterator> 16 #include <limits> 17 #include <map> 18 #include <memory> 19 #include <string> 20 #include <utility> 21 #include <vector> 22 23 #include "absl/flags/flag.h" 24 #include "absl/strings/string_view.h" 25 #include "api/array_view.h" 26 #include "api/audio/audio_device.h" 27 #include "api/audio/builtin_audio_processing_builder.h" 28 #include "api/audio_codecs/builtin_audio_encoder_factory.h" 29 #include "api/environment/environment.h" 30 #include "api/field_trials_view.h" 31 #include "api/make_ref_counted.h" 32 #include "api/numerics/samples_stats_counter.h" 33 #include "api/rtp_parameters.h" 34 #include "api/scoped_refptr.h" 35 #include "api/task_queue/pending_task_safety_flag.h" 36 #include "api/task_queue/task_queue_base.h" 37 #include "api/test/metrics/global_metrics_logger_and_exporter.h" 38 #include "api/test/metrics/metric.h" 39 #include "api/test/simulated_network.h" 40 #include "api/test/video/function_video_encoder_factory.h" 41 #include "api/transport/bitrate_settings.h" 42 #include "api/units/data_rate.h" 43 #include "api/units/time_delta.h" 44 #include "api/units/timestamp.h" 45 #include "api/video/builtin_video_bitrate_allocator_factory.h" 46 #include "api/video/video_bitrate_allocation.h" 47 #include "api/video/video_bitrate_allocator_factory.h" 48 #include "api/video/video_frame.h" 49 #include "api/video/video_sink_interface.h" 50 #include "api/video/video_source_interface.h" 51 #include "api/video_codecs/sdp_video_format.h" 52 #include "api/video_codecs/video_codec.h" 53 #include "api/video_codecs/video_encoder.h" 54 #include "api/video_codecs/video_encoder_factory.h" 55 #include "call/audio_receive_stream.h" 56 #include "call/audio_send_stream.h" 57 #include "call/audio_state.h" 58 #include "call/call.h" 59 #include "call/call_config.h" 60 #include "call/fake_network_pipe.h" 61 #include "call/video_receive_stream.h" 62 #include "call/video_send_stream.h" 63 #include "media/engine/internal_encoder_factory.h" 64 #include "media/engine/simulcast_encoder_adapter.h" 65 #include "modules/audio_device/include/test_audio_device.h" 66 #include "modules/audio_mixer/audio_mixer_impl.h" 67 #include "rtc_base/checks.h" 68 #include "rtc_base/event.h" 69 #include "rtc_base/logging.h" 70 #include "rtc_base/task_queue_for_test.h" 71 #include "rtc_base/thread.h" 72 #include "system_wrappers/include/metrics.h" 73 #include "test/call_test.h" 74 #include "test/drifting_clock.h" 75 #include "test/encoder_settings.h" 76 #include "test/fake_encoder.h" 77 #include "test/frame_generator_capturer.h" 78 #include "test/gtest.h" 79 #include "test/network/simulated_network.h" 80 #include "test/rtp_rtcp_observer.h" 81 #include "test/test_flags.h" 82 #include "test/video_encoder_proxy_factory.h" 83 #include "test/video_test_constants.h" 84 #include "video/config/video_encoder_config.h" 85 86 using webrtc::test::DriftingClock; 87 88 namespace webrtc { 89 namespace { 90 91 using test::GetGlobalMetricsLogger; 92 using test::ImprovementDirection; 93 using test::Unit; 94 95 enum : int { // The first valid value is 1. 96 kTransportSequenceNumberExtensionId = 1, 97 }; 98 99 } // namespace 100 101 class CallPerfTest : public test::CallTest { 102 public: 103 CallPerfTest() { 104 RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri, 105 kTransportSequenceNumberExtensionId)); 106 } 107 108 protected: 109 enum class FecMode { kOn, kOff }; 110 enum class CreateOrder { kAudioFirst, kVideoFirst }; 111 void TestAudioVideoSync(FecMode fec, 112 CreateOrder create_first, 113 float video_ntp_speed, 114 float video_rtp_speed, 115 float audio_rtp_speed, 116 absl::string_view test_label); 117 118 void TestMinTransmitBitrate(bool pad_to_min_bitrate); 119 120 void TestCaptureNtpTime(const BuiltInNetworkBehaviorConfig& net_config, 121 int threshold_ms, 122 int start_time_ms, 123 int run_time_ms); 124 void TestMinAudioVideoBitrate(int test_bitrate_from, 125 int test_bitrate_to, 126 int test_bitrate_step, 127 int min_bwe, 128 int start_bwe, 129 int max_bwe); 130 void TestEncodeFramerate(VideoEncoderFactory* encoder_factory, 131 absl::string_view payload_name, 132 const std::vector<int>& max_framerates); 133 }; 134 135 class VideoRtcpAndSyncObserver : public test::RtpRtcpObserver, 136 public VideoSinkInterface<VideoFrame> { 137 static const int kInSyncThresholdMs = 50; 138 static const int kStartupTimeMs = 2000; 139 static const int kMinRunTimeMs = 30000; 140 141 public: 142 explicit VideoRtcpAndSyncObserver(TaskQueueBase* task_queue, 143 Clock* clock, 144 absl::string_view test_label) 145 : test::RtpRtcpObserver(test::VideoTestConstants::kLongTimeout), 146 clock_(clock), 147 test_label_(test_label), 148 creation_time_ms_(clock_->TimeInMilliseconds()), 149 task_queue_(task_queue) {} 150 151 void OnFrame(const VideoFrame& /* video_frame */) override { 152 task_queue_->PostTask([this]() { CheckStats(); }); 153 } 154 155 void CheckStats() { 156 if (!receive_stream_) 157 return; 158 159 VideoReceiveStreamInterface::Stats stats = receive_stream_->GetStats(); 160 if (stats.sync_offset_ms == std::numeric_limits<int>::max()) 161 return; 162 163 Timestamp now = clock_->CurrentTime(); 164 int64_t time_since_creation = now.ms() - creation_time_ms_; 165 // During the first couple of seconds audio and video can falsely be 166 // estimated as being synchronized. We don't want to trigger on those. 167 if (time_since_creation < kStartupTimeMs) 168 return; 169 if (std::abs(stats.sync_offset_ms) < kInSyncThresholdMs) { 170 if (first_time_in_sync_ == -1) { 171 first_time_in_sync_ = now.ms(); 172 GetGlobalMetricsLogger()->LogSingleValueMetric( 173 "sync_convergence_time" + test_label_, "synchronization", 174 time_since_creation, Unit::kMilliseconds, 175 ImprovementDirection::kSmallerIsBetter); 176 } 177 if (time_since_creation > kMinRunTimeMs) 178 observation_complete_.Set(); 179 } 180 if (first_time_in_sync_ != -1) 181 sync_offset_ms_list_.AddSample( 182 {.value = static_cast<double>(stats.sync_offset_ms), .time = now}); 183 } 184 185 void set_receive_stream(VideoReceiveStreamInterface* receive_stream) { 186 RTC_DCHECK_EQ(task_queue_, TaskQueueBase::Current()); 187 // Note that receive_stream may be nullptr. 188 receive_stream_ = receive_stream; 189 } 190 191 void PrintResults() { 192 GetGlobalMetricsLogger()->LogMetric( 193 "stream_offset" + test_label_, "synchronization", sync_offset_ms_list_, 194 Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); 195 } 196 197 private: 198 Clock* const clock_; 199 const std::string test_label_; 200 const int64_t creation_time_ms_; 201 int64_t first_time_in_sync_ = -1; 202 VideoReceiveStreamInterface* receive_stream_ = nullptr; 203 SamplesStatsCounter sync_offset_ms_list_; 204 TaskQueueBase* const task_queue_; 205 }; 206 207 void CallPerfTest::TestAudioVideoSync(FecMode fec, 208 CreateOrder create_first, 209 float video_ntp_speed, 210 float video_rtp_speed, 211 float audio_rtp_speed, 212 absl::string_view test_label) { 213 const char* kSyncGroup = "av_sync"; 214 const uint32_t kAudioSendSsrc = 1234; 215 const uint32_t kAudioRecvSsrc = 5678; 216 217 BuiltInNetworkBehaviorConfig audio_net_config; 218 audio_net_config.queue_delay_ms = 500; 219 audio_net_config.loss_percent = 5; 220 221 auto observer = std::make_unique<VideoRtcpAndSyncObserver>( 222 task_queue(), Clock::GetRealTimeClock(), test_label); 223 224 std::map<uint8_t, MediaType> audio_pt_map; 225 std::map<uint8_t, MediaType> video_pt_map; 226 227 std::unique_ptr<test::PacketTransport> audio_send_transport; 228 std::unique_ptr<test::PacketTransport> video_send_transport; 229 std::unique_ptr<test::PacketTransport> receive_transport; 230 231 AudioSendStream* audio_send_stream; 232 AudioReceiveStreamInterface* audio_receive_stream; 233 std::unique_ptr<DriftingClock> drifting_clock; 234 235 SendTask(task_queue(), [&]() { 236 metrics::Reset(); 237 scoped_refptr<AudioDeviceModule> fake_audio_device = 238 TestAudioDeviceModule::Create( 239 env(), TestAudioDeviceModule::CreatePulsedNoiseCapturer(256, 48000), 240 TestAudioDeviceModule::CreateDiscardRenderer(48000), 241 audio_rtp_speed); 242 EXPECT_EQ(0, fake_audio_device->Init()); 243 244 AudioState::Config send_audio_state_config; 245 send_audio_state_config.audio_mixer = AudioMixerImpl::Create(); 246 send_audio_state_config.audio_processing = 247 BuiltinAudioProcessingBuilder().Build(env()); 248 send_audio_state_config.audio_device_module = fake_audio_device; 249 CallConfig sender_config = SendCallConfig(); 250 251 auto audio_state = AudioState::Create(send_audio_state_config); 252 fake_audio_device->RegisterAudioCallback(audio_state->audio_transport()); 253 sender_config.audio_state = audio_state; 254 CallConfig receiver_config = RecvCallConfig(); 255 receiver_config.audio_state = audio_state; 256 CreateCalls(std::move(sender_config), std::move(receiver_config)); 257 258 std::copy_if(std::begin(payload_type_map_), std::end(payload_type_map_), 259 std::inserter(audio_pt_map, audio_pt_map.end()), 260 [](const std::pair<const uint8_t, MediaType>& pair) { 261 return pair.second == MediaType::AUDIO; 262 }); 263 std::copy_if(std::begin(payload_type_map_), std::end(payload_type_map_), 264 std::inserter(video_pt_map, video_pt_map.end()), 265 [](const std::pair<const uint8_t, MediaType>& pair) { 266 return pair.second == MediaType::VIDEO; 267 }); 268 269 audio_send_transport = std::make_unique<test::PacketTransport>( 270 task_queue(), sender_call_.get(), observer.get(), 271 test::PacketTransport::kSender, audio_pt_map, 272 std::make_unique<FakeNetworkPipe>( 273 Clock::GetRealTimeClock(), 274 std::make_unique<SimulatedNetwork>(audio_net_config)), 275 GetRegisteredExtensions(), GetRegisteredExtensions()); 276 audio_send_transport->SetReceiver(receiver_call_->Receiver()); 277 278 video_send_transport = std::make_unique<test::PacketTransport>( 279 task_queue(), sender_call_.get(), observer.get(), 280 test::PacketTransport::kSender, video_pt_map, 281 std::make_unique<FakeNetworkPipe>( 282 Clock::GetRealTimeClock(), 283 std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig())), 284 GetRegisteredExtensions(), GetRegisteredExtensions()); 285 video_send_transport->SetReceiver(receiver_call_->Receiver()); 286 287 receive_transport = std::make_unique<test::PacketTransport>( 288 task_queue(), receiver_call_.get(), observer.get(), 289 test::PacketTransport::kReceiver, payload_type_map_, 290 std::make_unique<FakeNetworkPipe>( 291 Clock::GetRealTimeClock(), 292 std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig())), 293 GetRegisteredExtensions(), GetRegisteredExtensions()); 294 receive_transport->SetReceiver(sender_call_->Receiver()); 295 296 CreateSendConfig(1, 0, 0, video_send_transport.get()); 297 CreateMatchingReceiveConfigs(receive_transport.get()); 298 299 AudioSendStream::Config audio_send_config(audio_send_transport.get()); 300 audio_send_config.rtp.ssrc = kAudioSendSsrc; 301 // TODO(bugs.webrtc.org/14683): Let the tests fail with invalid config. 302 audio_send_config.send_codec_spec = AudioSendStream::Config::SendCodecSpec( 303 test::VideoTestConstants::kAudioSendPayloadType, {"OPUS", 48000, 2}); 304 audio_send_config.min_bitrate_bps = 6000; 305 audio_send_config.max_bitrate_bps = 510000; 306 audio_send_config.encoder_factory = CreateBuiltinAudioEncoderFactory(); 307 audio_send_stream = sender_call_->CreateAudioSendStream(audio_send_config); 308 309 GetVideoSendConfig()->rtp.nack.rtp_history_ms = 310 test::VideoTestConstants::kNackRtpHistoryMs; 311 if (fec == FecMode::kOn) { 312 GetVideoSendConfig()->rtp.ulpfec.red_payload_type = 313 test::VideoTestConstants::kRedPayloadType; 314 GetVideoSendConfig()->rtp.ulpfec.ulpfec_payload_type = 315 test::VideoTestConstants::kUlpfecPayloadType; 316 video_receive_configs_[0].rtp.red_payload_type = 317 test::VideoTestConstants::kRedPayloadType; 318 video_receive_configs_[0].rtp.ulpfec_payload_type = 319 test::VideoTestConstants::kUlpfecPayloadType; 320 } 321 video_receive_configs_[0].rtp.nack.rtp_history_ms = 1000; 322 video_receive_configs_[0].renderer = observer.get(); 323 video_receive_configs_[0].sync_group = kSyncGroup; 324 325 AudioReceiveStreamInterface::Config audio_recv_config; 326 audio_recv_config.rtp.remote_ssrc = kAudioSendSsrc; 327 audio_recv_config.rtp.local_ssrc = kAudioRecvSsrc; 328 audio_recv_config.rtcp_send_transport = receive_transport.get(); 329 audio_recv_config.sync_group = kSyncGroup; 330 audio_recv_config.decoder_factory = audio_decoder_factory_; 331 audio_recv_config.decoder_map = { 332 {test::VideoTestConstants::kAudioSendPayloadType, {"OPUS", 48000, 2}}}; 333 334 if (create_first == CreateOrder::kAudioFirst) { 335 audio_receive_stream = 336 receiver_call_->CreateAudioReceiveStream(audio_recv_config); 337 CreateVideoStreams(); 338 } else { 339 CreateVideoStreams(); 340 audio_receive_stream = 341 receiver_call_->CreateAudioReceiveStream(audio_recv_config); 342 } 343 EXPECT_EQ(1u, video_receive_streams_.size()); 344 observer->set_receive_stream(video_receive_streams_[0]); 345 drifting_clock = 346 std::make_unique<DriftingClock>(&env().clock(), video_ntp_speed); 347 CreateFrameGeneratorCapturerWithDrift( 348 drifting_clock.get(), video_rtp_speed, 349 test::VideoTestConstants::kDefaultFramerate, 350 test::VideoTestConstants::kDefaultWidth, 351 test::VideoTestConstants::kDefaultHeight); 352 353 Start(); 354 355 audio_send_stream->Start(); 356 audio_receive_stream->Start(); 357 }); 358 359 EXPECT_TRUE(observer->Wait()) 360 << "Timed out while waiting for audio and video to be synchronized."; 361 362 SendTask(task_queue(), [&]() { 363 // Clear the pointer to the receive stream since it will now be deleted. 364 observer->set_receive_stream(nullptr); 365 366 audio_send_stream->Stop(); 367 audio_receive_stream->Stop(); 368 369 Stop(); 370 371 DestroyStreams(); 372 373 sender_call_->DestroyAudioSendStream(audio_send_stream); 374 receiver_call_->DestroyAudioReceiveStream(audio_receive_stream); 375 376 DestroyCalls(); 377 // Call may post periodic rtcp packet to the transport on the process 378 // thread, thus transport should be destroyed after the call objects. 379 // Though transports keep pointers to the call objects, transports handle 380 // packets on the task_queue() and thus wouldn't create a race while current 381 // destruction happens in the same task as destruction of the call objects. 382 video_send_transport.reset(); 383 audio_send_transport.reset(); 384 receive_transport.reset(); 385 }); 386 387 observer->PrintResults(); 388 389 // In quick test synchronization may not be achieved in time. 390 if (!absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { 391 // TODO(bugs.webrtc.org/10417): Reenable this for iOS 392 #if !defined(WEBRTC_IOS) 393 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Video.AVSyncOffsetInMs")); 394 #endif 395 } 396 397 task_queue()->PostTask( 398 [to_delete = observer.release()]() { delete to_delete; }); 399 } 400 401 TEST_F(CallPerfTest, Synchronization_PlaysOutAudioAndVideoWithoutClockDrift) { 402 TestAudioVideoSync(FecMode::kOff, CreateOrder::kAudioFirst, 403 DriftingClock::kNoDrift, DriftingClock::kNoDrift, 404 DriftingClock::kNoDrift, "_video_no_drift"); 405 } 406 407 TEST_F(CallPerfTest, Synchronization_PlaysOutAudioAndVideoWithVideoNtpDrift) { 408 TestAudioVideoSync(FecMode::kOff, CreateOrder::kAudioFirst, 409 DriftingClock::PercentsFaster(10.0f), 410 DriftingClock::kNoDrift, DriftingClock::kNoDrift, 411 "_video_ntp_drift"); 412 } 413 414 TEST_F(CallPerfTest, 415 Synchronization_PlaysOutAudioAndVideoWithAudioFasterThanVideoDrift) { 416 TestAudioVideoSync(FecMode::kOff, CreateOrder::kAudioFirst, 417 DriftingClock::kNoDrift, 418 DriftingClock::PercentsSlower(30.0f), 419 DriftingClock::PercentsFaster(30.0f), "_audio_faster"); 420 } 421 422 TEST_F(CallPerfTest, 423 Synchronization_PlaysOutAudioAndVideoWithVideoFasterThanAudioDrift) { 424 TestAudioVideoSync(FecMode::kOn, CreateOrder::kVideoFirst, 425 DriftingClock::kNoDrift, 426 DriftingClock::PercentsFaster(30.0f), 427 DriftingClock::PercentsSlower(30.0f), "_video_faster"); 428 } 429 430 TEST_F(CallPerfTest, ReceivesCpuOveruseAndUnderuse) { 431 // Minimal normal usage at the start, then 30s overuse to allow filter to 432 // settle, and then 80s underuse to allow plenty of time for rampup again. 433 field_trials().Set("WebRTC-ForceSimulatedOveruseIntervalMs", "1-30000-80000"); 434 435 class LoadObserver : public test::SendTest, 436 public test::FrameGeneratorCapturer::SinkWantsObserver { 437 public: 438 LoadObserver() 439 : SendTest(test::VideoTestConstants::kLongTimeout), 440 test_phase_(TestPhase::kInit) {} 441 442 void OnFrameGeneratorCapturerCreated( 443 test::FrameGeneratorCapturer* frame_generator_capturer) override { 444 frame_generator_capturer->SetSinkWantsObserver(this); 445 // Set a high initial resolution to be sure that we can scale down. 446 frame_generator_capturer->ChangeResolution(1920, 1080); 447 } 448 449 // OnSinkWantsChanged is called when FrameGeneratorCapturer::AddOrUpdateSink 450 // is called. 451 // TODO(sprang): Add integration test for maintain-framerate mode? 452 void OnSinkWantsChanged(VideoSinkInterface<VideoFrame>* /* sink */, 453 const VideoSinkWants& wants) override { 454 RTC_LOG(LS_INFO) << "OnSinkWantsChanged fps:" << wants.max_framerate_fps 455 << " max_pixel_count " << wants.max_pixel_count 456 << " target_pixel_count" 457 << wants.target_pixel_count.value_or(-1); 458 // The sink wants can change either because an adaptation happened 459 // (i.e. the pixels or frame rate changed) or for other reasons, such 460 // as encoded resolutions being communicated (happens whenever we 461 // capture a new frame size). In this test, we only care about 462 // adaptations. 463 bool did_adapt = 464 last_wants_.max_pixel_count != wants.max_pixel_count || 465 last_wants_.target_pixel_count != wants.target_pixel_count || 466 last_wants_.max_framerate_fps != wants.max_framerate_fps; 467 last_wants_ = wants; 468 if (!did_adapt) { 469 if (test_phase_ == TestPhase::kInit) { 470 test_phase_ = TestPhase::kStart; 471 } 472 return; 473 } 474 // At kStart expect CPU overuse. Then expect CPU underuse when the encoder 475 // delay has been decreased. 476 switch (test_phase_) { 477 case TestPhase::kInit: 478 ADD_FAILURE() << "Got unexpected adaptation request, max res = " 479 << wants.max_pixel_count << ", target res = " 480 << wants.target_pixel_count.value_or(-1) 481 << ", max fps = " << wants.max_framerate_fps; 482 break; 483 case TestPhase::kStart: 484 if (wants.max_pixel_count < std::numeric_limits<int>::max()) { 485 // On adapting down, VideoStreamEncoder::VideoSourceProxy will set 486 // only the max pixel count, leaving the target unset. 487 test_phase_ = TestPhase::kAdaptedDown; 488 } else { 489 ADD_FAILURE() << "Got unexpected adaptation request, max res = " 490 << wants.max_pixel_count << ", target res = " 491 << wants.target_pixel_count.value_or(-1) 492 << ", max fps = " << wants.max_framerate_fps; 493 } 494 break; 495 case TestPhase::kAdaptedDown: 496 // On adapting up, the adaptation counter will again be at zero, and 497 // so all constraints will be reset. 498 if (wants.max_pixel_count == std::numeric_limits<int>::max() && 499 !wants.target_pixel_count) { 500 test_phase_ = TestPhase::kAdaptedUp; 501 observation_complete_.Set(); 502 } else { 503 ADD_FAILURE() << "Got unexpected adaptation request, max res = " 504 << wants.max_pixel_count << ", target res = " 505 << wants.target_pixel_count.value_or(-1) 506 << ", max fps = " << wants.max_framerate_fps; 507 } 508 break; 509 case TestPhase::kAdaptedUp: 510 ADD_FAILURE() << "Got unexpected adaptation request, max res = " 511 << wants.max_pixel_count << ", target res = " 512 << wants.target_pixel_count.value_or(-1) 513 << ", max fps = " << wants.max_framerate_fps; 514 } 515 } 516 517 void ModifyVideoConfigs( 518 VideoSendStream::Config* /* send_config */, 519 std::vector<VideoReceiveStreamInterface::Config>* /* receive_configs */, 520 VideoEncoderConfig* /* encoder_config */) override {} 521 522 void PerformTest() override { 523 EXPECT_TRUE(Wait()) << "Timed out before receiving an overuse callback."; 524 } 525 526 enum class TestPhase { 527 kInit, 528 kStart, 529 kAdaptedDown, 530 kAdaptedUp 531 } test_phase_; 532 533 private: 534 VideoSinkWants last_wants_; 535 } test; 536 537 RunBaseTest(&test); 538 } 539 540 void CallPerfTest::TestMinTransmitBitrate(bool pad_to_min_bitrate) { 541 static const int kMaxEncodeBitrateKbps = 30; 542 static const int kMinTransmitBitrateBps = 150000; 543 static const int kMinAcceptableTransmitBitrate = 130; 544 static const int kMaxAcceptableTransmitBitrate = 170; 545 static const int kNumBitrateObservationsInRange = 100; 546 static const int kAcceptableBitrateErrorMargin = 15; // +- 7 547 class BitrateObserver : public test::EndToEndTest { 548 public: 549 explicit BitrateObserver(const Environment& env, 550 bool using_min_transmit_bitrate, 551 TaskQueueBase* task_queue) 552 : EndToEndTest(test::VideoTestConstants::kLongTimeout), 553 env_(env), 554 send_stream_(nullptr), 555 converged_(false), 556 pad_to_min_bitrate_(using_min_transmit_bitrate), 557 min_acceptable_bitrate_(using_min_transmit_bitrate 558 ? kMinAcceptableTransmitBitrate 559 : (kMaxEncodeBitrateKbps - 560 kAcceptableBitrateErrorMargin / 2)), 561 max_acceptable_bitrate_(using_min_transmit_bitrate 562 ? kMaxAcceptableTransmitBitrate 563 : (kMaxEncodeBitrateKbps + 564 kAcceptableBitrateErrorMargin / 2)), 565 num_bitrate_observations_in_range_(0), 566 task_queue_(task_queue), 567 task_safety_flag_(PendingTaskSafetyFlag::CreateDetached()) {} 568 569 private: 570 // TODO(holmer): Run this with a timer instead of once per packet. 571 Action OnSendRtp(ArrayView<const uint8_t> /* packet */) override { 572 task_queue_->PostTask(SafeTask(task_safety_flag_, [this]() { 573 VideoSendStream::Stats stats = send_stream_->GetStats(); 574 575 if (!stats.substreams.empty()) { 576 RTC_DCHECK_EQ(1, stats.substreams.size()); 577 int bitrate_kbps = 578 stats.substreams.begin()->second.total_bitrate_bps / 1000; 579 if (bitrate_kbps > min_acceptable_bitrate_ && 580 bitrate_kbps < max_acceptable_bitrate_) { 581 converged_ = true; 582 ++num_bitrate_observations_in_range_; 583 if (num_bitrate_observations_in_range_ == 584 kNumBitrateObservationsInRange) 585 observation_complete_.Set(); 586 } 587 if (converged_) 588 bitrate_kbps_list_.AddSample( 589 {.value = static_cast<double>(bitrate_kbps), 590 .time = env_.clock().CurrentTime()}); 591 } 592 })); 593 return SEND_PACKET; 594 } 595 596 void OnVideoStreamsCreated(VideoSendStream* send_stream, 597 const std::vector<VideoReceiveStreamInterface*>& 598 /* receive_streams */) override { 599 send_stream_ = send_stream; 600 } 601 602 void OnStreamsStopped() override { task_safety_flag_->SetNotAlive(); } 603 604 void ModifyVideoConfigs( 605 VideoSendStream::Config* /* send_config */, 606 std::vector<VideoReceiveStreamInterface::Config>* /* receive_configs */, 607 VideoEncoderConfig* encoder_config) override { 608 if (pad_to_min_bitrate_) { 609 encoder_config->min_transmit_bitrate_bps = kMinTransmitBitrateBps; 610 } else { 611 RTC_DCHECK_EQ(0, encoder_config->min_transmit_bitrate_bps); 612 } 613 } 614 615 void PerformTest() override { 616 EXPECT_TRUE(Wait()) << "Timeout while waiting for send-bitrate stats."; 617 GetGlobalMetricsLogger()->LogMetric( 618 std::string("bitrate_stats_") + 619 (pad_to_min_bitrate_ ? "min_transmit_bitrate" 620 : "without_min_transmit_bitrate"), 621 "bitrate_kbps", bitrate_kbps_list_, Unit::kUnitless, 622 ImprovementDirection::kNeitherIsBetter); 623 } 624 625 Environment env_; 626 VideoSendStream* send_stream_; 627 bool converged_; 628 const bool pad_to_min_bitrate_; 629 const int min_acceptable_bitrate_; 630 const int max_acceptable_bitrate_; 631 int num_bitrate_observations_in_range_; 632 SamplesStatsCounter bitrate_kbps_list_; 633 TaskQueueBase* task_queue_; 634 scoped_refptr<PendingTaskSafetyFlag> task_safety_flag_; 635 } test(env(), pad_to_min_bitrate, task_queue()); 636 637 fake_encoder_max_bitrate_ = kMaxEncodeBitrateKbps; 638 RunBaseTest(&test); 639 } 640 641 TEST_F(CallPerfTest, Bitrate_Kbps_PadsToMinTransmitBitrate) { 642 TestMinTransmitBitrate(true); 643 } 644 645 TEST_F(CallPerfTest, Bitrate_Kbps_NoPadWithoutMinTransmitBitrate) { 646 TestMinTransmitBitrate(false); 647 } 648 649 // TODO(bugs.webrtc.org/8878) 650 #if defined(WEBRTC_MAC) 651 #define MAYBE_KeepsHighBitrateWhenReconfiguringSender \ 652 DISABLED_KeepsHighBitrateWhenReconfiguringSender 653 #else 654 #define MAYBE_KeepsHighBitrateWhenReconfiguringSender \ 655 KeepsHighBitrateWhenReconfiguringSender 656 #endif 657 TEST_F(CallPerfTest, MAYBE_KeepsHighBitrateWhenReconfiguringSender) { 658 static const uint32_t kInitialBitrateKbps = 400; 659 static const uint32_t kInitialBitrateOverheadKpbs = 6; 660 static const uint32_t kReconfigureThresholdKbps = 600; 661 662 class VideoStreamFactory 663 : public VideoEncoderConfig::VideoStreamFactoryInterface { 664 public: 665 VideoStreamFactory() {} 666 667 private: 668 std::vector<VideoStream> CreateEncoderStreams( 669 const FieldTrialsView& /*field_trials*/, 670 int frame_width, 671 int frame_height, 672 const VideoEncoderConfig& encoder_config) override { 673 std::vector<VideoStream> streams = 674 test::CreateVideoStreams(frame_width, frame_height, encoder_config); 675 streams[0].min_bitrate_bps = 50000; 676 streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000; 677 return streams; 678 } 679 }; 680 681 class BitrateObserver : public test::EndToEndTest, public test::FakeEncoder { 682 public: 683 explicit BitrateObserver(const Environment& env, TaskQueueBase* task_queue) 684 : EndToEndTest(test::VideoTestConstants::kDefaultTimeout), 685 FakeEncoder(env), 686 encoder_inits_(0), 687 last_set_bitrate_kbps_(0), 688 send_stream_(nullptr), 689 frame_generator_(nullptr), 690 encoder_factory_(this), 691 bitrate_allocator_factory_( 692 CreateBuiltinVideoBitrateAllocatorFactory()), 693 task_queue_(task_queue) {} 694 695 int32_t InitEncode(const VideoCodec* config, 696 const VideoEncoder::Settings& settings) override { 697 ++encoder_inits_; 698 if (encoder_inits_ == 1) { 699 // First time initialization. Frame size is known. 700 // `expected_bitrate` is affected by bandwidth estimation before the 701 // first frame arrives to the encoder. 702 uint32_t expected_bitrate = 703 last_set_bitrate_kbps_ > 0 704 ? last_set_bitrate_kbps_ 705 : kInitialBitrateKbps - kInitialBitrateOverheadKpbs; 706 EXPECT_EQ(expected_bitrate, config->startBitrate) 707 << "Encoder not initialized at expected bitrate."; 708 EXPECT_EQ(test::VideoTestConstants::kDefaultWidth, config->width); 709 EXPECT_EQ(test::VideoTestConstants::kDefaultHeight, config->height); 710 } else if (encoder_inits_ == 2) { 711 EXPECT_EQ(2 * test::VideoTestConstants::kDefaultWidth, config->width); 712 EXPECT_EQ(2 * test::VideoTestConstants::kDefaultHeight, config->height); 713 EXPECT_GE(last_set_bitrate_kbps_, kReconfigureThresholdKbps); 714 EXPECT_GT(config->startBitrate, kReconfigureThresholdKbps) 715 << "Encoder reconfigured with bitrate too far away from last set."; 716 observation_complete_.Set(); 717 } 718 return FakeEncoder::InitEncode(config, settings); 719 } 720 721 void SetRates(const RateControlParameters& parameters) override { 722 last_set_bitrate_kbps_ = parameters.bitrate.get_sum_kbps(); 723 if (encoder_inits_ == 1 && 724 parameters.bitrate.get_sum_kbps() > kReconfigureThresholdKbps) { 725 time_to_reconfigure_.Set(); 726 } 727 FakeEncoder::SetRates(parameters); 728 } 729 730 void ModifySenderBitrateConfig( 731 BitrateConstraints* bitrate_config) override { 732 bitrate_config->start_bitrate_bps = kInitialBitrateKbps * 1000; 733 } 734 735 void ModifyVideoConfigs( 736 VideoSendStream::Config* send_config, 737 std::vector<VideoReceiveStreamInterface::Config>* /* receive_configs */, 738 VideoEncoderConfig* encoder_config) override { 739 send_config->encoder_settings.encoder_factory = &encoder_factory_; 740 send_config->encoder_settings.bitrate_allocator_factory = 741 bitrate_allocator_factory_.get(); 742 encoder_config->max_bitrate_bps = 2 * kReconfigureThresholdKbps * 1000; 743 encoder_config->video_stream_factory = 744 make_ref_counted<VideoStreamFactory>(); 745 746 encoder_config_ = encoder_config->Copy(); 747 } 748 749 void OnVideoStreamsCreated(VideoSendStream* send_stream, 750 const std::vector<VideoReceiveStreamInterface*>& 751 /* receive_streams */) override { 752 send_stream_ = send_stream; 753 } 754 755 void OnFrameGeneratorCapturerCreated( 756 test::FrameGeneratorCapturer* frame_generator_capturer) override { 757 frame_generator_ = frame_generator_capturer; 758 } 759 760 void PerformTest() override { 761 ASSERT_TRUE( 762 time_to_reconfigure_.Wait(test::VideoTestConstants::kDefaultTimeout)) 763 << "Timed out before receiving an initial high bitrate."; 764 frame_generator_->ChangeResolution( 765 test::VideoTestConstants::kDefaultWidth * 2, 766 test::VideoTestConstants::kDefaultHeight * 2); 767 SendTask(task_queue_, [&]() { 768 send_stream_->ReconfigureVideoEncoder(encoder_config_.Copy()); 769 }); 770 EXPECT_TRUE(Wait()) 771 << "Timed out while waiting for a couple of high bitrate estimates " 772 "after reconfiguring the send stream."; 773 } 774 775 private: 776 Event time_to_reconfigure_; 777 int encoder_inits_; 778 uint32_t last_set_bitrate_kbps_; 779 VideoSendStream* send_stream_; 780 test::FrameGeneratorCapturer* frame_generator_; 781 test::VideoEncoderProxyFactory encoder_factory_; 782 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_; 783 VideoEncoderConfig encoder_config_; 784 TaskQueueBase* task_queue_; 785 } test(env(), task_queue()); 786 787 RunBaseTest(&test); 788 } 789 790 // Discovers the minimal supported audio+video bitrate. The test bitrate is 791 // considered supported if Rtt does not go above 400ms with the network 792 // contrained to the test bitrate. 793 // 794 // |test_bitrate_from test_bitrate_to| bitrate constraint range 795 // `test_bitrate_step` bitrate constraint update step during the test 796 // |min_bwe max_bwe| BWE range 797 // `start_bwe` initial BWE 798 void CallPerfTest::TestMinAudioVideoBitrate(int test_bitrate_from, 799 int test_bitrate_to, 800 int test_bitrate_step, 801 int min_bwe, 802 int start_bwe, 803 int max_bwe) { 804 static const std::string kAudioTrackId = "audio_track_0"; 805 static constexpr int kBitrateStabilizationMs = 10000; 806 static constexpr int kBitrateMeasurements = 10; 807 static constexpr int kBitrateMeasurementMs = 1000; 808 static constexpr int kShortDelayMs = 10; 809 static constexpr int kMinGoodRttMs = 400; 810 811 class MinVideoAndAudioBitrateTester : public test::EndToEndTest { 812 public: 813 MinVideoAndAudioBitrateTester(int test_bitrate_from, 814 int test_bitrate_to, 815 int test_bitrate_step, 816 int min_bwe, 817 int start_bwe, 818 int max_bwe, 819 TaskQueueBase* task_queue) 820 : EndToEndTest(), 821 test_bitrate_from_(test_bitrate_from), 822 test_bitrate_to_(test_bitrate_to), 823 test_bitrate_step_(test_bitrate_step), 824 min_bwe_(min_bwe), 825 start_bwe_(start_bwe), 826 max_bwe_(max_bwe), 827 task_queue_(task_queue) {} 828 829 protected: 830 BuiltInNetworkBehaviorConfig GetFakeNetworkPipeConfig() const { 831 BuiltInNetworkBehaviorConfig pipe_config; 832 pipe_config.link_capacity = DataRate::KilobitsPerSec(test_bitrate_from_); 833 return pipe_config; 834 } 835 836 BuiltInNetworkBehaviorConfig GetSendTransportConfig() const override { 837 return GetFakeNetworkPipeConfig(); 838 } 839 BuiltInNetworkBehaviorConfig GetReceiveTransportConfig() const override { 840 return GetFakeNetworkPipeConfig(); 841 } 842 843 void OnTransportCreated( 844 test::PacketTransport* /* to_receiver */, 845 SimulatedNetworkInterface* sender_network, 846 test::PacketTransport* /* to_sender */, 847 SimulatedNetworkInterface* receiver_network) override { 848 send_simulated_network_ = sender_network; 849 receive_simulated_network_ = receiver_network; 850 } 851 852 void PerformTest() override { 853 // Quick test mode, just to exercise all the code paths without actually 854 // caring about performance measurements. 855 const bool quick_perf_test = absl::GetFlag(FLAGS_webrtc_quick_perf_test); 856 857 int last_passed_test_bitrate = -1; 858 for (int test_bitrate = test_bitrate_from_; 859 test_bitrate_from_ < test_bitrate_to_ 860 ? test_bitrate <= test_bitrate_to_ 861 : test_bitrate >= test_bitrate_to_; 862 test_bitrate += test_bitrate_step_) { 863 BuiltInNetworkBehaviorConfig pipe_config; 864 pipe_config.link_capacity = DataRate::KilobitsPerSec(test_bitrate); 865 send_simulated_network_->SetConfig(pipe_config); 866 receive_simulated_network_->SetConfig(pipe_config); 867 868 Thread::SleepMs(quick_perf_test ? kShortDelayMs 869 : kBitrateStabilizationMs); 870 871 int64_t avg_rtt = 0; 872 for (int i = 0; i < kBitrateMeasurements; i++) { 873 Call::Stats call_stats; 874 SendTask(task_queue_, [this, &call_stats]() { 875 call_stats = sender_call_->GetStats(); 876 }); 877 avg_rtt += call_stats.rtt_ms; 878 Thread::SleepMs(quick_perf_test ? kShortDelayMs 879 : kBitrateMeasurementMs); 880 } 881 avg_rtt = avg_rtt / kBitrateMeasurements; 882 if (avg_rtt > kMinGoodRttMs) { 883 RTC_LOG(LS_WARNING) 884 << "Failed test bitrate: " << test_bitrate << " RTT: " << avg_rtt; 885 break; 886 } else { 887 RTC_LOG(LS_INFO) << "Passed test bitrate: " << test_bitrate 888 << " RTT: " << avg_rtt; 889 last_passed_test_bitrate = test_bitrate; 890 } 891 } 892 EXPECT_GT(last_passed_test_bitrate, -1) 893 << "Minimum supported bitrate out of the test scope"; 894 GetGlobalMetricsLogger()->LogSingleValueMetric( 895 "min_test_bitrate_", "min_bitrate", last_passed_test_bitrate, 896 Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); 897 } 898 899 void OnCallsCreated(Call* sender_call, Call* /* receiver_call */) override { 900 sender_call_ = sender_call; 901 BitrateConstraints bitrate_config; 902 bitrate_config.min_bitrate_bps = min_bwe_; 903 bitrate_config.start_bitrate_bps = start_bwe_; 904 bitrate_config.max_bitrate_bps = max_bwe_; 905 sender_call->GetTransportControllerSend()->SetSdpBitrateParameters( 906 bitrate_config); 907 } 908 909 size_t GetNumVideoStreams() const override { return 1; } 910 911 size_t GetNumAudioStreams() const override { return 1; } 912 913 private: 914 const int test_bitrate_from_; 915 const int test_bitrate_to_; 916 const int test_bitrate_step_; 917 const int min_bwe_; 918 const int start_bwe_; 919 const int max_bwe_; 920 SimulatedNetworkInterface* send_simulated_network_; 921 SimulatedNetworkInterface* receive_simulated_network_; 922 Call* sender_call_; 923 TaskQueueBase* const task_queue_; 924 } test(test_bitrate_from, test_bitrate_to, test_bitrate_step, min_bwe, 925 start_bwe, max_bwe, task_queue()); 926 927 RunBaseTest(&test); 928 } 929 930 TEST_F(CallPerfTest, Min_Bitrate_VideoAndAudio) { 931 TestMinAudioVideoBitrate(110, 40, -10, 10000, 70000, 200000); 932 } 933 934 void CallPerfTest::TestEncodeFramerate(VideoEncoderFactory* encoder_factory, 935 absl::string_view payload_name, 936 const std::vector<int>& max_framerates) { 937 static constexpr double kAllowedFpsDiff = 1.5; 938 static constexpr TimeDelta kMinGetStatsInterval = TimeDelta::Millis(400); 939 static constexpr TimeDelta kMinRunTime = TimeDelta::Seconds(15); 940 static constexpr DataRate kMaxBitrate = DataRate::KilobitsPerSec(1000); 941 942 class FramerateObserver 943 : public test::EndToEndTest, 944 public test::FrameGeneratorCapturer::SinkWantsObserver { 945 public: 946 FramerateObserver(VideoEncoderFactory* encoder_factory, 947 absl::string_view payload_name, 948 const std::vector<int>& max_framerates, 949 TaskQueueBase* task_queue) 950 : EndToEndTest(test::VideoTestConstants::kDefaultTimeout), 951 clock_(Clock::GetRealTimeClock()), 952 encoder_factory_(encoder_factory), 953 payload_name_(payload_name), 954 max_framerates_(max_framerates), 955 task_queue_(task_queue), 956 start_time_(clock_->CurrentTime()), 957 last_getstats_time_(start_time_), 958 send_stream_(nullptr) {} 959 960 void OnFrameGeneratorCapturerCreated( 961 test::FrameGeneratorCapturer* frame_generator_capturer) override { 962 frame_generator_capturer->ChangeResolution(640, 360); 963 } 964 965 void OnSinkWantsChanged(VideoSinkInterface<VideoFrame>* /* sink */, 966 const VideoSinkWants& /* wants */) override {} 967 968 void ModifySenderBitrateConfig( 969 BitrateConstraints* bitrate_config) override { 970 bitrate_config->start_bitrate_bps = kMaxBitrate.bps() / 2; 971 } 972 973 void OnVideoStreamsCreated(VideoSendStream* send_stream, 974 const std::vector<VideoReceiveStreamInterface*>& 975 /* receive_streams */) override { 976 send_stream_ = send_stream; 977 } 978 979 size_t GetNumVideoStreams() const override { 980 return max_framerates_.size(); 981 } 982 983 void ModifyVideoConfigs( 984 VideoSendStream::Config* send_config, 985 std::vector<VideoReceiveStreamInterface::Config>* /* receive_configs */, 986 VideoEncoderConfig* encoder_config) override { 987 send_config->encoder_settings.encoder_factory = encoder_factory_; 988 send_config->rtp.payload_name = payload_name_; 989 send_config->rtp.payload_type = 990 test::VideoTestConstants::kVideoSendPayloadType; 991 encoder_config->video_format.name = payload_name_; 992 encoder_config->codec_type = PayloadStringToCodecType(payload_name_); 993 encoder_config->max_bitrate_bps = kMaxBitrate.bps(); 994 for (size_t i = 0; i < max_framerates_.size(); ++i) { 995 encoder_config->simulcast_layers[i].max_framerate = max_framerates_[i]; 996 configured_framerates_[send_config->rtp.ssrcs[i]] = max_framerates_[i]; 997 } 998 } 999 1000 void PerformTest() override { 1001 EXPECT_TRUE(Wait()) << "Timeout while waiting for framerate stats."; 1002 } 1003 1004 void VerifyStats() const { 1005 const bool quick_perf_test = absl::GetFlag(FLAGS_webrtc_quick_perf_test); 1006 double input_fps = 0.0; 1007 for (const auto& configured_framerate : configured_framerates_) { 1008 input_fps = std::max(configured_framerate.second, input_fps); 1009 } 1010 for (const auto& encode_frame_rate_list : encode_frame_rate_lists_) { 1011 const SamplesStatsCounter& values = encode_frame_rate_list.second; 1012 GetGlobalMetricsLogger()->LogMetric( 1013 "substream_fps", "encode_frame_rate", values, Unit::kUnitless, 1014 ImprovementDirection::kNeitherIsBetter); 1015 if (values.IsEmpty()) { 1016 continue; 1017 } 1018 double average_fps = values.GetAverage(); 1019 uint32_t ssrc = encode_frame_rate_list.first; 1020 double expected_fps = configured_framerates_.find(ssrc)->second; 1021 if (quick_perf_test && expected_fps != input_fps) 1022 EXPECT_NEAR(expected_fps, average_fps, kAllowedFpsDiff); 1023 } 1024 } 1025 1026 Action OnSendRtp(ArrayView<const uint8_t> /* packet */) override { 1027 const Timestamp now = clock_->CurrentTime(); 1028 if (now - last_getstats_time_ > kMinGetStatsInterval) { 1029 last_getstats_time_ = now; 1030 task_queue_->PostTask([this, now]() { 1031 VideoSendStream::Stats stats = send_stream_->GetStats(); 1032 for (const auto& stat : stats.substreams) { 1033 encode_frame_rate_lists_[stat.first].AddSample( 1034 {.value = stat.second.encode_frame_rate, .time = now}); 1035 } 1036 if (now - start_time_ > kMinRunTime) { 1037 VerifyStats(); 1038 observation_complete_.Set(); 1039 } 1040 }); 1041 } 1042 return SEND_PACKET; 1043 } 1044 1045 Clock* const clock_; 1046 VideoEncoderFactory* const encoder_factory_; 1047 const std::string payload_name_; 1048 const std::vector<int> max_framerates_; 1049 TaskQueueBase* const task_queue_; 1050 const Timestamp start_time_; 1051 Timestamp last_getstats_time_; 1052 VideoSendStream* send_stream_; 1053 std::map<uint32_t, SamplesStatsCounter> encode_frame_rate_lists_; 1054 std::map<uint32_t, double> configured_framerates_; 1055 } test(encoder_factory, payload_name, max_framerates, task_queue()); 1056 1057 RunBaseTest(&test); 1058 } 1059 1060 TEST_F(CallPerfTest, TestEncodeFramerateVp8Simulcast) { 1061 InternalEncoderFactory internal_encoder_factory; 1062 test::FunctionVideoEncoderFactory encoder_factory( 1063 [&internal_encoder_factory](const Environment& env, 1064 const SdpVideoFormat& /* format */) { 1065 return std::make_unique<SimulcastEncoderAdapter>( 1066 env, &internal_encoder_factory, nullptr, SdpVideoFormat::VP8()); 1067 }); 1068 1069 TestEncodeFramerate(&encoder_factory, "VP8", 1070 /*max_framerates=*/{20, 30}); 1071 } 1072 1073 TEST_F(CallPerfTest, TestEncodeFramerateVp8SimulcastLowerInputFps) { 1074 InternalEncoderFactory internal_encoder_factory; 1075 test::FunctionVideoEncoderFactory encoder_factory( 1076 [&internal_encoder_factory](const Environment& env, 1077 const SdpVideoFormat& /* format */) { 1078 return std::make_unique<SimulcastEncoderAdapter>( 1079 env, &internal_encoder_factory, nullptr, SdpVideoFormat::VP8()); 1080 }); 1081 1082 TestEncodeFramerate(&encoder_factory, "VP8", 1083 /*max_framerates=*/{14, 20}); 1084 } 1085 1086 } // namespace webrtc