rtp_video_sender_unittest.cc (66909B)
1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "call/rtp_video_sender.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <map> 16 #include <memory> 17 #include <optional> 18 #include <vector> 19 20 #include "absl/strings/string_view.h" 21 #include "api/array_view.h" 22 #include "api/call/bitrate_allocation.h" 23 #include "api/call/transport.h" 24 #include "api/crypto/crypto_options.h" 25 #include "api/environment/environment.h" 26 #include "api/environment/environment_factory.h" 27 #include "api/field_trials.h" 28 #include "api/frame_transformer_interface.h" 29 #include "api/make_ref_counted.h" 30 #include "api/rtp_parameters.h" 31 #include "api/scoped_refptr.h" 32 #include "api/test/mock_frame_transformer.h" 33 #include "api/test/network_emulation/network_emulation_interfaces.h" 34 #include "api/transport/bitrate_settings.h" 35 #include "api/transport/rtp/dependency_descriptor.h" 36 #include "api/units/data_rate.h" 37 #include "api/units/data_size.h" 38 #include "api/units/time_delta.h" 39 #include "api/units/timestamp.h" 40 #include "api/video/encoded_image.h" 41 #include "api/video/video_codec_type.h" 42 #include "api/video/video_frame_type.h" 43 #include "api/video_codecs/video_encoder.h" 44 #include "call/rtp_config.h" 45 #include "call/rtp_transport_config.h" 46 #include "call/rtp_transport_controller_send.h" 47 #include "call/rtp_transport_controller_send_interface.h" 48 #include "call/video_send_stream.h" 49 #include "common_video/frame_counts.h" 50 #include "common_video/generic_frame_descriptor/generic_frame_info.h" 51 #include "modules/rtp_rtcp/include/report_block_data.h" 52 #include "modules/rtp_rtcp/include/rtcp_statistics.h" 53 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" 54 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 55 #include "modules/rtp_rtcp/source/byte_io.h" 56 #include "modules/rtp_rtcp/source/rtcp_packet/nack.h" 57 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" 58 #include "modules/rtp_rtcp/source/rtp_packet.h" 59 #include "modules/rtp_rtcp/source/rtp_sender_video.h" 60 #include "modules/video_coding/codecs/interface/common_constants.h" 61 #include "modules/video_coding/fec_controller_default.h" 62 #include "modules/video_coding/include/video_codec_interface.h" 63 #include "rtc_base/buffer.h" 64 #include "rtc_base/checks.h" 65 #include "rtc_base/rate_limiter.h" 66 #include "test/create_test_field_trials.h" 67 #include "test/gmock.h" 68 #include "test/gtest.h" 69 #include "test/mock_transport.h" 70 #include "test/scenario/scenario.h" 71 #include "test/scenario/scenario_config.h" 72 #include "test/time_controller/simulated_time_controller.h" 73 #include "video/config/video_encoder_config.h" 74 #include "video/send_statistics_proxy.h" 75 76 namespace webrtc { 77 namespace { 78 79 using ::testing::_; 80 using ::testing::Ge; 81 using ::testing::IsEmpty; 82 using ::testing::IsNull; 83 using ::testing::NiceMock; 84 using ::testing::NotNull; 85 using ::testing::SaveArg; 86 using ::testing::SizeIs; 87 88 constexpr int8_t kPayloadType = 96; 89 constexpr int8_t kPayloadType2 = 98; 90 constexpr uint32_t kSsrc1 = 12345; 91 constexpr uint32_t kSsrc2 = 23456; 92 constexpr uint32_t kRtxSsrc1 = 34567; 93 constexpr uint32_t kRtxSsrc2 = 45678; 94 constexpr int16_t kInitialPictureId1 = 222; 95 constexpr int16_t kInitialPictureId2 = 44; 96 constexpr int16_t kInitialTl0PicIdx1 = 99; 97 constexpr int16_t kInitialTl0PicIdx2 = 199; 98 constexpr int64_t kRetransmitWindowSizeMs = 500; 99 constexpr int kTransportsSequenceExtensionId = 7; 100 constexpr int kDependencyDescriptorExtensionId = 8; 101 102 class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver { 103 public: 104 MOCK_METHOD(void, OnReceivedIntraFrameRequest, (uint32_t), (override)); 105 }; 106 107 RtpSenderObservers CreateObservers( 108 RtcpIntraFrameObserver* intra_frame_callback, 109 ReportBlockDataObserver* report_block_data_observer, 110 StreamDataCountersCallback* rtp_stats, 111 BitrateStatisticsObserver* bitrate_observer, 112 FrameCountObserver* frame_count_observer, 113 RtcpPacketTypeCounterObserver* rtcp_type_observer) { 114 RtpSenderObservers observers; 115 observers.rtcp_rtt_stats = nullptr; 116 observers.intra_frame_callback = intra_frame_callback; 117 observers.rtcp_loss_notification_observer = nullptr; 118 observers.report_block_data_observer = report_block_data_observer; 119 observers.rtp_stats = rtp_stats; 120 observers.bitrate_observer = bitrate_observer; 121 observers.frame_count_observer = frame_count_observer; 122 observers.rtcp_type_observer = rtcp_type_observer; 123 observers.send_packet_observer = nullptr; 124 return observers; 125 } 126 127 BitrateConstraints GetBitrateConfig() { 128 BitrateConstraints bitrate_config; 129 bitrate_config.min_bitrate_bps = 30000; 130 bitrate_config.start_bitrate_bps = 300000; 131 bitrate_config.max_bitrate_bps = 3000000; 132 return bitrate_config; 133 } 134 135 VideoSendStream::Config CreateVideoSendStreamConfig( 136 Transport* transport, 137 const std::vector<uint32_t>& ssrcs, 138 const std::vector<uint32_t>& rtx_ssrcs, 139 int payload_type, 140 ArrayView<const int> payload_types) { 141 VideoSendStream::Config config(transport); 142 config.rtp.ssrcs = ssrcs; 143 config.rtp.rtx.ssrcs = rtx_ssrcs; 144 config.rtp.payload_type = payload_type; 145 config.rtp.rtx.payload_type = payload_type + 1; 146 config.rtp.nack.rtp_history_ms = 1000; 147 config.rtp.extensions.emplace_back(RtpExtension::kTransportSequenceNumberUri, 148 kTransportsSequenceExtensionId); 149 config.rtp.extensions.emplace_back(RtpDependencyDescriptorExtension::Uri(), 150 kDependencyDescriptorExtensionId); 151 config.rtp.extmap_allow_mixed = true; 152 153 if (!payload_types.empty()) { 154 RTC_CHECK_EQ(payload_types.size(), ssrcs.size()); 155 for (size_t i = 0; i < ssrcs.size(); ++i) { 156 auto& stream_config = config.rtp.stream_configs.emplace_back(); 157 stream_config.ssrc = ssrcs[i]; 158 stream_config.payload_type = payload_types[i]; 159 if (i < rtx_ssrcs.size()) { 160 auto& rtx = stream_config.rtx.emplace(); 161 rtx.ssrc = rtx_ssrcs[i]; 162 rtx.payload_type = payload_types[i] + 1; 163 } 164 } 165 } 166 return config; 167 } 168 169 class RtpVideoSenderTestFixture { 170 public: 171 RtpVideoSenderTestFixture( 172 const std::vector<uint32_t>& ssrcs, 173 const std::vector<uint32_t>& rtx_ssrcs, 174 int payload_type, 175 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states, 176 FrameCountObserver* frame_count_observer, 177 scoped_refptr<FrameTransformerInterface> frame_transformer, 178 const std::vector<int>& payload_types, 179 absl::string_view field_trials = "") 180 : field_trials_(CreateTestFieldTrials(field_trials)), 181 time_controller_(Timestamp::Millis(1000000)), 182 env_(CreateEnvironment(&field_trials_, 183 time_controller_.GetClock(), 184 time_controller_.CreateTaskQueueFactory())), 185 config_(CreateVideoSendStreamConfig(&transport_, 186 ssrcs, 187 rtx_ssrcs, 188 payload_type, 189 payload_types)), 190 bitrate_config_(GetBitrateConfig()), 191 transport_controller_( 192 RtpTransportConfig{.env = env_, .bitrate_config = bitrate_config_}), 193 stats_proxy_(time_controller_.GetClock(), 194 config_, 195 VideoEncoderConfig::ContentType::kRealtimeVideo, 196 env_.field_trials()), 197 retransmission_rate_limiter_(time_controller_.GetClock(), 198 kRetransmitWindowSizeMs) { 199 transport_controller_.EnsureStarted(); 200 std::map<uint32_t, RtpState> suspended_ssrcs; 201 router_ = std::make_unique<RtpVideoSender>( 202 env_, time_controller_.GetMainThread(), suspended_ssrcs, 203 suspended_payload_states, config_.rtp, config_.rtcp_report_interval_ms, 204 &transport_, 205 CreateObservers(&encoder_feedback_, &stats_proxy_, &stats_proxy_, 206 &stats_proxy_, frame_count_observer, &stats_proxy_), 207 &transport_controller_, &retransmission_rate_limiter_, 208 std::make_unique<FecControllerDefault>(env_), nullptr, CryptoOptions{}, 209 frame_transformer); 210 } 211 RtpVideoSenderTestFixture( 212 const std::vector<uint32_t>& ssrcs, 213 const std::vector<uint32_t>& rtx_ssrcs, 214 int payload_type, 215 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states, 216 FrameCountObserver* frame_count_observer, 217 scoped_refptr<FrameTransformerInterface> frame_transformer, 218 absl::string_view field_trials = "") 219 : RtpVideoSenderTestFixture(ssrcs, 220 rtx_ssrcs, 221 payload_type, 222 suspended_payload_states, 223 frame_count_observer, 224 frame_transformer, 225 /*payload_types=*/{}, 226 field_trials) {} 227 228 RtpVideoSenderTestFixture( 229 const std::vector<uint32_t>& ssrcs, 230 const std::vector<uint32_t>& rtx_ssrcs, 231 int payload_type, 232 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states, 233 FrameCountObserver* frame_count_observer, 234 absl::string_view field_trials = "") 235 : RtpVideoSenderTestFixture(ssrcs, 236 rtx_ssrcs, 237 payload_type, 238 suspended_payload_states, 239 frame_count_observer, 240 /*frame_transformer=*/nullptr, 241 /*payload_types=*/{}, 242 field_trials) {} 243 244 RtpVideoSenderTestFixture( 245 const std::vector<uint32_t>& ssrcs, 246 const std::vector<uint32_t>& rtx_ssrcs, 247 int payload_type, 248 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states, 249 absl::string_view field_trials = "") 250 : RtpVideoSenderTestFixture(ssrcs, 251 rtx_ssrcs, 252 payload_type, 253 suspended_payload_states, 254 /*frame_count_observer=*/nullptr, 255 /*frame_transformer=*/nullptr, 256 /*payload_types=*/{}, 257 field_trials) {} 258 259 ~RtpVideoSenderTestFixture() { SetSending(false); } 260 261 RtpVideoSender* router() { return router_.get(); } 262 MockTransport& transport() { return transport_; } 263 void AdvanceTime(TimeDelta delta) { time_controller_.AdvanceTime(delta); } 264 265 void SetSending(bool sending) { router_->SetSending(sending); } 266 267 private: 268 FieldTrials field_trials_; 269 NiceMock<MockTransport> transport_; 270 NiceMock<MockRtcpIntraFrameObserver> encoder_feedback_; 271 GlobalSimulatedTimeController time_controller_; 272 Environment env_; 273 VideoSendStream::Config config_; 274 BitrateConstraints bitrate_config_; 275 RtpTransportControllerSend transport_controller_; 276 SendStatisticsProxy stats_proxy_; 277 RateLimiter retransmission_rate_limiter_; 278 std::unique_ptr<RtpVideoSender> router_; 279 }; 280 281 BitrateAllocationUpdate CreateBitrateAllocationUpdate(int target_bitrate_bps) { 282 BitrateAllocationUpdate update; 283 update.target_bitrate = DataRate::BitsPerSec(target_bitrate_bps); 284 update.round_trip_time = TimeDelta::Zero(); 285 return update; 286 } 287 288 } // namespace 289 290 TEST(RtpVideoSenderTest, SendOnOneModule) { 291 constexpr uint8_t kPayload = 'a'; 292 EncodedImage encoded_image; 293 encoded_image.SetRtpTimestamp(1); 294 encoded_image.capture_time_ms_ = 2; 295 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 296 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); 297 298 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); 299 EXPECT_NE(EncodedImageCallback::Result::OK, 300 test.router()->OnEncodedImage(encoded_image, nullptr).error); 301 302 test.SetSending(true); 303 EXPECT_EQ(EncodedImageCallback::Result::OK, 304 test.router()->OnEncodedImage(encoded_image, nullptr).error); 305 306 test.SetSending(false); 307 EXPECT_NE(EncodedImageCallback::Result::OK, 308 test.router()->OnEncodedImage(encoded_image, nullptr).error); 309 310 test.SetSending(true); 311 EXPECT_EQ(EncodedImageCallback::Result::OK, 312 test.router()->OnEncodedImage(encoded_image, nullptr).error); 313 } 314 315 TEST(RtpVideoSenderTest, OnEncodedImageReturnOkWhenSendingTrue) { 316 constexpr uint8_t kPayload = 'a'; 317 EncodedImage encoded_image_1; 318 encoded_image_1.SetRtpTimestamp(1); 319 encoded_image_1.capture_time_ms_ = 2; 320 encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; 321 encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); 322 323 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 324 kPayloadType, {}); 325 326 CodecSpecificInfo codec_info; 327 codec_info.codecType = kVideoCodecVP8; 328 329 test.SetSending(true); 330 EXPECT_EQ(EncodedImageCallback::Result::OK, 331 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); 332 333 EncodedImage encoded_image_2(encoded_image_1); 334 encoded_image_2.SetSimulcastIndex(1); 335 EXPECT_EQ(EncodedImageCallback::Result::OK, 336 test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); 337 } 338 339 TEST(RtpVideoSenderTest, OnEncodedImageReturnErrorCodeWhenSendingFalse) { 340 constexpr uint8_t kPayload = 'a'; 341 EncodedImage encoded_image_1; 342 encoded_image_1.SetRtpTimestamp(1); 343 encoded_image_1.capture_time_ms_ = 2; 344 encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; 345 encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); 346 347 EncodedImage encoded_image_2(encoded_image_1); 348 encoded_image_2.SetSimulcastIndex(1); 349 350 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 351 kPayloadType, {}); 352 CodecSpecificInfo codec_info; 353 codec_info.codecType = kVideoCodecVP8; 354 355 // Setting rtp streams to inactive will turn the payload router to 356 // inactive. 357 test.SetSending(false); 358 // An incoming encoded image will not ask the module to send outgoing data 359 // because the payload router is inactive. 360 EXPECT_NE(EncodedImageCallback::Result::OK, 361 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); 362 EXPECT_NE(EncodedImageCallback::Result::OK, 363 test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); 364 } 365 366 TEST(RtpVideoSenderTest, 367 DiscardsHigherSimulcastFramesAfterLayerDisabledInVideoLayersAllocation) { 368 constexpr uint8_t kPayload = 'a'; 369 EncodedImage encoded_image_1; 370 encoded_image_1.SetRtpTimestamp(1); 371 encoded_image_1.capture_time_ms_ = 2; 372 encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; 373 encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); 374 EncodedImage encoded_image_2(encoded_image_1); 375 encoded_image_2.SetSimulcastIndex(1); 376 CodecSpecificInfo codec_info; 377 codec_info.codecType = kVideoCodecVP8; 378 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 379 kPayloadType, {}); 380 test.SetSending(true); 381 // A layer is sent on both rtp streams. 382 test.router()->OnVideoLayersAllocationUpdated( 383 {.active_spatial_layers = {{.rtp_stream_index = 0}, 384 {.rtp_stream_index = 1}}}); 385 386 EXPECT_EQ(EncodedImageCallback::Result::OK, 387 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); 388 EXPECT_EQ(EncodedImageCallback::Result::OK, 389 test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); 390 391 // Only rtp stream index 0 is configured to send a stream. 392 test.router()->OnVideoLayersAllocationUpdated( 393 {.active_spatial_layers = {{.rtp_stream_index = 0}}}); 394 test.AdvanceTime(TimeDelta::Millis(33)); 395 EXPECT_EQ(EncodedImageCallback::Result::OK, 396 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); 397 EXPECT_NE(EncodedImageCallback::Result::OK, 398 test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); 399 } 400 401 TEST(RtpVideoSenderTest, CreateWithNoPreviousStates) { 402 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 403 kPayloadType, {}); 404 test.SetSending(true); 405 406 std::map<uint32_t, RtpPayloadState> initial_states = 407 test.router()->GetRtpPayloadStates(); 408 EXPECT_EQ(2u, initial_states.size()); 409 EXPECT_NE(initial_states.find(kSsrc1), initial_states.end()); 410 EXPECT_NE(initial_states.find(kSsrc2), initial_states.end()); 411 } 412 413 TEST(RtpVideoSenderTest, CreateWithPreviousStates) { 414 const int64_t kState1SharedFrameId = 123; 415 const int64_t kState2SharedFrameId = 234; 416 RtpPayloadState state1; 417 state1.picture_id = kInitialPictureId1; 418 state1.tl0_pic_idx = kInitialTl0PicIdx1; 419 state1.shared_frame_id = kState1SharedFrameId; 420 RtpPayloadState state2; 421 state2.picture_id = kInitialPictureId2; 422 state2.tl0_pic_idx = kInitialTl0PicIdx2; 423 state2.shared_frame_id = kState2SharedFrameId; 424 std::map<uint32_t, RtpPayloadState> states = {{kSsrc1, state1}, 425 {kSsrc2, state2}}; 426 427 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 428 kPayloadType, states); 429 test.SetSending(true); 430 431 std::map<uint32_t, RtpPayloadState> initial_states = 432 test.router()->GetRtpPayloadStates(); 433 EXPECT_EQ(2u, initial_states.size()); 434 EXPECT_EQ(kInitialPictureId1, initial_states[kSsrc1].picture_id); 435 EXPECT_EQ(kInitialTl0PicIdx1, initial_states[kSsrc1].tl0_pic_idx); 436 EXPECT_EQ(kInitialPictureId2, initial_states[kSsrc2].picture_id); 437 EXPECT_EQ(kInitialTl0PicIdx2, initial_states[kSsrc2].tl0_pic_idx); 438 EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc1].shared_frame_id); 439 EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc2].shared_frame_id); 440 } 441 442 TEST(RtpVideoSenderTest, FrameCountCallbacks) { 443 class MockFrameCountObserver : public FrameCountObserver { 444 public: 445 MOCK_METHOD(void, 446 FrameCountUpdated, 447 (const FrameCounts& frame_counts, uint32_t ssrc), 448 (override)); 449 } callback; 450 451 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}, 452 &callback); 453 454 constexpr uint8_t kPayload = 'a'; 455 EncodedImage encoded_image; 456 encoded_image.SetRtpTimestamp(1); 457 encoded_image.capture_time_ms_ = 2; 458 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 459 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); 460 461 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 462 463 // No callbacks when not active. 464 EXPECT_CALL(callback, FrameCountUpdated).Times(0); 465 EXPECT_NE(EncodedImageCallback::Result::OK, 466 test.router()->OnEncodedImage(encoded_image, nullptr).error); 467 ::testing::Mock::VerifyAndClearExpectations(&callback); 468 469 test.SetSending(true); 470 471 FrameCounts frame_counts; 472 EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1)) 473 .WillOnce(SaveArg<0>(&frame_counts)); 474 EXPECT_EQ(EncodedImageCallback::Result::OK, 475 test.router()->OnEncodedImage(encoded_image, nullptr).error); 476 477 EXPECT_EQ(1, frame_counts.key_frames); 478 EXPECT_EQ(0, frame_counts.delta_frames); 479 480 ::testing::Mock::VerifyAndClearExpectations(&callback); 481 482 encoded_image._frameType = VideoFrameType::kVideoFrameDelta; 483 EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1)) 484 .WillOnce(SaveArg<0>(&frame_counts)); 485 EXPECT_EQ(EncodedImageCallback::Result::OK, 486 test.router()->OnEncodedImage(encoded_image, nullptr).error); 487 488 EXPECT_EQ(1, frame_counts.key_frames); 489 EXPECT_EQ(1, frame_counts.delta_frames); 490 } 491 492 // Integration test verifying that ack of packet via TransportFeedback means 493 // that the packet is removed from RtpPacketHistory and won't be retransmitted 494 // again. 495 TEST(RtpVideoSenderTest, DoesNotRetrasmitAckedPackets) { 496 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 497 kPayloadType, {}); 498 test.SetSending(true); 499 500 constexpr uint8_t kPayload = 'a'; 501 EncodedImage encoded_image; 502 encoded_image.SetRtpTimestamp(1); 503 encoded_image.capture_time_ms_ = 2; 504 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 505 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); 506 507 // Send two tiny images, mapping to two RTP packets. Capture sequence numbers. 508 std::vector<uint16_t> rtp_sequence_numbers; 509 std::vector<uint16_t> transport_sequence_numbers; 510 EXPECT_CALL(test.transport(), SendRtp) 511 .Times(2) 512 .WillRepeatedly( 513 [&rtp_sequence_numbers, &transport_sequence_numbers]( 514 ArrayView<const uint8_t> packet, const PacketOptions& options) { 515 RtpPacket rtp_packet; 516 EXPECT_TRUE(rtp_packet.Parse(packet)); 517 rtp_sequence_numbers.push_back(rtp_packet.SequenceNumber()); 518 transport_sequence_numbers.push_back(options.packet_id); 519 return true; 520 }); 521 EXPECT_EQ(EncodedImageCallback::Result::OK, 522 test.router()->OnEncodedImage(encoded_image, nullptr).error); 523 encoded_image.SetRtpTimestamp(2); 524 encoded_image.capture_time_ms_ = 3; 525 EXPECT_EQ(EncodedImageCallback::Result::OK, 526 test.router()->OnEncodedImage(encoded_image, nullptr).error); 527 528 test.AdvanceTime(TimeDelta::Millis(33)); 529 530 // Construct a NACK message for requesting retransmission of both packet. 531 rtcp::Nack nack; 532 nack.SetMediaSsrc(kSsrc1); 533 nack.SetPacketIds(rtp_sequence_numbers); 534 Buffer nack_buffer = nack.Build(); 535 536 std::vector<uint16_t> retransmitted_rtp_sequence_numbers; 537 EXPECT_CALL(test.transport(), SendRtp) 538 .Times(2) 539 .WillRepeatedly([&retransmitted_rtp_sequence_numbers]( 540 ArrayView<const uint8_t> packet, 541 const PacketOptions& options) { 542 RtpPacket rtp_packet; 543 EXPECT_TRUE(rtp_packet.Parse(packet)); 544 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); 545 // Capture the retransmitted sequence number from the RTX header. 546 ArrayView<const uint8_t> payload = rtp_packet.payload(); 547 retransmitted_rtp_sequence_numbers.push_back( 548 ByteReader<uint16_t>::ReadBigEndian(payload.data())); 549 return true; 550 }); 551 test.router()->DeliverRtcp(nack_buffer); 552 test.AdvanceTime(TimeDelta::Millis(33)); 553 554 // Verify that both packets were retransmitted. 555 EXPECT_EQ(retransmitted_rtp_sequence_numbers, rtp_sequence_numbers); 556 557 // Simulate transport feedback indicating fist packet received, next packet 558 // lost (not other way around as that would trigger early retransmit). 559 StreamFeedbackObserver::StreamPacketInfo lost_packet_feedback; 560 lost_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[0]; 561 lost_packet_feedback.ssrc = kSsrc1; 562 lost_packet_feedback.received = false; 563 lost_packet_feedback.is_retransmission = false; 564 565 StreamFeedbackObserver::StreamPacketInfo received_packet_feedback; 566 received_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[1]; 567 received_packet_feedback.ssrc = kSsrc1; 568 received_packet_feedback.received = true; 569 lost_packet_feedback.is_retransmission = false; 570 571 test.router()->OnPacketFeedbackVector( 572 {lost_packet_feedback, received_packet_feedback}); 573 574 // Advance time to make sure retransmission would be allowed and try again. 575 // This time the retransmission should not happen for the first packet since 576 // the history has been notified of the ack and removed the packet. The 577 // second packet, included in the feedback but not marked as received, should 578 // still be retransmitted. 579 test.AdvanceTime(TimeDelta::Millis(33)); 580 EXPECT_CALL(test.transport(), SendRtp) 581 .WillOnce([&lost_packet_feedback](ArrayView<const uint8_t> packet, 582 const PacketOptions& options) { 583 RtpPacket rtp_packet; 584 EXPECT_TRUE(rtp_packet.Parse(packet)); 585 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); 586 // Capture the retransmitted sequence number from the RTX header. 587 ArrayView<const uint8_t> payload = rtp_packet.payload(); 588 EXPECT_EQ(lost_packet_feedback.rtp_sequence_number, 589 ByteReader<uint16_t>::ReadBigEndian(payload.data())); 590 return true; 591 }); 592 test.router()->DeliverRtcp(nack_buffer); 593 test.AdvanceTime(TimeDelta::Millis(33)); 594 } 595 596 // This tests that we utilize transport wide feedback to retransmit lost 597 // packets. This is tested by dropping all ordinary packets from a "lossy" 598 // stream sent along with a secondary untouched stream. The transport wide 599 // feedback packets from the secondary stream allows the sending side to 600 // detect and retreansmit the lost packets from the lossy stream. 601 TEST(RtpVideoSenderTest, RetransmitsOnTransportWideLossInfo) { 602 int rtx_packets; 603 test::Scenario s(test_info_); 604 test::CallClientConfig call_conf; 605 // Keeping the bitrate fixed to avoid RTX due to probing. 606 call_conf.transport.rates.max_rate = DataRate::KilobitsPerSec(300); 607 call_conf.transport.rates.start_rate = DataRate::KilobitsPerSec(300); 608 test::NetworkSimulationConfig net_conf; 609 net_conf.bandwidth = DataRate::KilobitsPerSec(300); 610 auto send_node = s.CreateSimulationNode(net_conf); 611 auto* callee = s.CreateClient("return", call_conf); 612 auto* route = s.CreateRoutes(s.CreateClient("send", call_conf), {send_node}, 613 callee, {s.CreateSimulationNode(net_conf)}); 614 615 test::VideoStreamConfig lossy_config; 616 lossy_config.source.framerate = 5; 617 auto* lossy = s.CreateVideoStream(route->forward(), lossy_config); 618 // The secondary stream acts a driver for transport feedback messages, 619 // ensuring that lost packets on the lossy stream are retransmitted. 620 s.CreateVideoStream(route->forward(), test::VideoStreamConfig()); 621 622 send_node->router()->SetFilter([&](const EmulatedIpPacket& packet) { 623 RtpPacket rtp; 624 if (rtp.Parse(packet.data)) { 625 // Drops all regular packets for the lossy stream and counts all RTX 626 // packets. Since no packets are let trough, NACKs can't be triggered 627 // by the receiving side. 628 if (lossy->send()->UsingSsrc(rtp.Ssrc())) { 629 return false; 630 } else if (lossy->send()->UsingRtxSsrc(rtp.Ssrc())) { 631 ++rtx_packets; 632 } 633 } 634 return true; 635 }); 636 637 // Run for a short duration and reset counters to avoid counting RTX packets 638 // from initial probing. 639 s.RunFor(TimeDelta::Seconds(1)); 640 rtx_packets = 0; 641 int decoded_baseline = 0; 642 callee->SendTask([&decoded_baseline, &lossy]() { 643 decoded_baseline = lossy->receive()->GetStats().frames_decoded; 644 }); 645 s.RunFor(TimeDelta::Seconds(1)); 646 // We expect both that RTX packets were sent and that an appropriate number of 647 // frames were received. This is somewhat redundant but reduces the risk of 648 // false positives in future regressions (e.g. RTX is send due to probing). 649 EXPECT_GE(rtx_packets, 1); 650 int frames_decoded = 0; 651 callee->SendTask([&decoded_baseline, &frames_decoded, &lossy]() { 652 frames_decoded = 653 lossy->receive()->GetStats().frames_decoded - decoded_baseline; 654 }); 655 EXPECT_EQ(frames_decoded, 5); 656 } 657 658 // Integration test verifying that retransmissions are sent for packets which 659 // can be detected as lost early, using transport wide feedback. 660 TEST(RtpVideoSenderTest, EarlyRetransmits) { 661 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 662 kPayloadType, {}); 663 test.SetSending(true); 664 665 const uint8_t kPayload[1] = {'a'}; 666 EncodedImage encoded_image; 667 encoded_image.SetRtpTimestamp(1); 668 encoded_image.capture_time_ms_ = 2; 669 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 670 encoded_image.SetEncodedData( 671 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 672 encoded_image.SetSimulcastIndex(0); 673 674 CodecSpecificInfo codec_specific; 675 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; 676 677 // Send two tiny images, mapping to single RTP packets. Capture sequence 678 // numbers. 679 uint16_t frame1_rtp_sequence_number = 0; 680 uint16_t frame1_transport_sequence_number = 0; 681 EXPECT_CALL(test.transport(), SendRtp) 682 .WillOnce( 683 [&frame1_rtp_sequence_number, &frame1_transport_sequence_number]( 684 ArrayView<const uint8_t> packet, const PacketOptions& options) { 685 RtpPacket rtp_packet; 686 EXPECT_TRUE(rtp_packet.Parse(packet)); 687 frame1_rtp_sequence_number = rtp_packet.SequenceNumber(); 688 frame1_transport_sequence_number = options.packet_id; 689 EXPECT_EQ(rtp_packet.Ssrc(), kSsrc1); 690 return true; 691 }); 692 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 693 EncodedImageCallback::Result::OK); 694 695 test.AdvanceTime(TimeDelta::Millis(33)); 696 697 uint16_t frame2_rtp_sequence_number = 0; 698 uint16_t frame2_transport_sequence_number = 0; 699 encoded_image.SetSimulcastIndex(1); 700 EXPECT_CALL(test.transport(), SendRtp) 701 .WillOnce( 702 [&frame2_rtp_sequence_number, &frame2_transport_sequence_number]( 703 ArrayView<const uint8_t> packet, const PacketOptions& options) { 704 RtpPacket rtp_packet; 705 EXPECT_TRUE(rtp_packet.Parse(packet)); 706 frame2_rtp_sequence_number = rtp_packet.SequenceNumber(); 707 frame2_transport_sequence_number = options.packet_id; 708 EXPECT_EQ(rtp_packet.Ssrc(), kSsrc2); 709 return true; 710 }); 711 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 712 EncodedImageCallback::Result::OK); 713 test.AdvanceTime(TimeDelta::Millis(33)); 714 715 EXPECT_NE(frame1_transport_sequence_number, frame2_transport_sequence_number); 716 717 // Inject a transport feedback where the packet for the first frame is lost, 718 // expect a retransmission for it. 719 EXPECT_CALL(test.transport(), SendRtp) 720 .WillOnce([&frame1_rtp_sequence_number](ArrayView<const uint8_t> packet, 721 const PacketOptions& options) { 722 RtpPacket rtp_packet; 723 EXPECT_TRUE(rtp_packet.Parse(packet)); 724 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); 725 726 // Retransmitted sequence number from the RTX header should match 727 // the lost packet. 728 ArrayView<const uint8_t> payload = rtp_packet.payload(); 729 EXPECT_EQ(ByteReader<uint16_t>::ReadBigEndian(payload.data()), 730 frame1_rtp_sequence_number); 731 return true; 732 }); 733 734 StreamFeedbackObserver::StreamPacketInfo first_packet_feedback; 735 first_packet_feedback.rtp_sequence_number = frame1_rtp_sequence_number; 736 first_packet_feedback.ssrc = kSsrc1; 737 first_packet_feedback.received = false; 738 first_packet_feedback.is_retransmission = false; 739 740 StreamFeedbackObserver::StreamPacketInfo second_packet_feedback; 741 second_packet_feedback.rtp_sequence_number = frame2_rtp_sequence_number; 742 second_packet_feedback.ssrc = kSsrc2; 743 second_packet_feedback.received = true; 744 first_packet_feedback.is_retransmission = false; 745 746 test.router()->OnPacketFeedbackVector( 747 {first_packet_feedback, second_packet_feedback}); 748 749 // Wait for pacer to run and send the RTX packet. 750 test.AdvanceTime(TimeDelta::Millis(33)); 751 } 752 753 TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { 754 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); 755 test.SetSending(true); 756 757 RtpHeaderExtensionMap extensions; 758 extensions.Register<RtpDependencyDescriptorExtension>( 759 kDependencyDescriptorExtensionId); 760 std::vector<RtpPacket> sent_packets; 761 ON_CALL(test.transport(), SendRtp) 762 .WillByDefault( 763 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 764 sent_packets.emplace_back(&extensions); 765 EXPECT_TRUE(sent_packets.back().Parse(packet)); 766 return true; 767 }); 768 769 const uint8_t kPayload[1] = {'a'}; 770 EncodedImage encoded_image; 771 encoded_image.SetRtpTimestamp(1); 772 encoded_image.capture_time_ms_ = 2; 773 encoded_image.SetEncodedData( 774 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 775 776 CodecSpecificInfo codec_specific; 777 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; 778 codec_specific.template_structure.emplace(); 779 codec_specific.template_structure->num_decode_targets = 1; 780 codec_specific.template_structure->templates = { 781 FrameDependencyTemplate().T(0).Dtis("S"), 782 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), 783 FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), 784 }; 785 786 // Send two tiny images, mapping to single RTP packets. 787 // Send in key frame. 788 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 789 codec_specific.generic_frame_info = 790 GenericFrameInfo::Builder().T(0).Dtis("S").Build(); 791 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; 792 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 793 EncodedImageCallback::Result::OK); 794 test.AdvanceTime(TimeDelta::Millis(33)); 795 ASSERT_THAT(sent_packets, SizeIs(1)); 796 EXPECT_TRUE( 797 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>()); 798 799 // Send in delta frame. 800 encoded_image._frameType = VideoFrameType::kVideoFrameDelta; 801 codec_specific.template_structure = std::nullopt; 802 codec_specific.generic_frame_info = 803 GenericFrameInfo::Builder().T(1).Dtis("D").Build(); 804 codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}}; 805 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 806 EncodedImageCallback::Result::OK); 807 test.AdvanceTime(TimeDelta::Millis(33)); 808 ASSERT_THAT(sent_packets, SizeIs(2)); 809 EXPECT_TRUE( 810 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>()); 811 } 812 813 TEST(RtpVideoSenderTest, SimulcastIndependentFrameIds) { 814 absl::string_view field_trials = "WebRTC-GenericDescriptorAuth/Disabled/"; 815 const std::map<uint32_t, RtpPayloadState> kPayloadStates = { 816 {kSsrc1, {.frame_id = 100}}, {kSsrc2, {.frame_id = 200}}}; 817 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {}, kPayloadType, 818 kPayloadStates, field_trials); 819 test.SetSending(true); 820 821 RtpHeaderExtensionMap extensions; 822 extensions.Register<RtpDependencyDescriptorExtension>( 823 kDependencyDescriptorExtensionId); 824 std::vector<RtpPacket> sent_packets; 825 ON_CALL(test.transport(), SendRtp) 826 .WillByDefault( 827 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 828 sent_packets.emplace_back(&extensions); 829 EXPECT_TRUE(sent_packets.back().Parse(packet)); 830 return true; 831 }); 832 833 const uint8_t kPayload[1] = {'a'}; 834 EncodedImage encoded_image; 835 encoded_image.SetEncodedData( 836 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 837 838 CodecSpecificInfo codec_specific; 839 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; 840 codec_specific.template_structure.emplace(); 841 codec_specific.template_structure->num_decode_targets = 1; 842 codec_specific.template_structure->templates = { 843 FrameDependencyTemplate().T(0).Dtis("S"), 844 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({1}), 845 }; 846 codec_specific.generic_frame_info = 847 GenericFrameInfo::Builder().T(0).Dtis("S").Build(); 848 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 849 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; 850 851 encoded_image.SetSimulcastIndex(0); 852 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 853 EncodedImageCallback::Result::OK); 854 encoded_image.SetSimulcastIndex(1); 855 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 856 EncodedImageCallback::Result::OK); 857 858 test.AdvanceTime(TimeDelta::Millis(33)); 859 ASSERT_THAT(sent_packets, SizeIs(2)); 860 DependencyDescriptorMandatory dd_s0; 861 DependencyDescriptorMandatory dd_s1; 862 ASSERT_TRUE( 863 sent_packets[0].GetExtension<RtpDependencyDescriptorExtension>(&dd_s0)); 864 ASSERT_TRUE( 865 sent_packets[1].GetExtension<RtpDependencyDescriptorExtension>(&dd_s1)); 866 EXPECT_EQ(dd_s0.frame_number(), 100); 867 EXPECT_EQ(dd_s1.frame_number(), 200); 868 } 869 870 TEST(RtpVideoSenderTest, 871 SimulcastNoIndependentFrameIdsIfGenericDescriptorAuthIsEnabled) { 872 absl::string_view field_trials = "WebRTC-GenericDescriptorAuth/Enabled/"; 873 const std::map<uint32_t, RtpPayloadState> kPayloadStates = { 874 {kSsrc1, {.shared_frame_id = 1000, .frame_id = 100}}, 875 {kSsrc2, {.shared_frame_id = 1000, .frame_id = 200}}}; 876 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {}, kPayloadType, 877 kPayloadStates, field_trials); 878 test.SetSending(true); 879 880 RtpHeaderExtensionMap extensions; 881 extensions.Register<RtpDependencyDescriptorExtension>( 882 kDependencyDescriptorExtensionId); 883 std::vector<RtpPacket> sent_packets; 884 ON_CALL(test.transport(), SendRtp) 885 .WillByDefault( 886 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 887 sent_packets.emplace_back(&extensions); 888 EXPECT_TRUE(sent_packets.back().Parse(packet)); 889 return true; 890 }); 891 892 const uint8_t kPayload[1] = {'a'}; 893 EncodedImage encoded_image; 894 encoded_image.SetEncodedData( 895 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 896 897 CodecSpecificInfo codec_specific; 898 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; 899 codec_specific.template_structure.emplace(); 900 codec_specific.template_structure->num_decode_targets = 1; 901 codec_specific.template_structure->templates = { 902 FrameDependencyTemplate().T(0).Dtis("S"), 903 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({1}), 904 }; 905 codec_specific.generic_frame_info = 906 GenericFrameInfo::Builder().T(0).Dtis("S").Build(); 907 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 908 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; 909 910 encoded_image.SetSimulcastIndex(0); 911 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 912 EncodedImageCallback::Result::OK); 913 encoded_image.SetSimulcastIndex(1); 914 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 915 EncodedImageCallback::Result::OK); 916 917 test.AdvanceTime(TimeDelta::Millis(33)); 918 ASSERT_THAT(sent_packets, SizeIs(2)); 919 DependencyDescriptorMandatory dd_s0; 920 DependencyDescriptorMandatory dd_s1; 921 ASSERT_TRUE( 922 sent_packets[0].GetExtension<RtpDependencyDescriptorExtension>(&dd_s0)); 923 ASSERT_TRUE( 924 sent_packets[1].GetExtension<RtpDependencyDescriptorExtension>(&dd_s1)); 925 EXPECT_EQ(dd_s0.frame_number(), 1001); 926 EXPECT_EQ(dd_s1.frame_number(), 1002); 927 } 928 929 TEST(RtpVideoSenderTest, MixedCodecSimulcastPayloadType) { 930 // When multiple payload types are set, verify that the payload type switches 931 // corresponding to the simulcast index. 932 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 933 kPayloadType, {}, nullptr, nullptr, 934 {kPayloadType, kPayloadType2}); 935 test.SetSending(true); 936 937 std::vector<uint16_t> rtp_sequence_numbers; 938 std::vector<RtpPacket> sent_packets; 939 EXPECT_CALL(test.transport(), SendRtp) 940 .Times(3) 941 .WillRepeatedly([&](ArrayView<const uint8_t> packet, 942 const PacketOptions& options) -> bool { 943 RtpPacket& rtp_packet = sent_packets.emplace_back(); 944 EXPECT_TRUE(rtp_packet.Parse(packet)); 945 rtp_sequence_numbers.push_back(rtp_packet.SequenceNumber()); 946 return true; 947 }); 948 949 const uint8_t kPayload[1] = {'a'}; 950 EncodedImage encoded_image; 951 encoded_image.SetEncodedData( 952 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 953 954 CodecSpecificInfo codec_specific; 955 codec_specific.codecType = VideoCodecType::kVideoCodecVP8; 956 957 encoded_image.SetSimulcastIndex(0); 958 ASSERT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 959 EncodedImageCallback::Result::OK); 960 ASSERT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 961 EncodedImageCallback::Result::OK); 962 encoded_image.SetSimulcastIndex(1); 963 ASSERT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 964 EncodedImageCallback::Result::OK); 965 966 test.AdvanceTime(TimeDelta::Millis(33)); 967 ASSERT_THAT(sent_packets, SizeIs(3)); 968 EXPECT_EQ(sent_packets[0].PayloadType(), kPayloadType); 969 EXPECT_EQ(sent_packets[1].PayloadType(), kPayloadType); 970 EXPECT_EQ(sent_packets[2].PayloadType(), kPayloadType2); 971 972 // Verify that NACK is sent to the RTX payload type corresponding to the 973 // payload type. 974 rtcp::Nack nack1, nack2; 975 nack1.SetMediaSsrc(kSsrc1); 976 nack2.SetMediaSsrc(kSsrc2); 977 nack1.SetPacketIds({rtp_sequence_numbers[0], rtp_sequence_numbers[1]}); 978 nack2.SetPacketIds({rtp_sequence_numbers[2]}); 979 Buffer nack_buffer1 = nack1.Build(); 980 Buffer nack_buffer2 = nack2.Build(); 981 982 std::vector<RtpPacket> sent_rtx_packets; 983 EXPECT_CALL(test.transport(), SendRtp) 984 .Times(3) 985 .WillRepeatedly( 986 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 987 RtpPacket& rtp_packet = sent_rtx_packets.emplace_back(); 988 EXPECT_TRUE(rtp_packet.Parse(packet)); 989 return true; 990 }); 991 test.router()->DeliverRtcp(nack_buffer1); 992 test.router()->DeliverRtcp(nack_buffer2); 993 994 test.AdvanceTime(TimeDelta::Millis(33)); 995 996 ASSERT_THAT(sent_rtx_packets, SizeIs(3)); 997 EXPECT_EQ(sent_rtx_packets[0].PayloadType(), kPayloadType + 1); 998 EXPECT_EQ(sent_rtx_packets[1].PayloadType(), kPayloadType + 1); 999 EXPECT_EQ(sent_rtx_packets[2].PayloadType(), kPayloadType2 + 1); 1000 } 1001 1002 TEST(RtpVideoSenderTest, 1003 SupportsDependencyDescriptorForVp8NotProvidedByEncoder) { 1004 constexpr uint8_t kPayload[1] = {'a'}; 1005 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); 1006 RtpHeaderExtensionMap extensions; 1007 extensions.Register<RtpDependencyDescriptorExtension>( 1008 kDependencyDescriptorExtensionId); 1009 std::vector<RtpPacket> sent_packets; 1010 ON_CALL(test.transport(), SendRtp) 1011 .WillByDefault( 1012 [&](ArrayView<const uint8_t> packet, const PacketOptions&) { 1013 EXPECT_TRUE(sent_packets.emplace_back(&extensions).Parse(packet)); 1014 return true; 1015 }); 1016 test.SetSending(true); 1017 1018 EncodedImage key_frame_image; 1019 key_frame_image._frameType = VideoFrameType::kVideoFrameKey; 1020 key_frame_image.SetEncodedData( 1021 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1022 CodecSpecificInfo key_frame_info; 1023 key_frame_info.codecType = VideoCodecType::kVideoCodecVP8; 1024 ASSERT_EQ( 1025 test.router()->OnEncodedImage(key_frame_image, &key_frame_info).error, 1026 EncodedImageCallback::Result::OK); 1027 1028 EncodedImage delta_image; 1029 delta_image._frameType = VideoFrameType::kVideoFrameDelta; 1030 delta_image.SetEncodedData( 1031 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1032 CodecSpecificInfo delta_info; 1033 delta_info.codecType = VideoCodecType::kVideoCodecVP8; 1034 ASSERT_EQ(test.router()->OnEncodedImage(delta_image, &delta_info).error, 1035 EncodedImageCallback::Result::OK); 1036 1037 test.AdvanceTime(TimeDelta::Millis(123)); 1038 1039 DependencyDescriptor key_frame_dd; 1040 DependencyDescriptor delta_dd; 1041 ASSERT_THAT(sent_packets, SizeIs(2)); 1042 EXPECT_TRUE(sent_packets[0].GetExtension<RtpDependencyDescriptorExtension>( 1043 /*structure=*/nullptr, &key_frame_dd)); 1044 EXPECT_TRUE(sent_packets[1].GetExtension<RtpDependencyDescriptorExtension>( 1045 key_frame_dd.attached_structure.get(), &delta_dd)); 1046 } 1047 1048 TEST(RtpVideoSenderTest, SupportsDependencyDescriptorForVp9) { 1049 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); 1050 test.SetSending(true); 1051 1052 RtpHeaderExtensionMap extensions; 1053 extensions.Register<RtpDependencyDescriptorExtension>( 1054 kDependencyDescriptorExtensionId); 1055 std::vector<RtpPacket> sent_packets; 1056 ON_CALL(test.transport(), SendRtp) 1057 .WillByDefault( 1058 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 1059 sent_packets.emplace_back(&extensions); 1060 EXPECT_TRUE(sent_packets.back().Parse(packet)); 1061 return true; 1062 }); 1063 1064 const uint8_t kPayload[1] = {'a'}; 1065 EncodedImage encoded_image; 1066 encoded_image.SetRtpTimestamp(1); 1067 encoded_image.capture_time_ms_ = 2; 1068 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1069 encoded_image.SetEncodedData( 1070 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1071 1072 CodecSpecificInfo codec_specific; 1073 codec_specific.codecType = VideoCodecType::kVideoCodecVP9; 1074 codec_specific.template_structure.emplace(); 1075 codec_specific.template_structure->num_decode_targets = 2; 1076 codec_specific.template_structure->templates = { 1077 FrameDependencyTemplate().S(0).Dtis("SS"), 1078 FrameDependencyTemplate().S(1).Dtis("-S").FrameDiffs({1}), 1079 }; 1080 1081 // Send two tiny images, each mapping to single RTP packet. 1082 // Send in key frame for the base spatial layer. 1083 codec_specific.generic_frame_info = 1084 GenericFrameInfo::Builder().S(0).Dtis("SS").Build(); 1085 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; 1086 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1087 EncodedImageCallback::Result::OK); 1088 // Send in 2nd spatial layer. 1089 codec_specific.template_structure = std::nullopt; 1090 codec_specific.generic_frame_info = 1091 GenericFrameInfo::Builder().S(1).Dtis("-S").Build(); 1092 codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}, 1093 {1, false, true}}; 1094 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1095 EncodedImageCallback::Result::OK); 1096 1097 test.AdvanceTime(TimeDelta::Millis(33)); 1098 ASSERT_THAT(sent_packets, SizeIs(2)); 1099 EXPECT_TRUE(sent_packets[0].HasExtension<RtpDependencyDescriptorExtension>()); 1100 EXPECT_TRUE(sent_packets[1].HasExtension<RtpDependencyDescriptorExtension>()); 1101 } 1102 1103 TEST(RtpVideoSenderTest, 1104 SupportsDependencyDescriptorForVp9NotProvidedByEncoder) { 1105 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); 1106 test.SetSending(true); 1107 1108 RtpHeaderExtensionMap extensions; 1109 extensions.Register<RtpDependencyDescriptorExtension>( 1110 kDependencyDescriptorExtensionId); 1111 std::vector<RtpPacket> sent_packets; 1112 ON_CALL(test.transport(), SendRtp) 1113 .WillByDefault( 1114 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 1115 sent_packets.emplace_back(&extensions); 1116 EXPECT_TRUE(sent_packets.back().Parse(packet)); 1117 return true; 1118 }); 1119 1120 const uint8_t kPayload[1] = {'a'}; 1121 EncodedImage encoded_image; 1122 encoded_image.SetRtpTimestamp(1); 1123 encoded_image.capture_time_ms_ = 2; 1124 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1125 encoded_image._encodedWidth = 320; 1126 encoded_image._encodedHeight = 180; 1127 encoded_image.SetEncodedData( 1128 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1129 1130 CodecSpecificInfo codec_specific; 1131 codec_specific.codecType = VideoCodecType::kVideoCodecVP9; 1132 codec_specific.codecSpecific.VP9.num_spatial_layers = 1; 1133 codec_specific.codecSpecific.VP9.temporal_idx = kNoTemporalIdx; 1134 codec_specific.codecSpecific.VP9.first_frame_in_picture = true; 1135 codec_specific.end_of_picture = true; 1136 codec_specific.codecSpecific.VP9.inter_pic_predicted = false; 1137 1138 // Send two tiny images, each mapping to single RTP packet. 1139 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1140 EncodedImageCallback::Result::OK); 1141 1142 // Send in 2nd picture. 1143 encoded_image._frameType = VideoFrameType::kVideoFrameDelta; 1144 encoded_image.SetRtpTimestamp(3000); 1145 codec_specific.codecSpecific.VP9.inter_pic_predicted = true; 1146 codec_specific.codecSpecific.VP9.num_ref_pics = 1; 1147 codec_specific.codecSpecific.VP9.p_diff[0] = 1; 1148 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1149 EncodedImageCallback::Result::OK); 1150 1151 test.AdvanceTime(TimeDelta::Millis(33)); 1152 ASSERT_THAT(sent_packets, SizeIs(2)); 1153 EXPECT_TRUE(sent_packets[0].HasExtension<RtpDependencyDescriptorExtension>()); 1154 EXPECT_TRUE(sent_packets[1].HasExtension<RtpDependencyDescriptorExtension>()); 1155 } 1156 1157 TEST(RtpVideoSenderTest, 1158 SupportsDependencyDescriptorForH264NotProvidedByEncoder) { 1159 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); 1160 test.SetSending(true); 1161 1162 RtpHeaderExtensionMap extensions; 1163 extensions.Register<RtpDependencyDescriptorExtension>( 1164 kDependencyDescriptorExtensionId); 1165 std::vector<RtpPacket> sent_packets; 1166 EXPECT_CALL(test.transport(), SendRtp(_, _)) 1167 .Times(2) 1168 .WillRepeatedly([&](ArrayView<const uint8_t> packet, 1169 const PacketOptions& options) -> bool { 1170 sent_packets.emplace_back(&extensions); 1171 EXPECT_TRUE(sent_packets.back().Parse(packet)); 1172 return true; 1173 }); 1174 1175 const uint8_t kPayload[1] = {'a'}; 1176 EncodedImage encoded_image; 1177 encoded_image.SetRtpTimestamp(1); 1178 encoded_image.capture_time_ms_ = 2; 1179 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1180 encoded_image._encodedWidth = 320; 1181 encoded_image._encodedHeight = 180; 1182 encoded_image.SetEncodedData( 1183 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1184 1185 CodecSpecificInfo codec_specific; 1186 codec_specific.codecType = VideoCodecType::kVideoCodecH264; 1187 codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx; 1188 1189 // Send two tiny images, each mapping to single RTP packet. 1190 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1191 EncodedImageCallback::Result::OK); 1192 1193 // Send in 2nd picture. 1194 encoded_image._frameType = VideoFrameType::kVideoFrameDelta; 1195 encoded_image.SetRtpTimestamp(3000); 1196 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1197 EncodedImageCallback::Result::OK); 1198 1199 test.AdvanceTime(TimeDelta::Millis(33)); 1200 1201 ASSERT_THAT(sent_packets, SizeIs(2)); 1202 DependencyDescriptor dd_key; 1203 // Key frame should have attached structure. 1204 EXPECT_TRUE(sent_packets[0].GetExtension<RtpDependencyDescriptorExtension>( 1205 nullptr, &dd_key)); 1206 EXPECT_THAT(dd_key.attached_structure, NotNull()); 1207 // Delta frame does not have attached structure. 1208 DependencyDescriptor dd_delta; 1209 EXPECT_TRUE(sent_packets[1].GetExtension<RtpDependencyDescriptorExtension>( 1210 dd_key.attached_structure.get(), &dd_delta)); 1211 EXPECT_THAT(dd_delta.attached_structure, IsNull()); 1212 } 1213 1214 TEST(RtpVideoSenderTest, GenerateDependecyDescriptorForGenericCodecs) { 1215 absl::string_view field_trials = 1216 "WebRTC-GenericCodecDependencyDescriptor/Enabled/"; 1217 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}, field_trials); 1218 test.SetSending(true); 1219 1220 RtpHeaderExtensionMap extensions; 1221 extensions.Register<RtpDependencyDescriptorExtension>( 1222 kDependencyDescriptorExtensionId); 1223 std::vector<RtpPacket> sent_packets; 1224 ON_CALL(test.transport(), SendRtp) 1225 .WillByDefault( 1226 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 1227 sent_packets.emplace_back(&extensions); 1228 EXPECT_TRUE(sent_packets.back().Parse(packet)); 1229 return true; 1230 }); 1231 1232 const uint8_t kPayload[1] = {'a'}; 1233 EncodedImage encoded_image; 1234 encoded_image.SetRtpTimestamp(1); 1235 encoded_image.capture_time_ms_ = 2; 1236 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1237 encoded_image._encodedWidth = 320; 1238 encoded_image._encodedHeight = 180; 1239 encoded_image.SetEncodedData( 1240 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1241 1242 CodecSpecificInfo codec_specific; 1243 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; 1244 codec_specific.end_of_picture = true; 1245 1246 // Send two tiny images, each mapping to single RTP packet. 1247 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1248 EncodedImageCallback::Result::OK); 1249 1250 // Send in 2nd picture. 1251 encoded_image._frameType = VideoFrameType::kVideoFrameDelta; 1252 encoded_image.SetRtpTimestamp(3000); 1253 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1254 EncodedImageCallback::Result::OK); 1255 1256 test.AdvanceTime(TimeDelta::Millis(33)); 1257 ASSERT_THAT(sent_packets, SizeIs(2)); 1258 EXPECT_TRUE(sent_packets[0].HasExtension<RtpDependencyDescriptorExtension>()); 1259 EXPECT_TRUE(sent_packets[1].HasExtension<RtpDependencyDescriptorExtension>()); 1260 } 1261 1262 TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) { 1263 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); 1264 test.SetSending(true); 1265 1266 RtpHeaderExtensionMap extensions; 1267 extensions.Register<RtpDependencyDescriptorExtension>( 1268 kDependencyDescriptorExtensionId); 1269 std::vector<RtpPacket> sent_packets; 1270 ON_CALL(test.transport(), SendRtp) 1271 .WillByDefault( 1272 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 1273 sent_packets.emplace_back(&extensions); 1274 EXPECT_TRUE(sent_packets.back().Parse(packet)); 1275 return true; 1276 }); 1277 1278 const uint8_t kPayload[1] = {'a'}; 1279 EncodedImage encoded_image; 1280 encoded_image.SetRtpTimestamp(1); 1281 encoded_image.capture_time_ms_ = 2; 1282 encoded_image.SetEncodedData( 1283 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1284 1285 CodecSpecificInfo codec_specific; 1286 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; 1287 codec_specific.template_structure.emplace(); 1288 codec_specific.template_structure->num_decode_targets = 1; 1289 codec_specific.template_structure->templates = { 1290 FrameDependencyTemplate().T(0).Dtis("S"), 1291 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), 1292 FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), 1293 }; 1294 1295 // Send two tiny images, mapping to single RTP packets. 1296 // Send in a key frame. 1297 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1298 codec_specific.generic_frame_info = 1299 GenericFrameInfo::Builder().T(0).Dtis("S").Build(); 1300 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; 1301 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1302 EncodedImageCallback::Result::OK); 1303 test.AdvanceTime(TimeDelta::Millis(33)); 1304 ASSERT_THAT(sent_packets, SizeIs(1)); 1305 EXPECT_TRUE( 1306 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>()); 1307 1308 // Send in a new key frame without the support for the dependency descriptor. 1309 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1310 codec_specific.template_structure = std::nullopt; 1311 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, 1312 EncodedImageCallback::Result::OK); 1313 test.AdvanceTime(TimeDelta::Millis(33)); 1314 ASSERT_THAT(sent_packets, SizeIs(2)); 1315 EXPECT_FALSE( 1316 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>()); 1317 } 1318 1319 TEST(RtpVideoSenderTest, CanSetZeroBitrate) { 1320 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); 1321 test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(0), 1322 /*framerate*/ 0); 1323 } 1324 1325 TEST(RtpVideoSenderTest, SimulcastSenderRegistersFrameTransformers) { 1326 scoped_refptr<MockFrameTransformer> transformer = 1327 make_ref_counted<MockFrameTransformer>(); 1328 1329 EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc1)); 1330 EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc2)); 1331 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 1332 kPayloadType, {}, nullptr, transformer); 1333 1334 EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc1)); 1335 EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc2)); 1336 } 1337 1338 TEST(RtpVideoSenderTest, OverheadIsSubtractedFromTargetBitrate) { 1339 absl::string_view field_trials = 1340 "WebRTC-Video-UseFrameRateForOverhead/Enabled/"; 1341 1342 // TODO(jakobi): RTP header size should not be hard coded. 1343 constexpr uint32_t kRtpHeaderSizeBytes = 20; 1344 constexpr uint32_t kTransportPacketOverheadBytes = 40; 1345 constexpr uint32_t kOverheadPerPacketBytes = 1346 kRtpHeaderSizeBytes + kTransportPacketOverheadBytes; 1347 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}, field_trials); 1348 test.router()->OnTransportOverheadChanged(kTransportPacketOverheadBytes); 1349 test.SetSending(true); 1350 1351 { 1352 test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(300000), 1353 /*framerate*/ 15); 1354 // 1 packet per frame. 1355 EXPECT_EQ(test.router()->GetPayloadBitrateBps(), 1356 300000 - kOverheadPerPacketBytes * 8 * 30); 1357 } 1358 { 1359 test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(150000), 1360 /*framerate*/ 15); 1361 // 1 packet per frame. 1362 EXPECT_EQ(test.router()->GetPayloadBitrateBps(), 1363 150000 - kOverheadPerPacketBytes * 8 * 15); 1364 } 1365 { 1366 test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(1000000), 1367 /*framerate*/ 30); 1368 // 3 packets per frame. 1369 EXPECT_EQ(test.router()->GetPayloadBitrateBps(), 1370 1000000 - kOverheadPerPacketBytes * 8 * 30 * 3); 1371 } 1372 } 1373 1374 TEST(RtpVideoSenderTest, ClearsPendingPacketsOnInactivation) { 1375 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); 1376 test.SetSending(true); 1377 1378 RtpHeaderExtensionMap extensions; 1379 extensions.Register<RtpDependencyDescriptorExtension>( 1380 kDependencyDescriptorExtensionId); 1381 std::vector<RtpPacket> sent_packets; 1382 ON_CALL(test.transport(), SendRtp) 1383 .WillByDefault( 1384 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 1385 sent_packets.emplace_back(&extensions); 1386 EXPECT_TRUE(sent_packets.back().Parse(packet)); 1387 return true; 1388 }); 1389 1390 // Set a very low bitrate. 1391 test.router()->OnBitrateUpdated( 1392 CreateBitrateAllocationUpdate(/*target_bitrate_bps=*/10'000), 1393 /*framerate=*/30); 1394 1395 // Create and send a large keyframe. 1396 const size_t kImageSizeBytes = 10000; 1397 constexpr uint8_t kPayload[kImageSizeBytes] = {'a'}; 1398 EncodedImage encoded_image; 1399 encoded_image.SetRtpTimestamp(1); 1400 encoded_image.capture_time_ms_ = 2; 1401 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1402 encoded_image.SetEncodedData( 1403 EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); 1404 EXPECT_EQ(test.router() 1405 ->OnEncodedImage(encoded_image, /*codec_specific_info=*/nullptr) 1406 .error, 1407 EncodedImageCallback::Result::OK); 1408 1409 // Advance time a small amount, check that sent data is only part of the 1410 // image. 1411 test.AdvanceTime(TimeDelta::Millis(5)); 1412 DataSize transmittedPayload = DataSize::Zero(); 1413 for (const RtpPacket& packet : sent_packets) { 1414 transmittedPayload += DataSize::Bytes(packet.payload_size()); 1415 // Make sure we don't see the end of the frame. 1416 EXPECT_FALSE(packet.Marker()); 1417 } 1418 EXPECT_GT(transmittedPayload, DataSize::Zero()); 1419 EXPECT_LT(transmittedPayload, DataSize::Bytes(kImageSizeBytes / 3)); 1420 1421 // Record the RTP timestamp of the first frame. 1422 const uint32_t first_frame_timestamp = sent_packets[0].Timestamp(); 1423 sent_packets.clear(); 1424 1425 // Disable the sending module and advance time slightly. No packets should be 1426 // sent. 1427 test.SetSending(false); 1428 test.AdvanceTime(TimeDelta::Millis(20)); 1429 EXPECT_TRUE(sent_packets.empty()); 1430 1431 // Reactive the send module - any packets should have been removed, so nothing 1432 // should be transmitted. 1433 test.SetSending(true); 1434 test.AdvanceTime(TimeDelta::Millis(33)); 1435 EXPECT_TRUE(sent_packets.empty()); 1436 1437 // Send a new frame. 1438 encoded_image.SetRtpTimestamp(3); 1439 encoded_image.capture_time_ms_ = 4; 1440 EXPECT_EQ(test.router() 1441 ->OnEncodedImage(encoded_image, /*codec_specific_info=*/nullptr) 1442 .error, 1443 EncodedImageCallback::Result::OK); 1444 test.AdvanceTime(TimeDelta::Millis(33)); 1445 1446 // Advance time, check we get new packets - but only for the second frame. 1447 EXPECT_FALSE(sent_packets.empty()); 1448 EXPECT_NE(sent_packets[0].Timestamp(), first_frame_timestamp); 1449 } 1450 1451 TEST(RtpVideoSenderTest, 1452 ClearsPendingPacketsOnInactivationWithLayerAllocation) { 1453 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {}, kPayloadType, {}); 1454 test.SetSending(true); 1455 1456 RtpHeaderExtensionMap extensions; 1457 extensions.Register<RtpDependencyDescriptorExtension>( 1458 kDependencyDescriptorExtensionId); 1459 std::vector<RtpPacket> sent_packets; 1460 ON_CALL(test.transport(), SendRtp) 1461 .WillByDefault( 1462 [&](ArrayView<const uint8_t> packet, const PacketOptions& options) { 1463 sent_packets.emplace_back(&extensions); 1464 EXPECT_TRUE(sent_packets.back().Parse(packet)); 1465 return true; 1466 }); 1467 1468 // Set a very low bitrate. 1469 test.router()->OnBitrateUpdated( 1470 CreateBitrateAllocationUpdate(/*target_bitrate_bps=*/10'000), 1471 /*framerate=*/30); 1472 1473 // Create and send a large keyframe. 1474 constexpr uint8_t kImage[10'000] = {}; 1475 EncodedImage encoded_image; 1476 encoded_image.SetSimulcastIndex(0); 1477 encoded_image.SetRtpTimestamp(1); 1478 encoded_image.capture_time_ms_ = 2; 1479 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1480 encoded_image.SetEncodedData( 1481 EncodedImageBuffer::Create(kImage, std::size(kImage))); 1482 EXPECT_EQ(test.router() 1483 ->OnEncodedImage(encoded_image, /*codec_specific_info=*/nullptr) 1484 .error, 1485 EncodedImageCallback::Result::OK); 1486 1487 // Advance time a small amount, check that sent data is only part of the 1488 // image. 1489 test.AdvanceTime(TimeDelta::Millis(5)); 1490 DataSize transmitted_payload = DataSize::Zero(); 1491 for (const RtpPacket& packet : sent_packets) { 1492 transmitted_payload += DataSize::Bytes(packet.payload_size()); 1493 // Make sure we don't see the end of the frame. 1494 EXPECT_FALSE(packet.Marker()); 1495 } 1496 EXPECT_GT(transmitted_payload, DataSize::Zero()); 1497 EXPECT_LT(transmitted_payload, DataSize::Bytes(std::size(kImage)) / 3); 1498 1499 // Record the RTP timestamp of the first frame. 1500 const uint32_t first_frame_timestamp = sent_packets[0].Timestamp(); 1501 sent_packets.clear(); 1502 1503 // Disable the 1st sending module and advance time slightly. No packets should 1504 // be sent. 1505 test.router()->OnVideoLayersAllocationUpdated( 1506 {.active_spatial_layers = {{.rtp_stream_index = 1}}}); 1507 test.AdvanceTime(TimeDelta::Millis(20)); 1508 EXPECT_THAT(sent_packets, IsEmpty()); 1509 1510 // Reactive the send module - any packets should have been removed, so nothing 1511 // should be transmitted. 1512 test.router()->OnVideoLayersAllocationUpdated( 1513 {.active_spatial_layers = {{.rtp_stream_index = 0}, 1514 {.rtp_stream_index = 1}}}); 1515 test.AdvanceTime(TimeDelta::Millis(33)); 1516 EXPECT_THAT(sent_packets, IsEmpty()); 1517 1518 // Send a new frame. 1519 encoded_image.SetRtpTimestamp(3); 1520 encoded_image.capture_time_ms_ = 4; 1521 EXPECT_EQ(test.router() 1522 ->OnEncodedImage(encoded_image, /*codec_specific_info=*/nullptr) 1523 .error, 1524 EncodedImageCallback::Result::OK); 1525 test.AdvanceTime(TimeDelta::Millis(33)); 1526 1527 // Advance time, check we get new packets - but only for the second frame. 1528 ASSERT_THAT(sent_packets, SizeIs(Ge(1))); 1529 EXPECT_NE(sent_packets[0].Timestamp(), first_frame_timestamp); 1530 } 1531 1532 // Integration test verifying that when retransmission mode is set to 1533 // kRetransmitBaseLayer,only base layer is retransmitted. 1534 TEST(RtpVideoSenderTest, RetransmitsBaseLayerOnly) { 1535 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, 1536 kPayloadType, {}); 1537 test.SetSending(true); 1538 1539 test.router()->SetRetransmissionMode(kRetransmitBaseLayer); 1540 constexpr uint8_t kPayload = 'a'; 1541 EncodedImage encoded_image; 1542 encoded_image.SetRtpTimestamp(1); 1543 encoded_image.capture_time_ms_ = 2; 1544 encoded_image._frameType = VideoFrameType::kVideoFrameKey; 1545 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); 1546 1547 // Send two tiny images, mapping to two RTP packets. Capture sequence numbers. 1548 std::vector<uint16_t> rtp_sequence_numbers; 1549 std::vector<uint16_t> transport_sequence_numbers; 1550 std::vector<uint16_t> base_sequence_numbers; 1551 EXPECT_CALL(test.transport(), SendRtp) 1552 .Times(2) 1553 .WillRepeatedly( 1554 [&rtp_sequence_numbers, &transport_sequence_numbers]( 1555 ArrayView<const uint8_t> packet, const PacketOptions& options) { 1556 RtpPacket rtp_packet; 1557 EXPECT_TRUE(rtp_packet.Parse(packet)); 1558 rtp_sequence_numbers.push_back(rtp_packet.SequenceNumber()); 1559 transport_sequence_numbers.push_back(options.packet_id); 1560 return true; 1561 }); 1562 CodecSpecificInfo key_codec_info; 1563 key_codec_info.codecType = kVideoCodecVP8; 1564 key_codec_info.codecSpecific.VP8.temporalIdx = 0; 1565 EXPECT_EQ( 1566 EncodedImageCallback::Result::OK, 1567 test.router()->OnEncodedImage(encoded_image, &key_codec_info).error); 1568 encoded_image.SetRtpTimestamp(2); 1569 encoded_image.capture_time_ms_ = 3; 1570 encoded_image._frameType = VideoFrameType::kVideoFrameDelta; 1571 CodecSpecificInfo delta_codec_info; 1572 delta_codec_info.codecType = kVideoCodecVP8; 1573 delta_codec_info.codecSpecific.VP8.temporalIdx = 1; 1574 EXPECT_EQ( 1575 EncodedImageCallback::Result::OK, 1576 test.router()->OnEncodedImage(encoded_image, &delta_codec_info).error); 1577 1578 test.AdvanceTime(TimeDelta::Millis(33)); 1579 1580 // Construct a NACK message for requesting retransmission of both packet. 1581 rtcp::Nack nack; 1582 nack.SetMediaSsrc(kSsrc1); 1583 nack.SetPacketIds(rtp_sequence_numbers); 1584 Buffer nack_buffer = nack.Build(); 1585 1586 std::vector<uint16_t> retransmitted_rtp_sequence_numbers; 1587 EXPECT_CALL(test.transport(), SendRtp) 1588 .Times(1) 1589 .WillRepeatedly([&retransmitted_rtp_sequence_numbers]( 1590 ArrayView<const uint8_t> packet, 1591 const PacketOptions& options) { 1592 RtpPacket rtp_packet; 1593 EXPECT_TRUE(rtp_packet.Parse(packet)); 1594 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); 1595 // Capture the retransmitted sequence number from the RTX header. 1596 ArrayView<const uint8_t> payload = rtp_packet.payload(); 1597 retransmitted_rtp_sequence_numbers.push_back( 1598 ByteReader<uint16_t>::ReadBigEndian(payload.data())); 1599 return true; 1600 }); 1601 test.router()->DeliverRtcp(nack_buffer); 1602 test.AdvanceTime(TimeDelta::Millis(33)); 1603 1604 // Verify that only base layer packet was retransmitted. 1605 std::vector<uint16_t> base_rtp_sequence_numbers( 1606 rtp_sequence_numbers.begin(), rtp_sequence_numbers.begin() + 1); 1607 EXPECT_EQ(retransmitted_rtp_sequence_numbers, base_rtp_sequence_numbers); 1608 } 1609 1610 } // namespace webrtc