neteq_impl_unittest.cc (74624B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_coding/neteq/neteq_impl.h" 12 13 #include <algorithm> 14 #include <cstddef> 15 #include <cstdint> 16 #include <memory> 17 #include <optional> 18 #include <utility> 19 #include <vector> 20 21 #include "api/audio/audio_view.h" 22 #include "api/audio_codecs/audio_decoder.h" 23 #include "api/audio_codecs/audio_decoder_factory.h" 24 #include "api/audio_codecs/audio_format.h" 25 #include "api/audio_codecs/builtin_audio_decoder_factory.h" 26 #include "api/environment/environment.h" 27 #include "api/environment/environment_factory.h" 28 #include "api/make_ref_counted.h" 29 #include "api/neteq/default_neteq_controller_factory.h" 30 #include "api/neteq/default_neteq_factory.h" 31 #include "api/neteq/neteq.h" 32 #include "api/neteq/neteq_controller.h" 33 #include "api/neteq/tick_timer.h" 34 #include "api/rtp_headers.h" 35 #include "api/rtp_packet_info.h" 36 #include "api/scoped_refptr.h" 37 #include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" 38 #include "modules/audio_coding/neteq/decision_logic.h" 39 #include "modules/audio_coding/neteq/delay_manager.h" 40 #include "modules/audio_coding/neteq/expand.h" 41 #include "modules/audio_coding/neteq/mock/mock_decoder_database.h" 42 #include "modules/audio_coding/neteq/mock/mock_dtmf_buffer.h" 43 #include "modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h" 44 #include "modules/audio_coding/neteq/mock/mock_neteq_controller.h" 45 #include "modules/audio_coding/neteq/mock/mock_packet_buffer.h" 46 #include "modules/audio_coding/neteq/mock/mock_red_payload_splitter.h" 47 #include "modules/audio_coding/neteq/packet.h" 48 #include "modules/audio_coding/neteq/packet_buffer.h" 49 #include "modules/audio_coding/neteq/red_payload_splitter.h" 50 #include "modules/audio_coding/neteq/sync_buffer.h" 51 #include "modules/audio_coding/neteq/timestamp_scaler.h" 52 #include "rtc_base/checks.h" 53 #include "rtc_base/numerics/safe_conversions.h" 54 #include "system_wrappers/include/clock.h" 55 #include "test/audio_decoder_proxy_factory.h" 56 #include "test/function_audio_decoder_factory.h" 57 #include "test/gmock.h" 58 #include "test/gtest.h" 59 #include "test/mock_audio_decoder.h" 60 #include "test/mock_audio_decoder_factory.h" 61 62 using ::testing::_; 63 using ::testing::AtLeast; 64 using ::testing::DoAll; 65 using ::testing::ElementsAre; 66 using ::testing::InSequence; 67 using ::testing::Invoke; 68 using ::testing::IsEmpty; 69 using ::testing::IsNull; 70 using ::testing::Pointee; 71 using ::testing::Return; 72 using ::testing::ReturnNull; 73 using ::testing::SetArgPointee; 74 using ::testing::SetArrayArgument; 75 using ::testing::SizeIs; 76 using ::testing::WithArg; 77 78 namespace webrtc { 79 80 // This function is called when inserting a packet list into the mock packet 81 // buffer. The purpose is to delete all inserted packets properly, to avoid 82 // memory leaks in the test. 83 int DeletePacketsAndReturnOk(PacketList* packet_list) { 84 packet_list->clear(); 85 return PacketBuffer::kOK; 86 } 87 88 class NetEqImplTest : public ::testing::Test { 89 protected: 90 NetEqImplTest() : clock_(0), env_(CreateEnvironment(&clock_)) { 91 config_.sample_rate_hz = 8000; 92 } 93 94 void CreateInstance(scoped_refptr<AudioDecoderFactory> decoder_factory) { 95 ASSERT_TRUE(decoder_factory); 96 config_.enable_muted_state = enable_muted_state_; 97 NetEqImpl::Dependencies deps(env_, config_, std::move(decoder_factory), 98 DefaultNetEqControllerFactory()); 99 100 // Get a local pointer to NetEq's TickTimer object. 101 tick_timer_ = deps.tick_timer.get(); 102 103 if (use_mock_decoder_database_) { 104 std::unique_ptr<MockDecoderDatabase> mock(new MockDecoderDatabase); 105 mock_decoder_database_ = mock.get(); 106 EXPECT_CALL(*mock_decoder_database_, GetActiveCngDecoder()) 107 .WillOnce(ReturnNull()); 108 deps.decoder_database = std::move(mock); 109 } 110 decoder_database_ = deps.decoder_database.get(); 111 112 if (use_mock_dtmf_buffer_) { 113 std::unique_ptr<MockDtmfBuffer> mock( 114 new MockDtmfBuffer(config_.sample_rate_hz)); 115 mock_dtmf_buffer_ = mock.get(); 116 deps.dtmf_buffer = std::move(mock); 117 } 118 dtmf_buffer_ = deps.dtmf_buffer.get(); 119 120 if (use_mock_dtmf_tone_generator_) { 121 std::unique_ptr<MockDtmfToneGenerator> mock(new MockDtmfToneGenerator); 122 mock_dtmf_tone_generator_ = mock.get(); 123 deps.dtmf_tone_generator = std::move(mock); 124 } 125 dtmf_tone_generator_ = deps.dtmf_tone_generator.get(); 126 127 if (use_mock_packet_buffer_) { 128 std::unique_ptr<MockPacketBuffer> mock(new MockPacketBuffer( 129 config_.max_packets_in_buffer, tick_timer_, deps.stats.get())); 130 mock_packet_buffer_ = mock.get(); 131 deps.packet_buffer = std::move(mock); 132 } 133 packet_buffer_ = deps.packet_buffer.get(); 134 135 if (use_mock_neteq_controller_) { 136 std::unique_ptr<MockNetEqController> mock(new MockNetEqController()); 137 mock_neteq_controller_ = mock.get(); 138 deps.neteq_controller = std::move(mock); 139 } else { 140 NetEqController::Config controller_config; 141 controller_config.tick_timer = tick_timer_; 142 controller_config.base_min_delay_ms = config_.min_delay_ms; 143 controller_config.allow_time_stretching = true; 144 controller_config.max_packets_in_buffer = config_.max_packets_in_buffer; 145 auto delay_manager = std::make_unique<DelayManager>( 146 DelayManager::Config(env_.field_trials()), tick_timer_); 147 deps.neteq_controller = std::make_unique<DecisionLogic>( 148 env_, std::move(controller_config), std::move(delay_manager)); 149 } 150 neteq_controller_ = deps.neteq_controller.get(); 151 152 if (use_mock_payload_splitter_) { 153 std::unique_ptr<MockRedPayloadSplitter> mock(new MockRedPayloadSplitter); 154 mock_payload_splitter_ = mock.get(); 155 deps.red_payload_splitter = std::move(mock); 156 } 157 red_payload_splitter_ = deps.red_payload_splitter.get(); 158 159 deps.timestamp_scaler = std::unique_ptr<TimestampScaler>( 160 new TimestampScaler(*deps.decoder_database)); 161 162 neteq_.reset(new NetEqImpl(config_, std::move(deps))); 163 ASSERT_TRUE(neteq_ != nullptr); 164 } 165 166 void CreateInstance() { CreateInstance(CreateBuiltinAudioDecoderFactory()); } 167 168 void UseNoMocks() { 169 ASSERT_TRUE(neteq_ == nullptr) 170 << "Must call UseNoMocks before CreateInstance"; 171 use_mock_decoder_database_ = false; 172 use_mock_neteq_controller_ = false; 173 use_mock_dtmf_buffer_ = false; 174 use_mock_dtmf_tone_generator_ = false; 175 use_mock_packet_buffer_ = false; 176 use_mock_payload_splitter_ = false; 177 } 178 179 ~NetEqImplTest() override { 180 if (use_mock_decoder_database_) { 181 EXPECT_CALL(*mock_decoder_database_, Die()).Times(1); 182 } 183 if (use_mock_neteq_controller_) { 184 EXPECT_CALL(*mock_neteq_controller_, Die()).Times(1); 185 } 186 if (use_mock_dtmf_buffer_) { 187 EXPECT_CALL(*mock_dtmf_buffer_, Die()).Times(1); 188 } 189 if (use_mock_dtmf_tone_generator_) { 190 EXPECT_CALL(*mock_dtmf_tone_generator_, Die()).Times(1); 191 } 192 if (use_mock_packet_buffer_) { 193 EXPECT_CALL(*mock_packet_buffer_, Die()).Times(1); 194 } 195 } 196 197 void TestDtmfPacket(int sample_rate_hz) { 198 const size_t kPayloadLength = 4; 199 const uint8_t kPayloadType = 110; 200 const int kSampleRateHz = 16000; 201 config_.sample_rate_hz = kSampleRateHz; 202 UseNoMocks(); 203 CreateInstance(); 204 // Event: 2, E bit, Volume: 17, Length: 4336. 205 uint8_t payload[kPayloadLength] = {0x02, 0x80 + 0x11, 0x10, 0xF0}; 206 RTPHeader rtp_header; 207 rtp_header.payloadType = kPayloadType; 208 rtp_header.sequenceNumber = 0x1234; 209 rtp_header.timestamp = 0x12345678; 210 rtp_header.ssrc = 0x87654321; 211 212 EXPECT_TRUE(neteq_->RegisterPayloadType( 213 kPayloadType, SdpAudioFormat("telephone-event", sample_rate_hz, 1))); 214 215 // Insert first packet. 216 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 217 218 // Pull audio once. 219 const size_t kMaxOutputSize = 220 static_cast<size_t>(10 * kSampleRateHz / 1000); 221 AudioFrame output; 222 bool muted; 223 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 224 ASSERT_FALSE(muted); 225 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); 226 EXPECT_EQ(1u, output.num_channels_); 227 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); 228 229 // DTMF packets are immediately consumed by `InsertPacket()` and won't be 230 // returned by `GetAudio()`. 231 EXPECT_THAT(output.packet_infos_, IsEmpty()); 232 233 // Verify first 64 samples of actual output. 234 const std::vector<int16_t> kOutput( 235 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236 -1578, -2816, -3460, -3403, -2709, -1594, -363, 671, 1269, 1328, 237 908, 202, -513, -964, -955, -431, 504, 1617, 2602, 3164, 238 3101, 2364, 1073, -511, -2047, -3198, -3721, -3525, -2688, -1440, 239 -99, 1015, 1663, 1744, 1319, 588, -171, -680, -747, -315, 240 515, 1512, 2378, 2828, 2674, 1877, 568, -986, -2446, -3482, 241 -3864, -3516, -2534, -1163}); 242 ASSERT_GE(kMaxOutputSize, kOutput.size()); 243 EXPECT_TRUE(std::equal(kOutput.begin(), kOutput.end(), output.data())); 244 } 245 246 std::unique_ptr<NetEqImpl> neteq_; 247 NetEq::Config config_; 248 SimulatedClock clock_; 249 const Environment env_; 250 TickTimer* tick_timer_ = nullptr; 251 MockDecoderDatabase* mock_decoder_database_ = nullptr; 252 DecoderDatabase* decoder_database_ = nullptr; 253 bool use_mock_decoder_database_ = true; 254 MockNetEqController* mock_neteq_controller_ = nullptr; 255 NetEqController* neteq_controller_ = nullptr; 256 bool use_mock_neteq_controller_ = true; 257 MockDtmfBuffer* mock_dtmf_buffer_ = nullptr; 258 DtmfBuffer* dtmf_buffer_ = nullptr; 259 bool use_mock_dtmf_buffer_ = true; 260 MockDtmfToneGenerator* mock_dtmf_tone_generator_ = nullptr; 261 DtmfToneGenerator* dtmf_tone_generator_ = nullptr; 262 bool use_mock_dtmf_tone_generator_ = true; 263 MockPacketBuffer* mock_packet_buffer_ = nullptr; 264 PacketBuffer* packet_buffer_ = nullptr; 265 bool use_mock_packet_buffer_ = true; 266 MockRedPayloadSplitter* mock_payload_splitter_ = nullptr; 267 RedPayloadSplitter* red_payload_splitter_ = nullptr; 268 bool use_mock_payload_splitter_ = true; 269 bool enable_muted_state_ = false; 270 }; 271 272 // This tests the interface class NetEq. 273 // TODO(hlundin): Move to separate file? 274 TEST(NetEq, CreateAndDestroy) { 275 NetEq::Config config; 276 std::unique_ptr<NetEq> neteq = DefaultNetEqFactory().Create( 277 CreateEnvironment(), config, CreateBuiltinAudioDecoderFactory()); 278 } 279 280 TEST_F(NetEqImplTest, RegisterPayloadType) { 281 CreateInstance(); 282 constexpr int rtp_payload_type = 0; 283 const SdpAudioFormat format("pcmu", 8000, 1); 284 EXPECT_CALL(*mock_decoder_database_, 285 RegisterPayload(rtp_payload_type, format)); 286 neteq_->RegisterPayloadType(rtp_payload_type, format); 287 } 288 289 TEST_F(NetEqImplTest, CreateDecoder) { 290 UseNoMocks(); 291 CreateInstance(); 292 constexpr int rtp_payload_type = 0; 293 const SdpAudioFormat format("pcmu", 8000, 1); 294 EXPECT_TRUE(neteq_->RegisterPayloadType(rtp_payload_type, format)); 295 EXPECT_TRUE(neteq_->CreateDecoder(rtp_payload_type)); 296 } 297 298 TEST_F(NetEqImplTest, RemovePayloadType) { 299 CreateInstance(); 300 uint8_t rtp_payload_type = 0; 301 EXPECT_CALL(*mock_decoder_database_, Remove(rtp_payload_type)) 302 .WillOnce(Return(DecoderDatabase::kDecoderNotFound)); 303 // Check that kOK is returned when database returns kDecoderNotFound, because 304 // removing a payload type that was never registered is not an error. 305 EXPECT_EQ(NetEq::kOK, neteq_->RemovePayloadType(rtp_payload_type)); 306 } 307 308 TEST_F(NetEqImplTest, RemoveAllPayloadTypes) { 309 CreateInstance(); 310 EXPECT_CALL(*mock_decoder_database_, RemoveAll()).WillOnce(Return()); 311 neteq_->RemoveAllPayloadTypes(); 312 } 313 314 TEST_F(NetEqImplTest, InsertPacket) { 315 using ::testing::AllOf; 316 using ::testing::Field; 317 CreateInstance(); 318 const size_t kPayloadLength = 100; 319 const uint8_t kPayloadType = 0; 320 const uint16_t kFirstSequenceNumber = 0x1234; 321 const uint32_t kFirstTimestamp = 0x12345678; 322 const uint32_t kSsrc = 0x87654321; 323 uint8_t payload[kPayloadLength] = {0}; 324 RTPHeader rtp_header; 325 rtp_header.payloadType = kPayloadType; 326 rtp_header.sequenceNumber = kFirstSequenceNumber; 327 rtp_header.timestamp = kFirstTimestamp; 328 rtp_header.ssrc = kSsrc; 329 Packet fake_packet; 330 fake_packet.payload_type = kPayloadType; 331 fake_packet.sequence_number = kFirstSequenceNumber; 332 fake_packet.timestamp = kFirstTimestamp; 333 334 const Environment env = CreateEnvironment(); 335 auto mock_decoder_factory = make_ref_counted<MockAudioDecoderFactory>(); 336 EXPECT_CALL(*mock_decoder_factory, Create) 337 .WillOnce(WithArg<1>([&](const SdpAudioFormat& format) { 338 EXPECT_EQ("pcmu", format.name); 339 340 std::unique_ptr<MockAudioDecoder> mock_decoder(new MockAudioDecoder); 341 EXPECT_CALL(*mock_decoder, Channels()).WillRepeatedly(Return(1)); 342 EXPECT_CALL(*mock_decoder, SampleRateHz()).WillRepeatedly(Return(8000)); 343 EXPECT_CALL(*mock_decoder, Die()).Times(1); // Called when deleted. 344 345 return mock_decoder; 346 })); 347 DecoderDatabase::DecoderInfo info(env, SdpAudioFormat("pcmu", 8000, 1), 348 std::nullopt, mock_decoder_factory.get()); 349 350 // Expectations for decoder database. 351 EXPECT_CALL(*mock_decoder_database_, GetDecoderInfo(kPayloadType)) 352 .WillRepeatedly(Return(&info)); 353 354 // Expectations for packet buffer. 355 EXPECT_CALL(*mock_packet_buffer_, Empty()) 356 .WillOnce(Return(false)); // Called once after first packet is inserted. 357 EXPECT_CALL(*mock_packet_buffer_, Flush()).Times(1); 358 EXPECT_CALL(*mock_packet_buffer_, InsertPacket(_)) 359 .Times(2) 360 .WillRepeatedly(Return(PacketBuffer::kOK)); 361 EXPECT_CALL(*mock_packet_buffer_, PeekNextPacket()) 362 .Times(1) 363 .WillOnce(Return(&fake_packet)); 364 365 // Expectations for DTMF buffer. 366 EXPECT_CALL(*mock_dtmf_buffer_, Flush()).Times(1); 367 368 // Expectations for delay manager. 369 { 370 // All expectations within this block must be called in this specific order. 371 InSequence sequence; // Dummy variable. 372 // Expectations when the first packet is inserted. 373 EXPECT_CALL( 374 *mock_neteq_controller_, 375 PacketArrived( 376 /*fs_hz*/ 8000, 377 /*should_update_stats*/ _, 378 /*info*/ 379 AllOf( 380 Field(&NetEqController::PacketArrivedInfo::is_cng_or_dtmf, 381 false), 382 Field(&NetEqController::PacketArrivedInfo::main_sequence_number, 383 kFirstSequenceNumber), 384 Field(&NetEqController::PacketArrivedInfo::main_timestamp, 385 kFirstTimestamp)))); 386 EXPECT_CALL( 387 *mock_neteq_controller_, 388 PacketArrived( 389 /*fs_hz*/ 8000, 390 /*should_update_stats*/ _, 391 /*info*/ 392 AllOf( 393 Field(&NetEqController::PacketArrivedInfo::is_cng_or_dtmf, 394 false), 395 Field(&NetEqController::PacketArrivedInfo::main_sequence_number, 396 kFirstSequenceNumber + 1), 397 Field(&NetEqController::PacketArrivedInfo::main_timestamp, 398 kFirstTimestamp + 160)))); 399 } 400 401 // Insert first packet. 402 neteq_->InsertPacket(rtp_header, payload); 403 404 // Insert second packet. 405 rtp_header.timestamp += 160; 406 rtp_header.sequenceNumber += 1; 407 neteq_->InsertPacket(rtp_header, payload); 408 } 409 410 TEST_F(NetEqImplTest, CountStatsAfterFirstDecodedPacket) { 411 UseNoMocks(); 412 CreateInstance(); 413 const uint8_t kPayloadType = 17; // Just an arbitrary number. 414 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 415 SdpAudioFormat("l16", 8000, 1))); 416 const size_t kPayloadLengthSamples = 80; 417 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. 418 uint8_t payload[kPayloadLengthBytes] = {0}; 419 RTPHeader rtp_header; 420 rtp_header.payloadType = kPayloadType; 421 rtp_header.sequenceNumber = 0x1234; 422 rtp_header.timestamp = 0x12345678; 423 rtp_header.ssrc = 0x87654321; 424 AudioFrame frame; 425 // Get audio a couple of times to make sure that samples received remains 426 // zero. 427 for (int i = 0; i < 3; ++i) { 428 neteq_->GetAudio(&frame); 429 EXPECT_EQ(neteq_->GetLifetimeStatistics().concealed_samples, 0u); 430 EXPECT_EQ(neteq_->GetLifetimeStatistics().total_samples_received, 0u); 431 } 432 neteq_->InsertPacket(rtp_header, payload); 433 neteq_->GetAudio(&frame); 434 EXPECT_EQ(neteq_->GetLifetimeStatistics().concealed_samples, 0u); 435 EXPECT_EQ(neteq_->GetLifetimeStatistics().total_samples_received, 436 kPayloadLengthSamples); 437 } 438 439 TEST_F(NetEqImplTest, InsertPacketsUntilBufferIsFull) { 440 UseNoMocks(); 441 CreateInstance(); 442 443 const int kPayloadLengthSamples = 80; 444 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. 445 const uint8_t kPayloadType = 17; // Just an arbitrary number. 446 uint8_t payload[kPayloadLengthBytes] = {0}; 447 RTPHeader rtp_header; 448 rtp_header.payloadType = kPayloadType; 449 rtp_header.sequenceNumber = 0x1234; 450 rtp_header.timestamp = 0x12345678; 451 rtp_header.ssrc = 0x87654321; 452 453 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 454 SdpAudioFormat("l16", 8000, 1))); 455 456 // Insert packets. The buffer should not flush. 457 for (size_t i = 1; i <= config_.max_packets_in_buffer; ++i) { 458 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 459 rtp_header.timestamp += kPayloadLengthSamples; 460 rtp_header.sequenceNumber += 1; 461 EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer()); 462 } 463 464 // Insert one more packet and make sure the buffer got flushed. That is, it 465 // should only hold one single packet. 466 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 467 EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer()); 468 const Packet* test_packet = packet_buffer_->PeekNextPacket(); 469 EXPECT_EQ(rtp_header.timestamp, test_packet->timestamp); 470 EXPECT_EQ(rtp_header.sequenceNumber, test_packet->sequence_number); 471 } 472 473 TEST_F(NetEqImplTest, TestDtmfPacketAVT) { 474 TestDtmfPacket(8000); 475 } 476 477 TEST_F(NetEqImplTest, TestDtmfPacketAVT16kHz) { 478 TestDtmfPacket(16000); 479 } 480 481 TEST_F(NetEqImplTest, TestDtmfPacketAVT32kHz) { 482 TestDtmfPacket(32000); 483 } 484 485 TEST_F(NetEqImplTest, TestDtmfPacketAVT48kHz) { 486 TestDtmfPacket(48000); 487 } 488 489 // This test verifies that timestamps propagate from the incoming packets 490 // through to the sync buffer and to the playout timestamp. 491 TEST_F(NetEqImplTest, VerifyTimestampPropagation) { 492 const uint8_t kPayloadType = 17; // Just an arbitrary number. 493 const int kSampleRateHz = 8000; 494 const size_t kPayloadLengthSamples = 495 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms. 496 const size_t kPayloadLengthBytes = kPayloadLengthSamples; 497 uint8_t payload[kPayloadLengthBytes] = {0}; 498 RTPHeader rtp_header; 499 rtp_header.payloadType = kPayloadType; 500 rtp_header.sequenceNumber = 0x1234; 501 rtp_header.timestamp = 0x12345678; 502 rtp_header.ssrc = 0x87654321; 503 rtp_header.numCSRCs = 3; 504 rtp_header.arrOfCSRCs[0] = 43; 505 rtp_header.arrOfCSRCs[1] = 65; 506 rtp_header.arrOfCSRCs[2] = 17; 507 508 // This is a dummy decoder that produces as many output samples as the input 509 // has bytes. The output is an increasing series, starting at 1 for the first 510 // sample, and then increasing by 1 for each sample. 511 class CountingSamplesDecoder : public AudioDecoder { 512 public: 513 CountingSamplesDecoder() : next_value_(1) {} 514 515 // Produce as many samples as input bytes (`encoded_len`). 516 int DecodeInternal(const uint8_t* /* encoded */, 517 size_t encoded_len, 518 int /* sample_rate_hz */, 519 int16_t* decoded, 520 SpeechType* speech_type) override { 521 for (size_t i = 0; i < encoded_len; ++i) { 522 decoded[i] = next_value_++; 523 } 524 *speech_type = kSpeech; 525 return checked_cast<int>(encoded_len); 526 } 527 528 void Reset() override { next_value_ = 1; } 529 530 int SampleRateHz() const override { return kSampleRateHz; } 531 532 size_t Channels() const override { return 1; } 533 534 uint16_t next_value() const { return next_value_; } 535 536 private: 537 int16_t next_value_; 538 } decoder_; 539 540 auto decoder_factory = 541 make_ref_counted<test::AudioDecoderProxyFactory>(&decoder_); 542 543 UseNoMocks(); 544 CreateInstance(decoder_factory); 545 546 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 547 SdpAudioFormat("L16", 8000, 1))); 548 549 // Insert one packet. 550 clock_.AdvanceTimeMilliseconds(123456); 551 RtpPacketInfo expected_packet_info(rtp_header, clock_.CurrentTime()); 552 EXPECT_EQ(NetEq::kOK, 553 neteq_->InsertPacket(rtp_header, payload, expected_packet_info)); 554 555 // Pull audio once. 556 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000); 557 AudioFrame output; 558 bool muted; 559 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 560 ASSERT_FALSE(muted); 561 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); 562 EXPECT_EQ(1u, output.num_channels_); 563 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); 564 565 // Verify `output.packet_infos_`. 566 ASSERT_THAT(output.packet_infos_, SizeIs(1)); 567 EXPECT_EQ(output.packet_infos_[0], expected_packet_info); 568 569 // Start with a simple check that the fake decoder is behaving as expected. 570 EXPECT_EQ(kPayloadLengthSamples, 571 static_cast<size_t>(decoder_.next_value() - 1)); 572 573 // The value of the last of the output samples is the same as the number of 574 // samples played from the decoded packet. Thus, this number + the RTP 575 // timestamp should match the playout timestamp. 576 // Wrap the expected value in an std::optional to compare them as such. 577 EXPECT_EQ( 578 std::optional<uint32_t>(rtp_header.timestamp + 579 output.data()[output.samples_per_channel_ - 1]), 580 neteq_->GetPlayoutTimestamp()); 581 582 // Check the timestamp for the last value in the sync buffer. This should 583 // be one full frame length ahead of the RTP timestamp. 584 const SyncBuffer* sync_buffer = neteq_->sync_buffer_for_test(); 585 ASSERT_TRUE(sync_buffer != nullptr); 586 EXPECT_EQ(rtp_header.timestamp + kPayloadLengthSamples, 587 sync_buffer->end_timestamp()); 588 589 // Check that the number of samples still to play from the sync buffer add 590 // up with what was already played out. 591 EXPECT_EQ( 592 kPayloadLengthSamples - output.data()[output.samples_per_channel_ - 1], 593 sync_buffer->FutureLength()); 594 } 595 596 TEST_F(NetEqImplTest, ReorderedPacket) { 597 UseNoMocks(); 598 599 // Create a mock decoder object. 600 MockAudioDecoder mock_decoder; 601 602 CreateInstance( 603 make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder)); 604 605 const uint8_t kPayloadType = 17; // Just an arbitrary number. 606 const int kSampleRateHz = 8000; 607 const size_t kPayloadLengthSamples = 608 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms. 609 const size_t kPayloadLengthBytes = kPayloadLengthSamples; 610 uint8_t payload[kPayloadLengthBytes] = {0}; 611 RTPHeader rtp_header; 612 rtp_header.payloadType = kPayloadType; 613 rtp_header.sequenceNumber = 0x1234; 614 rtp_header.timestamp = 0x12345678; 615 rtp_header.ssrc = 0x87654321; 616 rtp_header.extension.set_audio_level( 617 AudioLevel(/*voice_activity=*/false, 42)); 618 619 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); 620 EXPECT_CALL(mock_decoder, SampleRateHz()) 621 .WillRepeatedly(Return(kSampleRateHz)); 622 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); 623 EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes)) 624 .WillRepeatedly(Return(checked_cast<int>(kPayloadLengthSamples))); 625 int16_t dummy_output[kPayloadLengthSamples] = {0}; 626 // The below expectation will make the mock decoder write 627 // `kPayloadLengthSamples` zeros to the output array, and mark it as speech. 628 EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(0), kPayloadLengthBytes, 629 kSampleRateHz, _, _)) 630 .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, 631 dummy_output + kPayloadLengthSamples), 632 SetArgPointee<4>(AudioDecoder::kSpeech), 633 Return(checked_cast<int>(kPayloadLengthSamples)))); 634 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 635 SdpAudioFormat("L16", 8000, 1))); 636 637 // Insert one packet. 638 clock_.AdvanceTimeMilliseconds(123456); 639 RtpPacketInfo expected_packet_info = 640 RtpPacketInfo(rtp_header, /*receive_time=*/clock_.CurrentTime()); 641 EXPECT_EQ(NetEq::kOK, 642 neteq_->InsertPacket(rtp_header, payload, expected_packet_info)); 643 644 // Pull audio once. 645 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000); 646 AudioFrame output; 647 bool muted; 648 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 649 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); 650 EXPECT_EQ(1u, output.num_channels_); 651 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); 652 653 // Verify `output.packet_infos_`. 654 ASSERT_THAT(output.packet_infos_, SizeIs(1)); 655 EXPECT_EQ(output.packet_infos_[0], expected_packet_info); 656 657 // Insert two more packets. The first one is out of order, and is already too 658 // old, the second one is the expected next packet. 659 rtp_header.sequenceNumber -= 1; 660 rtp_header.timestamp -= kPayloadLengthSamples; 661 rtp_header.extension.set_audio_level(AudioLevel(/*voice_activity=*/false, 1)); 662 payload[0] = 1; 663 clock_.AdvanceTimeMilliseconds(1000); 664 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 665 rtp_header.sequenceNumber += 2; 666 rtp_header.timestamp += 2 * kPayloadLengthSamples; 667 rtp_header.extension.set_audio_level(AudioLevel(/*voice_activity=*/false, 2)); 668 payload[0] = 2; 669 clock_.AdvanceTimeMilliseconds(2000); 670 expected_packet_info = 671 RtpPacketInfo(rtp_header, /*receive_time=*/clock_.CurrentTime()); 672 EXPECT_EQ(NetEq::kOK, 673 neteq_->InsertPacket(rtp_header, payload, expected_packet_info)); 674 675 // Expect only the second packet to be decoded (the one with "2" as the first 676 // payload byte). 677 EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(2), kPayloadLengthBytes, 678 kSampleRateHz, _, _)) 679 .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, 680 dummy_output + kPayloadLengthSamples), 681 SetArgPointee<4>(AudioDecoder::kSpeech), 682 Return(checked_cast<int>(kPayloadLengthSamples)))); 683 684 // Pull audio once. 685 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 686 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); 687 EXPECT_EQ(1u, output.num_channels_); 688 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); 689 690 // Now check the packet buffer, and make sure it is empty, since the 691 // out-of-order packet should have been discarded. 692 EXPECT_TRUE(packet_buffer_->Empty()); 693 694 // NetEq `packets_discarded` should capture this packet discard. 695 EXPECT_EQ(1u, neteq_->GetLifetimeStatistics().packets_discarded); 696 697 // Verify `output.packet_infos_`. Expect to only see the second packet. 698 ASSERT_THAT(output.packet_infos_, SizeIs(1)); 699 EXPECT_EQ(output.packet_infos_[0], expected_packet_info); 700 701 EXPECT_CALL(mock_decoder, Die()); 702 } 703 704 // This test verifies that NetEq can handle the situation where the first 705 // incoming packet is rejected. 706 TEST_F(NetEqImplTest, FirstPacketUnknown) { 707 UseNoMocks(); 708 CreateInstance(); 709 710 const uint8_t kPayloadType = 17; // Just an arbitrary number. 711 const int kSampleRateHz = 8000; 712 const size_t kPayloadLengthSamples = 713 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms. 714 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; 715 uint8_t payload[kPayloadLengthBytes] = {0}; 716 RTPHeader rtp_header; 717 rtp_header.payloadType = kPayloadType; 718 rtp_header.sequenceNumber = 0x1234; 719 rtp_header.timestamp = 0x12345678; 720 rtp_header.ssrc = 0x87654321; 721 722 // Insert one packet. Note that we have not registered any payload type, so 723 // this packet will be rejected. 724 EXPECT_EQ(NetEq::kFail, neteq_->InsertPacket(rtp_header, payload)); 725 726 // Pull audio once. 727 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000); 728 AudioFrame output; 729 bool muted; 730 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 731 ASSERT_LE(output.samples_per_channel_, kMaxOutputSize); 732 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); 733 EXPECT_EQ(1u, output.num_channels_); 734 EXPECT_EQ(AudioFrame::kPLC, output.speech_type_); 735 EXPECT_THAT(output.packet_infos_, IsEmpty()); 736 737 // Register the payload type. 738 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 739 SdpAudioFormat("l16", 8000, 1))); 740 741 // Insert 10 packets. 742 for (size_t i = 0; i < 10; ++i) { 743 rtp_header.sequenceNumber++; 744 rtp_header.timestamp += kPayloadLengthSamples; 745 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 746 EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer()); 747 } 748 749 // Pull audio repeatedly and make sure we get normal output, that is not PLC. 750 for (size_t i = 0; i < 3; ++i) { 751 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 752 ASSERT_LE(output.samples_per_channel_, kMaxOutputSize); 753 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); 754 EXPECT_EQ(1u, output.num_channels_); 755 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_) 756 << "NetEq did not decode the packets as expected."; 757 EXPECT_THAT(output.packet_infos_, SizeIs(1)); 758 } 759 } 760 761 std::vector<uint8_t> CreateRedPayload(size_t num_payloads, 762 int payload_type, 763 int payload_size, 764 int timestamp_offset) { 765 const size_t size = 766 payload_size + 1 + (num_payloads - 1) * (payload_size + kRedHeaderLength); 767 std::vector<uint8_t> payload(size, 0); 768 uint8_t* payload_ptr = payload.data(); 769 for (size_t i = 0; i < num_payloads; ++i) { 770 // Write the RED headers. 771 if (i == num_payloads - 1) { 772 // Special case for last payload. 773 *payload_ptr = payload_type & 0x7F; // F = 0; 774 ++payload_ptr; 775 break; 776 } 777 *payload_ptr = payload_type & 0x7F; 778 // Not the last block; set F = 1. 779 *payload_ptr |= 0x80; 780 ++payload_ptr; 781 const int this_offset = 782 checked_cast<int>((num_payloads - i - 1) * timestamp_offset); 783 *payload_ptr = this_offset >> 6; 784 ++payload_ptr; 785 RTC_DCHECK_LE(payload_size, 1023); // Max length described by 10 bits. 786 *payload_ptr = ((this_offset & 0x3F) << 2) | (payload_size >> 8); 787 ++payload_ptr; 788 *payload_ptr = payload_size & 0xFF; 789 ++payload_ptr; 790 } 791 return payload; 792 } 793 794 TEST_F(NetEqImplTest, InsertRedPayload) { 795 UseNoMocks(); 796 CreateInstance(); 797 constexpr int kRedPayloadType = 7; 798 neteq_->RegisterPayloadType(kRedPayloadType, SdpAudioFormat("red", 8000, 1)); 799 constexpr int kPayloadType = 8; 800 neteq_->RegisterPayloadType(kPayloadType, SdpAudioFormat("l16", 8000, 1)); 801 size_t frame_size = 80; // 10 ms. 802 size_t payload_size = frame_size * 2; 803 std::vector<uint8_t> payload = 804 CreateRedPayload(3, kPayloadType, payload_size, frame_size); 805 RTPHeader header; 806 header.payloadType = kRedPayloadType; 807 header.sequenceNumber = 0x1234; 808 header.timestamp = 0x12345678; 809 header.ssrc = 0x87654321; 810 AbsoluteCaptureTime capture_time; 811 capture_time.absolute_capture_timestamp = 1234; 812 header.extension.absolute_capture_time = capture_time; 813 header.extension.set_audio_level(AudioLevel(/*voice_activity=*/false, 12)); 814 header.numCSRCs = 1; 815 header.arrOfCSRCs[0] = 123; 816 neteq_->InsertPacket(header, payload); 817 AudioFrame frame; 818 bool muted; 819 neteq_->GetAudio(&frame, &muted); 820 // TODO(jakobi): Find a better way to test that the correct packet is decoded 821 // than using the timestamp. The fixed NetEq delay is an implementation 822 // detail that should not be tested. 823 constexpr int kNetEqFixedDelay = 5; 824 EXPECT_EQ(frame.timestamp_, 825 header.timestamp - frame_size * 2 - kNetEqFixedDelay); 826 EXPECT_TRUE(frame.packet_infos_.empty()); 827 neteq_->GetAudio(&frame, &muted); 828 EXPECT_EQ(frame.timestamp_, header.timestamp - frame_size - kNetEqFixedDelay); 829 EXPECT_TRUE(frame.packet_infos_.empty()); 830 neteq_->GetAudio(&frame, &muted); 831 EXPECT_EQ(frame.timestamp_, header.timestamp - kNetEqFixedDelay); 832 EXPECT_EQ(frame.packet_infos_.size(), 1u); 833 EXPECT_EQ(frame.packet_infos_.front().absolute_capture_time(), capture_time); 834 EXPECT_EQ(frame.packet_infos_.front().audio_level(), 835 header.extension.audio_level()->level()); 836 EXPECT_EQ(frame.packet_infos_.front().csrcs()[0], header.arrOfCSRCs[0]); 837 } 838 839 // This test verifies that audio interruption is not logged for the initial 840 // PLC period before the first packet is deocoded. 841 // TODO(henrik.lundin) Maybe move this test to neteq_network_stats_unittest.cc. 842 // Make the test parametrized, so that we can test with different initial 843 // sample rates in NetEq. 844 class NetEqImplTestSampleRateParameter 845 : public NetEqImplTest, 846 public testing::WithParamInterface<int> { 847 protected: 848 NetEqImplTestSampleRateParameter() 849 : NetEqImplTest(), initial_sample_rate_hz_(GetParam()) { 850 config_.sample_rate_hz = initial_sample_rate_hz_; 851 } 852 853 const int initial_sample_rate_hz_; 854 }; 855 856 class NetEqImplTestSdpFormatParameter 857 : public NetEqImplTest, 858 public testing::WithParamInterface<SdpAudioFormat> { 859 protected: 860 NetEqImplTestSdpFormatParameter() 861 : NetEqImplTest(), sdp_format_(GetParam()) {} 862 const SdpAudioFormat sdp_format_; 863 }; 864 865 // This test does the following: 866 // 0. Set up NetEq with initial sample rate given by test parameter, and a codec 867 // sample rate of 16000. 868 // 1. Start calling GetAudio before inserting any encoded audio. The audio 869 // produced will be PLC. 870 // 2. Insert a number of encoded audio packets. 871 // 3. Keep calling GetAudio and verify that no audio interruption was logged. 872 // Call GetAudio until NetEq runs out of data again; PLC starts. 873 // 4. Insert one more packet. 874 // 5. Call GetAudio until that packet is decoded and the PLC ends. 875 876 TEST_P(NetEqImplTestSampleRateParameter, 877 NoAudioInterruptionLoggedBeforeFirstDecode) { 878 UseNoMocks(); 879 CreateInstance(); 880 881 const uint8_t kPayloadType = 17; // Just an arbitrary number. 882 const int kPayloadSampleRateHz = 16000; 883 const size_t kPayloadLengthSamples = 884 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms. 885 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; 886 uint8_t payload[kPayloadLengthBytes] = {0}; 887 RTPHeader rtp_header; 888 rtp_header.payloadType = kPayloadType; 889 rtp_header.sequenceNumber = 0x1234; 890 rtp_header.timestamp = 0x12345678; 891 rtp_header.ssrc = 0x87654321; 892 893 // Register the payload type. 894 EXPECT_TRUE(neteq_->RegisterPayloadType( 895 kPayloadType, SdpAudioFormat("l16", kPayloadSampleRateHz, 1))); 896 897 // Pull audio several times. No packets have been inserted yet. 898 const size_t initial_output_size = 899 static_cast<size_t>(10 * initial_sample_rate_hz_ / 1000); // 10 ms 900 AudioFrame output; 901 bool muted; 902 for (int i = 0; i < 100; ++i) { 903 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 904 EXPECT_EQ(initial_output_size, output.samples_per_channel_); 905 EXPECT_EQ(1u, output.num_channels_); 906 EXPECT_NE(AudioFrame::kNormalSpeech, output.speech_type_); 907 EXPECT_THAT(output.packet_infos_, IsEmpty()); 908 } 909 910 // Lambda for inserting packets. 911 auto insert_packet = [&]() { 912 rtp_header.sequenceNumber++; 913 rtp_header.timestamp += kPayloadLengthSamples; 914 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 915 }; 916 // Insert 10 packets. 917 for (size_t i = 0; i < 10; ++i) { 918 insert_packet(); 919 EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer()); 920 } 921 922 // Pull audio repeatedly and make sure we get normal output, that is not PLC. 923 constexpr size_t kOutputSize = 924 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms 925 for (size_t i = 0; i < 3; ++i) { 926 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 927 EXPECT_EQ(kOutputSize, output.samples_per_channel_); 928 EXPECT_EQ(1u, output.num_channels_); 929 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_) 930 << "NetEq did not decode the packets as expected."; 931 EXPECT_THAT(output.packet_infos_, SizeIs(1)); 932 } 933 934 // Verify that no interruption was logged. 935 auto lifetime_stats = neteq_->GetLifetimeStatistics(); 936 EXPECT_EQ(0, lifetime_stats.interruption_count); 937 938 // Keep pulling audio data until a new PLC period is started. 939 size_t count_loops = 0; 940 while (output.speech_type_ == AudioFrame::kNormalSpeech) { 941 // Make sure we don't hang the test if we never go to PLC. 942 ASSERT_LT(++count_loops, 100u); 943 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 944 } 945 946 // Insert a few packets to avoid postpone decoding after expand. 947 for (size_t i = 0; i < 5; ++i) { 948 insert_packet(); 949 } 950 951 // Pull audio until the newly inserted packet is decoded and the PLC ends. 952 while (output.speech_type_ != AudioFrame::kNormalSpeech) { 953 // Make sure we don't hang the test if we never go to PLC. 954 ASSERT_LT(++count_loops, 100u); 955 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 956 } 957 958 // Verify that no interruption was logged. 959 lifetime_stats = neteq_->GetLifetimeStatistics(); 960 EXPECT_EQ(0, lifetime_stats.interruption_count); 961 } 962 963 // This test does the following: 964 // 0. Set up NetEq with initial sample rate given by test parameter, and a codec 965 // sample rate of 16000. 966 // 1. Insert a number of encoded audio packets. 967 // 2. Call GetAudio and verify that decoded audio is produced. 968 // 3. Keep calling GetAudio until NetEq runs out of data; PLC starts. 969 // 4. Keep calling GetAudio until PLC has been produced for at least 150 ms. 970 // 5. Insert one more packet. 971 // 6. Call GetAudio until that packet is decoded and the PLC ends. 972 // 7. Verify that an interruption was logged. 973 974 TEST_P(NetEqImplTestSampleRateParameter, AudioInterruptionLogged) { 975 UseNoMocks(); 976 CreateInstance(); 977 978 const uint8_t kPayloadType = 17; // Just an arbitrary number. 979 const int kPayloadSampleRateHz = 16000; 980 const size_t kPayloadLengthSamples = 981 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms. 982 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; 983 uint8_t payload[kPayloadLengthBytes] = {0}; 984 RTPHeader rtp_header; 985 rtp_header.payloadType = kPayloadType; 986 rtp_header.sequenceNumber = 0x1234; 987 rtp_header.timestamp = 0x12345678; 988 rtp_header.ssrc = 0x87654321; 989 990 // Register the payload type. 991 EXPECT_TRUE(neteq_->RegisterPayloadType( 992 kPayloadType, SdpAudioFormat("l16", kPayloadSampleRateHz, 1))); 993 994 // Lambda for inserting packets. 995 auto insert_packet = [&]() { 996 rtp_header.sequenceNumber++; 997 rtp_header.timestamp += kPayloadLengthSamples; 998 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 999 }; 1000 // Insert 10 packets. 1001 for (size_t i = 0; i < 10; ++i) { 1002 insert_packet(); 1003 EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer()); 1004 } 1005 1006 AudioFrame output; 1007 bool muted; 1008 // Keep pulling audio data until a new PLC period is started. 1009 size_t count_loops = 0; 1010 do { 1011 // Make sure we don't hang the test if we never go to PLC. 1012 ASSERT_LT(++count_loops, 100u); 1013 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1014 } while (output.speech_type_ == AudioFrame::kNormalSpeech); 1015 1016 // Pull audio 15 times, which produces 150 ms of output audio. This should 1017 // all be produced as PLC. The total length of the gap will then be 150 ms 1018 // plus an initial fraction of 10 ms at the start and the end of the PLC 1019 // period. In total, less than 170 ms. 1020 for (size_t i = 0; i < 15; ++i) { 1021 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1022 EXPECT_NE(AudioFrame::kNormalSpeech, output.speech_type_); 1023 } 1024 1025 // Insert a few packets to avoid postpone decoding after expand. 1026 for (size_t i = 0; i < 5; ++i) { 1027 insert_packet(); 1028 } 1029 1030 // Pull audio until the newly inserted packet is decoded and the PLC ends. 1031 while (output.speech_type_ != AudioFrame::kNormalSpeech) { 1032 // Make sure we don't hang the test if we never go to PLC. 1033 ASSERT_LT(++count_loops, 100u); 1034 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1035 } 1036 1037 // Verify that the interruption was logged. 1038 auto lifetime_stats = neteq_->GetLifetimeStatistics(); 1039 EXPECT_EQ(1, lifetime_stats.interruption_count); 1040 EXPECT_GT(lifetime_stats.total_interruption_duration_ms, 150); 1041 EXPECT_LT(lifetime_stats.total_interruption_duration_ms, 170); 1042 } 1043 1044 INSTANTIATE_TEST_SUITE_P(SampleRates, 1045 NetEqImplTestSampleRateParameter, 1046 testing::Values(8000, 16000, 32000, 48000)); 1047 1048 TEST_P(NetEqImplTestSdpFormatParameter, GetNackListScaledTimestamp) { 1049 UseNoMocks(); 1050 CreateInstance(); 1051 1052 neteq_->EnableNack(128); 1053 1054 const uint8_t kPayloadType = 17; // Just an arbitrary number. 1055 const int kPayloadSampleRateHz = sdp_format_.clockrate_hz; 1056 const size_t kPayloadLengthSamples = 1057 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms. 1058 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; 1059 std::vector<uint8_t> payload(kPayloadLengthBytes, 0); 1060 RTPHeader rtp_header; 1061 rtp_header.payloadType = kPayloadType; 1062 rtp_header.sequenceNumber = 0x1234; 1063 rtp_header.timestamp = 0x12345678; 1064 rtp_header.ssrc = 0x87654321; 1065 1066 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, sdp_format_)); 1067 1068 auto insert_packet = [&](bool lost = false) { 1069 rtp_header.sequenceNumber++; 1070 rtp_header.timestamp += kPayloadLengthSamples; 1071 if (!lost) 1072 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1073 }; 1074 1075 // Insert and decode 10 packets. 1076 for (size_t i = 0; i < 10; ++i) { 1077 insert_packet(); 1078 } 1079 AudioFrame output; 1080 size_t count_loops = 0; 1081 do { 1082 bool muted; 1083 // Make sure we don't hang the test if we never go to PLC. 1084 ASSERT_LT(++count_loops, 100u); 1085 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1086 } while (output.speech_type_ == AudioFrame::kNormalSpeech); 1087 1088 insert_packet(); 1089 1090 insert_packet(/*lost=*/true); 1091 1092 // Ensure packet gets marked as missing. 1093 for (int i = 0; i < 5; ++i) { 1094 insert_packet(); 1095 } 1096 1097 // Missing packet recoverable with 5ms RTT. 1098 EXPECT_THAT(neteq_->GetNackList(5), Not(IsEmpty())); 1099 1100 // No packets should have TimeToPlay > 500ms. 1101 EXPECT_THAT(neteq_->GetNackList(500), IsEmpty()); 1102 } 1103 1104 INSTANTIATE_TEST_SUITE_P(GetNackList, 1105 NetEqImplTestSdpFormatParameter, 1106 testing::Values(SdpAudioFormat("g722", 8000, 1), 1107 SdpAudioFormat("opus", 48000, 2))); 1108 1109 // This test verifies that NetEq can handle comfort noise and enters/quits codec 1110 // internal CNG mode properly. 1111 TEST_F(NetEqImplTest, CodecInternalCng) { 1112 UseNoMocks(); 1113 // Create a mock decoder object. 1114 MockAudioDecoder mock_decoder; 1115 CreateInstance( 1116 make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder)); 1117 1118 const uint8_t kPayloadType = 17; // Just an arbitrary number. 1119 const int kSampleRateKhz = 48; 1120 const size_t kPayloadLengthSamples = 1121 static_cast<size_t>(20 * kSampleRateKhz); // 20 ms. 1122 const size_t kPayloadLengthBytes = 10; 1123 uint8_t payload[kPayloadLengthBytes] = {0}; 1124 int16_t dummy_output[kPayloadLengthSamples] = {0}; 1125 1126 RTPHeader rtp_header; 1127 rtp_header.payloadType = kPayloadType; 1128 rtp_header.sequenceNumber = 0x1234; 1129 rtp_header.timestamp = 0x12345678; 1130 rtp_header.ssrc = 0x87654321; 1131 1132 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); 1133 EXPECT_CALL(mock_decoder, SampleRateHz()) 1134 .WillRepeatedly(Return(kSampleRateKhz * 1000)); 1135 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); 1136 EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes)) 1137 .WillRepeatedly(Return(checked_cast<int>(kPayloadLengthSamples))); 1138 // Packed duration when asking the decoder for more CNG data (without a new 1139 // packet). 1140 EXPECT_CALL(mock_decoder, PacketDuration(nullptr, 0)) 1141 .WillRepeatedly(Return(checked_cast<int>(kPayloadLengthSamples))); 1142 1143 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 1144 SdpAudioFormat("opus", 48000, 2))); 1145 1146 struct Packet { 1147 int sequence_number_delta; 1148 int timestamp_delta; 1149 AudioDecoder::SpeechType decoder_output_type; 1150 }; 1151 std::vector<Packet> packets = { 1152 {0, 0, AudioDecoder::kSpeech}, 1153 {1, kPayloadLengthSamples, AudioDecoder::kComfortNoise}, 1154 {2, 2 * kPayloadLengthSamples, AudioDecoder::kSpeech}, 1155 {1, kPayloadLengthSamples, AudioDecoder::kSpeech}}; 1156 1157 for (size_t i = 0; i < packets.size(); ++i) { 1158 rtp_header.sequenceNumber += packets[i].sequence_number_delta; 1159 rtp_header.timestamp += packets[i].timestamp_delta; 1160 payload[0] = i; 1161 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1162 1163 // Pointee(x) verifies that first byte of the payload equals x, this makes 1164 // it possible to verify that the correct payload is fed to Decode(). 1165 EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(i), kPayloadLengthBytes, 1166 kSampleRateKhz * 1000, _, _)) 1167 .WillOnce(DoAll(SetArrayArgument<3>( 1168 dummy_output, dummy_output + kPayloadLengthSamples), 1169 SetArgPointee<4>(packets[i].decoder_output_type), 1170 Return(checked_cast<int>(kPayloadLengthSamples)))); 1171 } 1172 1173 // Expect comfort noise to be returned by the decoder. 1174 EXPECT_CALL(mock_decoder, 1175 DecodeInternal(IsNull(), 0, kSampleRateKhz * 1000, _, _)) 1176 .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, 1177 dummy_output + kPayloadLengthSamples), 1178 SetArgPointee<4>(AudioDecoder::kComfortNoise), 1179 Return(checked_cast<int>(kPayloadLengthSamples)))); 1180 1181 std::vector<AudioFrame::SpeechType> expected_output = { 1182 AudioFrame::kNormalSpeech, AudioFrame::kCNG, AudioFrame::kNormalSpeech}; 1183 size_t output_index = 0; 1184 1185 int timeout_counter = 0; 1186 while (!packet_buffer_->Empty()) { 1187 ASSERT_LT(timeout_counter++, 20) << "Test timed out"; 1188 AudioFrame output; 1189 bool muted; 1190 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1191 if (output_index + 1 < expected_output.size() && 1192 output.speech_type_ == expected_output[output_index + 1]) { 1193 ++output_index; 1194 } else { 1195 EXPECT_EQ(output.speech_type_, expected_output[output_index]); 1196 } 1197 } 1198 1199 EXPECT_CALL(mock_decoder, Die()); 1200 } 1201 1202 TEST_F(NetEqImplTest, UnsupportedDecoder) { 1203 UseNoMocks(); 1204 ::testing::NiceMock<MockAudioDecoder> decoder; 1205 1206 CreateInstance(make_ref_counted<test::AudioDecoderProxyFactory>(&decoder)); 1207 static const size_t kNetEqMaxFrameSize = 5760; // 120 ms @ 48 kHz. 1208 static const size_t kChannels = 2; 1209 1210 const uint8_t kPayloadType = 17; // Just an arbitrary number. 1211 const int kSampleRateHz = 8000; 1212 1213 const size_t kPayloadLengthSamples = 1214 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms. 1215 const size_t kPayloadLengthBytes = 1; 1216 uint8_t payload[kPayloadLengthBytes] = {0}; 1217 int16_t dummy_output[kPayloadLengthSamples * kChannels] = {0}; 1218 RTPHeader rtp_header; 1219 rtp_header.payloadType = kPayloadType; 1220 rtp_header.sequenceNumber = 0x1234; 1221 rtp_header.timestamp = 0x12345678; 1222 rtp_header.ssrc = 0x87654321; 1223 1224 const uint8_t kFirstPayloadValue = 1; 1225 const uint8_t kSecondPayloadValue = 2; 1226 1227 EXPECT_CALL(decoder, 1228 PacketDuration(Pointee(kFirstPayloadValue), kPayloadLengthBytes)) 1229 .Times(AtLeast(1)) 1230 .WillRepeatedly(Return(checked_cast<int>(kNetEqMaxFrameSize + 1))); 1231 1232 EXPECT_CALL(decoder, DecodeInternal(Pointee(kFirstPayloadValue), _, _, _, _)) 1233 .Times(0); 1234 1235 EXPECT_CALL(decoder, DecodeInternal(Pointee(kSecondPayloadValue), 1236 kPayloadLengthBytes, kSampleRateHz, _, _)) 1237 .Times(1) 1238 .WillOnce(DoAll( 1239 SetArrayArgument<3>(dummy_output, 1240 dummy_output + kPayloadLengthSamples * kChannels), 1241 SetArgPointee<4>(AudioDecoder::kSpeech), 1242 Return(static_cast<int>(kPayloadLengthSamples * kChannels)))); 1243 1244 EXPECT_CALL(decoder, 1245 PacketDuration(Pointee(kSecondPayloadValue), kPayloadLengthBytes)) 1246 .Times(AtLeast(1)) 1247 .WillRepeatedly(Return(checked_cast<int>(kNetEqMaxFrameSize))); 1248 1249 EXPECT_CALL(decoder, SampleRateHz()).WillRepeatedly(Return(kSampleRateHz)); 1250 1251 EXPECT_CALL(decoder, Channels()).WillRepeatedly(Return(kChannels)); 1252 1253 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 1254 SdpAudioFormat("L16", 8000, 1))); 1255 1256 // Insert one packet. 1257 payload[0] = kFirstPayloadValue; // This will make Decode() fail. 1258 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1259 1260 // Insert another packet. 1261 payload[0] = kSecondPayloadValue; // This will make Decode() successful. 1262 rtp_header.sequenceNumber++; 1263 // The second timestamp needs to be at least 30 ms after the first to make 1264 // the second packet get decoded. 1265 rtp_header.timestamp += 3 * kPayloadLengthSamples; 1266 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1267 1268 AudioFrame output; 1269 bool muted; 1270 // First call to GetAudio will try to decode the "faulty" packet. 1271 // Expect kFail return value. 1272 EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); 1273 // Output size and number of channels should be correct. 1274 const size_t kExpectedOutputSize = 10 * (kSampleRateHz / 1000) * kChannels; 1275 EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels); 1276 EXPECT_EQ(kChannels, output.num_channels_); 1277 EXPECT_THAT(output.packet_infos_, IsEmpty()); 1278 1279 // Call GetAudio until the next packet is decoded. 1280 int calls = 0; 1281 int kTimeout = 10; 1282 while (output.packet_infos_.empty() && calls < kTimeout) { 1283 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1284 EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels); 1285 EXPECT_EQ(kChannels, output.num_channels_); 1286 } 1287 EXPECT_LT(calls, kTimeout); 1288 1289 // Die isn't called through NiceMock (since it's called by the 1290 // MockAudioDecoder constructor), so it needs to be mocked explicitly. 1291 EXPECT_CALL(decoder, Die()); 1292 } 1293 1294 // This test inserts packets until the buffer is flushed. After that, it asks 1295 // NetEq for the network statistics. The purpose of the test is to make sure 1296 // that even though the buffer size increment is negative (which it becomes when 1297 // the packet causing a flush is inserted), the packet length stored in the 1298 // decision logic remains valid. 1299 TEST_F(NetEqImplTest, FloodBufferAndGetNetworkStats) { 1300 UseNoMocks(); 1301 CreateInstance(); 1302 1303 const size_t kPayloadLengthSamples = 80; 1304 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. 1305 const uint8_t kPayloadType = 17; // Just an arbitrary number. 1306 uint8_t payload[kPayloadLengthBytes] = {0}; 1307 RTPHeader rtp_header; 1308 rtp_header.payloadType = kPayloadType; 1309 rtp_header.sequenceNumber = 0x1234; 1310 rtp_header.timestamp = 0x12345678; 1311 rtp_header.ssrc = 0x87654321; 1312 1313 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 1314 SdpAudioFormat("l16", 8000, 1))); 1315 1316 // Insert packets until the buffer flushes. 1317 for (size_t i = 0; i <= config_.max_packets_in_buffer; ++i) { 1318 EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer()); 1319 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1320 rtp_header.timestamp += checked_cast<uint32_t>(kPayloadLengthSamples); 1321 ++rtp_header.sequenceNumber; 1322 } 1323 EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer()); 1324 1325 // Ask for network statistics. This should not crash. 1326 NetEqNetworkStatistics stats; 1327 EXPECT_EQ(NetEq::kOK, neteq_->NetworkStatistics(&stats)); 1328 } 1329 1330 TEST_F(NetEqImplTest, DecodedPayloadTooShort) { 1331 UseNoMocks(); 1332 // Create a mock decoder object. 1333 MockAudioDecoder mock_decoder; 1334 1335 CreateInstance( 1336 make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder)); 1337 1338 const uint8_t kPayloadType = 17; // Just an arbitrary number. 1339 const int kSampleRateHz = 8000; 1340 const size_t kPayloadLengthSamples = 1341 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms. 1342 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; 1343 uint8_t payload[kPayloadLengthBytes] = {0}; 1344 RTPHeader rtp_header; 1345 rtp_header.payloadType = kPayloadType; 1346 rtp_header.sequenceNumber = 0x1234; 1347 rtp_header.timestamp = 0x12345678; 1348 rtp_header.ssrc = 0x87654321; 1349 1350 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); 1351 EXPECT_CALL(mock_decoder, SampleRateHz()) 1352 .WillRepeatedly(Return(kSampleRateHz)); 1353 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); 1354 EXPECT_CALL(mock_decoder, PacketDuration(_, _)) 1355 .WillRepeatedly(Return(checked_cast<int>(kPayloadLengthSamples))); 1356 int16_t dummy_output[kPayloadLengthSamples] = {0}; 1357 // The below expectation will make the mock decoder write 1358 // `kPayloadLengthSamples` - 5 zeros to the output array, and mark it as 1359 // speech. That is, the decoded length is 5 samples shorter than the expected. 1360 EXPECT_CALL(mock_decoder, 1361 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) 1362 .WillOnce( 1363 DoAll(SetArrayArgument<3>(dummy_output, 1364 dummy_output + kPayloadLengthSamples - 5), 1365 SetArgPointee<4>(AudioDecoder::kSpeech), 1366 Return(checked_cast<int>(kPayloadLengthSamples - 5)))); 1367 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 1368 SdpAudioFormat("L16", 8000, 1))); 1369 1370 // Insert one packet. 1371 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1372 1373 EXPECT_EQ(5u, neteq_->sync_buffer_for_test()->FutureLength()); 1374 1375 // Pull audio once. 1376 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000); 1377 AudioFrame output; 1378 bool muted; 1379 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1380 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); 1381 EXPECT_EQ(1u, output.num_channels_); 1382 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); 1383 EXPECT_THAT(output.packet_infos_, SizeIs(1)); 1384 1385 EXPECT_CALL(mock_decoder, Die()); 1386 } 1387 1388 // This test checks the behavior of NetEq when audio decoder fails. 1389 TEST_F(NetEqImplTest, DecodingError) { 1390 UseNoMocks(); 1391 // Create a mock decoder object. 1392 MockAudioDecoder mock_decoder; 1393 1394 CreateInstance( 1395 make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder)); 1396 1397 const uint8_t kPayloadType = 17; // Just an arbitrary number. 1398 const int kSampleRateHz = 8000; 1399 const int kDecoderErrorCode = -97; // Any negative number. 1400 1401 // We let decoder return 5 ms each time, and therefore, 2 packets make 10 ms. 1402 const size_t kFrameLengthSamples = 1403 static_cast<size_t>(5 * kSampleRateHz / 1000); 1404 1405 const size_t kPayloadLengthBytes = 1; // This can be arbitrary. 1406 1407 uint8_t payload[kPayloadLengthBytes] = {0}; 1408 1409 RTPHeader rtp_header; 1410 rtp_header.payloadType = kPayloadType; 1411 rtp_header.sequenceNumber = 0x1234; 1412 rtp_header.timestamp = 0x12345678; 1413 rtp_header.ssrc = 0x87654321; 1414 1415 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); 1416 EXPECT_CALL(mock_decoder, SampleRateHz()) 1417 .WillRepeatedly(Return(kSampleRateHz)); 1418 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); 1419 EXPECT_CALL(mock_decoder, PacketDuration(_, _)) 1420 .WillRepeatedly(Return(checked_cast<int>(kFrameLengthSamples))); 1421 EXPECT_CALL(mock_decoder, ErrorCode()).WillOnce(Return(kDecoderErrorCode)); 1422 EXPECT_CALL(mock_decoder, HasDecodePlc()).WillOnce(Return(false)); 1423 int16_t dummy_output[kFrameLengthSamples] = {0}; 1424 1425 { 1426 InSequence sequence; // Dummy variable. 1427 // Mock decoder works normally the first time. 1428 EXPECT_CALL(mock_decoder, 1429 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) 1430 .Times(3) 1431 .WillRepeatedly( 1432 DoAll(SetArrayArgument<3>(dummy_output, 1433 dummy_output + kFrameLengthSamples), 1434 SetArgPointee<4>(AudioDecoder::kSpeech), 1435 Return(checked_cast<int>(kFrameLengthSamples)))) 1436 .RetiresOnSaturation(); 1437 1438 // Then mock decoder fails. A common reason for failure can be buffer being 1439 // too short 1440 EXPECT_CALL(mock_decoder, 1441 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) 1442 .WillOnce(Return(-1)) 1443 .RetiresOnSaturation(); 1444 1445 // Mock decoder finally returns to normal. 1446 EXPECT_CALL(mock_decoder, 1447 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) 1448 .Times(2) 1449 .WillRepeatedly( 1450 DoAll(SetArrayArgument<3>(dummy_output, 1451 dummy_output + kFrameLengthSamples), 1452 SetArgPointee<4>(AudioDecoder::kSpeech), 1453 Return(checked_cast<int>(kFrameLengthSamples)))); 1454 } 1455 1456 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 1457 SdpAudioFormat("L16", 8000, 1))); 1458 1459 // Insert packets. 1460 for (int i = 0; i < 20; ++i) { 1461 rtp_header.sequenceNumber += 1; 1462 rtp_header.timestamp += kFrameLengthSamples; 1463 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1464 } 1465 1466 // Pull audio. 1467 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000); 1468 AudioFrame output; 1469 bool muted; 1470 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1471 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); 1472 EXPECT_EQ(1u, output.num_channels_); 1473 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); 1474 EXPECT_THAT(output.packet_infos_, SizeIs(2)); // 5 ms packets vs 10 ms output 1475 1476 // Pull audio again. Decoder fails. 1477 EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); 1478 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); 1479 EXPECT_EQ(1u, output.num_channels_); 1480 // We are not expecting anything for output.speech_type_, since an error was 1481 // returned. 1482 1483 // Pull audio again, should behave normal. 1484 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1485 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); 1486 EXPECT_EQ(1u, output.num_channels_); 1487 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); 1488 EXPECT_THAT(output.packet_infos_, SizeIs(2)); // 5 ms packets vs 10 ms output 1489 1490 EXPECT_CALL(mock_decoder, Die()); 1491 } 1492 1493 // Tests that the return value from last_output_sample_rate_hz() is equal to the 1494 // configured inital sample rate. 1495 TEST_F(NetEqImplTest, InitialLastOutputSampleRate) { 1496 UseNoMocks(); 1497 config_.sample_rate_hz = 48000; 1498 CreateInstance(); 1499 EXPECT_EQ(48000, neteq_->last_output_sample_rate_hz()); 1500 } 1501 1502 TEST_F(NetEqImplTest, TickTimerIncrement) { 1503 UseNoMocks(); 1504 CreateInstance(); 1505 ASSERT_TRUE(tick_timer_); 1506 EXPECT_EQ(0u, tick_timer_->ticks()); 1507 AudioFrame output; 1508 bool muted; 1509 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1510 EXPECT_EQ(1u, tick_timer_->ticks()); 1511 } 1512 1513 TEST_F(NetEqImplTest, SetBaseMinimumDelay) { 1514 UseNoMocks(); 1515 use_mock_neteq_controller_ = true; 1516 CreateInstance(); 1517 1518 EXPECT_CALL(*mock_neteq_controller_, SetBaseMinimumDelay(_)) 1519 .WillOnce(Return(true)) 1520 .WillOnce(Return(false)); 1521 1522 const int delay_ms = 200; 1523 1524 EXPECT_EQ(true, neteq_->SetBaseMinimumDelayMs(delay_ms)); 1525 EXPECT_EQ(false, neteq_->SetBaseMinimumDelayMs(delay_ms)); 1526 } 1527 1528 TEST_F(NetEqImplTest, GetBaseMinimumDelayMs) { 1529 UseNoMocks(); 1530 use_mock_neteq_controller_ = true; 1531 CreateInstance(); 1532 1533 const int delay_ms = 200; 1534 1535 EXPECT_CALL(*mock_neteq_controller_, GetBaseMinimumDelay()) 1536 .WillOnce(Return(delay_ms)); 1537 1538 EXPECT_EQ(delay_ms, neteq_->GetBaseMinimumDelayMs()); 1539 } 1540 1541 TEST_F(NetEqImplTest, TargetDelayMs) { 1542 UseNoMocks(); 1543 use_mock_neteq_controller_ = true; 1544 CreateInstance(); 1545 constexpr int kTargetLevelMs = 510; 1546 EXPECT_CALL(*mock_neteq_controller_, TargetLevelMs()) 1547 .WillOnce(Return(kTargetLevelMs)); 1548 EXPECT_EQ(510, neteq_->TargetDelayMs()); 1549 } 1550 1551 TEST_F(NetEqImplTest, InsertEmptyPacket) { 1552 UseNoMocks(); 1553 use_mock_neteq_controller_ = true; 1554 CreateInstance(); 1555 1556 RTPHeader rtp_header; 1557 rtp_header.payloadType = 17; 1558 rtp_header.sequenceNumber = 0x1234; 1559 rtp_header.timestamp = 0x12345678; 1560 rtp_header.ssrc = 0x87654321; 1561 1562 EXPECT_CALL(*mock_neteq_controller_, RegisterEmptyPacket()); 1563 neteq_->InsertEmptyPacket(rtp_header); 1564 } 1565 1566 TEST_F(NetEqImplTest, NotifyControllerOfReorderedPacket) { 1567 using ::testing::AllOf; 1568 using ::testing::Field; 1569 UseNoMocks(); 1570 use_mock_neteq_controller_ = true; 1571 CreateInstance(); 1572 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) 1573 .Times(1) 1574 .WillOnce(Return(NetEq::Operation::kNormal)); 1575 1576 const int kPayloadLengthSamples = 80; 1577 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. 1578 const uint8_t kPayloadType = 17; // Just an arbitrary number. 1579 uint8_t payload[kPayloadLengthBytes] = {0}; 1580 RTPHeader rtp_header; 1581 rtp_header.payloadType = kPayloadType; 1582 rtp_header.sequenceNumber = 0x1234; 1583 rtp_header.timestamp = 0x12345678; 1584 rtp_header.ssrc = 0x87654321; 1585 1586 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, 1587 SdpAudioFormat("l16", 8000, 1))); 1588 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1589 AudioFrame output; 1590 bool muted; 1591 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); 1592 1593 // Insert second packet that was sent before the first packet. 1594 rtp_header.sequenceNumber -= 1; 1595 rtp_header.timestamp -= kPayloadLengthSamples; 1596 EXPECT_CALL( 1597 *mock_neteq_controller_, 1598 PacketArrived( 1599 /*fs_hz*/ 8000, 1600 /*should_update_stats*/ true, 1601 /*info*/ 1602 AllOf( 1603 Field(&NetEqController::PacketArrivedInfo::packet_length_samples, 1604 kPayloadLengthSamples), 1605 Field(&NetEqController::PacketArrivedInfo::main_sequence_number, 1606 rtp_header.sequenceNumber), 1607 Field(&NetEqController::PacketArrivedInfo::main_timestamp, 1608 rtp_header.timestamp)))); 1609 1610 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1611 } 1612 1613 #if RTC_DCHECK_IS_ON 1614 #if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) 1615 TEST(NetEqImplDeathTest, CrashWith1000Channels) { 1616 EXPECT_DEATH(std::make_unique<AudioDecoderPcmU>(1000), ""); 1617 } 1618 #endif // GTEST_HAS_DEATH_TEST 1619 #endif 1620 1621 // When using a codec with kMaxNumberOfAudioChannels channels, there should be 1622 // no crashes. 1623 TEST_F(NetEqImplTest, NoCrashWithMaxChannels) { 1624 using ::testing::AllOf; 1625 using ::testing::Field; 1626 UseNoMocks(); 1627 use_mock_decoder_database_ = true; 1628 enable_muted_state_ = true; 1629 CreateInstance(); 1630 const size_t kPayloadLength = 100; 1631 const uint8_t kPayloadType = 0; 1632 const uint16_t kFirstSequenceNumber = 0x1234; 1633 const uint32_t kFirstTimestamp = 0x12345678; 1634 const uint32_t kSsrc = 0x87654321; 1635 uint8_t payload[kPayloadLength] = {0}; 1636 RTPHeader rtp_header; 1637 rtp_header.payloadType = kPayloadType; 1638 rtp_header.sequenceNumber = kFirstSequenceNumber; 1639 rtp_header.timestamp = kFirstTimestamp; 1640 rtp_header.ssrc = kSsrc; 1641 Packet fake_packet; 1642 fake_packet.payload_type = kPayloadType; 1643 fake_packet.sequence_number = kFirstSequenceNumber; 1644 fake_packet.timestamp = kFirstTimestamp; 1645 1646 AudioDecoder* decoder = nullptr; 1647 1648 const Environment env = CreateEnvironment(); 1649 auto mock_decoder_factory = make_ref_counted<MockAudioDecoderFactory>(); 1650 EXPECT_CALL(*mock_decoder_factory, Create) 1651 .WillOnce(WithArg<1>([&](const SdpAudioFormat& format) { 1652 EXPECT_EQ("pcmu", format.name); 1653 auto dec = 1654 std::make_unique<AudioDecoderPcmU>(kMaxNumberOfAudioChannels); 1655 decoder = dec.get(); 1656 return dec; 1657 })); 1658 DecoderDatabase::DecoderInfo info(env, SdpAudioFormat("pcmu", 8000, 1), 1659 std::nullopt, mock_decoder_factory.get()); 1660 // Expectations for decoder database. 1661 EXPECT_CALL(*mock_decoder_database_, GetDecoderInfo(kPayloadType)) 1662 .WillRepeatedly(Return(&info)); 1663 EXPECT_CALL(*mock_decoder_database_, GetActiveCngDecoder()) 1664 .WillRepeatedly(ReturnNull()); 1665 EXPECT_CALL(*mock_decoder_database_, GetActiveDecoder()) 1666 .WillRepeatedly(Return(decoder)); 1667 EXPECT_CALL(*mock_decoder_database_, SetActiveDecoder(_, _)) 1668 .WillOnce(Invoke([](uint8_t /* rtp_payload_type */, bool* new_decoder) { 1669 *new_decoder = true; 1670 return 0; 1671 })); 1672 1673 // Insert first packet. 1674 neteq_->InsertPacket(rtp_header, payload); 1675 1676 AudioFrame audio_frame; 1677 bool muted = false; 1678 bool got_error = false; 1679 1680 // Repeat 40 times to ensure we enter muted state. 1681 for (int i = 0; i < 40 && !muted; i++) { 1682 // GetAudio should return an error, and not crash, even in muted state. 1683 // EXPECT_NE(0, neteq_->GetAudio(&audio_frame, &muted)); 1684 if (neteq_->GetAudio(&audio_frame, &muted) == -1) 1685 got_error = true; 1686 } 1687 EXPECT_TRUE(got_error); 1688 EXPECT_TRUE(muted); 1689 } 1690 1691 // The test first inserts a packet with narrow-band CNG, then a packet with 1692 // wide-band speech. The expected behavior is to detect a change in sample rate, 1693 // even though no speech packet has been inserted before, and flush out the CNG 1694 // packet. 1695 TEST_F(NetEqImplTest, CngFirstThenSpeechWithNewSampleRate) { 1696 UseNoMocks(); 1697 CreateInstance(); 1698 constexpr int kCnPayloadType = 7; 1699 neteq_->RegisterPayloadType(kCnPayloadType, SdpAudioFormat("cn", 8000, 1)); 1700 constexpr int kSpeechPayloadType = 8; 1701 neteq_->RegisterPayloadType(kSpeechPayloadType, 1702 SdpAudioFormat("l16", 16000, 1)); 1703 1704 RTPHeader header; 1705 header.payloadType = kCnPayloadType; 1706 uint8_t payload[320] = {0}; 1707 1708 EXPECT_EQ(neteq_->InsertPacket(header, payload), NetEq::kOK); 1709 EXPECT_EQ(neteq_->GetLifetimeStatistics().packets_discarded, 0u); 1710 1711 header.payloadType = kSpeechPayloadType; 1712 header.timestamp += 160; 1713 EXPECT_EQ(neteq_->InsertPacket(header, payload), NetEq::kOK); 1714 // CN packet should be discarded, since it does not match the 1715 // new speech sample rate. 1716 EXPECT_EQ(neteq_->GetLifetimeStatistics().packets_discarded, 1u); 1717 1718 // Next decoded packet should be speech. 1719 AudioFrame audio_frame; 1720 bool muted; 1721 EXPECT_EQ(neteq_->GetAudio(&audio_frame, &muted), NetEq::kOK); 1722 EXPECT_EQ(audio_frame.sample_rate_hz(), 16000); 1723 EXPECT_EQ(audio_frame.speech_type_, AudioFrame::SpeechType::kNormalSpeech); 1724 } 1725 1726 TEST_F(NetEqImplTest, InsertPacketChangePayloadType) { 1727 UseNoMocks(); 1728 CreateInstance(); 1729 constexpr int kPcmuPayloadType = 7; 1730 neteq_->RegisterPayloadType(kPcmuPayloadType, 1731 SdpAudioFormat("pcmu", 8000, 1)); 1732 constexpr int kPcmaPayloadType = 8; 1733 neteq_->RegisterPayloadType(kPcmaPayloadType, 1734 SdpAudioFormat("pcma", 8000, 1)); 1735 1736 RTPHeader header; 1737 header.payloadType = kPcmuPayloadType; 1738 header.timestamp = 1234; 1739 uint8_t payload[160] = {0}; 1740 1741 std::optional<NetEq::DecoderFormat> decoder = 1742 neteq_->GetCurrentDecoderFormat(); 1743 EXPECT_FALSE(decoder.has_value()); 1744 1745 EXPECT_EQ(neteq_->InsertPacket(header, payload), NetEq::kOK); 1746 EXPECT_EQ(neteq_->GetLifetimeStatistics().packets_discarded, 0u); 1747 decoder = neteq_->GetCurrentDecoderFormat(); 1748 ASSERT_TRUE(decoder.has_value()); 1749 EXPECT_EQ(decoder->payload_type, kPcmuPayloadType); 1750 EXPECT_EQ(decoder->sdp_format.name, "pcmu"); 1751 1752 header.payloadType = kPcmaPayloadType; 1753 header.timestamp += 80; 1754 EXPECT_EQ(neteq_->InsertPacket(header, payload), NetEq::kOK); 1755 decoder = neteq_->GetCurrentDecoderFormat(); 1756 ASSERT_TRUE(decoder.has_value()); 1757 EXPECT_EQ(decoder->payload_type, kPcmaPayloadType); 1758 EXPECT_EQ(decoder->sdp_format.name, "pcma"); 1759 // The previous packet should be discarded since the codec changed. 1760 EXPECT_EQ(neteq_->GetLifetimeStatistics().packets_discarded, 1u); 1761 1762 // Next decoded packet should be speech. 1763 AudioFrame audio_frame; 1764 bool muted; 1765 EXPECT_EQ(neteq_->GetAudio(&audio_frame, &muted), NetEq::kOK); 1766 EXPECT_EQ(audio_frame.sample_rate_hz(), 8000); 1767 EXPECT_EQ(audio_frame.speech_type_, AudioFrame::SpeechType::kNormalSpeech); 1768 } 1769 1770 class Decoder120ms : public AudioDecoder { 1771 public: 1772 Decoder120ms(int sample_rate_hz, SpeechType speech_type) 1773 : sample_rate_hz_(sample_rate_hz), 1774 next_value_(1), 1775 speech_type_(speech_type) {} 1776 1777 int DecodeInternal(const uint8_t* /* encoded */, 1778 size_t /* encoded_len */, 1779 int sample_rate_hz, 1780 int16_t* decoded, 1781 SpeechType* speech_type) override { 1782 EXPECT_EQ(sample_rate_hz_, sample_rate_hz); 1783 size_t decoded_len = 1784 CheckedDivExact(sample_rate_hz, 1000) * 120 * Channels(); 1785 for (size_t i = 0; i < decoded_len; ++i) { 1786 decoded[i] = next_value_++; 1787 } 1788 *speech_type = speech_type_; 1789 return checked_cast<int>(decoded_len); 1790 } 1791 1792 void Reset() override { next_value_ = 1; } 1793 int SampleRateHz() const override { return sample_rate_hz_; } 1794 size_t Channels() const override { return 2; } 1795 1796 private: 1797 int sample_rate_hz_; 1798 int16_t next_value_; 1799 SpeechType speech_type_; 1800 }; 1801 1802 class NetEqImplTest120ms : public NetEqImplTest { 1803 protected: 1804 NetEqImplTest120ms() : NetEqImplTest() {} 1805 ~NetEqImplTest120ms() override {} 1806 1807 void CreateInstanceNoMocks() { 1808 UseNoMocks(); 1809 CreateInstance(decoder_factory_); 1810 EXPECT_TRUE(neteq_->RegisterPayloadType( 1811 kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}}))); 1812 } 1813 1814 void CreateInstanceWithDelayManagerMock() { 1815 UseNoMocks(); 1816 use_mock_neteq_controller_ = true; 1817 CreateInstance(decoder_factory_); 1818 EXPECT_TRUE(neteq_->RegisterPayloadType( 1819 kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}}))); 1820 } 1821 1822 uint32_t timestamp_diff_between_packets() const { 1823 return CheckedDivExact(kSamplingFreq_, 1000u) * 120; 1824 } 1825 1826 uint32_t first_timestamp() const { return 10u; } 1827 1828 void GetFirstPacket() { 1829 bool muted; 1830 for (int i = 0; i < 12; i++) { 1831 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1832 EXPECT_FALSE(muted); 1833 } 1834 } 1835 1836 void InsertPacket(uint32_t timestamp) { 1837 RTPHeader rtp_header; 1838 rtp_header.payloadType = kPayloadType; 1839 rtp_header.sequenceNumber = sequence_number_; 1840 rtp_header.timestamp = timestamp; 1841 rtp_header.ssrc = 15; 1842 const size_t kPayloadLengthBytes = 1; // This can be arbitrary. 1843 uint8_t payload[kPayloadLengthBytes] = {0}; 1844 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); 1845 sequence_number_++; 1846 } 1847 1848 void Register120msCodec(AudioDecoder::SpeechType speech_type) { 1849 const uint32_t sampling_freq = kSamplingFreq_; 1850 decoder_factory_ = make_ref_counted<test::FunctionAudioDecoderFactory>( 1851 [sampling_freq, speech_type]() { 1852 std::unique_ptr<AudioDecoder> decoder = 1853 std::make_unique<Decoder120ms>(sampling_freq, speech_type); 1854 RTC_CHECK_EQ(2, decoder->Channels()); 1855 return decoder; 1856 }); 1857 } 1858 1859 scoped_refptr<AudioDecoderFactory> decoder_factory_; 1860 AudioFrame output_; 1861 const uint32_t kPayloadType = 17; 1862 const uint32_t kSamplingFreq_ = 48000; 1863 uint16_t sequence_number_ = 1; 1864 }; 1865 1866 TEST_F(NetEqImplTest120ms, CodecInternalCng) { 1867 Register120msCodec(AudioDecoder::kComfortNoise); 1868 CreateInstanceNoMocks(); 1869 1870 InsertPacket(first_timestamp()); 1871 GetFirstPacket(); 1872 1873 bool muted; 1874 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1875 EXPECT_EQ(NetEq::Operation::kCodecInternalCng, 1876 neteq_->last_operation_for_test()); 1877 } 1878 1879 TEST_F(NetEqImplTest120ms, Normal) { 1880 Register120msCodec(AudioDecoder::kSpeech); 1881 CreateInstanceNoMocks(); 1882 1883 InsertPacket(first_timestamp()); 1884 GetFirstPacket(); 1885 1886 EXPECT_EQ(NetEq::Operation::kNormal, neteq_->last_operation_for_test()); 1887 } 1888 1889 TEST_F(NetEqImplTest120ms, Merge) { 1890 Register120msCodec(AudioDecoder::kSpeech); 1891 CreateInstanceWithDelayManagerMock(); 1892 1893 InsertPacket(first_timestamp()); 1894 1895 GetFirstPacket(); 1896 bool muted; 1897 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) 1898 .WillOnce(Return(NetEq::Operation::kExpand)); 1899 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1900 1901 InsertPacket(first_timestamp() + 2 * timestamp_diff_between_packets()); 1902 1903 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) 1904 .WillOnce(Return(NetEq::Operation::kMerge)); 1905 1906 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1907 EXPECT_EQ(NetEq::Operation::kMerge, neteq_->last_operation_for_test()); 1908 } 1909 1910 TEST_F(NetEqImplTest120ms, Expand) { 1911 Register120msCodec(AudioDecoder::kSpeech); 1912 CreateInstanceNoMocks(); 1913 1914 InsertPacket(first_timestamp()); 1915 GetFirstPacket(); 1916 1917 bool muted; 1918 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1919 EXPECT_EQ(NetEq::Operation::kExpand, neteq_->last_operation_for_test()); 1920 } 1921 1922 TEST_F(NetEqImplTest120ms, FastAccelerate) { 1923 Register120msCodec(AudioDecoder::kSpeech); 1924 CreateInstanceWithDelayManagerMock(); 1925 1926 InsertPacket(first_timestamp()); 1927 GetFirstPacket(); 1928 InsertPacket(first_timestamp() + timestamp_diff_between_packets()); 1929 1930 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) 1931 .Times(1) 1932 .WillOnce(Return(NetEq::Operation::kFastAccelerate)); 1933 1934 bool muted; 1935 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1936 EXPECT_EQ(NetEq::Operation::kFastAccelerate, 1937 neteq_->last_operation_for_test()); 1938 } 1939 1940 TEST_F(NetEqImplTest120ms, PreemptiveExpand) { 1941 Register120msCodec(AudioDecoder::kSpeech); 1942 CreateInstanceWithDelayManagerMock(); 1943 1944 InsertPacket(first_timestamp()); 1945 GetFirstPacket(); 1946 1947 InsertPacket(first_timestamp() + timestamp_diff_between_packets()); 1948 1949 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) 1950 .Times(1) 1951 .WillOnce(Return(NetEq::Operation::kPreemptiveExpand)); 1952 1953 bool muted; 1954 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1955 EXPECT_EQ(NetEq::Operation::kPreemptiveExpand, 1956 neteq_->last_operation_for_test()); 1957 } 1958 1959 TEST_F(NetEqImplTest120ms, Accelerate) { 1960 Register120msCodec(AudioDecoder::kSpeech); 1961 CreateInstanceWithDelayManagerMock(); 1962 1963 InsertPacket(first_timestamp()); 1964 GetFirstPacket(); 1965 1966 InsertPacket(first_timestamp() + timestamp_diff_between_packets()); 1967 1968 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) 1969 .Times(1) 1970 .WillOnce(Return(NetEq::Operation::kAccelerate)); 1971 1972 bool muted; 1973 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); 1974 EXPECT_EQ(NetEq::Operation::kAccelerate, neteq_->last_operation_for_test()); 1975 } 1976 1977 } // namespace webrtc