neteq_quality_test.cc (17967B)
1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_coding/neteq/tools/neteq_quality_test.h" 12 13 #include <algorithm> 14 #include <climits> 15 #include <cmath> 16 #include <cstdint> 17 #include <cstdio> 18 #include <cstdlib> 19 #include <fstream> 20 #include <iterator> 21 #include <memory> 22 #include <ostream> 23 #include <set> 24 #include <sstream> 25 #include <string> 26 #include <utility> 27 #include <vector> 28 29 #include "absl/flags/flag.h" 30 #include "absl/strings/string_view.h" 31 #include "api/array_view.h" 32 #include "api/audio_codecs/audio_decoder_factory.h" 33 #include "api/audio_codecs/audio_format.h" 34 #include "api/environment/environment_factory.h" 35 #include "api/neteq/default_neteq_factory.h" 36 #include "api/neteq/neteq.h" 37 #include "api/scoped_refptr.h" 38 #include "api/units/timestamp.h" 39 #include "modules/audio_coding/neteq/tools/neteq_quality_test.h" 40 #include "modules/audio_coding/neteq/tools/output_audio_file.h" 41 #include "modules/audio_coding/neteq/tools/output_wav_file.h" 42 #include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" 43 #include "modules/audio_coding/neteq/tools/rtp_generator.h" 44 #include "rtc_base/checks.h" 45 #include "rtc_base/string_encode.h" 46 #include "test/gtest.h" 47 #include "test/testsupport/file_utils.h" 48 49 ABSL_FLAG(std::string, 50 in_filename, 51 "audio_coding/speech_mono_16kHz.pcm", 52 "Path of the input file (relative to the resources/ directory) for " 53 "input audio (specify sample rate with --input_sample_rate, " 54 "and channels with --channels)."); 55 56 ABSL_FLAG(int, input_sample_rate, 16000, "Sample rate of input file in Hz."); 57 58 ABSL_FLAG(int, channels, 1, "Number of channels in input audio."); 59 60 ABSL_FLAG(std::string, 61 out_filename, 62 "neteq_quality_test_out.pcm", 63 "Name of output audio file, which will be saved in " + 64 ::webrtc::test::OutputPath()); 65 66 ABSL_FLAG( 67 int, 68 runtime_ms, 69 10000, 70 "Simulated runtime (milliseconds). -1 will consume the complete file."); 71 72 ABSL_FLAG(int, packet_loss_rate, 10, "Percentile of packet loss."); 73 74 ABSL_FLAG(int, 75 random_loss_mode, 76 ::webrtc::test::kUniformLoss, 77 "Random loss mode: 0--no loss, 1--uniform loss, 2--Gilbert Elliot " 78 "loss, 3--fixed loss."); 79 80 ABSL_FLAG(int, 81 burst_length, 82 30, 83 "Burst length in milliseconds, only valid for Gilbert Elliot loss."); 84 85 ABSL_FLAG(float, drift_factor, 0.0, "Time drift factor."); 86 87 ABSL_FLAG(int, 88 preload_packets, 89 1, 90 "Preload the buffer with this many packets."); 91 92 ABSL_FLAG(std::string, 93 loss_events, 94 "", 95 "List of loss events time and duration separated by comma: " 96 "<first_event_time> <first_event_duration>, <second_event_time> " 97 "<second_event_duration>, ..."); 98 99 namespace webrtc { 100 namespace test { 101 102 namespace { 103 104 std::unique_ptr<NetEq> CreateNetEq( 105 const NetEq::Config& config, 106 scoped_refptr<AudioDecoderFactory> decoder_factory) { 107 return DefaultNetEqFactory().Create(CreateEnvironment(), config, 108 std::move(decoder_factory)); 109 } 110 111 const std::string& GetInFilenamePath(absl::string_view file_name) { 112 std::vector<absl::string_view> name_parts = split(file_name, '.'); 113 RTC_CHECK_EQ(name_parts.size(), 2); 114 static const std::string path = 115 test::ResourcePath(name_parts[0], name_parts[1]); 116 return path; 117 } 118 119 const std::string& GetOutFilenamePath(absl::string_view file_name) { 120 static const std::string path = test::OutputPath() + std::string(file_name); 121 return path; 122 } 123 124 } // namespace 125 126 constexpr uint8_t kPayloadType = 95; 127 constexpr int kOutputSizeMs = 10; 128 constexpr int kInitSeed = 0x12345678; 129 constexpr int kPacketLossTimeUnitMs = 10; 130 131 // Common validator for file names. 132 static bool ValidateFilename(absl::string_view value, bool is_output) { 133 if (!is_output) { 134 RTC_CHECK_NE(value.substr(value.find_last_of('.') + 1), "wav") 135 << "WAV file input is not supported"; 136 } 137 FILE* fid = is_output ? fopen(std::string(value).c_str(), "wb") 138 : fopen(std::string(value).c_str(), "rb"); 139 if (fid == nullptr) 140 return false; 141 fclose(fid); 142 return true; 143 } 144 145 // ProbTrans00Solver() is to calculate the transition probability from no-loss 146 // state to itself in a modified Gilbert Elliot packet loss model. The result is 147 // to achieve the target packet loss rate `loss_rate`, when a packet is not 148 // lost only if all `units` drawings within the duration of the packet result in 149 // no-loss. 150 static double ProbTrans00Solver(int units, 151 double loss_rate, 152 double prob_trans_10) { 153 if (units == 1) 154 return prob_trans_10 / (1.0f - loss_rate) - prob_trans_10; 155 // 0 == prob_trans_00 ^ (units - 1) + (1 - loss_rate) / prob_trans_10 * 156 // prob_trans_00 - (1 - loss_rate) * (1 + 1 / prob_trans_10). 157 // There is a unique solution between 0.0 and 1.0, due to the monotonicity and 158 // an opposite sign at 0.0 and 1.0. 159 // For simplicity, we reformulate the equation as 160 // f(x) = x ^ (units - 1) + a x + b. 161 // Its derivative is 162 // f'(x) = (units - 1) x ^ (units - 2) + a. 163 // The derivative is strictly greater than 0 when x is between 0 and 1. 164 // We use Newton's method to solve the equation, iteration is 165 // x(k+1) = x(k) - f(x) / f'(x); 166 const double kPrecision = 0.001f; 167 const int kIterations = 100; 168 const double a = (1.0f - loss_rate) / prob_trans_10; 169 const double b = (loss_rate - 1.0f) * (1.0f + 1.0f / prob_trans_10); 170 double x = 0.0; // Starting point; 171 double f = b; 172 double f_p; 173 int iter = 0; 174 while ((f >= kPrecision || f <= -kPrecision) && iter < kIterations) { 175 f_p = (units - 1.0f) * std::pow(x, units - 2) + a; 176 x -= f / f_p; 177 if (x > 1.0f) { 178 x = 1.0f; 179 } else if (x < 0.0f) { 180 x = 0.0f; 181 } 182 f = std::pow(x, units - 1) + a * x + b; 183 iter++; 184 } 185 return x; 186 } 187 188 NetEqQualityTest::NetEqQualityTest( 189 int block_duration_ms, 190 int in_sampling_khz, 191 int out_sampling_khz, 192 const SdpAudioFormat& format, 193 const scoped_refptr<AudioDecoderFactory>& decoder_factory) 194 : audio_format_(format), 195 channels_(absl::GetFlag(FLAGS_channels)), 196 decoded_time_ms_(0), 197 decodable_time_ms_(0), 198 drift_factor_(absl::GetFlag(FLAGS_drift_factor)), 199 packet_loss_rate_(absl::GetFlag(FLAGS_packet_loss_rate)), 200 block_duration_ms_(block_duration_ms), 201 in_sampling_khz_(in_sampling_khz), 202 out_sampling_khz_(out_sampling_khz), 203 in_size_samples_( 204 static_cast<size_t>(in_sampling_khz_ * block_duration_ms_)), 205 payload_size_bytes_(0), 206 max_payload_bytes_(0), 207 in_file_(new ResampleInputAudioFile( 208 GetInFilenamePath(absl::GetFlag(FLAGS_in_filename)), 209 absl::GetFlag(FLAGS_input_sample_rate), 210 in_sampling_khz * 1000, 211 absl::GetFlag(FLAGS_runtime_ms) > 0)), 212 rtp_generator_( 213 new RtpGenerator(in_sampling_khz_, 0, 0, decodable_time_ms_)), 214 total_payload_size_bytes_(0) { 215 // Flag validation 216 RTC_CHECK(ValidateFilename( 217 GetInFilenamePath(absl::GetFlag(FLAGS_in_filename)), false)) 218 << "Invalid input filename."; 219 220 RTC_CHECK(absl::GetFlag(FLAGS_input_sample_rate) == 8000 || 221 absl::GetFlag(FLAGS_input_sample_rate) == 16000 || 222 absl::GetFlag(FLAGS_input_sample_rate) == 32000 || 223 absl::GetFlag(FLAGS_input_sample_rate) == 48000) 224 << "Invalid sample rate should be 8000, 16000, 32000 or 48000 Hz."; 225 226 RTC_CHECK_EQ(absl::GetFlag(FLAGS_channels), 1) 227 << "Invalid number of channels, current support only 1."; 228 229 RTC_CHECK(ValidateFilename( 230 GetOutFilenamePath(absl::GetFlag(FLAGS_out_filename)), true)) 231 << "Invalid output filename."; 232 233 RTC_CHECK(absl::GetFlag(FLAGS_packet_loss_rate) >= 0 && 234 absl::GetFlag(FLAGS_packet_loss_rate) <= 100) 235 << "Invalid packet loss percentile, should be between 0 and 100."; 236 237 RTC_CHECK(absl::GetFlag(FLAGS_random_loss_mode) >= 0 && 238 absl::GetFlag(FLAGS_random_loss_mode) < kLastLossMode) 239 << "Invalid random packet loss mode, should be between 0 and " 240 << kLastLossMode - 1 << "."; 241 242 RTC_CHECK_GE(absl::GetFlag(FLAGS_burst_length), kPacketLossTimeUnitMs) 243 << "Invalid burst length, should be greater than or equal to " 244 << kPacketLossTimeUnitMs << " ms."; 245 246 RTC_CHECK_GT(absl::GetFlag(FLAGS_drift_factor), -0.1) 247 << "Invalid drift factor, should be greater than -0.1."; 248 249 RTC_CHECK_GE(absl::GetFlag(FLAGS_preload_packets), 0) 250 << "Invalid number of packets to preload; must be non-negative."; 251 252 const std::string out_filename = 253 GetOutFilenamePath(absl::GetFlag(FLAGS_out_filename)); 254 const std::string log_filename = out_filename + ".log"; 255 log_file_.open(log_filename.c_str(), std::ofstream::out); 256 RTC_CHECK(log_file_.is_open()); 257 258 if (out_filename.size() >= 4 && 259 out_filename.substr(out_filename.size() - 4) == ".wav") { 260 // Open a wav file. 261 output_.reset( 262 new test::OutputWavFile(out_filename, 1000 * out_sampling_khz)); 263 } else { 264 // Open a pcm file. 265 output_.reset(new test::OutputAudioFile(out_filename)); 266 } 267 268 NetEq::Config config; 269 config.sample_rate_hz = out_sampling_khz_ * 1000; 270 neteq_ = CreateNetEq(config, decoder_factory); 271 max_payload_bytes_ = in_size_samples_ * channels_ * sizeof(int16_t); 272 in_data_.reset(new int16_t[in_size_samples_ * channels_]); 273 } 274 275 NetEqQualityTest::~NetEqQualityTest() { 276 log_file_.close(); 277 } 278 279 bool NoLoss::Lost(int /* now_ms */) { 280 return false; 281 } 282 283 UniformLoss::UniformLoss(double loss_rate) : loss_rate_(loss_rate) {} 284 285 bool UniformLoss::Lost(int /* now_ms */) { 286 int drop_this = rand(); 287 return (drop_this < loss_rate_ * RAND_MAX); 288 } 289 290 GilbertElliotLoss::GilbertElliotLoss(double prob_trans_11, double prob_trans_01) 291 : prob_trans_11_(prob_trans_11), 292 prob_trans_01_(prob_trans_01), 293 lost_last_(false), 294 uniform_loss_model_(new UniformLoss(0)) {} 295 296 GilbertElliotLoss::~GilbertElliotLoss() {} 297 298 bool GilbertElliotLoss::Lost(int now_ms) { 299 // Simulate bursty channel (Gilbert model). 300 // (1st order) Markov chain model with memory of the previous/last 301 // packet state (lost or received). 302 if (lost_last_) { 303 // Previous packet was not received. 304 uniform_loss_model_->set_loss_rate(prob_trans_11_); 305 return lost_last_ = uniform_loss_model_->Lost(now_ms); 306 } else { 307 uniform_loss_model_->set_loss_rate(prob_trans_01_); 308 return lost_last_ = uniform_loss_model_->Lost(now_ms); 309 } 310 } 311 312 FixedLossModel::FixedLossModel( 313 std::set<FixedLossEvent, FixedLossEventCmp> loss_events) 314 : loss_events_(loss_events) { 315 loss_events_it_ = loss_events_.begin(); 316 } 317 318 FixedLossModel::~FixedLossModel() {} 319 320 bool FixedLossModel::Lost(int now_ms) { 321 if (loss_events_it_ != loss_events_.end() && 322 now_ms > loss_events_it_->start_ms) { 323 if (now_ms <= loss_events_it_->start_ms + loss_events_it_->duration_ms) { 324 return true; 325 } else { 326 ++loss_events_it_; 327 return false; 328 } 329 } 330 return false; 331 } 332 333 void NetEqQualityTest::SetUp() { 334 ASSERT_TRUE(neteq_->RegisterPayloadType(kPayloadType, audio_format_)); 335 rtp_generator_->set_drift_factor(drift_factor_); 336 337 int units = block_duration_ms_ / kPacketLossTimeUnitMs; 338 switch (absl::GetFlag(FLAGS_random_loss_mode)) { 339 case kUniformLoss: { 340 // `unit_loss_rate` is the packet loss rate for each unit time interval 341 // (kPacketLossTimeUnitMs). Since a packet loss event is generated if any 342 // of |block_duration_ms_ / kPacketLossTimeUnitMs| unit time intervals of 343 // a full packet duration is drawn with a loss, `unit_loss_rate` fulfills 344 // (1 - unit_loss_rate) ^ (block_duration_ms_ / kPacketLossTimeUnitMs) == 345 // 1 - packet_loss_rate. 346 double unit_loss_rate = 347 (1.0 - std::pow(1.0 - 0.01 * packet_loss_rate_, 1.0 / units)); 348 loss_model_.reset(new UniformLoss(unit_loss_rate)); 349 break; 350 } 351 case kGilbertElliotLoss: { 352 // `FLAGS_burst_length` should be integer times of kPacketLossTimeUnitMs. 353 ASSERT_EQ(0, absl::GetFlag(FLAGS_burst_length) % kPacketLossTimeUnitMs); 354 355 // We do not allow 100 percent packet loss in Gilbert Elliot model, which 356 // makes no sense. 357 ASSERT_GT(100, packet_loss_rate_); 358 359 // To guarantee the overall packet loss rate, transition probabilities 360 // need to satisfy: 361 // pi_0 * (1 - prob_trans_01_) ^ units + 362 // pi_1 * prob_trans_10_ ^ (units - 1) == 1 - loss_rate 363 // pi_0 = prob_trans_10 / (prob_trans_10 + prob_trans_01_) 364 // is the stationary state probability of no-loss 365 // pi_1 = prob_trans_01_ / (prob_trans_10 + prob_trans_01_) 366 // is the stationary state probability of loss 367 // After a derivation prob_trans_00 should satisfy: 368 // prob_trans_00 ^ (units - 1) = (loss_rate - 1) / prob_trans_10 * 369 // prob_trans_00 + (1 - loss_rate) * (1 + 1 / prob_trans_10). 370 double loss_rate = 0.01f * packet_loss_rate_; 371 double prob_trans_10 = 372 1.0f * kPacketLossTimeUnitMs / absl::GetFlag(FLAGS_burst_length); 373 double prob_trans_00 = ProbTrans00Solver(units, loss_rate, prob_trans_10); 374 loss_model_.reset( 375 new GilbertElliotLoss(1.0f - prob_trans_10, 1.0f - prob_trans_00)); 376 break; 377 } 378 case kFixedLoss: { 379 std::istringstream loss_events_stream(absl::GetFlag(FLAGS_loss_events)); 380 std::string loss_event_string; 381 std::set<FixedLossEvent, FixedLossEventCmp> loss_events; 382 while (std::getline(loss_events_stream, loss_event_string, ',')) { 383 std::vector<int> loss_event_params; 384 std::istringstream loss_event_params_stream(loss_event_string); 385 std::copy(std::istream_iterator<int>(loss_event_params_stream), 386 std::istream_iterator<int>(), 387 std::back_inserter(loss_event_params)); 388 RTC_CHECK_EQ(loss_event_params.size(), 2); 389 auto result = loss_events.insert( 390 FixedLossEvent(loss_event_params[0], loss_event_params[1])); 391 RTC_CHECK(result.second); 392 } 393 RTC_CHECK_GT(loss_events.size(), 0); 394 loss_model_.reset(new FixedLossModel(loss_events)); 395 break; 396 } 397 default: { 398 loss_model_.reset(new NoLoss); 399 break; 400 } 401 } 402 403 // Make sure that the packet loss profile is same for all derived tests. 404 srand(kInitSeed); 405 } 406 407 std::ofstream& NetEqQualityTest::Log() { 408 return log_file_; 409 } 410 411 bool NetEqQualityTest::PacketLost() { 412 int cycles = block_duration_ms_ / kPacketLossTimeUnitMs; 413 414 // The loop is to make sure that codecs with different block lengths share the 415 // same packet loss profile. 416 bool lost = false; 417 for (int idx = 0; idx < cycles; idx++) { 418 if (loss_model_->Lost(decoded_time_ms_)) { 419 // The packet will be lost if any of the drawings indicates a loss, but 420 // the loop has to go on to make sure that codecs with different block 421 // lengths keep the same pace. 422 lost = true; 423 } 424 } 425 return lost; 426 } 427 428 int NetEqQualityTest::Transmit() { 429 int packet_input_time_ms = rtp_generator_->GetRtpHeader( 430 kPayloadType, in_size_samples_, &rtp_header_); 431 Log() << "Packet of size " << payload_size_bytes_ << " bytes, for frame at " 432 << packet_input_time_ms << " ms "; 433 if (payload_size_bytes_ > 0) { 434 if (!PacketLost()) { 435 int ret = neteq_->InsertPacket( 436 rtp_header_, 437 ArrayView<const uint8_t>(payload_.data(), payload_size_bytes_), 438 Timestamp::Millis(packet_input_time_ms)); 439 if (ret != NetEq::kOK) 440 return -1; 441 Log() << "was sent."; 442 } else { 443 Log() << "was lost."; 444 } 445 } 446 Log() << std::endl; 447 return packet_input_time_ms; 448 } 449 450 int NetEqQualityTest::DecodeBlock() { 451 bool muted; 452 int ret = neteq_->GetAudio(&out_frame_, &muted); 453 RTC_CHECK(!muted); 454 455 if (ret != NetEq::kOK) { 456 return -1; 457 } else { 458 RTC_DCHECK_EQ(out_frame_.num_channels_, channels_); 459 RTC_DCHECK_EQ(out_frame_.samples_per_channel_, 460 static_cast<size_t>(kOutputSizeMs * out_sampling_khz_)); 461 RTC_CHECK(output_->WriteArray( 462 out_frame_.data(), 463 out_frame_.samples_per_channel_ * out_frame_.num_channels_)); 464 return static_cast<int>(out_frame_.samples_per_channel_); 465 } 466 } 467 468 void NetEqQualityTest::Simulate() { 469 int audio_size_samples; 470 bool end_of_input = false; 471 int runtime_ms = absl::GetFlag(FLAGS_runtime_ms) >= 0 472 ? absl::GetFlag(FLAGS_runtime_ms) 473 : INT_MAX; 474 475 while (!end_of_input && decoded_time_ms_ < runtime_ms) { 476 // Preload the buffer if needed. 477 while (decodable_time_ms_ - 478 absl::GetFlag(FLAGS_preload_packets) * block_duration_ms_ < 479 decoded_time_ms_) { 480 if (!in_file_->Read(in_size_samples_ * channels_, &in_data_[0])) { 481 end_of_input = true; 482 ASSERT_TRUE(end_of_input && absl::GetFlag(FLAGS_runtime_ms) < 0); 483 break; 484 } 485 payload_.Clear(); 486 payload_size_bytes_ = EncodeBlock(&in_data_[0], in_size_samples_, 487 &payload_, max_payload_bytes_); 488 total_payload_size_bytes_ += payload_size_bytes_; 489 decodable_time_ms_ = Transmit() + block_duration_ms_; 490 } 491 audio_size_samples = DecodeBlock(); 492 if (audio_size_samples > 0) { 493 decoded_time_ms_ += audio_size_samples / out_sampling_khz_; 494 } 495 } 496 Log() << "Average bit rate was " 497 << 8.0f * total_payload_size_bytes_ / absl::GetFlag(FLAGS_runtime_ms) 498 << " kbps" << std::endl; 499 } 500 501 } // namespace test 502 } // namespace webrtc