neteq_test.cc (14893B)
1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_coding/neteq/tools/neteq_test.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <fstream> 16 #include <iomanip> 17 #include <iostream> 18 #include <memory> 19 #include <optional> 20 #include <utility> 21 22 #include "absl/strings/string_view.h" 23 #include "api/array_view.h" 24 #include "api/audio/audio_frame.h" 25 #include "api/audio_codecs/audio_decoder_factory.h" 26 #include "api/audio_codecs/audio_format.h" 27 #include "api/environment/environment.h" 28 #include "api/environment/environment_factory.h" 29 #include "api/field_trials.h" 30 #include "api/neteq/default_neteq_factory.h" 31 #include "api/neteq/neteq.h" 32 #include "api/neteq/neteq_factory.h" 33 #include "api/rtp_headers.h" 34 #include "api/scoped_refptr.h" 35 #include "api/test/neteq_simulator.h" 36 #include "api/units/timestamp.h" 37 #include "modules/audio_coding/neteq/tools/audio_sink.h" 38 #include "modules/audio_coding/neteq/tools/neteq_input.h" 39 #include "modules/rtp_rtcp/source/byte_io.h" 40 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 41 #include "rtc_base/checks.h" 42 #include "system_wrappers/include/clock.h" 43 44 namespace webrtc { 45 namespace test { 46 namespace { 47 48 std::optional<NetEq::Operation> ActionToOperations( 49 std::optional<NetEqSimulator::Action> a) { 50 if (!a) { 51 return std::nullopt; 52 } 53 switch (*a) { 54 case NetEqSimulator::Action::kAccelerate: 55 return std::make_optional(NetEq::Operation::kAccelerate); 56 case NetEqSimulator::Action::kExpand: 57 return std::make_optional(NetEq::Operation::kExpand); 58 case NetEqSimulator::Action::kNormal: 59 return std::make_optional(NetEq::Operation::kNormal); 60 case NetEqSimulator::Action::kPreemptiveExpand: 61 return std::make_optional(NetEq::Operation::kPreemptiveExpand); 62 } 63 } 64 65 std::unique_ptr<NetEq> CreateNetEq( 66 const Environment& env, 67 const NetEq::Config& config, 68 scoped_refptr<AudioDecoderFactory> decoder_factory) { 69 return DefaultNetEqFactory().Create(env, config, std::move(decoder_factory)); 70 } 71 72 } // namespace 73 74 void DefaultNetEqTestErrorCallback::OnInsertPacketError( 75 const RtpPacketReceived& packet) { 76 std::cerr << "InsertPacket returned an error." << std::endl; 77 std::cerr << "Packet data: " << NetEqInput::ToString(packet) << std::endl; 78 RTC_FATAL(); 79 } 80 81 void DefaultNetEqTestErrorCallback::OnGetAudioError() { 82 std::cerr << "GetAudio returned an error." << std::endl; 83 RTC_FATAL(); 84 } 85 86 NetEqTest::NetEqTest(const NetEq::Config& config, 87 scoped_refptr<AudioDecoderFactory> decoder_factory, 88 const DecoderMap& codecs, 89 std::unique_ptr<std::ofstream> text_log, 90 NetEqFactory* neteq_factory, 91 std::unique_ptr<NetEqInput> input, 92 std::unique_ptr<AudioSink> output, 93 Callbacks callbacks, 94 absl::string_view field_trials) 95 : input_(std::move(input)), 96 clock_(Timestamp::Millis(input_->NextEventTime().value_or(0))), 97 env_(CreateEnvironment(&clock_, 98 std::make_unique<FieldTrials>(field_trials))), 99 neteq_( 100 neteq_factory 101 ? neteq_factory->Create(env_, config, std::move(decoder_factory)) 102 : CreateNetEq(env_, config, std::move(decoder_factory))), 103 output_(std::move(output)), 104 callbacks_(callbacks), 105 sample_rate_hz_(config.sample_rate_hz), 106 text_log_(std::move(text_log)) { 107 RegisterDecoders(codecs); 108 } 109 110 NetEqTest::~NetEqTest() = default; 111 112 int64_t NetEqTest::Run() { 113 int64_t simulation_time = 0; 114 SimulationStepResult step_result; 115 do { 116 step_result = RunToNextGetAudio(); 117 simulation_time += step_result.simulation_step_ms; 118 } while (!step_result.is_simulation_finished); 119 if (callbacks_.simulation_ended_callback) { 120 callbacks_.simulation_ended_callback->SimulationEnded(simulation_time); 121 } 122 return simulation_time; 123 } 124 125 NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { 126 SimulationStepResult result; 127 const int64_t start_time_ms = *input_->NextEventTime(); 128 int64_t time_now_ms = clock_.CurrentTime().ms(); 129 current_state_.packet_iat_ms.clear(); 130 131 while (!input_->ended()) { 132 // Advance time to next event. 133 RTC_DCHECK(input_->NextEventTime()); 134 clock_.AdvanceTimeMilliseconds(*input_->NextEventTime() - time_now_ms); 135 time_now_ms = *input_->NextEventTime(); 136 // Check if it is time to insert packet. 137 if (input_->NextPacketTime() && time_now_ms >= *input_->NextPacketTime()) { 138 std::unique_ptr<RtpPacketReceived> packet_data = input_->PopPacket(); 139 RTC_CHECK(packet_data); 140 const size_t payload_data_length = packet_data->payload_size(); 141 RTPHeader rtp_header; 142 packet_data->GetHeader(&rtp_header); 143 if (payload_data_length != 0) { 144 int error = neteq_->InsertPacket(rtp_header, packet_data->payload(), 145 Timestamp::Millis(time_now_ms)); 146 if (error != NetEq::kOK && callbacks_.error_callback) { 147 callbacks_.error_callback->OnInsertPacketError(*packet_data); 148 } 149 if (callbacks_.post_insert_packet) { 150 callbacks_.post_insert_packet->AfterInsertPacket(*packet_data, 151 neteq_.get()); 152 } 153 } else { 154 neteq_->InsertEmptyPacket(rtp_header); 155 } 156 if (last_packet_time_ms_) { 157 current_state_.packet_iat_ms.push_back(time_now_ms - 158 *last_packet_time_ms_); 159 } 160 if (text_log_) { 161 const auto ops_state = neteq_->GetOperationsAndState(); 162 const auto delta_wallclock = 163 last_packet_time_ms_ ? (time_now_ms - *last_packet_time_ms_) : -1; 164 const auto delta_timestamp = 165 last_packet_timestamp_ 166 ? (static_cast<int64_t>(packet_data->Timestamp()) - 167 *last_packet_timestamp_) * 168 1000 / sample_rate_hz_ 169 : -1; 170 const auto packet_size_bytes = 171 packet_data->payload_size() == 12 172 ? ByteReader<uint32_t>::ReadLittleEndian( 173 &packet_data->payload()[8]) 174 : -1; 175 *text_log_ << "Packet - wallclock: " << std::setw(5) << time_now_ms 176 << ", delta wc: " << std::setw(4) << delta_wallclock 177 << ", seq_no: " << packet_data->SequenceNumber() 178 << ", timestamp: " << std::setw(10) 179 << packet_data->Timestamp() // 180 << ", delta ts: " << std::setw(4) << delta_timestamp // 181 << ", size: " << std::setw(5) << packet_size_bytes // 182 << ", frame size: " << std::setw(3) 183 << ops_state.current_frame_size_ms 184 << ", buffer size: " << std::setw(4) 185 << ops_state.current_buffer_size_ms << std::endl; 186 } 187 last_packet_time_ms_ = time_now_ms; 188 last_packet_timestamp_ = packet_data->Timestamp(); 189 } 190 191 if (input_->NextSetMinimumDelayInfo().has_value() && 192 time_now_ms >= input_->NextSetMinimumDelayInfo().value().timestamp_ms) { 193 neteq_->SetBaseMinimumDelayMs( 194 input_->NextSetMinimumDelayInfo().value().delay_ms); 195 input_->AdvanceSetMinimumDelay(); 196 } 197 198 // Check if it is time to get output audio. 199 if (input_->NextOutputEventTime() && 200 time_now_ms >= *input_->NextOutputEventTime()) { 201 if (callbacks_.get_audio_callback) { 202 callbacks_.get_audio_callback->BeforeGetAudio(neteq_.get()); 203 } 204 AudioFrame out_frame; 205 int error = neteq_->GetAudio(&out_frame, nullptr, nullptr, 206 ActionToOperations(next_action_)); 207 next_action_ = std::nullopt; 208 if (error != NetEq::kOK) { 209 if (callbacks_.error_callback) { 210 callbacks_.error_callback->OnGetAudioError(); 211 } 212 } else { 213 sample_rate_hz_ = out_frame.sample_rate_hz_; 214 } 215 if (callbacks_.get_audio_callback) { 216 callbacks_.get_audio_callback->AfterGetAudio( 217 time_now_ms, out_frame, out_frame.muted(), neteq_.get()); 218 } 219 220 if (output_) { 221 RTC_CHECK(output_->WriteArray( 222 out_frame.data(), 223 out_frame.samples_per_channel_ * out_frame.num_channels_)); 224 } 225 226 input_->AdvanceOutputEvent(); 227 result.simulation_step_ms = 228 input_->NextEventTime().value_or(time_now_ms) - start_time_ms; 229 const auto operations_state = neteq_->GetOperationsAndState(); 230 current_state_.current_delay_ms = operations_state.current_buffer_size_ms; 231 current_state_.packet_size_ms = operations_state.current_frame_size_ms; 232 current_state_.next_packet_available = 233 operations_state.next_packet_available; 234 current_state_.packet_buffer_flushed = 235 operations_state.packet_buffer_flushes > 236 prev_ops_state_.packet_buffer_flushes; 237 // TODO(ivoc): Add more accurate reporting by tracking the origin of 238 // samples in the sync buffer. 239 result.action_times_ms[Action::kExpand] = 0; 240 result.action_times_ms[Action::kAccelerate] = 0; 241 result.action_times_ms[Action::kPreemptiveExpand] = 0; 242 result.action_times_ms[Action::kNormal] = 0; 243 244 if (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC || 245 out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG) { 246 // Consider the whole frame to be the result of expansion. 247 result.action_times_ms[Action::kExpand] = 10; 248 } else if (operations_state.accelerate_samples - 249 prev_ops_state_.accelerate_samples > 250 0) { 251 // Consider the whole frame to be the result of acceleration. 252 result.action_times_ms[Action::kAccelerate] = 10; 253 } else if (operations_state.preemptive_samples - 254 prev_ops_state_.preemptive_samples > 255 0) { 256 // Consider the whole frame to be the result of preemptive expansion. 257 result.action_times_ms[Action::kPreemptiveExpand] = 10; 258 } else { 259 // Consider the whole frame to be the result of normal playout. 260 result.action_times_ms[Action::kNormal] = 10; 261 } 262 auto lifetime_stats = LifetimeStats(); 263 if (text_log_) { 264 const bool plc = 265 (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC) || 266 (out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG); 267 const bool cng = out_frame.speech_type_ == AudioFrame::SpeechType::kCNG; 268 const bool voice_concealed = 269 (lifetime_stats.concealed_samples - 270 lifetime_stats.silent_concealed_samples) > 271 (prev_lifetime_stats_.concealed_samples - 272 prev_lifetime_stats_.silent_concealed_samples); 273 *text_log_ << "GetAudio - wallclock: " << std::setw(5) << time_now_ms 274 << ", delta wc: " << std::setw(4) 275 << (input_->NextEventTime().value_or(time_now_ms) - 276 start_time_ms) 277 << ", CNG: " << cng << ", PLC: " << plc 278 << ", voice concealed: " << voice_concealed 279 << ", buffer size: " << std::setw(4) 280 << current_state_.current_delay_ms << std::endl; 281 if (lifetime_stats.packets_discarded > 282 prev_lifetime_stats_.packets_discarded) { 283 *text_log_ << "Discarded " 284 << (lifetime_stats.packets_discarded - 285 prev_lifetime_stats_.packets_discarded) 286 << " primary packets." << std::endl; 287 } 288 if (operations_state.packet_buffer_flushes > 289 prev_ops_state_.packet_buffer_flushes) { 290 *text_log_ << "Flushed packet buffer " 291 << (operations_state.packet_buffer_flushes - 292 prev_ops_state_.packet_buffer_flushes) 293 << " times." << std::endl; 294 } 295 } 296 prev_lifetime_stats_ = lifetime_stats; 297 const bool no_more_packets_to_decode = 298 !input_->NextPacketTime() && !operations_state.next_packet_available; 299 result.is_simulation_finished = 300 no_more_packets_to_decode || input_->ended(); 301 prev_ops_state_ = operations_state; 302 return result; 303 } 304 } 305 result.simulation_step_ms = 306 input_->NextEventTime().value_or(time_now_ms) - start_time_ms; 307 result.is_simulation_finished = true; 308 return result; 309 } 310 311 void NetEqTest::SetNextAction(NetEqTest::Action next_operation) { 312 next_action_ = std::optional<Action>(next_operation); 313 } 314 315 NetEqTest::NetEqState NetEqTest::GetNetEqState() { 316 return current_state_; 317 } 318 319 NetEqNetworkStatistics NetEqTest::SimulationStats() { 320 NetEqNetworkStatistics stats; 321 RTC_CHECK_EQ(neteq_->NetworkStatistics(&stats), 0); 322 return stats; 323 } 324 325 NetEqLifetimeStatistics NetEqTest::LifetimeStats() const { 326 return neteq_->GetLifetimeStatistics(); 327 } 328 329 NetEqTest::DecoderMap NetEqTest::StandardDecoderMap() { 330 DecoderMap codecs = {{0, SdpAudioFormat("pcmu", 8000, 1)}, 331 {8, SdpAudioFormat("pcma", 8000, 1)}, 332 #ifdef WEBRTC_CODEC_OPUS 333 {111, SdpAudioFormat("opus", 48000, 2)}, 334 {63, SdpAudioFormat("red", 48000, 2)}, 335 #endif 336 {93, SdpAudioFormat("l16", 8000, 1)}, 337 {94, SdpAudioFormat("l16", 16000, 1)}, 338 {95, SdpAudioFormat("l16", 32000, 1)}, 339 {96, SdpAudioFormat("l16", 48000, 1)}, 340 {9, SdpAudioFormat("g722", 8000, 1)}, 341 {106, SdpAudioFormat("telephone-event", 8000, 1)}, 342 {114, SdpAudioFormat("telephone-event", 16000, 1)}, 343 {115, SdpAudioFormat("telephone-event", 32000, 1)}, 344 {116, SdpAudioFormat("telephone-event", 48000, 1)}, 345 {117, SdpAudioFormat("red", 8000, 1)}, 346 {13, SdpAudioFormat("cn", 8000, 1)}, 347 {98, SdpAudioFormat("cn", 16000, 1)}, 348 {99, SdpAudioFormat("cn", 32000, 1)}, 349 {100, SdpAudioFormat("cn", 48000, 1)}}; 350 return codecs; 351 } 352 353 void NetEqTest::RegisterDecoders(const DecoderMap& codecs) { 354 for (const auto& c : codecs) { 355 RTC_CHECK(neteq_->RegisterPayloadType(c.first, c.second)) 356 << "Cannot register " << c.second.name << " to payload type " 357 << c.first; 358 } 359 } 360 361 } // namespace test 362 } // namespace webrtc