neteq_delay_analyzer.cc (12182B)
1 /* 2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" 12 13 #include <algorithm> 14 #include <cstddef> 15 #include <cstdint> 16 #include <fstream> 17 #include <ios> 18 #include <limits> 19 #include <ostream> // no-presubmit-check TODO(webrtc:8982) 20 #include <string> 21 #include <utility> 22 #include <vector> 23 24 #include "absl/strings/string_view.h" 25 #include "api/audio/audio_frame.h" 26 #include "api/neteq/neteq.h" 27 #include "api/rtp_packet_info.h" 28 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 29 #include "rtc_base/checks.h" 30 #include "rtc_base/numerics/sequence_number_unwrapper.h" 31 32 namespace webrtc { 33 namespace test { 34 namespace { 35 constexpr char kArrivalDelayX[] = "arrival_delay_x"; 36 constexpr char kArrivalDelayY[] = "arrival_delay_y"; 37 constexpr char kTargetDelayX[] = "target_delay_x"; 38 constexpr char kTargetDelayY[] = "target_delay_y"; 39 constexpr char kPlayoutDelayX[] = "playout_delay_x"; 40 constexpr char kPlayoutDelayY[] = "playout_delay_y"; 41 42 // Helper function for NetEqDelayAnalyzer::CreateGraphs. Returns the 43 // interpolated value of a function at the point x. Vector x_vec contains the 44 // sample points, and y_vec contains the function values at these points. The 45 // return value is a linear interpolation between y_vec values. 46 double LinearInterpolate(double x, 47 const std::vector<int64_t>& x_vec, 48 const std::vector<int64_t>& y_vec) { 49 // Find first element which is larger than x. 50 auto it = std::upper_bound(x_vec.begin(), x_vec.end(), x); 51 if (it == x_vec.end()) { 52 --it; 53 } 54 const size_t upper_ix = it - x_vec.begin(); 55 56 size_t lower_ix; 57 if (upper_ix == 0 || x_vec[upper_ix] <= x) { 58 lower_ix = upper_ix; 59 } else { 60 lower_ix = upper_ix - 1; 61 } 62 double y; 63 if (lower_ix == upper_ix) { 64 y = y_vec[lower_ix]; 65 } else { 66 RTC_DCHECK_NE(x_vec[lower_ix], x_vec[upper_ix]); 67 y = (x - x_vec[lower_ix]) * (y_vec[upper_ix] - y_vec[lower_ix]) / 68 (x_vec[upper_ix] - x_vec[lower_ix]) + 69 y_vec[lower_ix]; 70 } 71 return y; 72 } 73 74 void PrintDelays(const NetEqDelayAnalyzer::Delays& delays, 75 int64_t ref_time_ms, 76 absl::string_view var_name_x, 77 absl::string_view var_name_y, 78 std::ofstream& output, 79 absl::string_view terminator = "") { 80 output << var_name_x << " = [ "; 81 for (const std::pair<int64_t, float>& delay : delays) { 82 output << (delay.first - ref_time_ms) / 1000.f << ", "; 83 } 84 output << "]" << terminator << std::endl; 85 86 output << var_name_y << " = [ "; 87 for (const std::pair<int64_t, float>& delay : delays) { 88 output << delay.second << ", "; 89 } 90 output << "]" << terminator << std::endl; 91 } 92 93 } // namespace 94 95 void NetEqDelayAnalyzer::AfterInsertPacket(const RtpPacketReceived& packet, 96 NetEq* /* neteq */) { 97 data_.insert(std::make_pair(packet.Timestamp(), 98 TimingData(packet.arrival_time().ms()))); 99 ssrcs_.insert(packet.Ssrc()); 100 payload_types_.insert(packet.PayloadType()); 101 } 102 103 void NetEqDelayAnalyzer::BeforeGetAudio(NetEq* neteq) { 104 last_sync_buffer_ms_ = neteq->SyncBufferSizeMs(); 105 } 106 107 void NetEqDelayAnalyzer::AfterGetAudio(int64_t time_now_ms, 108 const AudioFrame& audio_frame, 109 bool /*muted*/, 110 NetEq* neteq) { 111 get_audio_time_ms_.push_back(time_now_ms); 112 for (const RtpPacketInfo& info : audio_frame.packet_infos_) { 113 auto it = data_.find(info.rtp_timestamp()); 114 if (it == data_.end()) { 115 // This is a packet that was split out from another packet. Skip it. 116 continue; 117 } 118 auto& it_timing = it->second; 119 RTC_CHECK(!it_timing.decode_get_audio_count) 120 << "Decode time already written"; 121 it_timing.decode_get_audio_count = get_audio_count_; 122 RTC_CHECK(!it_timing.sync_delay_ms) << "Decode time already written"; 123 it_timing.sync_delay_ms = last_sync_buffer_ms_; 124 it_timing.target_delay_ms = neteq->TargetDelayMs(); 125 it_timing.current_delay_ms = neteq->FilteredCurrentDelayMs(); 126 } 127 last_sample_rate_hz_ = audio_frame.sample_rate_hz_; 128 ++get_audio_count_; 129 } 130 131 void NetEqDelayAnalyzer::CreateGraphs(Delays* arrival_delay_ms, 132 Delays* corrected_arrival_delay_ms, 133 Delays* playout_delay_ms, 134 Delays* target_delay_ms) const { 135 if (get_audio_time_ms_.empty()) { 136 return; 137 } 138 // Create nominal_get_audio_time_ms, a vector starting at 139 // get_audio_time_ms_[0] and increasing by 10 for each element. 140 std::vector<int64_t> nominal_get_audio_time_ms(get_audio_time_ms_.size()); 141 nominal_get_audio_time_ms[0] = get_audio_time_ms_[0]; 142 std::transform( 143 nominal_get_audio_time_ms.begin(), nominal_get_audio_time_ms.end() - 1, 144 nominal_get_audio_time_ms.begin() + 1, [](int64_t& x) { return x + 10; }); 145 RTC_DCHECK( 146 std::is_sorted(get_audio_time_ms_.begin(), get_audio_time_ms_.end())); 147 148 std::vector<double> rtp_timestamps_ms; 149 double offset = std::numeric_limits<double>::max(); 150 RtpTimestampUnwrapper unwrapper; 151 // This loop traverses data_ and populates rtp_timestamps_ms as well as 152 // calculates the base offset. 153 for (auto& d : data_) { 154 rtp_timestamps_ms.push_back(static_cast<double>(unwrapper.Unwrap(d.first)) / 155 CheckedDivExact(last_sample_rate_hz_, 1000)); 156 offset = 157 std::min(offset, d.second.arrival_time_ms - rtp_timestamps_ms.back()); 158 } 159 160 // This loop traverses the data again and populates the graph vectors. The 161 // reason to have two loops and traverse twice is that the offset cannot be 162 // known until the first traversal is done. Meanwhile, the final offset must 163 // be known already at the start of this second loop. 164 size_t i = 0; 165 for (const auto& data : data_) { 166 const double offset_send_time_ms = rtp_timestamps_ms[i++] + offset; 167 const auto& timing = data.second; 168 corrected_arrival_delay_ms->push_back(std::make_pair( 169 timing.arrival_time_ms, 170 LinearInterpolate(timing.arrival_time_ms, get_audio_time_ms_, 171 nominal_get_audio_time_ms) - 172 offset_send_time_ms)); 173 arrival_delay_ms->push_back(std::make_pair( 174 timing.arrival_time_ms, timing.arrival_time_ms - offset_send_time_ms)); 175 176 if (timing.decode_get_audio_count) { 177 // This packet was decoded. 178 RTC_DCHECK(timing.sync_delay_ms); 179 const int64_t get_audio_time = 180 *timing.decode_get_audio_count * 10 + get_audio_time_ms_[0]; 181 const float playout_ms = 182 get_audio_time + *timing.sync_delay_ms - offset_send_time_ms; 183 playout_delay_ms->push_back(std::make_pair(get_audio_time, playout_ms)); 184 RTC_DCHECK(timing.target_delay_ms); 185 RTC_DCHECK(timing.current_delay_ms); 186 const float target = 187 playout_ms - *timing.current_delay_ms + *timing.target_delay_ms; 188 target_delay_ms->push_back(std::make_pair(get_audio_time, target)); 189 } 190 } 191 } 192 193 void NetEqDelayAnalyzer::CreateMatlabScript( 194 absl::string_view script_name) const { 195 Delays arrival_delay_ms; 196 Delays corrected_arrival_delay_ms; 197 Delays playout_delay_ms; 198 Delays target_delay_ms; 199 CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, 200 &playout_delay_ms, &target_delay_ms); 201 202 // Maybe better to find the actually smallest timestamp, to surely avoid 203 // x-axis starting from negative. 204 const int64_t ref_time_ms = arrival_delay_ms.front().first; 205 206 // Create an output file stream to Matlab script file. 207 std::ofstream output(std::string{script_name}); 208 209 PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, 210 kArrivalDelayY, output, ";"); 211 212 // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, 213 // kCorrectedArrivalDelayY, output); 214 215 PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, 216 output, ";"); 217 218 PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, 219 output, ";"); 220 221 output << "h=plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " 222 << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " 223 << kPlayoutDelayX << ", " << kPlayoutDelayY << ");" << std::endl; 224 output << "set(h(1),'color',0.75*[1 1 1]);" << std::endl; 225 output << "set(h(2),'markersize',6);" << std::endl; 226 output << "set(h(3),'linew',1.5);" << std::endl; 227 output << "ax1=axis;" << std::endl; 228 output << "axis tight" << std::endl; 229 output << "ax2=axis;" << std::endl; 230 output << "axis([ax2(1:3) ax1(4)])" << std::endl; 231 output << "xlabel('time [s]');" << std::endl; 232 output << "ylabel('relative delay [ms]');" << std::endl; 233 if (!ssrcs_.empty()) { 234 auto ssrc_it = ssrcs_.cbegin(); 235 output << "title('SSRC: 0x" << std::hex << static_cast<int64_t>(*ssrc_it++); 236 while (ssrc_it != ssrcs_.end()) { 237 output << ", 0x" << std::hex << static_cast<int64_t>(*ssrc_it++); 238 } 239 output << std::dec; 240 auto pt_it = payload_types_.cbegin(); 241 output << "; Payload Types: " << *pt_it++; 242 while (pt_it != payload_types_.end()) { 243 output << ", " << *pt_it++; 244 } 245 output << "');" << std::endl; 246 } 247 } 248 249 void NetEqDelayAnalyzer::CreatePythonScript( 250 absl::string_view script_name) const { 251 Delays arrival_delay_ms; 252 Delays corrected_arrival_delay_ms; 253 Delays playout_delay_ms; 254 Delays target_delay_ms; 255 CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, 256 &playout_delay_ms, &target_delay_ms); 257 258 // Maybe better to find the actually smallest timestamp, to surely avoid 259 // x-axis starting from negative. 260 const int64_t ref_time_ms = arrival_delay_ms.front().first; 261 262 // Create an output file stream to the python script file. 263 std::ofstream output(std::string{script_name}); 264 265 // Necessary includes 266 output << "import numpy as np" << std::endl; 267 output << "import matplotlib.pyplot as plt" << std::endl; 268 269 PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, 270 kArrivalDelayY, output); 271 272 // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, 273 // kCorrectedArrivalDelayY, output); 274 275 PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, 276 output); 277 278 PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, 279 output); 280 281 output << "if __name__ == '__main__':" << std::endl; 282 output << " h=plt.plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " 283 << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " 284 << kPlayoutDelayX << ", " << kPlayoutDelayY << ")" << std::endl; 285 output << " plt.setp(h[0],'color',[.75, .75, .75])" << std::endl; 286 output << " plt.setp(h[1],'markersize',6)" << std::endl; 287 output << " plt.setp(h[2],'linewidth',1.5)" << std::endl; 288 output << " plt.axis('tight')" << std::endl; 289 output << " plt.xlabel('time [s]')" << std::endl; 290 output << " plt.ylabel('relative delay [ms]')" << std::endl; 291 if (!ssrcs_.empty()) { 292 auto ssrc_it = ssrcs_.cbegin(); 293 output << " plt.legend((\"arrival delay\", \"target delay\", \"playout " 294 "delay\"))" 295 << std::endl; 296 output << " plt.title('SSRC: 0x" << std::hex 297 << static_cast<int64_t>(*ssrc_it++); 298 while (ssrc_it != ssrcs_.end()) { 299 output << ", 0x" << std::hex << static_cast<int64_t>(*ssrc_it++); 300 } 301 output << std::dec; 302 auto pt_it = payload_types_.cbegin(); 303 output << "; Payload Types: " << *pt_it++; 304 while (pt_it != payload_types_.end()) { 305 output << ", " << *pt_it++; 306 } 307 output << "')" << std::endl; 308 } 309 output << " plt.show()" << std::endl; 310 } 311 312 } // namespace test 313 } // namespace webrtc