frame_combiner_unittest.cc (13076B)
1 /* 2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_mixer/frame_combiner.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <initializer_list> 16 #include <numeric> 17 #include <string> 18 #include <vector> 19 20 #include "api/array_view.h" 21 #include "api/audio/audio_frame.h" 22 #include "api/audio/audio_view.h" 23 #include "api/audio/channel_layout.h" 24 #include "api/rtp_packet_info.h" 25 #include "api/rtp_packet_infos.h" 26 #include "api/units/timestamp.h" 27 #include "audio/utility/audio_frame_operations.h" 28 #include "modules/audio_mixer/gain_change_calculator.h" 29 #include "modules/audio_mixer/sine_wave_generator.h" 30 #include "rtc_base/checks.h" 31 #include "rtc_base/strings/string_builder.h" 32 #include "test/gmock.h" 33 #include "test/gtest.h" 34 35 namespace webrtc { 36 37 namespace { 38 39 using ::testing::ElementsAreArray; 40 using ::testing::IsEmpty; 41 using ::testing::UnorderedElementsAreArray; 42 43 struct FrameCombinerConfig { 44 bool use_limiter; 45 int sample_rate_hz; 46 int number_of_channels; 47 float wave_frequency; 48 }; 49 50 std::string ProduceDebugText(int sample_rate_hz, 51 int number_of_channels, 52 int number_of_sources) { 53 StringBuilder ss; 54 ss << "Sample rate: " << sample_rate_hz << " ,"; 55 ss << "number of channels: " << number_of_channels << " ,"; 56 ss << "number of sources: " << number_of_sources; 57 return ss.Release(); 58 } 59 60 std::string ProduceDebugText(const FrameCombinerConfig& config) { 61 StringBuilder ss; 62 ss << "Sample rate: " << config.sample_rate_hz << " ,"; 63 ss << "number of channels: " << config.number_of_channels << " ,"; 64 ss << "limiter active: " << (config.use_limiter ? "on" : "off") << " ,"; 65 ss << "wave frequency: " << config.wave_frequency << " ,"; 66 return ss.Release(); 67 } 68 69 AudioFrame frame1; 70 AudioFrame frame2; 71 72 void SetUpFrames(int sample_rate_hz, int number_of_channels) { 73 RtpPacketInfo packet_info1(/*ssrc=*/1001, /*csrcs=*/{}, 74 /*rtp_timestamp=*/1000, 75 /*receive_time=*/Timestamp::Millis(1)); 76 RtpPacketInfo packet_info2(/*ssrc=*/4004, /*csrcs=*/{}, 77 /*rtp_timestamp=*/1234, 78 /*receive_time=*/Timestamp::Millis(2)); 79 RtpPacketInfo packet_info3(/*ssrc=*/7007, /*csrcs=*/{}, 80 /*rtp_timestamp=*/1333, 81 /*receive_time=*/Timestamp::Millis(2)); 82 83 frame1.packet_infos_ = RtpPacketInfos({packet_info1}); 84 frame2.packet_infos_ = RtpPacketInfos({packet_info2, packet_info3}); 85 86 for (auto* frame : {&frame1, &frame2}) { 87 frame->UpdateFrame(0, nullptr, CheckedDivExact(sample_rate_hz, 100), 88 sample_rate_hz, AudioFrame::kNormalSpeech, 89 AudioFrame::kVadActive, number_of_channels); 90 } 91 } 92 } // namespace 93 94 // The limiter requires sample rate divisible by 2000. 95 TEST(FrameCombiner, BasicApiCallsLimiter) { 96 FrameCombiner combiner(true); 97 for (const int rate : {8000, 18000, 34000, 48000}) { 98 for (const int number_of_channels : {1, 2, 4, 8}) { 99 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; 100 SetUpFrames(rate, number_of_channels); 101 102 for (const int number_of_frames : {0, 1, 2}) { 103 SCOPED_TRACE( 104 ProduceDebugText(rate, number_of_channels, number_of_frames)); 105 const std::vector<AudioFrame*> frames_to_combine( 106 all_frames.begin(), all_frames.begin() + number_of_frames); 107 AudioFrame audio_frame_for_mixing; 108 combiner.Combine(frames_to_combine, number_of_channels, rate, 109 frames_to_combine.size(), &audio_frame_for_mixing); 110 } 111 } 112 } 113 } 114 115 // The RtpPacketInfos field of the mixed packet should contain the union of the 116 // RtpPacketInfos from the frames that were actually mixed. 117 TEST(FrameCombiner, ContainsAllRtpPacketInfos) { 118 static constexpr int kSampleRateHz = 48000; 119 static constexpr int kNumChannels = 1; 120 FrameCombiner combiner(true); 121 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; 122 SetUpFrames(kSampleRateHz, kNumChannels); 123 124 for (const int number_of_frames : {0, 1, 2}) { 125 SCOPED_TRACE( 126 ProduceDebugText(kSampleRateHz, kNumChannels, number_of_frames)); 127 const std::vector<AudioFrame*> frames_to_combine( 128 all_frames.begin(), all_frames.begin() + number_of_frames); 129 130 std::vector<RtpPacketInfo> packet_infos; 131 for (const auto& frame : frames_to_combine) { 132 packet_infos.insert(packet_infos.end(), frame->packet_infos_.begin(), 133 frame->packet_infos_.end()); 134 } 135 136 AudioFrame audio_frame_for_mixing; 137 combiner.Combine(frames_to_combine, kNumChannels, kSampleRateHz, 138 frames_to_combine.size(), &audio_frame_for_mixing); 139 EXPECT_THAT(audio_frame_for_mixing.packet_infos_, 140 UnorderedElementsAreArray(packet_infos)); 141 } 142 } 143 144 #if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) 145 // There are CHECKs in place to check for invalid parameters. 146 TEST(FrameCombinerDeathTest, BuildCrashesWithManyChannels) { 147 FrameCombiner combiner(true); 148 for (const int rate : {8000, 18000, 34000, 48000}) { 149 for (const int number_of_channels : {10, 20, 21}) { 150 RTC_DCHECK_LE(number_of_channels, kMaxNumberOfAudioChannels); 151 if (static_cast<size_t>(rate / 100 * number_of_channels) > 152 AudioFrame::kMaxDataSizeSamples) { 153 continue; 154 } 155 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; 156 SetUpFrames(rate, number_of_channels); 157 158 const int number_of_frames = 2; 159 SCOPED_TRACE( 160 ProduceDebugText(rate, number_of_channels, number_of_frames)); 161 const std::vector<AudioFrame*> frames_to_combine( 162 all_frames.begin(), all_frames.begin() + number_of_frames); 163 AudioFrame audio_frame_for_mixing; 164 EXPECT_DEATH( 165 combiner.Combine(frames_to_combine, number_of_channels, rate, 166 frames_to_combine.size(), &audio_frame_for_mixing), 167 ""); 168 } 169 } 170 } 171 #endif // GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) 172 173 TEST(FrameCombinerDeathTest, DebugBuildCrashesWithHighRate) { 174 FrameCombiner combiner(true); 175 for (const int rate : {50000, 96000, 128000, 196000}) { 176 for (const int number_of_channels : {1, 2, 3}) { 177 if (static_cast<size_t>(rate / 100 * number_of_channels) > 178 AudioFrame::kMaxDataSizeSamples) { 179 continue; 180 } 181 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; 182 SetUpFrames(rate, number_of_channels); 183 184 const int number_of_frames = 2; 185 SCOPED_TRACE( 186 ProduceDebugText(rate, number_of_channels, number_of_frames)); 187 const std::vector<AudioFrame*> frames_to_combine( 188 all_frames.begin(), all_frames.begin() + number_of_frames); 189 AudioFrame audio_frame_for_mixing; 190 #if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) 191 EXPECT_DEATH( 192 combiner.Combine(frames_to_combine, number_of_channels, rate, 193 frames_to_combine.size(), &audio_frame_for_mixing), 194 "") 195 << "number_of_channels=" << number_of_channels << ", rate=" << rate 196 << ", frames to combine=" << frames_to_combine.size(); 197 #endif 198 } 199 } 200 } 201 202 // With no limiter, the rate has to be divisible by 100 since we use 203 // 10 ms frames. 204 TEST(FrameCombiner, BasicApiCallsNoLimiter) { 205 FrameCombiner combiner(false); 206 for (const int rate : {8000, 10000, 11000, 32000, 44100}) { 207 for (const int number_of_channels : {1, 2, 4, 8}) { 208 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; 209 SetUpFrames(rate, number_of_channels); 210 211 for (const int number_of_frames : {0, 1, 2}) { 212 SCOPED_TRACE( 213 ProduceDebugText(rate, number_of_channels, number_of_frames)); 214 const std::vector<AudioFrame*> frames_to_combine( 215 all_frames.begin(), all_frames.begin() + number_of_frames); 216 AudioFrame audio_frame_for_mixing; 217 combiner.Combine(frames_to_combine, number_of_channels, rate, 218 frames_to_combine.size(), &audio_frame_for_mixing); 219 } 220 } 221 } 222 } 223 224 TEST(FrameCombiner, CombiningZeroFramesShouldProduceSilence) { 225 FrameCombiner combiner(false); 226 for (const int rate : {8000, 10000, 11000, 32000, 44100}) { 227 for (const int number_of_channels : {1, 2}) { 228 SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 0)); 229 230 AudioFrame audio_frame_for_mixing; 231 232 const std::vector<AudioFrame*> frames_to_combine; 233 combiner.Combine(frames_to_combine, number_of_channels, rate, 234 frames_to_combine.size(), &audio_frame_for_mixing); 235 const int16_t* audio_frame_for_mixing_data = 236 audio_frame_for_mixing.data(); 237 const std::vector<int16_t> mixed_data( 238 audio_frame_for_mixing_data, 239 audio_frame_for_mixing_data + number_of_channels * rate / 100); 240 241 const std::vector<int16_t> expected(number_of_channels * rate / 100, 0); 242 EXPECT_EQ(mixed_data, expected); 243 EXPECT_THAT(audio_frame_for_mixing.packet_infos_, IsEmpty()); 244 } 245 } 246 } 247 248 TEST(FrameCombiner, CombiningOneFrameShouldNotChangeFrame) { 249 FrameCombiner combiner(false); 250 for (const int rate : {8000, 10000, 11000, 32000, 44100}) { 251 // kMaxConcurrentChannels is 8. 252 for (const int number_of_channels : {1, 2, 4, kMaxConcurrentChannels}) { 253 SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 1)); 254 255 AudioFrame audio_frame_for_mixing; 256 257 SetUpFrames(rate, number_of_channels); 258 int16_t* frame1_data = frame1.mutable_data(); 259 std::iota(frame1_data, frame1_data + number_of_channels * rate / 100, 0); 260 const std::vector<AudioFrame*> frames_to_combine = {&frame1}; 261 combiner.Combine(frames_to_combine, number_of_channels, rate, 262 frames_to_combine.size(), &audio_frame_for_mixing); 263 264 const int16_t* audio_frame_for_mixing_data = 265 audio_frame_for_mixing.data(); 266 const std::vector<int16_t> mixed_data( 267 audio_frame_for_mixing_data, 268 audio_frame_for_mixing_data + number_of_channels * rate / 100); 269 270 std::vector<int16_t> expected(number_of_channels * rate / 100); 271 std::iota(expected.begin(), expected.end(), 0); 272 EXPECT_EQ(mixed_data, expected); 273 EXPECT_THAT(audio_frame_for_mixing.packet_infos_, 274 ElementsAreArray(frame1.packet_infos_)); 275 } 276 } 277 } 278 279 // Send a sine wave through the FrameCombiner, and check that the 280 // difference between input and output varies smoothly. Also check 281 // that it is inside reasonable bounds. This is to catch issues like 282 // chromium:695993 and chromium:816875. 283 TEST(FrameCombiner, GainCurveIsSmoothForAlternatingNumberOfStreams) { 284 // Rates are divisible by 2000 when limiter is active. 285 std::vector<FrameCombinerConfig> configs = { 286 {false, 30100, 2, 50.f}, {false, 16500, 1, 3200.f}, 287 {true, 8000, 1, 3200.f}, {true, 16000, 1, 50.f}, 288 {true, 18000, 8, 3200.f}, {true, 10000, 2, 50.f}, 289 }; 290 291 for (const auto& config : configs) { 292 SCOPED_TRACE(ProduceDebugText(config)); 293 294 FrameCombiner combiner(config.use_limiter); 295 296 constexpr int16_t wave_amplitude = 30000; 297 SineWaveGenerator wave_generator(config.wave_frequency, wave_amplitude); 298 299 GainChangeCalculator change_calculator; 300 float cumulative_change = 0.f; 301 302 constexpr size_t iterations = 100; 303 304 for (size_t i = 0; i < iterations; ++i) { 305 SetUpFrames(config.sample_rate_hz, config.number_of_channels); 306 wave_generator.GenerateNextFrame(&frame1); 307 AudioFrameOperations::Mute(&frame2); 308 309 std::vector<AudioFrame*> frames_to_combine = {&frame1}; 310 if (i % 2 == 0) { 311 frames_to_combine.push_back(&frame2); 312 } 313 const size_t number_of_samples = 314 frame1.samples_per_channel_ * config.number_of_channels; 315 316 // Ensures limiter is on if 'use_limiter'. 317 constexpr size_t number_of_streams = 2; 318 AudioFrame audio_frame_for_mixing; 319 combiner.Combine(frames_to_combine, config.number_of_channels, 320 config.sample_rate_hz, number_of_streams, 321 &audio_frame_for_mixing); 322 cumulative_change += change_calculator.CalculateGainChange( 323 ArrayView<const int16_t>(frame1.data(), number_of_samples), 324 ArrayView<const int16_t>(audio_frame_for_mixing.data(), 325 number_of_samples)); 326 } 327 328 // Check that the gain doesn't vary too much. 329 EXPECT_LT(cumulative_change, 10); 330 331 // Check that the latest gain is within reasonable bounds. It 332 // should be slightly less that 1. 333 EXPECT_LT(0.9f, change_calculator.LatestGain()); 334 EXPECT_LT(change_calculator.LatestGain(), 1.01f); 335 } 336 } 337 } // namespace webrtc