merge.cc (17427B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_coding/neteq/merge.h" 12 13 #include <algorithm> // min, max 14 #include <cstdint> 15 #include <cstring> // memmove, memcpy, memset, size_t 16 #include <limits> 17 #include <memory> 18 19 #include "api/array_view.h" 20 #include "common_audio/signal_processing/dot_product_with_scale.h" 21 #include "common_audio/signal_processing/include/signal_processing_library.h" 22 #include "common_audio/signal_processing/include/spl_inl.h" 23 #include "modules/audio_coding/neteq/audio_multi_vector.h" 24 #include "modules/audio_coding/neteq/cross_correlation.h" 25 #include "modules/audio_coding/neteq/dsp_helper.h" 26 #include "modules/audio_coding/neteq/expand.h" 27 #include "modules/audio_coding/neteq/sync_buffer.h" 28 #include "rtc_base/checks.h" 29 #include "rtc_base/numerics/safe_conversions.h" 30 #include "rtc_base/numerics/safe_minmax.h" 31 32 namespace webrtc { 33 34 Merge::Merge(int fs_hz, 35 size_t num_channels, 36 Expand* expand, 37 SyncBuffer* sync_buffer) 38 : fs_hz_(fs_hz), 39 num_channels_(num_channels), 40 fs_mult_(fs_hz_ / 8000), 41 timestamps_per_call_(static_cast<size_t>(fs_hz_ / 100)), 42 expand_(expand), 43 sync_buffer_(sync_buffer), 44 expanded_(num_channels_) { 45 RTC_DCHECK_GT(num_channels_, 0); 46 } 47 48 Merge::~Merge() = default; 49 50 size_t Merge::Process(int16_t* input, 51 size_t input_length, 52 AudioMultiVector* output) { 53 // TODO(hlundin): Change to an enumerator and skip assert. 54 RTC_DCHECK(fs_hz_ == 8000 || fs_hz_ == 16000 || fs_hz_ == 32000 || 55 fs_hz_ == 48000); 56 RTC_DCHECK_LE(fs_hz_, kMaxSampleRate); // Should not be possible. 57 if (input_length == 0) { 58 return 0; 59 } 60 61 size_t old_length; 62 size_t expand_period; 63 // Get expansion data to overlap and mix with. 64 size_t expanded_length = GetExpandedSignal(&old_length, &expand_period); 65 66 // Transfer input signal to an AudioMultiVector. 67 AudioMultiVector input_vector(num_channels_); 68 input_vector.PushBackInterleaved( 69 ArrayView<const int16_t>(input, input_length)); 70 size_t input_length_per_channel = input_vector.Size(); 71 RTC_DCHECK_EQ(input_length_per_channel, input_length / num_channels_); 72 73 size_t best_correlation_index = 0; 74 size_t output_length = 0; 75 76 std::unique_ptr<int16_t[]> input_channel( 77 new int16_t[input_length_per_channel]); 78 std::unique_ptr<int16_t[]> expanded_channel(new int16_t[expanded_length]); 79 for (size_t channel = 0; channel < num_channels_; ++channel) { 80 input_vector[channel].CopyTo(input_length_per_channel, 0, 81 input_channel.get()); 82 expanded_[channel].CopyTo(expanded_length, 0, expanded_channel.get()); 83 84 const int16_t new_mute_factor = std::min<int16_t>( 85 16384, SignalScaling(input_channel.get(), input_length_per_channel, 86 expanded_channel.get())); 87 88 if (channel == 0) { 89 // Downsample, correlate, and find strongest correlation period for the 90 // reference (i.e., first) channel only. 91 // Downsample to 4kHz sample rate. 92 Downsample(input_channel.get(), input_length_per_channel, 93 expanded_channel.get(), expanded_length); 94 95 // Calculate the lag of the strongest correlation period. 96 best_correlation_index = CorrelateAndPeakSearch( 97 old_length, input_length_per_channel, expand_period); 98 } 99 100 temp_data_.resize(input_length_per_channel + best_correlation_index); 101 int16_t* decoded_output = temp_data_.data() + best_correlation_index; 102 103 // Mute the new decoded data if needed (and unmute it linearly). 104 // This is the overlapping part of expanded_signal. 105 size_t interpolation_length = 106 std::min(kMaxCorrelationLength * fs_mult_, 107 expanded_length - best_correlation_index); 108 interpolation_length = 109 std::min(interpolation_length, input_length_per_channel); 110 111 RTC_DCHECK_LE(new_mute_factor, 16384); 112 int16_t mute_factor = 113 std::max(expand_->MuteFactor(channel), new_mute_factor); 114 RTC_DCHECK_GE(mute_factor, 0); 115 116 if (mute_factor < 16384) { 117 // Set a suitable muting slope (Q20). 0.004 for NB, 0.002 for WB, 118 // and so on, or as fast as it takes to come back to full gain within the 119 // frame length. 120 const int back_to_fullscale_inc = static_cast<int>( 121 ((16384 - mute_factor) << 6) / input_length_per_channel); 122 const int increment = std::max(4194 / fs_mult_, back_to_fullscale_inc); 123 mute_factor = static_cast<int16_t>(DspHelper::RampSignal( 124 input_channel.get(), interpolation_length, mute_factor, increment)); 125 DspHelper::UnmuteSignal(&input_channel[interpolation_length], 126 input_length_per_channel - interpolation_length, 127 &mute_factor, increment, 128 &decoded_output[interpolation_length]); 129 } else { 130 // No muting needed. 131 memmove( 132 &decoded_output[interpolation_length], 133 &input_channel[interpolation_length], 134 sizeof(int16_t) * (input_length_per_channel - interpolation_length)); 135 } 136 137 // Do overlap and mix linearly. 138 int16_t increment = 139 static_cast<int16_t>(16384 / (interpolation_length + 1)); // In Q14. 140 int16_t local_mute_factor = 16384 - increment; 141 memmove(temp_data_.data(), expanded_channel.get(), 142 sizeof(int16_t) * best_correlation_index); 143 DspHelper::CrossFade(&expanded_channel[best_correlation_index], 144 input_channel.get(), interpolation_length, 145 &local_mute_factor, increment, decoded_output); 146 147 output_length = best_correlation_index + input_length_per_channel; 148 if (channel == 0) { 149 RTC_DCHECK(output->Empty()); // Output should be empty at this point. 150 output->AssertSize(output_length); 151 } else { 152 RTC_DCHECK_EQ(output->Size(), output_length); 153 } 154 (*output)[channel].OverwriteAt(temp_data_.data(), output_length, 0); 155 } 156 157 // Copy back the first part of the data to `sync_buffer_` and remove it from 158 // `output`. 159 sync_buffer_->ReplaceAtIndex(*output, old_length, sync_buffer_->next_index()); 160 output->PopFront(old_length); 161 162 // Return new added length. `old_length` samples were borrowed from 163 // `sync_buffer_`. 164 RTC_DCHECK_GE(output_length, old_length); 165 return output_length - old_length; 166 } 167 168 size_t Merge::GetExpandedSignal(size_t* old_length, size_t* expand_period) { 169 // Check how much data that is left since earlier. 170 *old_length = sync_buffer_->FutureLength(); 171 // Should never be less than overlap_length. 172 RTC_DCHECK_GE(*old_length, expand_->overlap_length()); 173 // Generate data to merge the overlap with using expand. 174 expand_->SetParametersForMergeAfterExpand(); 175 176 if (*old_length >= 210 * kMaxSampleRate / 8000) { 177 // TODO(hlundin): Write test case for this. 178 // The number of samples available in the sync buffer is more than what fits 179 // in expanded_signal. Keep the first 210 * kMaxSampleRate / 8000 samples, 180 // but shift them towards the end of the buffer. This is ok, since all of 181 // the buffer will be expand data anyway, so as long as the beginning is 182 // left untouched, we're fine. 183 size_t length_diff = *old_length - 210 * kMaxSampleRate / 8000; 184 sync_buffer_->InsertZerosAtIndex(length_diff, sync_buffer_->next_index()); 185 *old_length = 210 * kMaxSampleRate / 8000; 186 // This is the truncated length. 187 } 188 // This assert should always be true thanks to the if statement above. 189 RTC_DCHECK_GE(210 * kMaxSampleRate / 8000, *old_length); 190 191 AudioMultiVector expanded_temp(num_channels_); 192 expand_->Process(&expanded_temp); 193 *expand_period = expanded_temp.Size(); // Samples per channel. 194 195 expanded_.Clear(); 196 // Copy what is left since earlier into the expanded vector. 197 expanded_.PushBackFromIndex(*sync_buffer_, sync_buffer_->next_index()); 198 RTC_DCHECK_EQ(expanded_.Size(), *old_length); 199 RTC_DCHECK_GT(expanded_temp.Size(), 0); 200 // Do "ugly" copy and paste from the expanded in order to generate more data 201 // to correlate (but not interpolate) with. 202 const size_t required_length = static_cast<size_t>((120 + 80 + 2) * fs_mult_); 203 if (expanded_.Size() < required_length) { 204 while (expanded_.Size() < required_length) { 205 // Append one more pitch period each time. 206 expanded_.PushBack(expanded_temp); 207 } 208 // Trim the length to exactly `required_length`. 209 expanded_.PopBack(expanded_.Size() - required_length); 210 } 211 RTC_DCHECK_GE(expanded_.Size(), required_length); 212 return required_length; 213 } 214 215 int16_t Merge::SignalScaling(const int16_t* input, 216 size_t input_length, 217 const int16_t* expanded_signal) const { 218 // Adjust muting factor if new vector is more or less of the BGN energy. 219 const auto mod_input_length = 220 SafeMin<size_t>(64 * dchecked_cast<size_t>(fs_mult_), input_length); 221 const int16_t expanded_max = 222 WebRtcSpl_MaxAbsValueW16(expanded_signal, mod_input_length); 223 int32_t factor = 224 (expanded_max * expanded_max) / (std::numeric_limits<int32_t>::max() / 225 static_cast<int32_t>(mod_input_length)); 226 const int expanded_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); 227 int32_t energy_expanded = WebRtcSpl_DotProductWithScale( 228 expanded_signal, expanded_signal, mod_input_length, expanded_shift); 229 230 // Calculate energy of input signal. 231 const int16_t input_max = WebRtcSpl_MaxAbsValueW16(input, mod_input_length); 232 factor = (input_max * input_max) / (std::numeric_limits<int32_t>::max() / 233 static_cast<int32_t>(mod_input_length)); 234 const int input_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); 235 int32_t energy_input = WebRtcSpl_DotProductWithScale( 236 input, input, mod_input_length, input_shift); 237 238 // Align to the same Q-domain. 239 if (input_shift > expanded_shift) { 240 energy_expanded = energy_expanded >> (input_shift - expanded_shift); 241 } else { 242 energy_input = energy_input >> (expanded_shift - input_shift); 243 } 244 245 // Calculate muting factor to use for new frame. 246 int16_t mute_factor; 247 if (energy_input > energy_expanded) { 248 // Normalize `energy_input` to 14 bits. 249 int16_t temp_shift = WebRtcSpl_NormW32(energy_input) - 17; 250 energy_input = WEBRTC_SPL_SHIFT_W32(energy_input, temp_shift); 251 // Put `energy_expanded` in a domain 14 higher, so that 252 // energy_expanded / energy_input is in Q14. 253 energy_expanded = WEBRTC_SPL_SHIFT_W32(energy_expanded, temp_shift + 14); 254 // Calculate sqrt(energy_expanded / energy_input) in Q14. 255 mute_factor = static_cast<int16_t>( 256 WebRtcSpl_SqrtFloor((energy_expanded / energy_input) << 14)); 257 } else { 258 // Set to 1 (in Q14) when `expanded` has higher energy than `input`. 259 mute_factor = 16384; 260 } 261 262 return mute_factor; 263 } 264 265 // TODO(hlundin): There are some parameter values in this method that seem 266 // strange. Compare with Expand::Correlation. 267 void Merge::Downsample(const int16_t* input, 268 size_t input_length, 269 const int16_t* expanded_signal, 270 size_t expanded_length) { 271 const int16_t* filter_coefficients; 272 size_t num_coefficients; 273 int decimation_factor = fs_hz_ / 4000; 274 static const size_t kCompensateDelay = 0; 275 size_t length_limit = static_cast<size_t>(fs_hz_ / 100); // 10 ms in samples. 276 if (fs_hz_ == 8000) { 277 filter_coefficients = DspHelper::kDownsample8kHzTbl; 278 num_coefficients = 3; 279 } else if (fs_hz_ == 16000) { 280 filter_coefficients = DspHelper::kDownsample16kHzTbl; 281 num_coefficients = 5; 282 } else if (fs_hz_ == 32000) { 283 filter_coefficients = DspHelper::kDownsample32kHzTbl; 284 num_coefficients = 7; 285 } else { // fs_hz_ == 48000 286 filter_coefficients = DspHelper::kDownsample48kHzTbl; 287 num_coefficients = 7; 288 } 289 size_t signal_offset = num_coefficients - 1; 290 WebRtcSpl_DownsampleFast( 291 &expanded_signal[signal_offset], expanded_length - signal_offset, 292 expanded_downsampled_, kExpandDownsampLength, filter_coefficients, 293 num_coefficients, decimation_factor, kCompensateDelay); 294 if (input_length <= length_limit) { 295 // Not quite long enough, so we have to cheat a bit. 296 // If the input is shorter than the offset, we consider the input to be 0 297 // length. This will cause us to skip the downsampling since it makes no 298 // sense anyway, and input_downsampled_ will be filled with zeros. This is 299 // clearly a pathological case, and the signal quality will suffer, but 300 // there is not much we can do. 301 const size_t temp_len = 302 input_length > signal_offset ? input_length - signal_offset : 0; 303 // TODO(hlundin): Should `downsamp_temp_len` be corrected for round-off 304 // errors? I.e., (temp_len + decimation_factor - 1) / decimation_factor? 305 size_t downsamp_temp_len = temp_len / decimation_factor; 306 if (downsamp_temp_len > 0) { 307 WebRtcSpl_DownsampleFast(&input[signal_offset], temp_len, 308 input_downsampled_, downsamp_temp_len, 309 filter_coefficients, num_coefficients, 310 decimation_factor, kCompensateDelay); 311 } 312 memset(&input_downsampled_[downsamp_temp_len], 0, 313 sizeof(int16_t) * (kInputDownsampLength - downsamp_temp_len)); 314 } else { 315 WebRtcSpl_DownsampleFast( 316 &input[signal_offset], input_length - signal_offset, input_downsampled_, 317 kInputDownsampLength, filter_coefficients, num_coefficients, 318 decimation_factor, kCompensateDelay); 319 } 320 } 321 322 size_t Merge::CorrelateAndPeakSearch(size_t start_position, 323 size_t input_length, 324 size_t expand_period) const { 325 // Calculate correlation without any normalization. 326 const size_t max_corr_length = kMaxCorrelationLength; 327 size_t stop_position_downsamp = 328 std::min(max_corr_length, expand_->max_lag() / (fs_mult_ * 2) + 1); 329 330 int32_t correlation[kMaxCorrelationLength]; 331 CrossCorrelationWithAutoShift(input_downsampled_, expanded_downsampled_, 332 kInputDownsampLength, stop_position_downsamp, 1, 333 correlation); 334 335 // Normalize correlation to 14 bits and copy to a 16-bit array. 336 const size_t pad_length = expand_->overlap_length() - 1; 337 const size_t correlation_buffer_size = 2 * pad_length + kMaxCorrelationLength; 338 std::unique_ptr<int16_t[]> correlation16( 339 new int16_t[correlation_buffer_size]); 340 memset(correlation16.get(), 0, correlation_buffer_size * sizeof(int16_t)); 341 int16_t* correlation_ptr = &correlation16[pad_length]; 342 int32_t max_correlation = 343 WebRtcSpl_MaxAbsValueW32(correlation, stop_position_downsamp); 344 int norm_shift = std::max(0, 17 - WebRtcSpl_NormW32(max_correlation)); 345 WebRtcSpl_VectorBitShiftW32ToW16(correlation_ptr, stop_position_downsamp, 346 correlation, norm_shift); 347 348 // Calculate allowed starting point for peak finding. 349 // The peak location bestIndex must fulfill two criteria: 350 // (1) w16_bestIndex + input_length < 351 // timestamps_per_call_ + expand_->overlap_length(); 352 // (2) w16_bestIndex + input_length < start_position. 353 size_t start_index = timestamps_per_call_ + expand_->overlap_length(); 354 start_index = std::max(start_position, start_index); 355 start_index = (input_length > start_index) ? 0 : (start_index - input_length); 356 // Downscale starting index to 4kHz domain. (fs_mult_ * 2 = fs_hz_ / 4000.) 357 size_t start_index_downsamp = start_index / (fs_mult_ * 2); 358 359 // Calculate a modified `stop_position_downsamp` to account for the increased 360 // start index `start_index_downsamp` and the effective array length. 361 size_t modified_stop_pos = 362 std::min(stop_position_downsamp, 363 kMaxCorrelationLength + pad_length - start_index_downsamp); 364 size_t best_correlation_index; 365 int16_t best_correlation; 366 static const size_t kNumCorrelationCandidates = 1; 367 DspHelper::PeakDetection(&correlation_ptr[start_index_downsamp], 368 modified_stop_pos, kNumCorrelationCandidates, 369 fs_mult_, &best_correlation_index, 370 &best_correlation); 371 // Compensate for modified start index. 372 best_correlation_index += start_index; 373 374 // Ensure that underrun does not occur for 10ms case => we have to get at 375 // least 10ms + overlap . (This should never happen thanks to the above 376 // modification of peak-finding starting point.) 377 while (((best_correlation_index + input_length) < 378 (timestamps_per_call_ + expand_->overlap_length())) || 379 ((best_correlation_index + input_length) < start_position)) { 380 RTC_DCHECK_NOTREACHED(); // Should never happen. 381 best_correlation_index += expand_period; // Jump one lag ahead. 382 } 383 return best_correlation_index; 384 } 385 386 size_t Merge::RequiredFutureSamples() { 387 return fs_hz_ / 100 * num_channels_; // 10 ms. 388 } 389 390 } // namespace webrtc