cubeb_resampler_internal.h (24225B)
1 /* 2 * Copyright © 2016 Mozilla Foundation 3 * 4 * This program is made available under an ISC-style license. See the 5 * accompanying file LICENSE for details. 6 */ 7 8 #if !defined(CUBEB_RESAMPLER_INTERNAL) 9 #define CUBEB_RESAMPLER_INTERNAL 10 11 #include <algorithm> 12 #include <cassert> 13 #include <cmath> 14 #include <memory> 15 #ifdef CUBEB_GECKO_BUILD 16 #include "mozilla/UniquePtr.h" 17 // In libc++, symbols such as std::unique_ptr may be defined in std::__1. 18 // The _LIBCPP_BEGIN_NAMESPACE_STD and _LIBCPP_END_NAMESPACE_STD macros 19 // will expand to the correct namespace. 20 #ifdef _LIBCPP_BEGIN_NAMESPACE_STD 21 #define MOZ_BEGIN_STD_NAMESPACE _LIBCPP_BEGIN_NAMESPACE_STD 22 #define MOZ_END_STD_NAMESPACE _LIBCPP_END_NAMESPACE_STD 23 #else 24 #define MOZ_BEGIN_STD_NAMESPACE namespace std { 25 #define MOZ_END_STD_NAMESPACE } 26 #endif 27 MOZ_BEGIN_STD_NAMESPACE 28 using mozilla::DefaultDelete; 29 using mozilla::UniquePtr; 30 #define default_delete DefaultDelete 31 #define unique_ptr UniquePtr 32 MOZ_END_STD_NAMESPACE 33 #endif 34 #include "cubeb-speex-resampler.h" 35 #include "cubeb/cubeb.h" 36 #include "cubeb_log.h" 37 #include "cubeb_resampler.h" 38 #include "cubeb_utils.h" 39 #include <stdio.h> 40 41 /* This header file contains the internal C++ API of the resamplers, for 42 * testing. */ 43 44 // When dropping audio input frames to prevent building 45 // an input delay, this function returns the number of frames 46 // to keep in the buffer. 47 // @parameter sample_rate The sample rate of the stream. 48 // @return A number of frames to keep. 49 uint32_t 50 min_buffered_audio_frame(uint32_t sample_rate); 51 52 int 53 to_speex_quality(cubeb_resampler_quality q); 54 55 struct cubeb_resampler { 56 virtual long fill(void * input_buffer, long * input_frames_count, 57 void * output_buffer, long frames_needed) = 0; 58 virtual long latency() = 0; 59 virtual cubeb_resampler_stats stats() = 0; 60 virtual ~cubeb_resampler() {} 61 }; 62 63 /** Base class for processors. This is just used to share methods for now. */ 64 class processor { 65 public: 66 explicit processor(uint32_t channels) : channels(channels) {} 67 68 protected: 69 size_t frames_to_samples(size_t frames) const { return frames * channels; } 70 size_t samples_to_frames(size_t samples) const 71 { 72 assert(!(samples % channels)); 73 return samples / channels; 74 } 75 /** The number of channel of the audio buffers to be resampled. */ 76 const uint32_t channels; 77 }; 78 79 template <typename T> 80 class passthrough_resampler : public cubeb_resampler, public processor { 81 public: 82 passthrough_resampler(cubeb_stream * s, cubeb_data_callback cb, void * ptr, 83 uint32_t input_channels, uint32_t sample_rate); 84 85 virtual long fill(void * input_buffer, long * input_frames_count, 86 void * output_buffer, long output_frames); 87 88 virtual long latency() { return 0; } 89 90 virtual cubeb_resampler_stats stats() 91 { 92 cubeb_resampler_stats stats; 93 stats.input_input_buffer_size = internal_input_buffer.length(); 94 stats.input_output_buffer_size = 0; 95 stats.output_input_buffer_size = 0; 96 stats.output_output_buffer_size = 0; 97 return stats; 98 } 99 100 void drop_audio_if_needed() 101 { 102 uint32_t to_keep = min_buffered_audio_frame(sample_rate); 103 uint32_t available = samples_to_frames(internal_input_buffer.length()); 104 if (available > to_keep) { 105 ALOGV("Dropping %u frames", available - to_keep); 106 internal_input_buffer.pop(nullptr, 107 frames_to_samples(available - to_keep)); 108 } 109 } 110 111 private: 112 cubeb_stream * const stream; 113 const cubeb_data_callback data_callback; 114 void * const user_ptr; 115 /* This allows to buffer some input to account for the fact that we buffer 116 * some inputs. */ 117 auto_array<T> internal_input_buffer; 118 uint32_t sample_rate; 119 }; 120 121 /** Bidirectional resampler, can resample an input and an output stream, or just 122 * an input stream or output stream. In this case a delay is inserted in the 123 * opposite direction to keep the streams synchronized. */ 124 template <typename T, typename InputProcessing, typename OutputProcessing> 125 class cubeb_resampler_speex : public cubeb_resampler { 126 public: 127 cubeb_resampler_speex(InputProcessing * input_processor, 128 OutputProcessing * output_processor, cubeb_stream * s, 129 cubeb_data_callback cb, void * ptr); 130 131 virtual ~cubeb_resampler_speex(); 132 133 virtual long fill(void * input_buffer, long * input_frames_count, 134 void * output_buffer, long output_frames_needed); 135 136 virtual cubeb_resampler_stats stats() 137 { 138 cubeb_resampler_stats stats = {}; 139 if (input_processor) { 140 stats.input_input_buffer_size = input_processor->input_buffer_size(); 141 stats.input_output_buffer_size = input_processor->output_buffer_size(); 142 } 143 if (output_processor) { 144 stats.output_input_buffer_size = output_processor->input_buffer_size(); 145 stats.output_output_buffer_size = output_processor->output_buffer_size(); 146 } 147 return stats; 148 } 149 150 virtual long latency() 151 { 152 if (input_processor && output_processor) { 153 assert(input_processor->latency() == output_processor->latency()); 154 return input_processor->latency(); 155 } else if (input_processor) { 156 return input_processor->latency(); 157 } else { 158 return output_processor->latency(); 159 } 160 } 161 162 private: 163 typedef long (cubeb_resampler_speex::*processing_callback)( 164 T * input_buffer, long * input_frames_count, T * output_buffer, 165 long output_frames_needed); 166 167 long fill_internal_duplex(T * input_buffer, long * input_frames_count, 168 T * output_buffer, long output_frames_needed); 169 long fill_internal_input(T * input_buffer, long * input_frames_count, 170 T * output_buffer, long output_frames_needed); 171 long fill_internal_output(T * input_buffer, long * input_frames_count, 172 T * output_buffer, long output_frames_needed); 173 174 std::unique_ptr<InputProcessing> input_processor; 175 std::unique_ptr<OutputProcessing> output_processor; 176 processing_callback fill_internal; 177 cubeb_stream * const stream; 178 const cubeb_data_callback data_callback; 179 void * const user_ptr; 180 bool draining = false; 181 }; 182 183 /** Handles one way of a (possibly) duplex resampler, working on interleaved 184 * audio buffers of type T. This class is designed so that the number of frames 185 * coming out of the resampler can be precisely controled. It manages its own 186 * input buffer, and can use the caller's output buffer, or allocate its own. */ 187 template <typename T> class cubeb_resampler_speex_one_way : public processor { 188 public: 189 /** The sample type of this resampler, either 16-bit integers or 32-bit 190 * floats. */ 191 typedef T sample_type; 192 /** Construct a resampler resampling from #source_rate to #target_rate, that 193 * can be arbitrary, strictly positive number. 194 * @parameter channels The number of channels this resampler will resample. 195 * @parameter source_rate The sample-rate of the audio input. 196 * @parameter target_rate The sample-rate of the audio output. 197 * @parameter quality A number between 0 (fast, low quality) and 10 (slow, 198 * high quality). */ 199 cubeb_resampler_speex_one_way(uint32_t channels, uint32_t source_rate, 200 uint32_t target_rate, int quality) 201 : processor(channels), 202 resampling_ratio(static_cast<float>(source_rate) / target_rate), 203 source_rate(source_rate), additional_latency(0), leftover_samples(0) 204 { 205 int r; 206 speex_resampler = 207 speex_resampler_init(channels, source_rate, target_rate, quality, &r); 208 assert(r == RESAMPLER_ERR_SUCCESS && "resampler allocation failure"); 209 210 uint32_t input_latency = speex_resampler_get_input_latency(speex_resampler); 211 const size_t LATENCY_SAMPLES = 8192; 212 T input_buffer[LATENCY_SAMPLES] = {}; 213 T output_buffer[LATENCY_SAMPLES] = {}; 214 uint32_t input_frame_count = input_latency; 215 uint32_t output_frame_count = LATENCY_SAMPLES; 216 assert(input_latency * channels <= LATENCY_SAMPLES); 217 speex_resample(input_buffer, &input_frame_count, output_buffer, 218 &output_frame_count); 219 } 220 221 /** Destructor, deallocate the resampler */ 222 virtual ~cubeb_resampler_speex_one_way() 223 { 224 speex_resampler_destroy(speex_resampler); 225 } 226 227 /* Fill the resampler with `input_frame_count` frames. */ 228 void input(T * input_buffer, size_t input_frame_count) 229 { 230 resampling_in_buffer.push(input_buffer, 231 frames_to_samples(input_frame_count)); 232 } 233 234 /** Outputs exactly `output_frame_count` into `output_buffer`. 235 * `output_buffer` has to be at least `output_frame_count` long. */ 236 size_t output(T * output_buffer, size_t output_frame_count) 237 { 238 uint32_t in_len = samples_to_frames(resampling_in_buffer.length()); 239 uint32_t out_len = output_frame_count; 240 241 speex_resample(resampling_in_buffer.data(), &in_len, output_buffer, 242 &out_len); 243 244 /* This shifts back any unresampled samples to the beginning of the input 245 buffer. */ 246 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len)); 247 248 return out_len; 249 } 250 251 size_t output_for_input(uint32_t input_frames) 252 { 253 return (size_t)floorf( 254 (input_frames + samples_to_frames(resampling_in_buffer.length())) / 255 resampling_ratio); 256 } 257 258 /** Returns a buffer containing exactly `output_frame_count` resampled frames. 259 * The consumer should not hold onto the pointer. */ 260 T * output(size_t output_frame_count, size_t * input_frames_used) 261 { 262 if (resampling_out_buffer.capacity() < 263 frames_to_samples(output_frame_count)) { 264 resampling_out_buffer.reserve(frames_to_samples(output_frame_count)); 265 } 266 267 uint32_t in_len = samples_to_frames(resampling_in_buffer.length()); 268 uint32_t out_len = output_frame_count; 269 270 speex_resample(resampling_in_buffer.data(), &in_len, 271 resampling_out_buffer.data(), &out_len); 272 273 if (out_len < output_frame_count) { 274 LOGV("underrun during resampling: got %u frames, expected %zu", 275 (unsigned)out_len, output_frame_count); 276 // silence the rightmost part 277 T * data = resampling_out_buffer.data(); 278 for (uint32_t i = frames_to_samples(out_len); 279 i < frames_to_samples(output_frame_count); i++) { 280 data[i] = 0; 281 } 282 } 283 284 /* This shifts back any unresampled samples to the beginning of the input 285 buffer. */ 286 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len)); 287 *input_frames_used = in_len; 288 289 return resampling_out_buffer.data(); 290 } 291 292 /** Get the latency of the resampler, in output frames. */ 293 uint32_t latency() const 294 { 295 /* The documentation of the resampler talks about "samples" here, but it 296 * only consider a single channel here so it's the same number of frames. */ 297 int latency = 0; 298 299 latency = speex_resampler_get_output_latency(speex_resampler) + 300 additional_latency; 301 302 assert(latency >= 0); 303 304 return latency; 305 } 306 307 /** Returns the number of frames to pass in the input of the resampler to have 308 * at least `output_frame_count` resampled frames. */ 309 uint32_t input_needed_for_output(int32_t output_frame_count) const 310 { 311 assert(output_frame_count >= 0); // Check overflow 312 int32_t unresampled_frames_left = 313 samples_to_frames(resampling_in_buffer.length()); 314 float input_frames_needed_frac = 315 static_cast<float>(output_frame_count) * resampling_ratio; 316 // speex_resample()` can be irregular in its consumption of input samples. 317 // Provide one more frame than the number that would be required with 318 // regular consumption, to make the speex resampler behave more regularly, 319 // and so predictably. 320 auto input_frame_needed = 321 1 + static_cast<int32_t>(ceilf(input_frames_needed_frac)); 322 input_frame_needed -= std::min(unresampled_frames_left, input_frame_needed); 323 return input_frame_needed; 324 } 325 326 /** Returns a pointer to the input buffer, that contains empty space for at 327 * least `frame_count` elements. This is useful so that consumer can 328 * directly write into the input buffer of the resampler. The pointer 329 * returned is adjusted so that leftover data are not overwritten. 330 */ 331 T * input_buffer(size_t frame_count) 332 { 333 leftover_samples = resampling_in_buffer.length(); 334 resampling_in_buffer.reserve(leftover_samples + 335 frames_to_samples(frame_count)); 336 return resampling_in_buffer.data() + leftover_samples; 337 } 338 339 /** This method works with `input_buffer`, and allows to inform the 340 processor how much frames have been written in the provided buffer. */ 341 void written(size_t written_frames) 342 { 343 resampling_in_buffer.set_length(leftover_samples + 344 frames_to_samples(written_frames)); 345 } 346 347 void drop_audio_if_needed() 348 { 349 // Keep at most 100ms buffered. 350 uint32_t available = samples_to_frames(resampling_in_buffer.length()); 351 uint32_t to_keep = min_buffered_audio_frame(source_rate); 352 if (available > to_keep) { 353 ALOGV("Dropping %u frames", available - to_keep); 354 resampling_in_buffer.pop(nullptr, frames_to_samples(available - to_keep)); 355 } 356 } 357 358 size_t input_buffer_size() const { return resampling_in_buffer.length(); } 359 size_t output_buffer_size() const { return resampling_out_buffer.length(); } 360 361 private: 362 /** Wrapper for the speex resampling functions to have a typed 363 * interface. */ 364 void speex_resample(float * input_buffer, uint32_t * input_frame_count, 365 float * output_buffer, uint32_t * output_frame_count) 366 { 367 #ifndef NDEBUG 368 int rv; 369 rv = 370 #endif 371 speex_resampler_process_interleaved_float( 372 speex_resampler, input_buffer, input_frame_count, output_buffer, 373 output_frame_count); 374 assert(rv == RESAMPLER_ERR_SUCCESS); 375 } 376 377 void speex_resample(short * input_buffer, uint32_t * input_frame_count, 378 short * output_buffer, uint32_t * output_frame_count) 379 { 380 #ifndef NDEBUG 381 int rv; 382 rv = 383 #endif 384 speex_resampler_process_interleaved_int( 385 speex_resampler, input_buffer, input_frame_count, output_buffer, 386 output_frame_count); 387 assert(rv == RESAMPLER_ERR_SUCCESS); 388 } 389 390 /** The state for the speex resampler used internaly. */ 391 SpeexResamplerState * speex_resampler; 392 /** Source rate / target rate. */ 393 const float resampling_ratio; 394 const uint32_t source_rate; 395 /** Storage for the input frames, to be resampled. Also contains 396 * any unresampled frames after resampling. */ 397 auto_array<T> resampling_in_buffer; 398 /* Storage for the resampled frames, to be passed back to the caller. */ 399 auto_array<T> resampling_out_buffer; 400 /** Additional latency inserted into the pipeline for synchronisation. */ 401 uint32_t additional_latency; 402 /** When `input_buffer` is called, this allows tracking the number of 403 samples that were in the buffer. */ 404 uint32_t leftover_samples; 405 }; 406 407 /** This class allows delaying an audio stream by `frames` frames. */ 408 template <typename T> class delay_line : public processor { 409 public: 410 /** Constructor 411 * @parameter frames the number of frames of delay. 412 * @parameter channels the number of channels of this delay line. 413 * @parameter sample_rate sample-rate of the audio going through this delay 414 * line */ 415 delay_line(uint32_t frames, uint32_t channels, uint32_t sample_rate) 416 : processor(channels), length(frames), leftover_samples(0), 417 sample_rate(sample_rate) 418 { 419 /* Fill the delay line with some silent frames to add latency. */ 420 delay_input_buffer.push_silence(frames * channels); 421 } 422 /** Push some frames into the delay line. 423 * @parameter buffer the frames to push. 424 * @parameter frame_count the number of frames in #buffer. */ 425 void input(T * buffer, uint32_t frame_count) 426 { 427 delay_input_buffer.push(buffer, frames_to_samples(frame_count)); 428 } 429 /** Pop some frames from the internal buffer, into a internal output buffer. 430 * @parameter frames_needed the number of frames to be returned. 431 * @return a buffer containing the delayed frames. The consumer should not 432 * hold onto the pointer. */ 433 T * output(uint32_t frames_needed, size_t * input_frames_used) 434 { 435 if (delay_output_buffer.capacity() < frames_to_samples(frames_needed)) { 436 delay_output_buffer.reserve(frames_to_samples(frames_needed)); 437 } 438 439 delay_output_buffer.clear(); 440 delay_output_buffer.push(delay_input_buffer.data(), 441 frames_to_samples(frames_needed)); 442 delay_input_buffer.pop(nullptr, frames_to_samples(frames_needed)); 443 *input_frames_used = frames_needed; 444 445 return delay_output_buffer.data(); 446 } 447 /** Get a pointer to the first writable location in the input buffer> 448 * @parameter frames_needed the number of frames the user needs to write 449 * into the buffer. 450 * @returns a pointer to a location in the input buffer where #frames_needed 451 * can be writen. */ 452 T * input_buffer(uint32_t frames_needed) 453 { 454 leftover_samples = delay_input_buffer.length(); 455 delay_input_buffer.reserve(leftover_samples + 456 frames_to_samples(frames_needed)); 457 return delay_input_buffer.data() + leftover_samples; 458 } 459 /** This method works with `input_buffer`, and allows to inform the 460 processor how much frames have been written in the provided buffer. */ 461 void written(size_t frames_written) 462 { 463 delay_input_buffer.set_length(leftover_samples + 464 frames_to_samples(frames_written)); 465 } 466 /** Drains the delay line, emptying the buffer. 467 * @parameter output_buffer the buffer in which the frames are written. 468 * @parameter frames_needed the maximum number of frames to write. 469 * @return the actual number of frames written. */ 470 size_t output(T * output_buffer, uint32_t frames_needed) 471 { 472 uint32_t in_len = samples_to_frames(delay_input_buffer.length()); 473 uint32_t out_len = frames_needed; 474 475 uint32_t to_pop = std::min(in_len, out_len); 476 477 delay_input_buffer.pop(output_buffer, frames_to_samples(to_pop)); 478 479 return to_pop; 480 } 481 /** Returns the number of frames one needs to input into the delay line to 482 * get #frames_needed frames back. 483 * @parameter frames_needed the number of frames one want to write into the 484 * delay_line 485 * @returns the number of frames one will get. */ 486 uint32_t input_needed_for_output(int32_t frames_needed) const 487 { 488 assert(frames_needed >= 0); // Check overflow 489 return frames_needed; 490 } 491 /** Returns the number of frames produces for `input_frames` frames in input 492 */ 493 size_t output_for_input(uint32_t input_frames) { return input_frames; } 494 /** The number of frames this delay line delays the stream by. 495 * @returns The number of frames of delay. */ 496 size_t latency() { return length; } 497 498 void drop_audio_if_needed() 499 { 500 uint32_t available = samples_to_frames(delay_input_buffer.length()); 501 uint32_t to_keep = min_buffered_audio_frame(sample_rate); 502 if (available > to_keep) { 503 ALOGV("Dropping %u frames", available - to_keep); 504 505 delay_input_buffer.pop(nullptr, frames_to_samples(available - to_keep)); 506 } 507 } 508 509 size_t input_buffer_size() const { return delay_input_buffer.length(); } 510 size_t output_buffer_size() const { return delay_output_buffer.length(); } 511 512 private: 513 /** The length, in frames, of this delay line */ 514 uint32_t length; 515 /** When `input_buffer` is called, this allows tracking the number of 516 samples that where in the buffer. */ 517 uint32_t leftover_samples; 518 /** The input buffer, where the delay is applied. */ 519 auto_array<T> delay_input_buffer; 520 /** The output buffer. This is only ever used if using the ::output with a 521 * single argument. */ 522 auto_array<T> delay_output_buffer; 523 uint32_t sample_rate; 524 }; 525 526 /** This sits behind the C API and is more typed. */ 527 template <typename T> 528 cubeb_resampler * 529 cubeb_resampler_create_internal(cubeb_stream * stream, 530 cubeb_stream_params * input_params, 531 cubeb_stream_params * output_params, 532 unsigned int target_rate, 533 cubeb_data_callback callback, void * user_ptr, 534 cubeb_resampler_quality quality, 535 cubeb_resampler_reclock reclock) 536 { 537 std::unique_ptr<cubeb_resampler_speex_one_way<T>> input_resampler = nullptr; 538 std::unique_ptr<cubeb_resampler_speex_one_way<T>> output_resampler = nullptr; 539 std::unique_ptr<delay_line<T>> input_delay = nullptr; 540 std::unique_ptr<delay_line<T>> output_delay = nullptr; 541 542 assert((input_params || output_params) && 543 "need at least one valid parameter pointer."); 544 545 /* All the streams we have have a sample rate that matches the target 546 sample rate, use a no-op resampler, that simply forwards the buffers to 547 the callback. */ 548 if (((input_params && input_params->rate == target_rate) && 549 (output_params && output_params->rate == target_rate)) || 550 (input_params && !output_params && (input_params->rate == target_rate)) || 551 (output_params && !input_params && 552 (output_params->rate == target_rate))) { 553 LOG("Input and output sample-rate match, target rate of %dHz", target_rate); 554 return new passthrough_resampler<T>( 555 stream, callback, user_ptr, input_params ? input_params->channels : 0, 556 target_rate); 557 } 558 559 /* Determine if we need to resampler one or both directions, and create the 560 resamplers. */ 561 if (output_params && (output_params->rate != target_rate)) { 562 output_resampler.reset(new cubeb_resampler_speex_one_way<T>( 563 output_params->channels, target_rate, output_params->rate, 564 to_speex_quality(quality))); 565 if (!output_resampler) { 566 return NULL; 567 } 568 } 569 570 if (input_params && (input_params->rate != target_rate)) { 571 input_resampler.reset(new cubeb_resampler_speex_one_way<T>( 572 input_params->channels, input_params->rate, target_rate, 573 to_speex_quality(quality))); 574 if (!input_resampler) { 575 return NULL; 576 } 577 } 578 579 /* If we resample only one direction but we have a duplex stream, insert a 580 * delay line with a length equal to the resampler latency of the 581 * other direction so that the streams are synchronized. */ 582 if (input_resampler && !output_resampler && input_params && output_params) { 583 output_delay.reset(new delay_line<T>(input_resampler->latency(), 584 output_params->channels, 585 output_params->rate)); 586 if (!output_delay) { 587 return NULL; 588 } 589 } else if (output_resampler && !input_resampler && input_params && 590 output_params) { 591 input_delay.reset(new delay_line<T>(output_resampler->latency(), 592 input_params->channels, 593 output_params->rate)); 594 if (!input_delay) { 595 return NULL; 596 } 597 } 598 599 if (input_resampler && output_resampler) { 600 LOG("Resampling input (%d) and output (%d) to target rate of %dHz", 601 input_params->rate, output_params->rate, target_rate); 602 return new cubeb_resampler_speex<T, cubeb_resampler_speex_one_way<T>, 603 cubeb_resampler_speex_one_way<T>>( 604 input_resampler.release(), output_resampler.release(), stream, callback, 605 user_ptr); 606 } else if (input_resampler) { 607 LOG("Resampling input (%d) to target and output rate of %dHz", 608 input_params->rate, target_rate); 609 return new cubeb_resampler_speex<T, cubeb_resampler_speex_one_way<T>, 610 delay_line<T>>(input_resampler.release(), 611 output_delay.release(), 612 stream, callback, user_ptr); 613 } else { 614 LOG("Resampling output (%dHz) to target and input rate of %dHz", 615 output_params->rate, target_rate); 616 return new cubeb_resampler_speex<T, delay_line<T>, 617 cubeb_resampler_speex_one_way<T>>( 618 input_delay.release(), output_resampler.release(), stream, callback, 619 user_ptr); 620 } 621 } 622 623 #endif /* CUBEB_RESAMPLER_INTERNAL */