tor-browser

DynamicResampler.h (14071B)

---

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef DOM_MEDIA_DRIFTCONTROL_DYNAMICRESAMPLER_H_
#define DOM_MEDIA_DRIFTCONTROL_DYNAMICRESAMPLER_H_

#include <speex/speex_resampler.h>

#include "AudioRingBuffer.h"
#include "AudioSegment.h"
#include "TimeUnits.h"
#include "WavDumper.h"

namespace mozilla {

const uint32_t STEREO = 2;

/**
* DynamicResampler allows updating on the fly the output sample rate and the
* number of channels. In addition to that, it maintains an internal buffer for
* the input data and allows pre-buffering as well. The Resample() method
* strives to provide the requested number of output frames by using the input
* data including any pre-buffering. If there are fewer frames in the internal
* buffer than is requested, the internal buffer is padded with enough silence
* to allow the requested to be resampled and returned.
*
* Input data buffering makes use of the AudioRingBuffer. The capacity of the
* buffer is initially 100ms of audio and it is pre-allocated during
* SetSampleFormat(). Should the input data grow beyond that, the input buffer
* is re-allocated on the fly. In addition to that, due to special feature of
* AudioRingBuffer, no extra copies take place when the input data is fed to the
* resampler.
*
* The sample format must be set before using any method.
*
* The DynamicResampler is not thread-safe, so all the methods appart from the
* constructor must be called on the same thread.
*/
class DynamicResampler final {
public:
 /**
  * Provide the initial input and output rate and the amount of pre-buffering.
  * The channel count will be set to stereo. Memory allocation will take
  * place. The input buffer is non-interleaved.
  */
 DynamicResampler(uint32_t aInRate, uint32_t aOutRate,
                  uint32_t aInputPreBufferFrameCount = 0);
 ~DynamicResampler();

 /**
  * Set the sample format type to float or short.
  */
 void SetSampleFormat(AudioSampleFormat aFormat);
 uint32_t GetInRate() const { return mInRate; }
 uint32_t GetChannels() const { return mChannels; }

 /**
  * Append `aInFrames` number of frames from `aInBuffer` to the internal input
  * buffer. Memory copy/move takes place.
  */
 void AppendInput(Span<const float* const> aInBuffer, uint32_t aInFrames);
 void AppendInput(Span<const int16_t* const> aInBuffer, uint32_t aInFrames);
 /**
  * Append `aInFrames` number of frames of silence to the internal input
  * buffer. Memory copy/move takes place.
  */
 void AppendInputSilence(const uint32_t aInFrames);
 /**
  * Return the number of frames the internal input buffer can store.
  */
 uint32_t InFramesBufferSize() const;
 /**
  * Return the number of frames stored in the internal input buffer.
  */
 uint32_t InFramesBuffered(uint32_t aChannelIndex) const;

 /**
  * Prepends existing input data with a silent pre-buffer if not already done.
  * Data will be prepended so that after resampling aDuration of data,
  * the buffering level will be as close as possible to
  * mInputPreBufferFrameCount, which is the desired buffering level.
  */
 void EnsurePreBuffer(media::TimeUnit aDuration);

 /**
  * Set the number of frames that should be used for input pre-buffering.
  */
 void SetInputPreBufferFrameCount(uint32_t aInputPreBufferFrameCount);

 /*
  * Resample as much frames as needed from the internal input buffer to the
  * `aOutBuffer` in order to provide all `aOutFrames`.
  *
  * On first call, prepends the input buffer with silence so that after
  * resampling aOutFrames frames of data, the input buffer holds data as close
  * as possible to the configured pre-buffer size.
  *
  * If there are not enough input frames to provide the requested output
  * frames, the input buffer is padded with enough silence to allow the
  * requested frames to be resampled, and the pre-buffer is reset so that the
  * next call will be treated as the first.
  *
  * Returns true if the internal input buffer underran and had to be padded
  * with silence, otherwise false.
  */
 bool Resample(float* aOutBuffer, uint32_t aOutFrames, uint32_t aChannelIndex);
 bool Resample(int16_t* aOutBuffer, uint32_t aOutFrames,
               uint32_t aChannelIndex);

 /**
  * Update the output rate or/and the channel count. If a value is not updated
  * compared to the current one nothing happens. Changing the `aInRate`
  * results in recalculation in the resampler. Changing `aChannels` results in
  * the reallocation of the internal input buffer with the exception of
  * changes between mono to stereo and vice versa where no reallocation takes
  * place. A stereo internal input buffer is always maintained even if the
  * sound is mono.
  */
 void UpdateResampler(uint32_t aInRate, uint32_t aChannels);

private:
 template <typename T>
 void AppendInputInternal(Span<const T* const>& aInBuffer,
                          uint32_t aInFrames) {
   MOZ_ASSERT(aInBuffer.Length() == (uint32_t)mChannels);
   for (uint32_t i = 0; i < mChannels; ++i) {
     PushInFrames(aInBuffer[i], aInFrames, i);
   }
 }

 void ResampleInternal(const float* aInBuffer, uint32_t* aInFrames,
                       float* aOutBuffer, uint32_t* aOutFrames,
                       uint32_t aChannelIndex);
 void ResampleInternal(const int16_t* aInBuffer, uint32_t* aInFrames,
                       int16_t* aOutBuffer, uint32_t* aOutFrames,
                       uint32_t aChannelIndex);

 template <typename T>
 bool ResampleInternal(T* aOutBuffer, uint32_t aOutFrames,
                       uint32_t aChannelIndex) {
   MOZ_ASSERT(mInRate);
   MOZ_ASSERT(mOutRate);
   MOZ_ASSERT(mChannels);
   MOZ_ASSERT(aChannelIndex < mChannels);
   MOZ_ASSERT(aChannelIndex < mInternalInBuffer.Length());
   MOZ_ASSERT(aOutFrames);

   uint32_t outFramesNeeded = aOutFrames;
   T* nextOutFrame = aOutBuffer;
   if (mInRate == mOutRate) {
     if (!mResamplerIsBypassed) {
       uint32_t latency = speex_resampler_get_input_latency(mResampler);
       mInternalInBuffer[aChannelIndex].ReadNoCopy(
           [&](const Span<const T>& aInBuffer) -> uint32_t {
             // Although unlikely with the sample rates used with this class,
             // the resampler input latency may temporarily be higher than
             // indicated, after a change in resampling rate that reduces the
             // indicated latency. The resampler's "magic" samples cause
             // this. All frames in the resampler are extracted when
             // `latency` output frames have been extracted.
             uint32_t outFramesResampled = std::min(outFramesNeeded, latency);
             uint32_t inFrames = aInBuffer.Length();
             ResampleInternal(aInBuffer.Elements(), &inFrames, nextOutFrame,
                              &outFramesResampled, aChannelIndex);
             nextOutFrame += outFramesResampled;
             outFramesNeeded -= outFramesResampled;
             if (outFramesResampled == latency) {
               mResamplerIsBypassed = true;
               // The last `latency` frames of input to the resampler will not
               // be extracted from the resampler. Leave them in
               // mInternalInBuffer to be copied directly to nextOutFrame.
               MOZ_ASSERT(inFrames >= latency);
               return inFrames - latency;
             }
             return inFrames;
           });
     }
     bool underrun = false;
     if (uint32_t buffered = mInternalInBuffer[aChannelIndex].AvailableRead();
         buffered < outFramesNeeded) {
       underrun = true;
       mIsPreBufferSet = false;
       mInternalInBuffer[aChannelIndex].WriteSilence(outFramesNeeded -
                                                     buffered);
     }
     DebugOnly<uint32_t> numFramesRead = mInternalInBuffer[aChannelIndex].Read(
         Span(nextOutFrame, outFramesNeeded));
     MOZ_ASSERT(numFramesRead == outFramesNeeded);
     // Workaround to avoid discontinuity when the speex resampler operates
     // again. Feed it with the last 20 frames to warm up the internal memory
     // of the resampler and then skip memory equals to resampler's input
     // latency.
     mInputTail[aChannelIndex].StoreTail<T>(aOutBuffer, aOutFrames);
     if (aChannelIndex == 0 && !mIsWarmingUp) {
       mInputStreamFile.Write(nextOutFrame, outFramesNeeded);
       mOutputStreamFile.Write(nextOutFrame, outFramesNeeded);
     }
     return underrun;
   }

   auto resample = [&](const T* aInBuffer, uint32_t aInLength) -> uint32_t {
     uint32_t outFramesResampled = outFramesNeeded;
     uint32_t inFrames = aInLength;
     ResampleInternal(aInBuffer, &inFrames, nextOutFrame, &outFramesResampled,
                      aChannelIndex);
     nextOutFrame += outFramesResampled;
     outFramesNeeded -= outFramesResampled;
     mInputTail[aChannelIndex].StoreTail<T>(aInBuffer, inFrames);
     return inFrames;
   };

   MOZ_ASSERT(!mResamplerIsBypassed);
   mInternalInBuffer[aChannelIndex].ReadNoCopy(
       [&](const Span<const T>& aInBuffer) -> uint32_t {
         if (!outFramesNeeded) {
           return 0;
         }
         return resample(aInBuffer.Elements(), aInBuffer.Length());
       });

   if (outFramesNeeded == 0) {
     return false;
   }

   while (outFramesNeeded > 0) {
     MOZ_ASSERT(mInternalInBuffer[aChannelIndex].AvailableRead() == 0);
     // Round up.
     uint32_t totalInFramesNeeded =
         ((CheckedUint32(outFramesNeeded) * mInRate + mOutRate - 1) / mOutRate)
             .value();
     resample(nullptr, totalInFramesNeeded);
   }
   mIsPreBufferSet = false;
   return true;
 }

 template <typename T>
 void PushInFrames(const T* aInBuffer, const uint32_t aInFrames,
                   uint32_t aChannelIndex) {
   MOZ_ASSERT(aInBuffer);
   MOZ_ASSERT(aInFrames);
   MOZ_ASSERT(mChannels);
   MOZ_ASSERT(aChannelIndex < mChannels);
   MOZ_ASSERT(aChannelIndex < mInternalInBuffer.Length());
   EnsureInputBufferSizeInFrames(
       mInternalInBuffer[aChannelIndex].AvailableRead() + aInFrames);
   mInternalInBuffer[aChannelIndex].Write(Span(aInBuffer, aInFrames));
 }

 void WarmUpResampler(bool aSkipLatency);

 bool EnsureInputBufferSizeInFrames(uint32_t aSizeInFrames) {
   uint32_t sampleSize = 0;
   if (mSampleFormat == AUDIO_FORMAT_FLOAT32) {
     sampleSize = sizeof(float);
   } else if (mSampleFormat == AUDIO_FORMAT_S16) {
     sampleSize = sizeof(short);
   }

   if (sampleSize == 0) {
     // No sample format set, we wouldn't know how many bytes to allocate.
     return true;
   }

   uint32_t sizeInFrames = InFramesBufferSize();
   if (aSizeInFrames <= sizeInFrames) {
     // Buffer size is sufficient.
     return true;  // no reallocation necessary
   }

   // 5 second cap.
   const uint32_t cap = 5 * mInRate;
   if (sizeInFrames >= cap) {
     // Already at the cap.
     return false;
   }

   // As a backoff strategy, at least double the previous size.
   sizeInFrames *= 2;

   if (aSizeInFrames > sizeInFrames) {
     // A larger buffer than the normal backoff strategy provides is needed, or
     // this is the first time setting the buffer size. Add another 50ms, as
     // some jitter is expected.
     sizeInFrames = aSizeInFrames + mInRate / 20;
   }

   // mInputPreBufferFrameCount is an indication of the desired average
   // buffering.  Provide for at least twice this.
   sizeInFrames = std::max(sizeInFrames, mInputPreBufferFrameCount * 2);

   sizeInFrames = std::min(cap, sizeInFrames);

   bool success = true;
   for (auto& b : mInternalInBuffer) {
     success = success && b.EnsureLengthBytes(sampleSize * sizeInFrames);
   }

   if (success) {
     // All buffers have the new size.
     return true;
   }

   // Allocating an input buffer failed. We stick with the old buffer size.
   NS_WARNING(nsPrintfCString("Failed to allocate a buffer of %u bytes (%u "
                              "frames). Expect glitches.",
                              sampleSize * sizeInFrames, sizeInFrames)
                  .get());
   return false;
 }

public:
 const uint32_t mOutRate;

private:
 bool mIsPreBufferSet = false;
 bool mIsWarmingUp = false;
 // The resampler can be bypassed when the input and output rates match and
 // any frames buffered in the resampler have been extracted.  This initial
 // value is reset on construction by UpdateResampler() if the rates differ.
 bool mResamplerIsBypassed = true;
 uint32_t mInputPreBufferFrameCount;
 uint32_t mChannels = 0;
 uint32_t mInRate;

 AutoTArray<AudioRingBuffer, STEREO> mInternalInBuffer;

 SpeexResamplerState* mResampler = nullptr;
 AudioSampleFormat mSampleFormat = AUDIO_FORMAT_SILENCE;

 class TailBuffer {
  public:
   template <typename T>
   T* Buffer() {
     return reinterpret_cast<T*>(mBuffer);
   }
   /* Store the MAXSIZE last elements of the buffer. */
   template <typename T>
   void StoreTail(const Span<const T>& aInBuffer) {
     StoreTail(aInBuffer.data(), aInBuffer.size());
   }
   template <typename T>
   void StoreTail(const T* aInBuffer, uint32_t aInFrames) {
     const T* inBuffer = aInBuffer;
     mSize = std::min(aInFrames, MAXSIZE);
     if (inBuffer) {
       PodCopy(Buffer<T>(), inBuffer + aInFrames - mSize, mSize);
     } else {
       std::fill_n(Buffer<T>(), mSize, static_cast<T>(0));
     }
   }
   uint32_t Length() { return mSize; }
   static constexpr uint32_t MAXSIZE = 20;

  private:
   float mBuffer[MAXSIZE] = {};
   uint32_t mSize = 0;
 };
 AutoTArray<TailBuffer, STEREO> mInputTail;

 WavDumper mInputStreamFile;
 WavDumper mOutputStreamFile;
};

}  // namespace mozilla

#endif  // DOM_MEDIA_DRIFTCONTROL_DYNAMICRESAMPLER_H_