tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

AudioDecoderInputTrack.h (9116B)

      1 /* This Source Code Form is subject to the terms of the Mozilla Public
      2 * License, v. 2.0. If a copy of the MPL was not distributed with this
      3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      4 
      5 #ifndef AudioDecoderInputTrack_h
      6 #define AudioDecoderInputTrack_h
      7 
      8 #include "AudioSegment.h"
      9 #include "MediaEventSource.h"
     10 #include "MediaSegment.h"
     11 #include "MediaTimer.h"
     12 #include "MediaTrackGraph.h"
     13 #include "TimeUnits.h"
     14 #include "mozilla/SPSCQueue.h"
     15 #include "mozilla/StateMirroring.h"
     16 #include "mozilla/TimeStamp.h"
     17 #include "nsISerialEventTarget.h"
     18 
     19 namespace mozilla {
     20 
     21 class AudioData;
     22 class AudioInfo;
     23 class RLBoxSoundTouch;
     24 
     25 /**
     26 * AudioDecoderInputTrack is used as a source for the audio decoder data, which
     27 * supports adjusting playback rate and preserve pitch.
     28 * The owner of this track would be responsible to push audio data via
     29 * `AppendData()` into a SPSC queue, which is a thread-safe queue between the
     30 * decoder thread (producer) and the graph thread (consumer). MediaTrackGraph
     31 * requires data via `ProcessInput()`, then AudioDecoderInputTrack would convert
     32 * (based on sample rate and playback rate) and append the amount of needed
     33 * audio frames onto the output segment that would be used by MediaTrackGraph.
     34 */
     35 class AudioDecoderInputTrack final : public ProcessedMediaTrack {
     36 public:
     37  static AudioDecoderInputTrack* Create(MediaTrackGraph* aGraph,
     38                                        nsISerialEventTarget* aDecoderThread,
     39                                        const AudioInfo& aInfo,
     40                                        float aPlaybackRate, float aVolume,
     41                                        bool aPreservesPitch);
     42 
     43  // SPSCData suppports filling different supported type variants, and is used
     44  // to achieve a thread-safe information exchange between the decoder thread
     45  // and the graph thread.
     46  struct SPSCData final {
     47    struct Empty {};
     48    struct ClearFutureData {};
     49    struct DecodedData {
     50      DecodedData()
     51          : mStartTime(media::TimeUnit::Invalid()),
     52            mEndTime(media::TimeUnit::Invalid()) {}
     53      DecodedData(DecodedData&& aDecodedData)
     54          : mSegment(std::move(aDecodedData.mSegment)) {
     55        mStartTime = aDecodedData.mStartTime;
     56        mEndTime = aDecodedData.mEndTime;
     57        aDecodedData.Clear();
     58      }
     59      DecodedData(media::TimeUnit aStartTime, media::TimeUnit aEndTime)
     60          : mStartTime(aStartTime), mEndTime(aEndTime) {}
     61      DecodedData(const DecodedData&) = delete;
     62      DecodedData& operator=(const DecodedData&) = delete;
     63      void Clear() {
     64        mSegment.Clear();
     65        mStartTime = media::TimeUnit::Invalid();
     66        mEndTime = media::TimeUnit::Invalid();
     67      }
     68      AudioSegment mSegment;
     69      media::TimeUnit mStartTime;
     70      media::TimeUnit mEndTime;
     71    };
     72    struct EOS {};
     73 
     74    SPSCData() : mData(Empty()) {};
     75    explicit SPSCData(ClearFutureData&& aArg) : mData(std::move(aArg)) {};
     76    explicit SPSCData(DecodedData&& aArg) : mData(std::move(aArg)) {};
     77    explicit SPSCData(EOS&& aArg) : mData(std::move(aArg)) {};
     78 
     79    bool HasData() const { return !mData.is<Empty>(); }
     80    bool IsClearFutureData() const { return mData.is<ClearFutureData>(); }
     81    bool IsDecodedData() const { return mData.is<DecodedData>(); }
     82    bool IsEOS() const { return mData.is<EOS>(); }
     83 
     84    DecodedData* AsDecodedData() {
     85      return IsDecodedData() ? &mData.as<DecodedData>() : nullptr;
     86    }
     87 
     88    Variant<Empty, ClearFutureData, DecodedData, EOS> mData;
     89  };
     90 
     91  // Decoder thread API
     92  void AppendData(AudioData* aAudio, const PrincipalHandle& aPrincipalHandle);
     93  void AppendData(nsTArray<RefPtr<AudioData>>& aAudioArray,
     94                  const PrincipalHandle& aPrincipalHandle);
     95  void NotifyEndOfStream();
     96  void ClearFutureData();
     97  void SetVolume(float aVolume);
     98  void SetPlaybackRate(float aPlaybackRate);
     99  void SetPreservesPitch(bool aPreservesPitch);
    100  // After calling this, the track are not expected to receive any new data.
    101  void Close();
    102  bool HasBatchedData() const;
    103 
    104  MediaEventSource<int64_t>& OnOutput() { return mOnOutput; }
    105  MediaEventSource<void>& OnEnd() { return mOnEnd; }
    106 
    107  // Graph Thread API
    108  void DestroyImpl() override;
    109  void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
    110  uint32_t NumberOfChannels() const override;
    111 
    112  // The functions below are only used for testing.
    113  TrackTime WrittenFrames() const {
    114    AssertOnGraphThread();
    115    return mWrittenFrames;
    116  }
    117  float Volume() const {
    118    AssertOnGraphThread();
    119    return mVolume;
    120  }
    121  float PlaybackRate() const {
    122    AssertOnGraphThread();
    123    return mPlaybackRate;
    124  }
    125 
    126 protected:
    127  ~AudioDecoderInputTrack();
    128 
    129 private:
    130  AudioDecoderInputTrack(nsISerialEventTarget* aDecoderThread,
    131                         TrackRate aGraphRate, const AudioInfo& aInfo,
    132                         float aPlaybackRate, float aVolume,
    133                         bool aPreservesPitch);
    134 
    135  // Return false if the converted segment contains zero duration.
    136  bool ConvertAudioDataToSegment(AudioData* aAudio, AudioSegment& aSegment,
    137                                 const PrincipalHandle& aPrincipalHandle);
    138 
    139  void HandleSPSCData(SPSCData& aData);
    140 
    141  // These methods would return the total frames that we consumed from
    142  // `mBufferedData`.
    143  TrackTime AppendBufferedDataToOutput(TrackTime aExpectedDuration);
    144  TrackTime FillDataToTimeStretcher(TrackTime aExpectedDuration);
    145  TrackTime AppendTimeStretchedDataToSegment(TrackTime aExpectedDuration,
    146                                             AudioSegment& aOutput);
    147  TrackTime AppendUnstretchedDataToSegment(TrackTime aExpectedDuration,
    148                                           AudioSegment& aOutput);
    149 
    150  // Return the total frames that we retrieve from the time stretcher.
    151  TrackTime DrainStretchedDataIfNeeded(TrackTime aExpectedDuration,
    152                                       AudioSegment& aOutput);
    153  TrackTime GetDataFromTimeStretcher(TrackTime aExpectedDuration,
    154                                     AudioSegment& aOutput);
    155  void NotifyInTheEndOfProcessInput(TrackTime aFillDuration);
    156 
    157  bool HasSentAllData() const;
    158 
    159  bool ShouldBatchData() const;
    160  void BatchData(AudioData* aAudio, const PrincipalHandle& aPrincipalHandle);
    161  void DispatchPushBatchedDataIfNeeded();
    162  void PushBatchedDataIfNeeded();
    163  void PushDataToSPSCQueue(SPSCData& data);
    164 
    165  void SetVolumeImpl(float aVolume);
    166  void SetPlaybackRateImpl(float aPlaybackRate);
    167  void SetPreservesPitchImpl(bool aPreservesPitch);
    168 
    169  void EnsureTimeStretcher();
    170  void SetTempoAndRateForTimeStretcher();
    171  uint32_t GetChannelCountForTimeStretcher() const;
    172 
    173  inline void AssertOnDecoderThread() const {
    174    MOZ_ASSERT(mDecoderThread->IsOnCurrentThread());
    175  }
    176 
    177  const RefPtr<nsISerialEventTarget> mDecoderThread;
    178 
    179  // Notify the amount of audio frames which have been sent to the track.
    180  MediaEventProducer<int64_t> mOnOutput;
    181  // Notify when the track is ended.
    182  MediaEventProducer<void> mOnEnd;
    183 
    184  // These variables are ONLY used in the decoder thread.
    185  nsAutoRef<SpeexResamplerState> mResampler;
    186  uint32_t mResamplerChannelCount;
    187  const uint32_t mInitialInputChannels;
    188  TrackRate mInputSampleRate;
    189  DelayedScheduler<TimeStamp> mDelayedScheduler;
    190  bool mShutdownSPSCQueue = false;
    191 
    192  // These attributes are ONLY used in the graph thread.
    193  bool mReceivedEOS = false;
    194  TrackTime mWrittenFrames = 0;
    195  float mPlaybackRate;
    196  float mVolume;
    197  bool mPreservesPitch;
    198 
    199  // A thread-safe queue shared by the decoder thread and the graph thread.
    200  // The decoder thread is the producer side, and the graph thread is the
    201  // consumer side. This queue should NEVER get full. In order to achieve that,
    202  // we would batch input samples when SPSC queue doesn't have many available
    203  // capacity.
    204  // In addition, as the media track isn't guaranteed to be destroyed on the
    205  // graph thread (it could be destroyed on the main thread as well) so we might
    206  // not clear all data in SPSC queue when the track's `DestroyImpl()` gets
    207  // called. We leave to destroy the queue later when the track gets destroyed.
    208  SPSCQueue<SPSCData> mSPSCQueue{40};
    209 
    210  // When the graph requires the less amount of audio frames than the amount of
    211  // frames an audio data has, then the remaining part of frames would be stored
    212  // and used in next iteration.
    213  // This is ONLY used in the graph thread.
    214  AudioSegment mBufferedData;
    215 
    216  // In order to prevent SPSC queue from being full, we want to batch multiple
    217  // data into one to control the density of SPSC queue, the length of batched
    218  // data would be dynamically adjusted by queue's available capacity.
    219  // This is ONLY used in the decoder thread.
    220  SPSCData::DecodedData mBatchedData;
    221 
    222  // True if we've sent all data to the graph, then the track will be marked as
    223  // ended in the next iteration.
    224  bool mSentAllData = false;
    225 
    226  // This is used to adjust the playback rate and pitch.
    227  RLBoxSoundTouch* mTimeStretcher = nullptr;
    228 
    229  // Buffers that would be used for the time stretching.
    230  AutoTArray<AudioDataValue, 2> mInterleavedBuffer;
    231 };
    232 
    233 }  // namespace mozilla
    234 
    235 #endif  // AudioDecoderInputTrack_h