audio_ingress.h (5604B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef AUDIO_VOIP_AUDIO_INGRESS_H_ 12 #define AUDIO_VOIP_AUDIO_INGRESS_H_ 13 14 #include <algorithm> 15 #include <atomic> 16 #include <cstdint> 17 #include <map> 18 #include <memory> 19 #include <optional> 20 21 #include "api/array_view.h" 22 #include "api/audio/audio_mixer.h" 23 #include "api/audio_codecs/audio_decoder_factory.h" 24 #include "api/audio_codecs/audio_format.h" 25 #include "api/environment/environment.h" 26 #include "api/neteq/neteq.h" 27 #include "api/scoped_refptr.h" 28 #include "api/voip/voip_statistics.h" 29 #include "audio/audio_level.h" 30 #include "modules/audio_coding/acm2/acm_resampler.h" 31 #include "modules/audio_coding/include/audio_coding_module.h" 32 #include "modules/audio_coding/include/audio_coding_module_typedefs.h" 33 #include "modules/rtp_rtcp/include/receive_statistics.h" 34 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" 35 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" 36 #include "rtc_base/numerics/safe_conversions.h" 37 #include "rtc_base/numerics/sequence_number_unwrapper.h" 38 #include "rtc_base/synchronization/mutex.h" 39 #include "rtc_base/thread_annotations.h" 40 41 namespace webrtc { 42 43 // AudioIngress handles incoming RTP/RTCP packets from the remote 44 // media endpoint. Received RTP packets are injected into AcmReceiver and 45 // when audio output thread requests for audio samples to play through system 46 // output such as speaker device, AudioIngress provides the samples via its 47 // implementation on AudioMixer::Source interface. 48 // 49 // Note that this class is originally based on ChannelReceive in 50 // audio/channel_receive.cc with non-audio related logic trimmed as aimed for 51 // smaller footprint. 52 class AudioIngress : public AudioMixer::Source { 53 public: 54 AudioIngress(const Environment& env, 55 RtpRtcpInterface* rtp_rtcp, 56 ReceiveStatistics* receive_statistics, 57 scoped_refptr<AudioDecoderFactory> decoder_factory); 58 ~AudioIngress() override; 59 60 // Start or stop receiving operation of AudioIngress. 61 bool StartPlay(); 62 void StopPlay() { 63 playing_ = false; 64 output_audio_level_.ResetLevelFullRange(); 65 } 66 67 // Query the state of the AudioIngress. 68 bool IsPlaying() const { return playing_; } 69 70 // Set the decoder formats and payload type for AcmReceiver where the 71 // key type (int) of the map is the payload type of SdpAudioFormat. 72 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs); 73 74 // APIs to handle received RTP/RTCP packets from caller. 75 void ReceivedRTPPacket(ArrayView<const uint8_t> rtp_packet); 76 void ReceivedRTCPPacket(ArrayView<const uint8_t> rtcp_packet); 77 78 // See comments on LevelFullRange, TotalEnergy, TotalDuration from 79 // audio/audio_level.h. 80 int GetOutputAudioLevel() const { 81 return output_audio_level_.LevelFullRange(); 82 } 83 double GetOutputTotalEnergy() { return output_audio_level_.TotalEnergy(); } 84 double GetOutputTotalDuration() { 85 return output_audio_level_.TotalDuration(); 86 } 87 88 NetworkStatistics GetNetworkStatistics() const; 89 90 ChannelStatistics GetChannelStatistics(); 91 92 // Implementation of AudioMixer::Source interface. 93 AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( 94 int sampling_rate, 95 AudioFrame* audio_frame) override; 96 int Ssrc() const override { return dchecked_cast<int>(remote_ssrc_.load()); } 97 int PreferredSampleRate() const override { 98 std::optional<NetEq::DecoderFormat> decoder = 99 neteq_->GetCurrentDecoderFormat(); 100 101 // If we haven't received any RTP packet from remote and thus 102 // last_packet_sampling_rate is not available then use NetEq's sampling 103 // rate as that would be what would be used for audio output sample. 104 return std::max(decoder ? decoder->sample_rate_hz : 0, 105 neteq_->last_output_sample_rate_hz()); 106 } 107 108 private: 109 const Environment env_; 110 111 // Indicates AudioIngress status as caller invokes Start/StopPlaying. 112 // If not playing, incoming RTP data processing is skipped, thus 113 // producing no data to output device. 114 std::atomic<bool> playing_; 115 116 // Currently active remote ssrc from remote media endpoint. 117 std::atomic<uint32_t> remote_ssrc_; 118 119 // The first rtp timestamp of the output audio frame that is used to 120 // calculate elasped time for subsequent audio frames. 121 std::atomic<int64_t> first_rtp_timestamp_; 122 123 // Synchronizaton is handled internally by ReceiveStatistics. 124 ReceiveStatistics* const rtp_receive_statistics_; 125 126 // Synchronizaton is handled internally by RtpRtcpInterface. 127 RtpRtcpInterface* const rtp_rtcp_; 128 129 // Synchronizaton is handled internally by NetEq. 130 const std::unique_ptr<NetEq> neteq_; 131 132 // Synchronizaton is handled internally by voe::AudioLevel. 133 voe::AudioLevel output_audio_level_; 134 135 Mutex lock_; 136 137 RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(lock_); 138 139 // For receiving RTP statistics, this tracks the sampling rate value 140 // per payload type set when caller set via SetReceiveCodecs. 141 std::map<int, int> receive_codec_info_ RTC_GUARDED_BY(lock_); 142 143 RtpTimestampUnwrapper timestamp_wrap_handler_ RTC_GUARDED_BY(lock_); 144 145 // Resampler for the output audio. 146 acm2::ResamplerHelper resampler_helper_ RTC_GUARDED_BY(lock_); 147 }; 148 149 } // namespace webrtc 150 151 #endif // AUDIO_VOIP_AUDIO_INGRESS_H_