tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

audio_ingress.cc (13294B)

      1 /*
      2 *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
      3 *
      4 *  Use of this source code is governed by a BSD-style license
      5 *  that can be found in the LICENSE file in the root of the source
      6 *  tree. An additional intellectual property rights grant can be found
      7 *  in the file PATENTS.  All contributing project authors may
      8 *  be found in the AUTHORS file in the root of the source tree.
      9 */
     10 
     11 #include "audio/voip/audio_ingress.h"
     12 
     13 #include <cstdint>
     14 #include <ctime>
     15 #include <map>
     16 #include <optional>
     17 #include <utility>
     18 #include <vector>
     19 
     20 #include "api/array_view.h"
     21 #include "api/audio/audio_mixer.h"
     22 #include "api/audio_codecs/audio_decoder_factory.h"
     23 #include "api/audio_codecs/audio_format.h"
     24 #include "api/environment/environment.h"
     25 #include "api/neteq/default_neteq_factory.h"
     26 #include "api/neteq/neteq.h"
     27 #include "api/scoped_refptr.h"
     28 #include "api/units/time_delta.h"
     29 #include "api/voip/voip_statistics.h"
     30 #include "audio/utility/audio_frame_operations.h"
     31 #include "modules/audio_coding/include/audio_coding_module.h"
     32 #include "modules/audio_coding/include/audio_coding_module_typedefs.h"
     33 #include "modules/rtp_rtcp/include/receive_statistics.h"
     34 #include "modules/rtp_rtcp/include/report_block_data.h"
     35 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
     36 #include "modules/rtp_rtcp/source/byte_io.h"
     37 #include "modules/rtp_rtcp/source/rtcp_packet/common_header.h"
     38 #include "modules/rtp_rtcp/source/rtcp_packet/receiver_report.h"
     39 #include "modules/rtp_rtcp/source/rtcp_packet/sender_report.h"
     40 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
     41 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h"
     42 #include "rtc_base/checks.h"
     43 #include "rtc_base/logging.h"
     44 #include "rtc_base/synchronization/mutex.h"
     45 
     46 namespace webrtc {
     47 
     48 namespace {
     49 
     50 NetEq::Config CreateNetEqConfig() {
     51  NetEq::Config config;
     52  config.enable_muted_state = true;
     53  return config;
     54 }
     55 
     56 }  // namespace
     57 
     58 AudioIngress::AudioIngress(const Environment& env,
     59                           RtpRtcpInterface* rtp_rtcp,
     60                           ReceiveStatistics* receive_statistics,
     61                           scoped_refptr<AudioDecoderFactory> decoder_factory)
     62    : env_(env),
     63      playing_(false),
     64      remote_ssrc_(0),
     65      first_rtp_timestamp_(-1),
     66      rtp_receive_statistics_(receive_statistics),
     67      rtp_rtcp_(rtp_rtcp),
     68      neteq_(DefaultNetEqFactory().Create(env,
     69                                          CreateNetEqConfig(),
     70                                          decoder_factory)),
     71      ntp_estimator_(&env_.clock()) {}
     72 
     73 AudioIngress::~AudioIngress() = default;
     74 
     75 AudioMixer::Source::AudioFrameInfo AudioIngress::GetAudioFrameWithInfo(
     76    int sampling_rate,
     77    AudioFrame* audio_frame) {
     78  // Get 10ms raw PCM data from the ACM.
     79  bool muted = false;
     80  {
     81    MutexLock lock(&lock_);
     82    if ((neteq_->GetAudio(audio_frame, &muted) != NetEq::kOK) ||
     83        !resampler_helper_.MaybeResample(sampling_rate, audio_frame)) {
     84      RTC_DLOG(LS_ERROR) << "GetAudio() failed!";
     85      // In all likelihood, the audio in this frame is garbage. We return an
     86      // error so that the audio mixer module doesn't add it to the mix. As
     87      // a result, it won't be played out and the actions skipped here are
     88      // irrelevant.
     89      return AudioMixer::Source::AudioFrameInfo::kError;
     90    }
     91  }
     92 
     93  if (muted) {
     94    AudioFrameOperations::Mute(audio_frame);
     95  }
     96 
     97  // Measure audio level.
     98  constexpr double kAudioSampleDurationSeconds = 0.01;
     99  output_audio_level_.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds);
    100 
    101  // If caller invoked StopPlay(), then mute the frame.
    102  if (!playing_) {
    103    AudioFrameOperations::Mute(audio_frame);
    104    muted = true;
    105  }
    106 
    107  // Set first rtp timestamp with first audio frame with valid timestamp.
    108  if (first_rtp_timestamp_ < 0 && audio_frame->timestamp_ != 0) {
    109    first_rtp_timestamp_ = audio_frame->timestamp_;
    110  }
    111 
    112  if (first_rtp_timestamp_ >= 0) {
    113    // Compute elapsed and NTP times.
    114    int64_t unwrap_timestamp;
    115    {
    116      MutexLock lock(&lock_);
    117      unwrap_timestamp =
    118          timestamp_wrap_handler_.Unwrap(audio_frame->timestamp_);
    119      audio_frame->ntp_time_ms_ =
    120          ntp_estimator_.Estimate(audio_frame->timestamp_);
    121    }
    122    // For clock rate, default to the playout sampling rate if we haven't
    123    // received any packets yet.
    124    std::optional<NetEq::DecoderFormat> decoder =
    125        neteq_->GetCurrentDecoderFormat();
    126    int clock_rate = decoder ? decoder->sdp_format.clockrate_hz
    127                             : neteq_->last_output_sample_rate_hz();
    128    RTC_DCHECK_GT(clock_rate, 0);
    129    audio_frame->elapsed_time_ms_ =
    130        (unwrap_timestamp - first_rtp_timestamp_) / (clock_rate / 1000);
    131  }
    132 
    133  return muted ? AudioMixer::Source::AudioFrameInfo::kMuted
    134               : AudioMixer::Source::AudioFrameInfo::kNormal;
    135 }
    136 
    137 bool AudioIngress::StartPlay() {
    138  {
    139    MutexLock lock(&lock_);
    140    if (receive_codec_info_.empty()) {
    141      RTC_DLOG(LS_WARNING) << "Receive codecs have not been set yet";
    142      return false;
    143    }
    144  }
    145  playing_ = true;
    146  return true;
    147 }
    148 
    149 void AudioIngress::SetReceiveCodecs(
    150    const std::map<int, SdpAudioFormat>& codecs) {
    151  {
    152    MutexLock lock(&lock_);
    153    for (const auto& kv : codecs) {
    154      receive_codec_info_[kv.first] = kv.second.clockrate_hz;
    155    }
    156  }
    157  neteq_->SetCodecs(codecs);
    158 }
    159 
    160 void AudioIngress::ReceivedRTPPacket(ArrayView<const uint8_t> rtp_packet) {
    161  RtpPacketReceived rtp_packet_received;
    162  rtp_packet_received.Parse(rtp_packet.data(), rtp_packet.size());
    163 
    164  // Set payload type's sampling rate before we feed it into ReceiveStatistics.
    165  {
    166    MutexLock lock(&lock_);
    167    const auto& it =
    168        receive_codec_info_.find(rtp_packet_received.PayloadType());
    169    // If sampling rate info is not available in our received codec set, it
    170    // would mean that remote media endpoint is sending incorrect payload id
    171    // which can't be processed correctly especially on payload type id in
    172    // dynamic range.
    173    if (it == receive_codec_info_.end()) {
    174      RTC_DLOG(LS_WARNING) << "Unexpected payload id received: "
    175                           << rtp_packet_received.PayloadType();
    176      return;
    177    }
    178    rtp_packet_received.set_payload_type_frequency(it->second);
    179  }
    180 
    181  // Track current remote SSRC.
    182  if (rtp_packet_received.Ssrc() != remote_ssrc_) {
    183    rtp_rtcp_->SetRemoteSSRC(rtp_packet_received.Ssrc());
    184    remote_ssrc_.store(rtp_packet_received.Ssrc());
    185  }
    186 
    187  rtp_receive_statistics_->OnRtpPacket(rtp_packet_received);
    188 
    189  RTPHeader header;
    190  rtp_packet_received.GetHeader(&header);
    191 
    192  size_t packet_length = rtp_packet_received.size();
    193  if (packet_length < header.headerLength ||
    194      (packet_length - header.headerLength) < header.paddingLength) {
    195    RTC_DLOG(LS_ERROR) << "Packet length(" << packet_length << ") header("
    196                       << header.headerLength << ") padding("
    197                       << header.paddingLength << ")";
    198    return;
    199  }
    200 
    201  const uint8_t* payload = rtp_packet_received.data() + header.headerLength;
    202  size_t payload_length = packet_length - header.headerLength;
    203  size_t payload_data_length = payload_length - header.paddingLength;
    204  auto data_view = ArrayView<const uint8_t>(payload, payload_data_length);
    205 
    206  // Push the incoming payload (parsed and ready for decoding) into the ACM.
    207  if (data_view.empty()) {
    208    neteq_->InsertEmptyPacket(header);
    209  } else if (neteq_->InsertPacket(header, data_view,
    210                                  env_.clock().CurrentTime()) < 0) {
    211    RTC_DLOG(LS_ERROR) << "ChannelReceive::OnReceivedPayloadData() unable to "
    212                          "insert packet into NetEq";
    213  }
    214 }
    215 
    216 void AudioIngress::ReceivedRTCPPacket(ArrayView<const uint8_t> rtcp_packet) {
    217  rtcp::CommonHeader rtcp_header;
    218  if (rtcp_header.Parse(rtcp_packet.data(), rtcp_packet.size()) &&
    219      (rtcp_header.type() == rtcp::SenderReport::kPacketType ||
    220       rtcp_header.type() == rtcp::ReceiverReport::kPacketType)) {
    221    RTC_DCHECK_GE(rtcp_packet.size(), 8);
    222 
    223    uint32_t sender_ssrc =
    224        ByteReader<uint32_t>::ReadBigEndian(rtcp_packet.data() + 4);
    225 
    226    // If we don't have remote ssrc at this point, it's likely that remote
    227    // endpoint is receive-only or it could have restarted the media.
    228    if (sender_ssrc != remote_ssrc_) {
    229      rtp_rtcp_->SetRemoteSSRC(sender_ssrc);
    230      remote_ssrc_.store(sender_ssrc);
    231    }
    232  }
    233 
    234  // Deliver RTCP packet to RTP/RTCP module for parsing and processing.
    235  rtp_rtcp_->IncomingRtcpPacket(rtcp_packet);
    236 
    237  std::optional<TimeDelta> rtt = rtp_rtcp_->LastRtt();
    238  if (!rtt.has_value()) {
    239    // Waiting for valid RTT.
    240    return;
    241  }
    242 
    243  std::optional<RtpRtcpInterface::SenderReportStats> last_sr =
    244      rtp_rtcp_->GetSenderReportStats();
    245  if (!last_sr.has_value()) {
    246    // Waiting for RTCP.
    247    return;
    248  }
    249 
    250  {
    251    MutexLock lock(&lock_);
    252    ntp_estimator_.UpdateRtcpTimestamp(*rtt, last_sr->last_remote_ntp_timestamp,
    253                                       last_sr->last_remote_rtp_timestamp);
    254  }
    255 }
    256 
    257 NetworkStatistics AudioIngress::GetNetworkStatistics() const {
    258  NetworkStatistics stats;
    259  stats.currentExpandRate = 0;
    260  stats.currentSpeechExpandRate = 0;
    261  stats.currentPreemptiveRate = 0;
    262  stats.currentAccelerateRate = 0;
    263  stats.currentSecondaryDecodedRate = 0;
    264  stats.currentSecondaryDiscardedRate = 0;
    265  stats.meanWaitingTimeMs = -1;
    266  stats.maxWaitingTimeMs = 1;
    267 
    268  NetEqNetworkStatistics neteq_stat = neteq_->CurrentNetworkStatistics();
    269  stats.currentBufferSize = neteq_stat.current_buffer_size_ms;
    270  stats.preferredBufferSize = neteq_stat.preferred_buffer_size_ms;
    271  stats.jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false;
    272 
    273  NetEqLifetimeStatistics neteq_lifetime_stat = neteq_->GetLifetimeStatistics();
    274  stats.totalSamplesReceived = neteq_lifetime_stat.total_samples_received;
    275  stats.concealedSamples = neteq_lifetime_stat.concealed_samples;
    276  stats.silentConcealedSamples = neteq_lifetime_stat.silent_concealed_samples;
    277  stats.concealmentEvents = neteq_lifetime_stat.concealment_events;
    278  stats.jitterBufferDelayMs = neteq_lifetime_stat.jitter_buffer_delay_ms;
    279  stats.jitterBufferTargetDelayMs =
    280      neteq_lifetime_stat.jitter_buffer_target_delay_ms;
    281  stats.jitterBufferMinimumDelayMs =
    282      neteq_lifetime_stat.jitter_buffer_minimum_delay_ms;
    283  stats.jitterBufferEmittedCount =
    284      neteq_lifetime_stat.jitter_buffer_emitted_count;
    285  stats.delayedPacketOutageSamples =
    286      neteq_lifetime_stat.delayed_packet_outage_samples;
    287  stats.relativePacketArrivalDelayMs =
    288      neteq_lifetime_stat.relative_packet_arrival_delay_ms;
    289  stats.interruptionCount = neteq_lifetime_stat.interruption_count;
    290  stats.totalInterruptionDurationMs =
    291      neteq_lifetime_stat.total_interruption_duration_ms;
    292  stats.insertedSamplesForDeceleration =
    293      neteq_lifetime_stat.inserted_samples_for_deceleration;
    294  stats.removedSamplesForAcceleration =
    295      neteq_lifetime_stat.removed_samples_for_acceleration;
    296  stats.fecPacketsReceived = neteq_lifetime_stat.fec_packets_received;
    297  stats.fecPacketsDiscarded = neteq_lifetime_stat.fec_packets_discarded;
    298  stats.totalProcessingDelayUs = neteq_lifetime_stat.total_processing_delay_us;
    299  stats.packetsDiscarded = neteq_lifetime_stat.packets_discarded;
    300 
    301  NetEqOperationsAndState neteq_operations_and_state =
    302      neteq_->GetOperationsAndState();
    303  stats.packetBufferFlushes = neteq_operations_and_state.packet_buffer_flushes;
    304 
    305  return stats;
    306 }
    307 
    308 ChannelStatistics AudioIngress::GetChannelStatistics() {
    309  ChannelStatistics channel_stats;
    310 
    311  // Get clockrate for current decoder ahead of jitter calculation.
    312  auto decoder = neteq_->GetCurrentDecoderFormat();
    313  const uint32_t clockrate_hz = decoder ? decoder->sdp_format.clockrate_hz : 0;
    314 
    315  StreamStatistician* statistician =
    316      rtp_receive_statistics_->GetStatistician(remote_ssrc_);
    317  if (statistician) {
    318    RtpReceiveStats stats = statistician->GetStats();
    319    channel_stats.packets_lost = stats.packets_lost;
    320    channel_stats.packets_received = stats.packet_counter.packets;
    321    channel_stats.bytes_received = stats.packet_counter.payload_bytes;
    322    channel_stats.remote_ssrc = remote_ssrc_;
    323    if (clockrate_hz > 0) {
    324      channel_stats.jitter = static_cast<double>(stats.jitter) / clockrate_hz;
    325    }
    326  }
    327 
    328  // Get RTCP report using remote SSRC.
    329  const std::vector<ReportBlockData>& report_data =
    330      rtp_rtcp_->GetLatestReportBlockData();
    331  for (const ReportBlockData& rtcp_report : report_data) {
    332    if (rtp_rtcp_->SSRC() != rtcp_report.source_ssrc() ||
    333        remote_ssrc_ != rtcp_report.sender_ssrc()) {
    334      continue;
    335    }
    336    RemoteRtcpStatistics remote_stat;
    337    remote_stat.packets_lost = rtcp_report.cumulative_lost();
    338    remote_stat.fraction_lost = rtcp_report.fraction_lost();
    339    if (clockrate_hz > 0) {
    340      remote_stat.jitter = rtcp_report.jitter(clockrate_hz).seconds<double>();
    341    }
    342    if (rtcp_report.has_rtt()) {
    343      remote_stat.round_trip_time = rtcp_report.last_rtt().seconds<double>();
    344    }
    345    remote_stat.last_report_received_timestamp_ms =
    346        rtcp_report.report_block_timestamp_utc().ms();
    347    channel_stats.remote_rtcp = remote_stat;
    348 
    349    // Receive only channel won't send any RTP packets.
    350    if (!channel_stats.remote_ssrc.has_value()) {
    351      channel_stats.remote_ssrc = remote_ssrc_;
    352    }
    353    break;
    354  }
    355 
    356  return channel_stats;
    357 }
    358 
    359 }  // namespace webrtc