tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

decision_logic.cc (13670B)

      1 /*
      2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3 *
      4 *  Use of this source code is governed by a BSD-style license
      5 *  that can be found in the LICENSE file in the root of the source
      6 *  tree. An additional intellectual property rights grant can be found
      7 *  in the file PATENTS.  All contributing project authors may
      8 *  be found in the AUTHORS file in the root of the source tree.
      9 */
     10 
     11 #include "modules/audio_coding/neteq/decision_logic.h"
     12 
     13 #include <cstdint>
     14 #include <cstdio>
     15 #include <memory>
     16 #include <optional>
     17 #include <utility>
     18 
     19 #include "api/environment/environment.h"
     20 #include "api/neteq/delay_manager_interface.h"
     21 #include "api/neteq/neteq.h"
     22 #include "api/neteq/neteq_controller.h"
     23 #include "modules/audio_coding/neteq/buffer_level_filter.h"
     24 #include "modules/audio_coding/neteq/packet_arrival_history.h"
     25 #include "modules/audio_coding/neteq/packet_buffer.h"
     26 #include "rtc_base/checks.h"
     27 #include "rtc_base/numerics/safe_conversions.h"
     28 
     29 namespace webrtc {
     30 
     31 namespace {
     32 
     33 constexpr int kPostponeDecodingLevel = 50;
     34 constexpr int kTargetLevelWindowMs = 100;
     35 // The granularity of delay adjustments (accelerate/preemptive expand) is 15ms,
     36 // but round up since the clock has a granularity of 10ms.
     37 constexpr int kDelayAdjustmentGranularityMs = 20;
     38 constexpr int kPacketHistorySizeMs = 2000;
     39 constexpr size_t kCngTimeoutMs = 1000;
     40 
     41 bool IsTimestretch(NetEq::Mode mode) {
     42  return mode == NetEq::Mode::kAccelerateSuccess ||
     43         mode == NetEq::Mode::kAccelerateLowEnergy ||
     44         mode == NetEq::Mode::kPreemptiveExpandSuccess ||
     45         mode == NetEq::Mode::kPreemptiveExpandLowEnergy;
     46 }
     47 
     48 bool IsCng(NetEq::Mode mode) {
     49  return mode == NetEq::Mode::kRfc3389Cng ||
     50         mode == NetEq::Mode::kCodecInternalCng;
     51 }
     52 
     53 bool IsExpand(NetEq::Mode mode) {
     54  return mode == NetEq::Mode::kExpand || mode == NetEq::Mode::kCodecPlc;
     55 }
     56 
     57 }  // namespace
     58 
     59 DecisionLogic::DecisionLogic(
     60    const Environment& env,
     61    NetEqController::Config config,
     62    std::unique_ptr<DelayManagerInterface> delay_manager)
     63    : DecisionLogic(config,
     64                    std::move(delay_manager),
     65                    std::make_unique<BufferLevelFilter>()) {}
     66 
     67 DecisionLogic::DecisionLogic(
     68    NetEqController::Config config,
     69    std::unique_ptr<DelayManagerInterface> delay_manager,
     70    std::unique_ptr<BufferLevelFilter> buffer_level_filter,
     71    std::unique_ptr<PacketArrivalHistory> packet_arrival_history)
     72    : delay_manager_(std::move(delay_manager)),
     73      delay_constraints_(config.max_packets_in_buffer,
     74                         config.base_min_delay_ms),
     75      buffer_level_filter_(std::move(buffer_level_filter)),
     76      packet_arrival_history_(
     77          packet_arrival_history
     78              ? std::move(packet_arrival_history)
     79              : std::make_unique<PacketArrivalHistory>(config.tick_timer,
     80                                                       kPacketHistorySizeMs)),
     81      tick_timer_(config.tick_timer),
     82      disallow_time_stretching_(!config.allow_time_stretching),
     83      timescale_countdown_(
     84          tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1)) {}
     85 
     86 DecisionLogic::~DecisionLogic() = default;
     87 
     88 void DecisionLogic::SoftReset() {
     89  packet_length_samples_ = 0;
     90  sample_memory_ = 0;
     91  prev_time_scale_ = false;
     92  timescale_countdown_ =
     93      tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1);
     94  time_stretched_cn_samples_ = 0;
     95  delay_manager_->Reset();
     96  buffer_level_filter_->Reset();
     97  packet_arrival_history_->Reset();
     98 }
     99 
    100 void DecisionLogic::SetSampleRate(int fs_hz, size_t output_size_samples) {
    101  // TODO(hlundin): Change to an enumerator and skip assert.
    102  RTC_DCHECK(fs_hz == 8000 || fs_hz == 16000 || fs_hz == 32000 ||
    103             fs_hz == 48000);
    104  sample_rate_khz_ = fs_hz / 1000;
    105  output_size_samples_ = output_size_samples;
    106  packet_arrival_history_->set_sample_rate(fs_hz);
    107 }
    108 
    109 NetEq::Operation DecisionLogic::GetDecision(const NetEqStatus& status,
    110                                            bool* /* reset_decoder */) {
    111  prev_time_scale_ = prev_time_scale_ && IsTimestretch(status.last_mode);
    112  if (prev_time_scale_) {
    113    timescale_countdown_ = tick_timer_->GetNewCountdown(kMinTimescaleInterval);
    114  }
    115  if (!IsCng(status.last_mode) && !IsExpand(status.last_mode)) {
    116    FilterBufferLevel(status.packet_buffer_info.span_samples);
    117  }
    118 
    119  // Guard for errors, to avoid getting stuck in error mode.
    120  if (status.last_mode == NetEq::Mode::kError) {
    121    if (!status.next_packet) {
    122      return NetEq::Operation::kExpand;
    123    } else {
    124      // Use kUndefined to flag for a reset.
    125      return NetEq::Operation::kUndefined;
    126    }
    127  }
    128 
    129  if (status.next_packet && status.next_packet->is_cng) {
    130    return CngOperation(status);
    131  }
    132 
    133  // Handle the case with no packet at all available (except maybe DTMF).
    134  if (!status.next_packet) {
    135    return NoPacket(status);
    136  }
    137 
    138  if (PostponeDecode(status)) {
    139    return NoPacket(status);
    140  }
    141 
    142  const uint32_t five_seconds_samples =
    143      static_cast<uint32_t>(5000 * sample_rate_khz_);
    144  // Check if the required packet is available.
    145  if (status.target_timestamp == status.next_packet->timestamp) {
    146    return ExpectedPacketAvailable(status);
    147  }
    148  if (!PacketBuffer::IsObsoleteTimestamp(status.next_packet->timestamp,
    149                                         status.target_timestamp,
    150                                         five_seconds_samples)) {
    151    return FuturePacketAvailable(status);
    152  }
    153  // This implies that available_timestamp < target_timestamp, which can
    154  // happen when a new stream or codec is received. Signal for a reset.
    155  return NetEq::Operation::kUndefined;
    156 }
    157 
    158 int DecisionLogic::TargetLevelMs() const {
    159  return delay_constraints_.Clamp(UnlimitedTargetLevelMs());
    160 }
    161 
    162 int DecisionLogic::UnlimitedTargetLevelMs() const {
    163  return delay_manager_->TargetDelayMs();
    164 }
    165 
    166 int DecisionLogic::GetFilteredBufferLevel() const {
    167  return buffer_level_filter_->filtered_current_level();
    168 }
    169 
    170 std::optional<int> DecisionLogic::PacketArrived(int fs_hz,
    171                                                bool should_update_stats,
    172                                                const PacketArrivedInfo& info) {
    173  buffer_flush_ = buffer_flush_ || info.buffer_flush;
    174  if (!should_update_stats || info.is_cng_or_dtmf) {
    175    return std::nullopt;
    176  }
    177  if (info.packet_length_samples > 0 && fs_hz > 0 &&
    178      info.packet_length_samples != packet_length_samples_) {
    179    packet_length_samples_ = info.packet_length_samples;
    180    delay_constraints_.SetPacketAudioLength(packet_length_samples_ * 1000 /
    181                                            fs_hz);
    182  }
    183  bool inserted = packet_arrival_history_->Insert(info.main_timestamp,
    184                                                  info.packet_length_samples);
    185  if (!inserted || packet_arrival_history_->size() < 2) {
    186    // No meaningful delay estimate unless at least 2 packets have arrived.
    187    return std::nullopt;
    188  }
    189  int arrival_delay_ms =
    190      packet_arrival_history_->GetDelayMs(info.main_timestamp);
    191  bool reordered =
    192      !packet_arrival_history_->IsNewestRtpTimestamp(info.main_timestamp);
    193  delay_manager_->Update(arrival_delay_ms, reordered, info);
    194  return arrival_delay_ms;
    195 }
    196 
    197 void DecisionLogic::FilterBufferLevel(size_t buffer_size_samples) {
    198  buffer_level_filter_->SetTargetBufferLevel(TargetLevelMs());
    199 
    200  int time_stretched_samples = time_stretched_cn_samples_;
    201  if (prev_time_scale_) {
    202    time_stretched_samples += sample_memory_;
    203  }
    204 
    205  if (buffer_flush_) {
    206    buffer_level_filter_->SetFilteredBufferLevel(buffer_size_samples);
    207    buffer_flush_ = false;
    208  } else {
    209    buffer_level_filter_->Update(buffer_size_samples, time_stretched_samples);
    210  }
    211  prev_time_scale_ = false;
    212  time_stretched_cn_samples_ = 0;
    213 }
    214 
    215 NetEq::Operation DecisionLogic::CngOperation(
    216    NetEqController::NetEqStatus status) {
    217  // Signed difference between target and available timestamp.
    218  int32_t timestamp_diff = static_cast<int32_t>(
    219      static_cast<uint32_t>(status.generated_noise_samples +
    220                            status.target_timestamp) -
    221      status.next_packet->timestamp);
    222  int optimal_level_samp = TargetLevelMs() * sample_rate_khz_;
    223  const int64_t excess_waiting_time_samp =
    224      -static_cast<int64_t>(timestamp_diff) - optimal_level_samp;
    225 
    226  if (excess_waiting_time_samp > optimal_level_samp / 2) {
    227    // The waiting time for this packet will be longer than 1.5
    228    // times the wanted buffer delay. Apply fast-forward to cut the
    229    // waiting time down to the optimal.
    230    noise_fast_forward_ =
    231        saturated_cast<size_t>(noise_fast_forward_ + excess_waiting_time_samp);
    232    timestamp_diff =
    233        saturated_cast<int32_t>(timestamp_diff + excess_waiting_time_samp);
    234  }
    235 
    236  if (timestamp_diff < 0 && status.last_mode == NetEq::Mode::kRfc3389Cng) {
    237    // Not time to play this packet yet. Wait another round before using this
    238    // packet. Keep on playing CNG from previous CNG parameters.
    239    return NetEq::Operation::kRfc3389CngNoPacket;
    240  } else {
    241    // Otherwise, go for the CNG packet now.
    242    noise_fast_forward_ = 0;
    243    return NetEq::Operation::kRfc3389Cng;
    244  }
    245 }
    246 
    247 NetEq::Operation DecisionLogic::NoPacket(NetEqController::NetEqStatus status) {
    248  switch (status.last_mode) {
    249    case NetEq::Mode::kRfc3389Cng:
    250      return NetEq::Operation::kRfc3389CngNoPacket;
    251    case NetEq::Mode::kCodecInternalCng: {
    252      // Stop CNG after a timeout.
    253      if (status.generated_noise_samples > kCngTimeoutMs * sample_rate_khz_) {
    254        return NetEq::Operation::kExpand;
    255      }
    256      return NetEq::Operation::kCodecInternalCng;
    257    }
    258    default:
    259      return status.play_dtmf ? NetEq::Operation::kDtmf
    260                              : NetEq::Operation::kExpand;
    261  }
    262 }
    263 
    264 NetEq::Operation DecisionLogic::ExpectedPacketAvailable(
    265    NetEqController::NetEqStatus status) {
    266  if (!disallow_time_stretching_ && status.last_mode != NetEq::Mode::kExpand &&
    267      !status.play_dtmf) {
    268    const int playout_delay_ms = GetPlayoutDelayMs(status);
    269    const int64_t low_limit = TargetLevelMs();
    270    const int64_t high_limit = low_limit +
    271                               packet_arrival_history_->GetMaxDelayMs() +
    272                               kDelayAdjustmentGranularityMs;
    273    if (playout_delay_ms >= high_limit * 4) {
    274      return NetEq::Operation::kFastAccelerate;
    275    }
    276    if (TimescaleAllowed()) {
    277      if (playout_delay_ms >= high_limit) {
    278        return NetEq::Operation::kAccelerate;
    279      }
    280      if (playout_delay_ms < low_limit) {
    281        return NetEq::Operation::kPreemptiveExpand;
    282      }
    283    }
    284  }
    285  return NetEq::Operation::kNormal;
    286 }
    287 
    288 NetEq::Operation DecisionLogic::FuturePacketAvailable(
    289    NetEqController::NetEqStatus status) {
    290  // Required packet is not available, but a future packet is.
    291  // Check if we should continue with an ongoing concealment because the new
    292  // packet is too far into the future.
    293  const int buffer_delay_samples =
    294      status.packet_buffer_info.span_samples_wait_time;
    295  const int buffer_delay_ms = buffer_delay_samples / sample_rate_khz_;
    296  const int high_limit = TargetLevelMs() + kTargetLevelWindowMs / 2;
    297  const bool above_target_delay = buffer_delay_ms > high_limit;
    298  if ((PacketTooEarly(status) && !above_target_delay)) {
    299    return NoPacket(status);
    300  }
    301  uint32_t timestamp_leap =
    302      status.next_packet->timestamp - status.target_timestamp;
    303  if (timestamp_leap != status.generated_noise_samples) {
    304    // The delay was adjusted, reinitialize the buffer level filter.
    305    buffer_level_filter_->SetFilteredBufferLevel(buffer_delay_samples);
    306  }
    307 
    308  // Time to play the next packet.
    309  switch (status.last_mode) {
    310    case NetEq::Mode::kExpand:
    311      return NetEq::Operation::kMerge;
    312    case NetEq::Mode::kCodecPlc:
    313    case NetEq::Mode::kRfc3389Cng:
    314    case NetEq::Mode::kCodecInternalCng:
    315      return NetEq::Operation::kNormal;
    316    default:
    317      return status.play_dtmf ? NetEq::Operation::kDtmf
    318                              : NetEq::Operation::kExpand;
    319  }
    320 }
    321 
    322 bool DecisionLogic::UnderTargetLevel() const {
    323  return buffer_level_filter_->filtered_current_level() <
    324         TargetLevelMs() * sample_rate_khz_;
    325 }
    326 
    327 bool DecisionLogic::PostponeDecode(NetEqController::NetEqStatus status) const {
    328  // Make sure we don't restart audio too soon after CNG or expand to avoid
    329  // running out of data right away again.
    330  const size_t min_buffer_level_samples =
    331      TargetLevelMs() * sample_rate_khz_ * kPostponeDecodingLevel / 100;
    332  const size_t buffer_level_samples =
    333      status.packet_buffer_info.span_samples_wait_time;
    334  if (buffer_level_samples >= min_buffer_level_samples) {
    335    return false;
    336  }
    337  // Don't postpone decoding if there is a future DTX packet in the packet
    338  // buffer.
    339  if (status.packet_buffer_info.dtx_or_cng) {
    340    return false;
    341  }
    342  // Continue CNG until the buffer is at least at the minimum level.
    343  if (IsCng(status.last_mode)) {
    344    return true;
    345  }
    346  // Only continue expand if the mute factor is low enough (otherwise the
    347  // expansion was short enough to not be noticable). Note that the MuteFactor
    348  // is in Q14, so a value of 16384 corresponds to 1.
    349  if (IsExpand(status.last_mode) && status.expand_mutefactor < 16384 / 2) {
    350    return true;
    351  }
    352  return false;
    353 }
    354 
    355 bool DecisionLogic::PacketTooEarly(NetEqController::NetEqStatus status) const {
    356  const uint32_t timestamp_leap =
    357      status.next_packet->timestamp - status.target_timestamp;
    358  return timestamp_leap > status.generated_noise_samples;
    359 }
    360 
    361 int DecisionLogic::GetPlayoutDelayMs(
    362    NetEqController::NetEqStatus status) const {
    363  uint32_t playout_timestamp =
    364      status.target_timestamp - status.sync_buffer_samples;
    365  return packet_arrival_history_->GetDelayMs(playout_timestamp);
    366 }
    367 
    368 }  // namespace webrtc