tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

frame_buffer.cc (11258B)

      1 /*
      2 *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
      3 *
      4 *  Use of this source code is governed by a BSD-style license
      5 *  that can be found in the LICENSE file in the root of the source
      6 *  tree. An additional intellectual property rights grant can be found
      7 *  in the file PATENTS.  All contributing project authors may
      8 *  be found in the AUTHORS file in the root of the source tree.
      9 */
     10 
     11 #include "api/video/frame_buffer.h"
     12 
     13 #include <algorithm>
     14 #include <cstddef>
     15 #include <cstdint>
     16 #include <iterator>
     17 #include <memory>
     18 #include <optional>
     19 #include <utility>
     20 
     21 #include "absl/algorithm/container.h"
     22 #include "absl/container/inlined_vector.h"
     23 #include "api/array_view.h"
     24 #include "api/field_trials_view.h"
     25 #include "api/video/encoded_frame.h"
     26 #include "rtc_base/logging.h"
     27 #include "rtc_base/numerics/sequence_number_util.h"
     28 #include "rtc_base/trace_event.h"
     29 
     30 namespace webrtc {
     31 namespace {
     32 bool ValidReferences(const EncodedFrame& frame) {
     33  // All references must point backwards, and duplicates are not allowed.
     34  for (size_t i = 0; i < frame.num_references; ++i) {
     35    if (frame.references[i] >= frame.Id())
     36      return false;
     37 
     38    for (size_t j = i + 1; j < frame.num_references; ++j) {
     39      if (frame.references[i] == frame.references[j])
     40        return false;
     41    }
     42  }
     43 
     44  return true;
     45 }
     46 
     47 // Since FrameBuffer::FrameInfo is private it can't be used in the function
     48 // signature, hence the FrameIteratorT type.
     49 template <typename FrameIteratorT>
     50 ArrayView<const int64_t> GetReferences(const FrameIteratorT& it) {
     51  return {it->second.encoded_frame->references,
     52          std::min<size_t>(it->second.encoded_frame->num_references,
     53                           EncodedFrame::kMaxFrameReferences)};
     54 }
     55 
     56 template <typename FrameIteratorT>
     57 int64_t GetFrameId(const FrameIteratorT& it) {
     58  return it->first;
     59 }
     60 
     61 template <typename FrameIteratorT>
     62 uint32_t GetTimestamp(const FrameIteratorT& it) {
     63  return it->second.encoded_frame->RtpTimestamp();
     64 }
     65 
     66 template <typename FrameIteratorT>
     67 bool IsLastFrameInTemporalUnit(const FrameIteratorT& it) {
     68  return it->second.encoded_frame->is_last_spatial_layer;
     69 }
     70 }  // namespace
     71 
     72 FrameBuffer::FrameBuffer(int max_size,
     73                         int max_decode_history,
     74                         const FieldTrialsView& field_trials)
     75    : legacy_frame_id_jump_behavior_(
     76          !field_trials.IsDisabled("WebRTC-LegacyFrameIdJumpBehavior")),
     77      max_size_(max_size),
     78      decoded_frame_history_(max_decode_history) {}
     79 
     80 bool FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
     81  const uint32_t ssrc =
     82      frame->PacketInfos().empty() ? 0 : frame->PacketInfos()[0].ssrc();
     83  if (!ValidReferences(*frame)) {
     84    TRACE_EVENT2("webrtc",
     85                 "FrameBuffer::InsertFrame Frame dropped (Invalid references)",
     86                 "remote_ssrc", ssrc, "frame_id", frame->Id());
     87    RTC_DLOG(LS_WARNING) << "Frame " << frame->Id()
     88                         << " has invalid references, dropping frame.";
     89    return false;
     90  }
     91 
     92  if (frame->Id() <= decoded_frame_history_.GetLastDecodedFrameId()) {
     93    if (legacy_frame_id_jump_behavior_ && frame->is_keyframe() &&
     94        AheadOf(frame->RtpTimestamp(),
     95                *decoded_frame_history_.GetLastDecodedFrameTimestamp())) {
     96      TRACE_EVENT2("webrtc",
     97                   "FrameBuffer::InsertFrame Frames dropped (OOO + PicId jump)",
     98                   "remote_ssrc", ssrc, "frame_id", frame->Id());
     99      RTC_DLOG(LS_WARNING)
    100          << "Keyframe " << frame->Id()
    101          << " has newer timestamp but older picture id, clearing buffer.";
    102      Clear();
    103    } else {
    104      // Already decoded past this frame.
    105      TRACE_EVENT2("webrtc",
    106                   "FrameBuffer::InsertFrame Frame dropped (Out of order)",
    107                   "remote_ssrc", ssrc, "frame_id", frame->Id());
    108      return false;
    109    }
    110  }
    111 
    112  if (frames_.size() == max_size_) {
    113    if (frame->is_keyframe()) {
    114      TRACE_EVENT2("webrtc",
    115                   "FrameBuffer::InsertFrame Frames dropped (KF + Full buffer)",
    116                   "remote_ssrc", ssrc, "frame_id", frame->Id());
    117      RTC_DLOG(LS_WARNING) << "Keyframe " << frame->Id()
    118                           << " inserted into full buffer, clearing buffer.";
    119      Clear();
    120    } else {
    121      // No space for this frame.
    122      TRACE_EVENT2("webrtc",
    123                   "FrameBuffer::InsertFrame Frame dropped (Full buffer)",
    124                   "remote_ssrc", ssrc, "frame_id", frame->Id());
    125      return false;
    126    }
    127  }
    128 
    129  const int64_t frame_id = frame->Id();
    130  auto insert_res =
    131      frames_.emplace(frame_id, FrameInfo{.encoded_frame = std::move(frame)});
    132  if (!insert_res.second) {
    133    // Frame has already been inserted.
    134    return false;
    135  }
    136 
    137  if (frames_.size() == max_size_) {
    138    RTC_DLOG(LS_WARNING) << "Frame " << frame_id
    139                         << " inserted, buffer is now full.";
    140  }
    141 
    142  PropagateContinuity(insert_res.first);
    143  FindNextAndLastDecodableTemporalUnit();
    144  return true;
    145 }
    146 
    147 absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4>
    148 FrameBuffer::ExtractNextDecodableTemporalUnit() {
    149  absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> res;
    150  if (!next_decodable_temporal_unit_) {
    151    return res;
    152  }
    153 
    154  auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
    155  for (auto it = next_decodable_temporal_unit_->first_frame; it != end_it;
    156       ++it) {
    157    decoded_frame_history_.InsertDecoded(GetFrameId(it), GetTimestamp(it));
    158    res.push_back(std::move(it->second.encoded_frame));
    159  }
    160 
    161  DropNextDecodableTemporalUnit();
    162  return res;
    163 }
    164 
    165 void FrameBuffer::DropNextDecodableTemporalUnit() {
    166  if (!next_decodable_temporal_unit_) {
    167    return;
    168  }
    169 
    170  auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
    171 
    172  UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), end_it);
    173 
    174  frames_.erase(frames_.begin(), end_it);
    175  FindNextAndLastDecodableTemporalUnit();
    176 }
    177 
    178 void FrameBuffer::UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it,
    179                                                         FrameIterator end_it) {
    180  uint32_t dropped_ssrc = 0;
    181  int64_t dropped_frame_id = 0;
    182  unsigned int num_discarded_packets = 0;
    183  unsigned int num_dropped_frames =
    184      std::count_if(begin_it, end_it, [&](const auto& f) {
    185        if (f.second.encoded_frame) {
    186          const auto& packetInfos = f.second.encoded_frame->PacketInfos();
    187          dropped_frame_id = f.first;
    188          if (!packetInfos.empty()) {
    189            dropped_ssrc = packetInfos[0].ssrc();
    190          }
    191          num_discarded_packets += packetInfos.size();
    192        }
    193        return f.second.encoded_frame != nullptr;
    194      });
    195 
    196  if (num_dropped_frames > 0) {
    197    TRACE_EVENT2("webrtc", "FrameBuffer Dropping Old Frames", "remote_ssrc",
    198                 dropped_ssrc, "frame_id", dropped_frame_id);
    199  }
    200  if (num_discarded_packets > 0) {
    201    TRACE_EVENT2("webrtc", "FrameBuffer Discarding Old Packets", "remote_ssrc",
    202                 dropped_ssrc, "frame_id", dropped_frame_id);
    203  }
    204 
    205  num_dropped_frames_ += num_dropped_frames;
    206  num_discarded_packets_ += num_discarded_packets;
    207 }
    208 
    209 std::optional<int64_t> FrameBuffer::LastContinuousFrameId() const {
    210  return last_continuous_frame_id_;
    211 }
    212 
    213 std::optional<int64_t> FrameBuffer::LastContinuousTemporalUnitFrameId() const {
    214  return last_continuous_temporal_unit_frame_id_;
    215 }
    216 
    217 std::optional<FrameBuffer::DecodabilityInfo>
    218 FrameBuffer::DecodableTemporalUnitsInfo() const {
    219  return decodable_temporal_units_info_;
    220 }
    221 
    222 int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const {
    223  return num_continuous_temporal_units_;
    224 }
    225 int FrameBuffer::GetTotalNumberOfDroppedFrames() const {
    226  return num_dropped_frames_;
    227 }
    228 int FrameBuffer::GetTotalNumberOfDiscardedPackets() const {
    229  return num_discarded_packets_;
    230 }
    231 
    232 size_t FrameBuffer::CurrentSize() const {
    233  return frames_.size();
    234 }
    235 
    236 bool FrameBuffer::IsContinuous(const FrameIterator& it) const {
    237  for (int64_t reference : GetReferences(it)) {
    238    if (decoded_frame_history_.WasDecoded(reference)) {
    239      continue;
    240    }
    241 
    242    auto reference_frame_it = frames_.find(reference);
    243    if (reference_frame_it != frames_.end() &&
    244        reference_frame_it->second.continuous) {
    245      continue;
    246    }
    247 
    248    return false;
    249  }
    250 
    251  return true;
    252 }
    253 
    254 void FrameBuffer::PropagateContinuity(const FrameIterator& frame_it) {
    255  for (auto it = frame_it; it != frames_.end(); ++it) {
    256    if (!it->second.continuous) {
    257      if (IsContinuous(it)) {
    258        it->second.continuous = true;
    259        if (last_continuous_frame_id_ < GetFrameId(it)) {
    260          last_continuous_frame_id_ = GetFrameId(it);
    261        }
    262        if (IsLastFrameInTemporalUnit(it)) {
    263          num_continuous_temporal_units_++;
    264          if (last_continuous_temporal_unit_frame_id_ < GetFrameId(it)) {
    265            last_continuous_temporal_unit_frame_id_ = GetFrameId(it);
    266          }
    267        }
    268      }
    269    }
    270  }
    271 }
    272 
    273 void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
    274  next_decodable_temporal_unit_.reset();
    275  decodable_temporal_units_info_.reset();
    276 
    277  if (!last_continuous_temporal_unit_frame_id_) {
    278    return;
    279  }
    280 
    281  FrameIterator first_frame_it = frames_.begin();
    282  FrameIterator last_frame_it = frames_.begin();
    283  absl::InlinedVector<int64_t, 4> frames_in_temporal_unit;
    284  uint32_t last_decodable_temporal_unit_timestamp;
    285  for (auto frame_it = frames_.begin(); frame_it != frames_.end();) {
    286    if (GetFrameId(frame_it) > *last_continuous_temporal_unit_frame_id_) {
    287      break;
    288    }
    289 
    290    if (GetTimestamp(frame_it) != GetTimestamp(first_frame_it)) {
    291      frames_in_temporal_unit.clear();
    292      first_frame_it = frame_it;
    293    }
    294 
    295    frames_in_temporal_unit.push_back(GetFrameId(frame_it));
    296 
    297    last_frame_it = frame_it++;
    298 
    299    if (IsLastFrameInTemporalUnit(last_frame_it)) {
    300      bool temporal_unit_decodable = true;
    301      for (auto it = first_frame_it; it != frame_it && temporal_unit_decodable;
    302           ++it) {
    303        for (int64_t reference : GetReferences(it)) {
    304          if (!decoded_frame_history_.WasDecoded(reference) &&
    305              !absl::c_linear_search(frames_in_temporal_unit, reference)) {
    306            // A frame in the temporal unit has a non-decoded reference outside
    307            // the temporal unit, so it's not yet ready to be decoded.
    308            temporal_unit_decodable = false;
    309            break;
    310          }
    311        }
    312      }
    313 
    314      if (temporal_unit_decodable) {
    315        if (!next_decodable_temporal_unit_) {
    316          next_decodable_temporal_unit_ = {.first_frame = first_frame_it,
    317                                           .last_frame = last_frame_it};
    318        }
    319 
    320        last_decodable_temporal_unit_timestamp = GetTimestamp(first_frame_it);
    321      }
    322    }
    323  }
    324 
    325  if (next_decodable_temporal_unit_) {
    326    decodable_temporal_units_info_ = {
    327        .next_rtp_timestamp =
    328            GetTimestamp(next_decodable_temporal_unit_->first_frame),
    329        .last_rtp_timestamp = last_decodable_temporal_unit_timestamp};
    330  }
    331 }
    332 
    333 void FrameBuffer::Clear() {
    334  UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), frames_.end());
    335  frames_.clear();
    336  next_decodable_temporal_unit_.reset();
    337  decodable_temporal_units_info_.reset();
    338  last_continuous_frame_id_.reset();
    339  last_continuous_temporal_unit_frame_id_.reset();
    340  decoded_frame_history_.Clear();
    341 }
    342 
    343 }  // namespace webrtc