rtp_vp8_ref_finder.cc (9471B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/video_coding/rtp_vp8_ref_finder.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <memory> 16 #include <utility> 17 18 #include "api/video/video_frame_type.h" 19 #include "modules/rtp_rtcp/source/frame_object.h" 20 #include "modules/video_coding/codecs/interface/common_constants.h" 21 #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" 22 #include "modules/video_coding/rtp_frame_reference_finder.h" 23 #include "rtc_base/logging.h" 24 #include "rtc_base/numerics/mod_ops.h" 25 #include "rtc_base/numerics/sequence_number_util.h" 26 27 namespace webrtc { 28 29 RtpFrameReferenceFinder::ReturnVector RtpVp8RefFinder::ManageFrame( 30 std::unique_ptr<RtpFrameObject> frame) { 31 const RTPVideoHeaderVP8& codec_header = 32 std::get<RTPVideoHeaderVP8>(frame->GetRtpVideoHeader().video_type_header); 33 34 if (codec_header.temporalIdx != kNoTemporalIdx) 35 frame->SetTemporalIndex(codec_header.temporalIdx); 36 37 int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0PicIdx & 0xFF); 38 FrameDecision decision = 39 ManageFrameInternal(frame.get(), codec_header, unwrapped_tl0); 40 41 RtpFrameReferenceFinder::ReturnVector res; 42 switch (decision) { 43 case kStash: 44 if (stashed_frames_.size() > kMaxStashedFrames) { 45 stashed_frames_.pop_back(); 46 } 47 stashed_frames_.push_front( 48 {.unwrapped_tl0 = unwrapped_tl0, .frame = std::move(frame)}); 49 return res; 50 case kHandOff: 51 res.push_back(std::move(frame)); 52 RetryStashedFrames(res); 53 return res; 54 case kDrop: 55 return res; 56 } 57 58 return res; 59 } 60 61 RtpVp8RefFinder::FrameDecision RtpVp8RefFinder::ManageFrameInternal( 62 RtpFrameObject* frame, 63 const RTPVideoHeaderVP8& codec_header, 64 int64_t unwrapped_tl0) { 65 // Protect against corrupted packets with arbitrary large temporal idx. 66 if (codec_header.temporalIdx >= kMaxTemporalLayers) 67 return kDrop; 68 69 frame->SetSpatialIndex(0); 70 frame->SetId(codec_header.pictureId & 0x7FFF); 71 72 if (last_picture_id_ == -1) 73 last_picture_id_ = frame->Id(); 74 75 // Clean up info about not yet received frames that are too old. 76 uint16_t old_picture_id = 77 Subtract<kFrameIdLength>(frame->Id(), kMaxNotYetReceivedFrames); 78 auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id); 79 not_yet_received_frames_.erase(not_yet_received_frames_.begin(), 80 clean_frames_to); 81 // Avoid re-adding picture ids that were just erased. 82 if (AheadOf<uint16_t, kFrameIdLength>(old_picture_id, last_picture_id_)) { 83 last_picture_id_ = old_picture_id; 84 } 85 // Find if there has been a gap in fully received frames and save the picture 86 // id of those frames in `not_yet_received_frames_`. 87 if (AheadOf<uint16_t, kFrameIdLength>(frame->Id(), last_picture_id_)) { 88 do { 89 last_picture_id_ = Add<kFrameIdLength>(last_picture_id_, 1); 90 not_yet_received_frames_.insert(last_picture_id_); 91 } while (last_picture_id_ != frame->Id()); 92 } 93 94 // Clean up info for base layers that are too old. 95 int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo; 96 auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx); 97 layer_info_.erase(layer_info_.begin(), clean_layer_info_to); 98 99 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { 100 if (codec_header.temporalIdx != 0) { 101 return kDrop; 102 } 103 frame->num_references = 0; 104 layer_info_[unwrapped_tl0].fill(-1); 105 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); 106 return kHandOff; 107 } 108 109 auto layer_info_it = layer_info_.find( 110 codec_header.temporalIdx == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0); 111 112 // If we don't have the base layer frame yet, stash this frame. 113 if (layer_info_it == layer_info_.end()) 114 return kStash; 115 116 // A non keyframe base layer frame has been received, copy the layer info 117 // from the previous base layer frame and set a reference to the previous 118 // base layer frame. 119 if (codec_header.temporalIdx == 0) { 120 layer_info_it = 121 layer_info_.emplace(unwrapped_tl0, layer_info_it->second).first; 122 frame->num_references = 1; 123 int64_t last_pid_on_layer = layer_info_it->second[0]; 124 125 // Is this an old frame that has already been used to update the state? If 126 // so, drop it. 127 if (AheadOrAt<uint16_t, kFrameIdLength>(last_pid_on_layer, frame->Id())) { 128 return kDrop; 129 } 130 131 frame->references[0] = last_pid_on_layer; 132 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); 133 return kHandOff; 134 } 135 136 // Layer sync frame, this frame only references its base layer frame. 137 if (codec_header.layerSync) { 138 frame->num_references = 1; 139 int64_t last_pid_on_layer = layer_info_it->second[codec_header.temporalIdx]; 140 141 // Is this an old frame that has already been used to update the state? If 142 // so, drop it. 143 if (last_pid_on_layer != -1 && 144 AheadOrAt<uint16_t, kFrameIdLength>(last_pid_on_layer, frame->Id())) { 145 return kDrop; 146 } 147 148 frame->references[0] = layer_info_it->second[0]; 149 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); 150 return kHandOff; 151 } 152 153 // Find all references for this frame. 154 frame->num_references = 0; 155 for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) { 156 // If we have not yet received a previous frame on this temporal layer, 157 // stash this frame. 158 if (layer_info_it->second[layer] == -1) 159 return kStash; 160 161 // If the last frame on this layer is ahead of this frame it means that 162 // a layer sync frame has been received after this frame for the same 163 // base layer frame, drop this frame. 164 if (AheadOf<uint16_t, kFrameIdLength>(layer_info_it->second[layer], 165 frame->Id())) { 166 return kDrop; 167 } 168 169 // If we have not yet received a frame between this frame and the referenced 170 // frame then we have to wait for that frame to be completed first. 171 auto not_received_frame_it = 172 not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); 173 if (not_received_frame_it != not_yet_received_frames_.end() && 174 AheadOf<uint16_t, kFrameIdLength>(frame->Id(), 175 *not_received_frame_it)) { 176 return kStash; 177 } 178 179 if (!(AheadOf<uint16_t, kFrameIdLength>(frame->Id(), 180 layer_info_it->second[layer]))) { 181 RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->Id() 182 << " and packet range [" << frame->first_seq_num() 183 << ", " << frame->last_seq_num() 184 << "] already received, " 185 " dropping frame."; 186 return kDrop; 187 } 188 189 ++frame->num_references; 190 frame->references[layer] = layer_info_it->second[layer]; 191 } 192 193 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); 194 return kHandOff; 195 } 196 197 void RtpVp8RefFinder::UpdateLayerInfoVp8(RtpFrameObject* frame, 198 int64_t unwrapped_tl0, 199 uint8_t temporal_idx) { 200 auto layer_info_it = layer_info_.find(unwrapped_tl0); 201 202 // Update this layer info and newer. 203 while (layer_info_it != layer_info_.end()) { 204 if (layer_info_it->second[temporal_idx] != -1 && 205 AheadOf<uint16_t, kFrameIdLength>(layer_info_it->second[temporal_idx], 206 frame->Id())) { 207 // The frame was not newer, then no subsequent layer info have to be 208 // update. 209 break; 210 } 211 212 layer_info_it->second[temporal_idx] = frame->Id(); 213 ++unwrapped_tl0; 214 layer_info_it = layer_info_.find(unwrapped_tl0); 215 } 216 not_yet_received_frames_.erase(frame->Id()); 217 218 UnwrapPictureIds(frame); 219 } 220 221 void RtpVp8RefFinder::RetryStashedFrames( 222 RtpFrameReferenceFinder::ReturnVector& res) { 223 bool complete_frame = false; 224 do { 225 complete_frame = false; 226 for (auto it = stashed_frames_.begin(); it != stashed_frames_.end();) { 227 const RTPVideoHeaderVP8& codec_header = std::get<RTPVideoHeaderVP8>( 228 it->frame->GetRtpVideoHeader().video_type_header); 229 FrameDecision decision = 230 ManageFrameInternal(it->frame.get(), codec_header, it->unwrapped_tl0); 231 232 switch (decision) { 233 case kStash: 234 ++it; 235 break; 236 case kHandOff: 237 complete_frame = true; 238 res.push_back(std::move(it->frame)); 239 [[fallthrough]]; 240 case kDrop: 241 it = stashed_frames_.erase(it); 242 } 243 } 244 } while (complete_frame); 245 } 246 247 void RtpVp8RefFinder::UnwrapPictureIds(RtpFrameObject* frame) { 248 for (size_t i = 0; i < frame->num_references; ++i) 249 frame->references[i] = unwrapper_.Unwrap(frame->references[i]); 250 frame->SetId(unwrapper_.Unwrap(frame->Id())); 251 } 252 253 void RtpVp8RefFinder::ClearTo(uint16_t seq_num) { 254 auto it = stashed_frames_.begin(); 255 while (it != stashed_frames_.end()) { 256 if (AheadOf<uint16_t>(seq_num, it->frame->first_seq_num())) { 257 it = stashed_frames_.erase(it); 258 } else { 259 ++it; 260 } 261 } 262 } 263 264 } // namespace webrtc