generic_decoder.cc (15042B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/video_coding/generic_decoder.h" 12 13 #include <algorithm> 14 #include <cstddef> 15 #include <cstdint> 16 #include <iterator> 17 #include <optional> 18 #include <tuple> 19 #include <utility> 20 21 #include "absl/algorithm/container.h" 22 #include "api/field_trials_view.h" 23 #include "api/units/time_delta.h" 24 #include "api/units/timestamp.h" 25 #include "api/video/color_space.h" 26 #include "api/video/corruption_detection/frame_instrumentation_data.h" 27 #include "api/video/encoded_frame.h" 28 #include "api/video/encoded_image.h" 29 #include "api/video/video_content_type.h" 30 #include "api/video/video_frame.h" 31 #include "api/video/video_frame_type.h" 32 #include "api/video/video_timing.h" 33 #include "api/video_codecs/video_decoder.h" 34 #include "common_video/include/corruption_score_calculator.h" 35 #include "modules/include/module_common_types_public.h" 36 #include "modules/video_coding/encoded_frame.h" 37 #include "modules/video_coding/include/video_coding_defines.h" 38 #include "modules/video_coding/include/video_error_codes.h" 39 #include "modules/video_coding/timing/timing.h" 40 #include "rtc_base/checks.h" 41 #include "rtc_base/logging.h" 42 #include "rtc_base/synchronization/mutex.h" 43 #include "rtc_base/trace_event.h" 44 #include "system_wrappers/include/clock.h" 45 #include "system_wrappers/include/metrics.h" 46 47 namespace webrtc { 48 49 namespace { 50 51 constexpr size_t kDecoderFrameMemoryLength = 10; 52 53 } // namespace 54 55 VCMDecodedFrameCallback::VCMDecodedFrameCallback( 56 VCMTiming* timing, 57 Clock* clock, 58 const FieldTrialsView& /* field_trials */, 59 CorruptionScoreCalculator* corruption_score_calculator) 60 : _clock(clock), 61 _timing(timing), 62 corruption_score_calculator_(corruption_score_calculator) { 63 ntp_offset_ = 64 _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); 65 } 66 67 VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {} 68 69 void VCMDecodedFrameCallback::SetUserReceiveCallback( 70 VCMReceiveCallback* receiveCallback) { 71 RTC_DCHECK(construction_thread_.IsCurrent()); 72 RTC_DCHECK((!_receiveCallback && receiveCallback) || 73 (_receiveCallback && !receiveCallback)); 74 _receiveCallback = receiveCallback; 75 } 76 77 VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() { 78 // Called on the decode thread via VCMCodecDataBase::GetDecoder. 79 // The callback must always have been set before this happens. 80 RTC_DCHECK(_receiveCallback); 81 return _receiveCallback; 82 } 83 84 int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) { 85 // This function may be called on the decode TaskQueue, but may also be called 86 // on an OS provided queue such as on iOS (see e.g. b/153465112). 87 return Decoded(decodedImage, -1); 88 } 89 90 int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, 91 int64_t decode_time_ms) { 92 Decoded(decodedImage, 93 decode_time_ms >= 0 ? std::optional<int32_t>(decode_time_ms) 94 : std::nullopt, 95 std::nullopt); 96 return WEBRTC_VIDEO_CODEC_OK; 97 } 98 99 std::pair<std::optional<FrameInfo>, size_t> 100 VCMDecodedFrameCallback::FindFrameInfo(uint32_t rtp_timestamp) { 101 std::optional<FrameInfo> frame_info; 102 103 auto it = absl::c_find_if(frame_infos_, [rtp_timestamp](const auto& entry) { 104 return entry.rtp_timestamp == rtp_timestamp || 105 IsNewerTimestamp(entry.rtp_timestamp, rtp_timestamp); 106 }); 107 size_t dropped_frames = std::distance(frame_infos_.begin(), it); 108 109 if (it != frame_infos_.end() && it->rtp_timestamp == rtp_timestamp) { 110 // Frame was found and should also be removed from the queue. 111 frame_info = std::move(*it); 112 ++it; 113 } 114 115 frame_infos_.erase(frame_infos_.begin(), it); 116 return std::make_pair(std::move(frame_info), dropped_frames); 117 } 118 119 void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, 120 std::optional<int32_t> decode_time_ms, 121 std::optional<uint8_t> qp) { 122 RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point"; 123 TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded", 124 TRACE_EVENT_SCOPE_GLOBAL, "timestamp", 125 decodedImage.rtp_timestamp()); 126 // TODO(holmer): We should improve this so that we can handle multiple 127 // callbacks from one call to Decode(). 128 std::optional<FrameInfo> frame_info; 129 int timestamp_map_size = 0; 130 int dropped_frames = 0; 131 { 132 MutexLock lock(&lock_); 133 std::tie(frame_info, dropped_frames) = 134 FindFrameInfo(decodedImage.rtp_timestamp()); 135 timestamp_map_size = frame_infos_.size(); 136 } 137 if (dropped_frames > 0) { 138 _receiveCallback->OnDroppedFrames(dropped_frames); 139 } 140 141 if (!frame_info) { 142 RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping " 143 "frame with timestamp " 144 << decodedImage.rtp_timestamp(); 145 return; 146 } 147 148 decodedImage.set_ntp_time_ms(frame_info->ntp_time_ms); 149 decodedImage.set_packet_infos(frame_info->packet_infos); 150 decodedImage.set_rotation(frame_info->rotation); 151 decodedImage.set_color_space(frame_info->color_space); 152 VideoFrame::RenderParameters render_parameters = _timing->RenderParameters(); 153 if (render_parameters.max_composition_delay_in_frames) { 154 // Subtract frames that are in flight. 155 render_parameters.max_composition_delay_in_frames = 156 std::max(0, *render_parameters.max_composition_delay_in_frames - 157 timestamp_map_size); 158 } 159 decodedImage.set_render_parameters(render_parameters); 160 161 RTC_DCHECK(frame_info->decode_start); 162 const Timestamp now = _clock->CurrentTime(); 163 const TimeDelta decode_time = decode_time_ms 164 ? TimeDelta::Millis(*decode_time_ms) 165 : now - *frame_info->decode_start; 166 _timing->StopDecodeTimer(decode_time, now); 167 decodedImage.set_processing_time( 168 {.start = *frame_info->decode_start, 169 .finish = *frame_info->decode_start + decode_time}); 170 171 // Report timing information. 172 TimingFrameInfo timing_frame_info; 173 if (frame_info->timing.flags != VideoSendTiming::kInvalid) { 174 int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_; 175 // Convert remote timestamps to local time from ntp timestamps. 176 frame_info->timing.encode_start_ms -= ntp_offset_; 177 frame_info->timing.encode_finish_ms -= ntp_offset_; 178 frame_info->timing.packetization_finish_ms -= ntp_offset_; 179 frame_info->timing.pacer_exit_ms -= ntp_offset_; 180 frame_info->timing.network_timestamp_ms -= ntp_offset_; 181 frame_info->timing.network2_timestamp_ms -= ntp_offset_; 182 183 int64_t sender_delta_ms = 0; 184 if (decodedImage.ntp_time_ms() < 0) { 185 // Sender clock is not estimated yet. Make sure that sender times are all 186 // negative to indicate that. Yet they still should be relatively correct. 187 sender_delta_ms = 188 std::max({capture_time_ms, frame_info->timing.encode_start_ms, 189 frame_info->timing.encode_finish_ms, 190 frame_info->timing.packetization_finish_ms, 191 frame_info->timing.pacer_exit_ms, 192 frame_info->timing.network_timestamp_ms, 193 frame_info->timing.network2_timestamp_ms}) + 194 1; 195 } 196 197 timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms; 198 timing_frame_info.encode_start_ms = 199 frame_info->timing.encode_start_ms - sender_delta_ms; 200 timing_frame_info.encode_finish_ms = 201 frame_info->timing.encode_finish_ms - sender_delta_ms; 202 timing_frame_info.packetization_finish_ms = 203 frame_info->timing.packetization_finish_ms - sender_delta_ms; 204 timing_frame_info.pacer_exit_ms = 205 frame_info->timing.pacer_exit_ms - sender_delta_ms; 206 timing_frame_info.network_timestamp_ms = 207 frame_info->timing.network_timestamp_ms - sender_delta_ms; 208 timing_frame_info.network2_timestamp_ms = 209 frame_info->timing.network2_timestamp_ms - sender_delta_ms; 210 RTC_HISTOGRAM_COUNTS_1000( 211 "WebRTC.Video.GenericDecoder.CaptureToEncodeDelay", 212 timing_frame_info.encode_start_ms - timing_frame_info.capture_time_ms); 213 RTC_HISTOGRAM_COUNTS_1000( 214 "WebRTC.Video.GenericDecoder.EncodeDelay", 215 timing_frame_info.encode_finish_ms - timing_frame_info.encode_start_ms); 216 RTC_HISTOGRAM_COUNTS_1000( 217 "WebRTC.Video.GenericDecoder.PacerAndPacketizationDelay", 218 timing_frame_info.pacer_exit_ms - timing_frame_info.encode_finish_ms); 219 } 220 221 timing_frame_info.flags = frame_info->timing.flags; 222 timing_frame_info.decode_start_ms = frame_info->decode_start->ms(); 223 timing_frame_info.decode_finish_ms = now.ms(); 224 timing_frame_info.render_time_ms = 225 frame_info->render_time ? frame_info->render_time->ms() : -1; 226 timing_frame_info.rtp_timestamp = decodedImage.rtp_timestamp(); 227 timing_frame_info.receive_start_ms = frame_info->timing.receive_start_ms; 228 timing_frame_info.receive_finish_ms = frame_info->timing.receive_finish_ms; 229 RTC_HISTOGRAM_COUNTS_1000( 230 "WebRTC.Video.GenericDecoder.PacketReceiveDelay", 231 timing_frame_info.receive_finish_ms - timing_frame_info.receive_start_ms); 232 RTC_HISTOGRAM_COUNTS_1000( 233 "WebRTC.Video.GenericDecoder.JitterBufferDelay", 234 timing_frame_info.decode_start_ms - timing_frame_info.receive_finish_ms); 235 RTC_HISTOGRAM_COUNTS_1000( 236 "WebRTC.Video.GenericDecoder.DecodeDelay", 237 timing_frame_info.decode_finish_ms - timing_frame_info.decode_start_ms); 238 _timing->SetTimingFrameInfo(timing_frame_info); 239 240 decodedImage.set_timestamp_us( 241 frame_info->render_time ? frame_info->render_time->us() : -1); 242 _receiveCallback->OnFrameToRender({.video_frame = decodedImage, 243 .qp = qp, 244 .decode_time = decode_time, 245 .content_type = frame_info->content_type, 246 .frame_type = frame_info->frame_type}); 247 248 if (corruption_score_calculator_ && 249 frame_info->frame_instrumentation_data.has_value()) { 250 corruption_score_calculator_->CalculateCorruptionScore( 251 decodedImage, *frame_info->frame_instrumentation_data, 252 frame_info->content_type); 253 } 254 } 255 256 void VCMDecodedFrameCallback::OnDecoderInfoChanged( 257 const VideoDecoder::DecoderInfo& decoder_info) { 258 _receiveCallback->OnDecoderInfoChanged(decoder_info); 259 } 260 261 void VCMDecodedFrameCallback::Map(FrameInfo frameInfo) { 262 int dropped_frames = 0; 263 { 264 MutexLock lock(&lock_); 265 int initial_size = frame_infos_.size(); 266 if (initial_size == kDecoderFrameMemoryLength) { 267 frame_infos_.pop_front(); 268 dropped_frames = 1; 269 } 270 frame_infos_.push_back(std::move(frameInfo)); 271 // If no frame is dropped, the new size should be `initial_size` + 1 272 } 273 if (dropped_frames > 0) { 274 _receiveCallback->OnDroppedFrames(dropped_frames); 275 } 276 } 277 278 void VCMDecodedFrameCallback::ClearTimestampMap() { 279 int dropped_frames = 0; 280 { 281 MutexLock lock(&lock_); 282 dropped_frames = frame_infos_.size(); 283 frame_infos_.clear(); 284 } 285 if (dropped_frames > 0) { 286 _receiveCallback->OnDroppedFrames(dropped_frames); 287 } 288 } 289 290 VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder) 291 : _callback(nullptr), 292 decoder_(decoder), 293 _last_keyframe_content_type(VideoContentType::UNSPECIFIED) { 294 RTC_DCHECK(decoder_); 295 } 296 297 VCMGenericDecoder::~VCMGenericDecoder() { 298 decoder_->Release(); 299 } 300 301 bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) { 302 TRACE_EVENT0("webrtc", "VCMGenericDecoder::Configure"); 303 304 bool ok = decoder_->Configure(settings); 305 decoder_info_ = decoder_->GetDecoderInfo(); 306 RTC_LOG(LS_INFO) << "Decoder implementation: " << decoder_info_.ToString(); 307 if (_callback) { 308 _callback->OnDecoderInfoChanged(decoder_info_); 309 } 310 return ok; 311 } 312 313 int32_t VCMGenericDecoder::Decode(const EncodedFrame& frame, Timestamp now) { 314 return Decode(frame, now, frame.RenderTimeMs(), 315 frame.CodecSpecific()->frame_instrumentation_data); 316 } 317 318 int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { 319 return Decode(frame, now, frame.RenderTimeMs(), 320 frame.CodecSpecific()->frame_instrumentation_data); 321 } 322 323 int32_t VCMGenericDecoder::Decode( 324 const EncodedImage& frame, 325 Timestamp now, 326 int64_t render_time_ms, 327 const std::optional<FrameInstrumentationData>& frame_instrumentation_data) { 328 TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", 329 frame.RtpTimestamp()); 330 FrameInfo frame_info; 331 frame_info.rtp_timestamp = frame.RtpTimestamp(); 332 frame_info.decode_start = now; 333 frame_info.render_time = 334 render_time_ms >= 0 335 ? std::make_optional(Timestamp::Millis(render_time_ms)) 336 : std::nullopt; 337 frame_info.rotation = frame.rotation(); 338 frame_info.timing = frame.video_timing(); 339 frame_info.ntp_time_ms = frame.ntp_time_ms_; 340 frame_info.packet_infos = frame.PacketInfos(); 341 frame_info.frame_instrumentation_data = frame_instrumentation_data; 342 const webrtc::ColorSpace* color_space = frame.ColorSpace(); 343 if (color_space) 344 frame_info.color_space = *color_space; 345 346 // Set correctly only for key frames. Thus, use latest key frame 347 // content type. If the corresponding key frame was lost, decode will fail 348 // and content type will be ignored. 349 if (frame.FrameType() == VideoFrameType::kVideoFrameKey) { 350 frame_info.content_type = frame.contentType(); 351 _last_keyframe_content_type = frame.contentType(); 352 } else { 353 frame_info.content_type = _last_keyframe_content_type; 354 } 355 frame_info.frame_type = frame.FrameType(); 356 _callback->Map(std::move(frame_info)); 357 358 int32_t ret = decoder_->Decode(frame, render_time_ms); 359 VideoDecoder::DecoderInfo decoder_info = decoder_->GetDecoderInfo(); 360 if (decoder_info != decoder_info_) { 361 RTC_LOG(LS_INFO) << "Changed decoder implementation to: " 362 << decoder_info.ToString(); 363 decoder_info_ = decoder_info; 364 if (decoder_info.implementation_name.empty()) { 365 decoder_info.implementation_name = "unknown"; 366 } 367 _callback->OnDecoderInfoChanged(std::move(decoder_info)); 368 } 369 if (ret < WEBRTC_VIDEO_CODEC_OK || ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT) { 370 _callback->ClearTimestampMap(); 371 } 372 return ret; 373 } 374 375 int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback( 376 VCMDecodedFrameCallback* callback) { 377 _callback = callback; 378 int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback); 379 if (callback && !decoder_info_.implementation_name.empty()) { 380 callback->OnDecoderInfoChanged(decoder_info_); 381 } 382 return ret; 383 } 384 385 } // namespace webrtc