video_receiver.cc (9928B)
1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <cstddef> 12 #include <cstdint> 13 #include <vector> 14 15 #include "api/field_trials_view.h" 16 #include "api/rtp_headers.h" 17 #include "api/sequence_checker.h" 18 #include "api/units/timestamp.h" 19 #include "api/video/video_frame_type.h" 20 #include "api/video_codecs/video_decoder.h" 21 #include "modules/rtp_rtcp/source/rtp_video_header.h" 22 #include "modules/video_coding/deprecated/jitter_buffer.h" 23 #include "modules/video_coding/deprecated/packet.h" 24 #include "modules/video_coding/deprecated/receiver.h" 25 #include "modules/video_coding/encoded_frame.h" 26 #include "modules/video_coding/generic_decoder.h" 27 #include "modules/video_coding/include/video_coding_defines.h" 28 #include "modules/video_coding/internal_defines.h" 29 #include "modules/video_coding/timing/timing.h" 30 #include "modules/video_coding/video_coding_impl.h" 31 #include "rtc_base/checks.h" 32 #include "rtc_base/logging.h" 33 #include "rtc_base/one_time_event.h" 34 #include "rtc_base/synchronization/mutex.h" 35 #include "rtc_base/trace_event.h" 36 #include "system_wrappers/include/clock.h" 37 38 namespace webrtc { 39 namespace vcm { 40 41 VideoReceiver::VideoReceiver(Clock* clock, 42 VCMTiming* timing, 43 const FieldTrialsView& field_trials) 44 : clock_(clock), 45 _timing(timing), 46 _receiver(_timing, clock_, field_trials), 47 _decodedFrameCallback(_timing, 48 clock_, 49 field_trials, 50 /*corruption_score_calculator=*/nullptr), 51 _frameTypeCallback(nullptr), 52 _packetRequestCallback(nullptr), 53 _scheduleKeyRequest(false), 54 drop_frames_until_keyframe_(false), 55 max_nack_list_size_(0), 56 _codecDataBase(), 57 _retransmissionTimer(10, clock_), 58 _keyRequestTimer(500, clock_) { 59 decoder_thread_checker_.Detach(); 60 module_thread_checker_.Detach(); 61 } 62 63 VideoReceiver::~VideoReceiver() { 64 RTC_DCHECK_RUN_ON(&construction_thread_checker_); 65 } 66 67 void VideoReceiver::Process() { 68 RTC_DCHECK_RUN_ON(&module_thread_checker_); 69 70 // Key frame requests 71 if (_keyRequestTimer.TimeUntilProcess() == 0) { 72 _keyRequestTimer.Processed(); 73 bool request_key_frame = _frameTypeCallback != nullptr; 74 if (request_key_frame) { 75 MutexLock lock(&process_mutex_); 76 request_key_frame = _scheduleKeyRequest; 77 } 78 if (request_key_frame) 79 RequestKeyFrame(); 80 } 81 82 // Packet retransmission requests 83 // TODO(holmer): Add API for changing Process interval and make sure it's 84 // disabled when NACK is off. 85 if (_retransmissionTimer.TimeUntilProcess() == 0) { 86 _retransmissionTimer.Processed(); 87 bool callback_registered = _packetRequestCallback != nullptr; 88 uint16_t length = max_nack_list_size_; 89 if (callback_registered && length > 0) { 90 // Collect sequence numbers from the default receiver. 91 bool request_key_frame = false; 92 std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame); 93 int32_t ret = VCM_OK; 94 if (request_key_frame) { 95 ret = RequestKeyFrame(); 96 } 97 if (ret == VCM_OK && !nackList.empty()) { 98 MutexLock lock(&process_mutex_); 99 if (_packetRequestCallback != nullptr) { 100 _packetRequestCallback->ResendPackets(&nackList[0], nackList.size()); 101 } 102 } 103 } 104 } 105 } 106 107 // Register a receive callback. Will be called whenever there is a new frame 108 // ready for rendering. 109 int32_t VideoReceiver::RegisterReceiveCallback( 110 VCMReceiveCallback* receiveCallback) { 111 RTC_DCHECK_RUN_ON(&construction_thread_checker_); 112 // This value is set before the decoder thread starts and unset after 113 // the decoder thread has been stopped. 114 _decodedFrameCallback.SetUserReceiveCallback(receiveCallback); 115 return VCM_OK; 116 } 117 118 // Register an externally defined decoder object. 119 void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder, 120 uint8_t payloadType) { 121 RTC_DCHECK_RUN_ON(&construction_thread_checker_); 122 if (externalDecoder == nullptr) { 123 RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType)); 124 return; 125 } 126 _codecDataBase.RegisterExternalDecoder(payloadType, externalDecoder); 127 } 128 129 // Register a frame type request callback. 130 int32_t VideoReceiver::RegisterFrameTypeCallback( 131 VCMFrameTypeCallback* frameTypeCallback) { 132 RTC_DCHECK_RUN_ON(&construction_thread_checker_); 133 // This callback is used on the module thread, but since we don't get 134 // callbacks on the module thread while the decoder thread isn't running 135 // (and this function must not be called when the decoder is running), 136 // we don't need a lock here. 137 _frameTypeCallback = frameTypeCallback; 138 return VCM_OK; 139 } 140 141 int32_t VideoReceiver::RegisterPacketRequestCallback( 142 VCMPacketRequestCallback* callback) { 143 RTC_DCHECK_RUN_ON(&construction_thread_checker_); 144 // This callback is used on the module thread, but since we don't get 145 // callbacks on the module thread while the decoder thread isn't running 146 // (and this function must not be called when the decoder is running), 147 // we don't need a lock here. 148 _packetRequestCallback = callback; 149 return VCM_OK; 150 } 151 152 // Decode next frame, blocking. 153 // Should be called as often as possible to get the most out of the decoder. 154 int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { 155 RTC_DCHECK_RUN_ON(&decoder_thread_checker_); 156 VCMEncodedFrame* frame = _receiver.FrameForDecoding(maxWaitTimeMs, true); 157 158 if (!frame) 159 return VCM_FRAME_NOT_READY; 160 161 bool drop_frame = false; 162 { 163 MutexLock lock(&process_mutex_); 164 if (drop_frames_until_keyframe_) { 165 // Still getting delta frames, schedule another keyframe request as if 166 // decode failed. 167 if (frame->FrameType() != VideoFrameType::kVideoFrameKey) { 168 drop_frame = true; 169 _scheduleKeyRequest = true; 170 } else { 171 drop_frames_until_keyframe_ = false; 172 } 173 } 174 } 175 176 if (drop_frame) { 177 _receiver.ReleaseFrame(frame); 178 return VCM_FRAME_NOT_READY; 179 } 180 181 // If this frame was too late, we should adjust the delay accordingly 182 if (frame->RenderTimeMs() > 0) 183 _timing->UpdateCurrentDelay(Timestamp::Millis(frame->RenderTimeMs()), 184 clock_->CurrentTime()); 185 186 if (first_frame_received_()) { 187 RTC_LOG(LS_INFO) << "Received first complete decodable video frame"; 188 } 189 190 const int32_t ret = Decode(*frame); 191 _receiver.ReleaseFrame(frame); 192 return ret; 193 } 194 195 int32_t VideoReceiver::RequestKeyFrame() { 196 RTC_DCHECK_RUN_ON(&module_thread_checker_); 197 198 TRACE_EVENT0("webrtc", "RequestKeyFrame"); 199 if (_frameTypeCallback != nullptr) { 200 const int32_t ret = _frameTypeCallback->RequestKeyFrame(); 201 if (ret < 0) { 202 return ret; 203 } 204 MutexLock lock(&process_mutex_); 205 _scheduleKeyRequest = false; 206 } else { 207 return VCM_MISSING_CALLBACK; 208 } 209 return VCM_OK; 210 } 211 212 // Must be called from inside the receive side critical section. 213 int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) { 214 RTC_DCHECK_RUN_ON(&decoder_thread_checker_); 215 TRACE_EVENT0("webrtc", "VideoReceiver::Decode"); 216 // Change decoder if payload type has changed 217 VCMGenericDecoder* decoder = 218 _codecDataBase.GetDecoder(frame, &_decodedFrameCallback); 219 if (decoder == nullptr) { 220 return VCM_NO_CODEC_REGISTERED; 221 } 222 return decoder->Decode(frame, clock_->CurrentTime()); 223 } 224 225 // Register possible receive codecs, can be called multiple times 226 void VideoReceiver::RegisterReceiveCodec( 227 uint8_t payload_type, 228 const VideoDecoder::Settings& settings) { 229 RTC_DCHECK_RUN_ON(&construction_thread_checker_); 230 _codecDataBase.RegisterReceiveCodec(payload_type, settings); 231 } 232 233 // Incoming packet from network parsed and ready for decode, non blocking. 234 int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload, 235 size_t payloadLength, 236 const RTPHeader& rtp_header, 237 const RTPVideoHeader& video_header) { 238 RTC_DCHECK_RUN_ON(&module_thread_checker_); 239 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) { 240 TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum", 241 rtp_header.sequenceNumber); 242 } 243 if (incomingPayload == nullptr) { 244 // The jitter buffer doesn't handle non-zero payload lengths for packets 245 // without payload. 246 // TODO(holmer): We should fix this in the jitter buffer. 247 payloadLength = 0; 248 } 249 // Callers don't provide any ntp time. 250 const VCMPacket packet(incomingPayload, payloadLength, rtp_header, 251 video_header, /*ntp_time_ms=*/0, 252 clock_->CurrentTime()); 253 int32_t ret = _receiver.InsertPacket(packet); 254 255 // TODO(holmer): Investigate if this somehow should use the key frame 256 // request scheduling to throttle the requests. 257 if (ret == VCM_FLUSH_INDICATOR) { 258 { 259 MutexLock lock(&process_mutex_); 260 drop_frames_until_keyframe_ = true; 261 } 262 RequestKeyFrame(); 263 } else if (ret < 0) { 264 return ret; 265 } 266 return VCM_OK; 267 } 268 269 void VideoReceiver::SetNackSettings(size_t max_nack_list_size, 270 int max_packet_age_to_nack, 271 int max_incomplete_time_ms) { 272 RTC_DCHECK_RUN_ON(&construction_thread_checker_); 273 if (max_nack_list_size != 0) { 274 max_nack_list_size_ = max_nack_list_size; 275 } 276 _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, 277 max_incomplete_time_ms); 278 } 279 280 } // namespace vcm 281 } // namespace webrtc