receiver.cc (7364B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/video_coding/deprecated/receiver.h" 12 13 #include <cstdint> 14 #include <cstdlib> 15 #include <memory> 16 #include <optional> 17 #include <utility> 18 #include <vector> 19 20 #include "absl/memory/memory.h" 21 #include "api/field_trials_view.h" 22 #include "api/units/time_delta.h" 23 #include "api/units/timestamp.h" 24 #include "api/video/encoded_image.h" 25 #include "api/video/video_timing.h" 26 #include "modules/video_coding/deprecated/event_wrapper.h" 27 #include "modules/video_coding/deprecated/jitter_buffer_common.h" 28 #include "modules/video_coding/deprecated/packet.h" 29 #include "modules/video_coding/encoded_frame.h" 30 #include "modules/video_coding/include/video_coding_defines.h" 31 #include "modules/video_coding/internal_defines.h" 32 #include "modules/video_coding/timing/timing.h" 33 #include "rtc_base/logging.h" 34 #include "rtc_base/numerics/safe_conversions.h" 35 #include "rtc_base/trace_event.h" 36 #include "system_wrappers/include/clock.h" 37 38 namespace webrtc { 39 40 enum { kMaxReceiverDelayMs = 10000 }; 41 42 VCMReceiver::VCMReceiver(VCMTiming* timing, 43 Clock* clock, 44 const FieldTrialsView& field_trials) 45 : VCMReceiver::VCMReceiver(timing, 46 clock, 47 absl::WrapUnique(EventWrapper::Create()), 48 absl::WrapUnique(EventWrapper::Create()), 49 field_trials) {} 50 51 VCMReceiver::VCMReceiver(VCMTiming* timing, 52 Clock* clock, 53 std::unique_ptr<EventWrapper> receiver_event, 54 std::unique_ptr<EventWrapper> jitter_buffer_event, 55 const FieldTrialsView& field_trials) 56 : clock_(clock), 57 jitter_buffer_(clock_, std::move(jitter_buffer_event), field_trials), 58 timing_(timing), 59 render_wait_event_(std::move(receiver_event)), 60 max_video_delay_ms_(kMaxVideoDelayMs) { 61 jitter_buffer_.Start(); 62 } 63 64 VCMReceiver::~VCMReceiver() { 65 render_wait_event_->Set(); 66 } 67 68 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet) { 69 // Insert the packet into the jitter buffer. The packet can either be empty or 70 // contain media at this point. 71 bool retransmitted = false; 72 const VCMFrameBufferEnum ret = 73 jitter_buffer_.InsertPacket(packet, &retransmitted); 74 if (ret == kOldPacket) { 75 return VCM_OK; 76 } else if (ret == kFlushIndicator) { 77 return VCM_FLUSH_INDICATOR; 78 } else if (ret < 0) { 79 return VCM_JITTER_BUFFER_ERROR; 80 } 81 if (ret == kCompleteSession && !retransmitted) { 82 // We don't want to include timestamps which have suffered from 83 // retransmission here, since we compensate with extra retransmission 84 // delay within the jitter estimate. 85 timing_->IncomingTimestamp(packet.timestamp, clock_->CurrentTime()); 86 } 87 return VCM_OK; 88 } 89 90 VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms, 91 bool prefer_late_decoding) { 92 const int64_t start_time_ms = clock_->TimeInMilliseconds(); 93 int64_t render_time_ms = 0; 94 // Exhaust wait time to get a complete frame for decoding. 95 VCMEncodedFrame* found_frame = 96 jitter_buffer_.NextCompleteFrame(max_wait_time_ms); 97 98 if (found_frame == nullptr) { 99 return nullptr; 100 } 101 uint32_t frame_timestamp = found_frame->RtpTimestamp(); 102 103 if (std::optional<VideoPlayoutDelay> playout_delay = 104 found_frame->EncodedImage().PlayoutDelay()) { 105 timing_->set_playout_delay(*playout_delay); 106 } 107 108 // We have a frame - Set timing and render timestamp. 109 timing_->SetJitterDelay( 110 TimeDelta::Millis(jitter_buffer_.EstimatedJitterMs())); 111 const Timestamp now = clock_->CurrentTime(); 112 const int64_t now_ms = now.ms(); 113 timing_->UpdateCurrentDelay(frame_timestamp); 114 render_time_ms = timing_->RenderTime(frame_timestamp, now).ms(); 115 // Check render timing. 116 bool timing_error = false; 117 // Assume that render timing errors are due to changes in the video stream. 118 if (render_time_ms < 0) { 119 timing_error = true; 120 } else if (std::abs(render_time_ms - now_ms) > max_video_delay_ms_) { 121 int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms)); 122 RTC_LOG(LS_WARNING) 123 << "A frame about to be decoded is out of the configured " 124 "delay bounds (" 125 << frame_delay << " > " << max_video_delay_ms_ 126 << "). Resetting the video jitter buffer."; 127 timing_error = true; 128 } else if (static_cast<int>(timing_->TargetVideoDelay().ms()) > 129 max_video_delay_ms_) { 130 RTC_LOG(LS_WARNING) << "The video target delay has grown larger than " 131 << max_video_delay_ms_ 132 << " ms. Resetting jitter buffer."; 133 timing_error = true; 134 } 135 136 if (timing_error) { 137 // Timing error => reset timing and flush the jitter buffer. 138 jitter_buffer_.Flush(); 139 timing_->Reset(); 140 return nullptr; 141 } 142 143 if (prefer_late_decoding) { 144 // Decode frame as close as possible to the render timestamp. 145 const int32_t available_wait_time = 146 max_wait_time_ms - 147 static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms); 148 uint16_t new_max_wait_time = 149 static_cast<uint16_t>(VCM_MAX(available_wait_time, 0)); 150 uint32_t wait_time_ms = saturated_cast<uint32_t>( 151 timing_ 152 ->MaxWaitingTime(Timestamp::Millis(render_time_ms), 153 clock_->CurrentTime(), 154 /*too_many_frames_queued=*/false) 155 .ms()); 156 if (new_max_wait_time < wait_time_ms) { 157 // We're not allowed to wait until the frame is supposed to be rendered, 158 // waiting as long as we're allowed to avoid busy looping, and then return 159 // NULL. Next call to this function might return the frame. 160 render_wait_event_->Wait(new_max_wait_time); 161 return nullptr; 162 } 163 // Wait until it's time to render. 164 render_wait_event_->Wait(wait_time_ms); 165 } 166 167 // Extract the frame from the jitter buffer and set the render time. 168 VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp); 169 if (frame == nullptr) { 170 return nullptr; 171 } 172 frame->SetRenderTime(render_time_ms); 173 TRACE_EVENT_ASYNC_STEP_INTO1("webrtc", "Video", frame->RtpTimestamp(), 174 "SetRenderTS", "render_time", 175 frame->RenderTimeMs()); 176 return frame; 177 } 178 179 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) { 180 jitter_buffer_.ReleaseFrame(frame); 181 } 182 183 void VCMReceiver::SetNackSettings(size_t max_nack_list_size, 184 int max_packet_age_to_nack, 185 int max_incomplete_time_ms) { 186 jitter_buffer_.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, 187 max_incomplete_time_ms); 188 } 189 190 std::vector<uint16_t> VCMReceiver::NackList(bool* request_key_frame) { 191 return jitter_buffer_.GetNackList(request_key_frame); 192 } 193 194 } // namespace webrtc