jitter_buffer.h (11547B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_DEPRECATED_JITTER_BUFFER_H_ 12 #define MODULES_VIDEO_CODING_DEPRECATED_JITTER_BUFFER_H_ 13 14 #include <cstddef> 15 #include <cstdint> 16 #include <list> 17 #include <map> 18 #include <memory> 19 #include <set> 20 #include <vector> 21 22 #include "api/field_trials_view.h" 23 #include "modules/include/module_common_types_public.h" 24 #include "modules/video_coding/deprecated/decoding_state.h" 25 #include "modules/video_coding/deprecated/event_wrapper.h" 26 #include "modules/video_coding/deprecated/jitter_buffer_common.h" 27 #include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h" 28 #include "modules/video_coding/timing/jitter_estimator.h" 29 #include "rtc_base/synchronization/mutex.h" 30 #include "rtc_base/thread_annotations.h" 31 32 namespace webrtc { 33 34 // forward declarations 35 class Clock; 36 class VCMFrameBuffer; 37 class VCMPacket; 38 class VCMEncodedFrame; 39 40 typedef std::list<VCMFrameBuffer*> UnorderedFrameList; 41 42 struct VCMJitterSample { 43 VCMJitterSample() : timestamp(0), frame_size(0), latest_packet_time(-1) {} 44 uint32_t timestamp; 45 uint32_t frame_size; 46 int64_t latest_packet_time; 47 }; 48 49 class TimestampLessThan { 50 public: 51 bool operator()(uint32_t timestamp1, uint32_t timestamp2) const { 52 return IsNewerTimestamp(timestamp2, timestamp1); 53 } 54 }; 55 56 class FrameList 57 : public std::map<uint32_t, VCMFrameBuffer*, TimestampLessThan> { 58 public: 59 void InsertFrame(VCMFrameBuffer* frame); 60 VCMFrameBuffer* PopFrame(uint32_t timestamp); 61 VCMFrameBuffer* Front() const; 62 VCMFrameBuffer* Back() const; 63 int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it, 64 UnorderedFrameList* free_frames); 65 void CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state, 66 UnorderedFrameList* free_frames); 67 void Reset(UnorderedFrameList* free_frames); 68 }; 69 70 class VCMJitterBuffer { 71 public: 72 VCMJitterBuffer(Clock* clock, 73 std::unique_ptr<EventWrapper> event, 74 const FieldTrialsView& field_trials); 75 76 ~VCMJitterBuffer(); 77 78 VCMJitterBuffer(const VCMJitterBuffer&) = delete; 79 VCMJitterBuffer& operator=(const VCMJitterBuffer&) = delete; 80 81 // Initializes and starts jitter buffer. 82 void Start() RTC_LOCKS_EXCLUDED(mutex_); 83 84 // Signals all internal events and stops the jitter buffer. 85 void Stop() RTC_LOCKS_EXCLUDED(mutex_); 86 87 // Returns true if the jitter buffer is running. 88 bool Running() const RTC_LOCKS_EXCLUDED(mutex_); 89 90 // Empty the jitter buffer of all its data. 91 void Flush() RTC_LOCKS_EXCLUDED(mutex_); 92 93 // Gets number of packets received. 94 int num_packets() const RTC_LOCKS_EXCLUDED(mutex_); 95 96 // Gets number of duplicated packets received. 97 int num_duplicated_packets() const RTC_LOCKS_EXCLUDED(mutex_); 98 99 // Wait `max_wait_time_ms` for a complete frame to arrive. 100 // If found, a pointer to the frame is returned. Returns nullptr otherwise. 101 VCMEncodedFrame* NextCompleteFrame(uint32_t max_wait_time_ms) 102 RTC_LOCKS_EXCLUDED(mutex_); 103 104 // Extract frame corresponding to input timestamp. 105 // Frame will be set to a decoding state. 106 VCMEncodedFrame* ExtractAndSetDecode(uint32_t timestamp) 107 RTC_LOCKS_EXCLUDED(mutex_); 108 109 // Releases a frame returned from the jitter buffer, should be called when 110 // done with decoding. 111 void ReleaseFrame(VCMEncodedFrame* frame) RTC_LOCKS_EXCLUDED(mutex_); 112 113 // Returns the time in ms when the latest packet was inserted into the frame. 114 // Retransmitted is set to true if any of the packets belonging to the frame 115 // has been retransmitted. 116 int64_t LastPacketTime(const VCMEncodedFrame* frame, 117 bool* retransmitted) const RTC_LOCKS_EXCLUDED(mutex_); 118 119 // Inserts a packet into a frame returned from GetFrame(). 120 // If the return value is <= 0, `frame` is invalidated and the pointer must 121 // be dropped after this function returns. 122 VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted) 123 RTC_LOCKS_EXCLUDED(mutex_); 124 125 // Returns the estimated jitter in milliseconds. 126 uint32_t EstimatedJitterMs() RTC_LOCKS_EXCLUDED(mutex_); 127 128 void SetNackSettings(size_t max_nack_list_size, 129 int max_packet_age_to_nack, 130 int max_incomplete_time_ms) RTC_LOCKS_EXCLUDED(mutex_); 131 132 // Returns a list of the sequence numbers currently missing. 133 std::vector<uint16_t> GetNackList(bool* request_key_frame) 134 RTC_LOCKS_EXCLUDED(mutex_); 135 136 private: 137 class SequenceNumberLessThan { 138 public: 139 bool operator()(const uint16_t& sequence_number1, 140 const uint16_t& sequence_number2) const { 141 return IsNewerSequenceNumber(sequence_number2, sequence_number1); 142 } 143 }; 144 typedef std::set<uint16_t, SequenceNumberLessThan> SequenceNumberSet; 145 146 // Gets the frame assigned to the timestamp of the packet. May recycle 147 // existing frames if no free frames are available. Returns an error code if 148 // failing, or kNoError on success. `frame_list` contains which list the 149 // packet was in, or NULL if it was not in a FrameList (a new frame). 150 VCMFrameBufferEnum GetFrame(const VCMPacket& packet, 151 VCMFrameBuffer** frame, 152 FrameList** frame_list) 153 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 154 155 // Returns true if `frame` is continuous in `decoding_state`, not taking 156 // decodable frames into account. 157 bool IsContinuousInState(const VCMFrameBuffer& frame, 158 const VCMDecodingState& decoding_state) const 159 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 160 // Returns true if `frame` is continuous in the `last_decoded_state_`, taking 161 // all decodable frames into account. 162 bool IsContinuous(const VCMFrameBuffer& frame) const 163 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 164 // Looks for frames in `incomplete_frames_` which are continuous in the 165 // provided `decoded_state`. Starts the search from the timestamp of 166 // `decoded_state`. 167 void FindAndInsertContinuousFramesWithState( 168 const VCMDecodingState& decoded_state) 169 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 170 // Looks for frames in `incomplete_frames_` which are continuous in 171 // `last_decoded_state_` taking all decodable frames into account. Starts 172 // the search from `new_frame`. 173 void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame) 174 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 175 VCMFrameBuffer* NextFrame() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 176 // Returns true if the NACK list was updated to cover sequence numbers up to 177 // `sequence_number`. If false a key frame is needed to get into a state where 178 // we can continue decoding. 179 bool UpdateNackList(uint16_t sequence_number) 180 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 181 bool TooLargeNackList() const; 182 // Returns true if the NACK list was reduced without problem. If false a key 183 // frame is needed to get into a state where we can continue decoding. 184 bool HandleTooLargeNackList() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 185 bool MissingTooOldPacket(uint16_t latest_sequence_number) const 186 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 187 // Returns true if the too old packets was successfully removed from the NACK 188 // list. If false, a key frame is needed to get into a state where we can 189 // continue decoding. 190 bool HandleTooOldPackets(uint16_t latest_sequence_number) 191 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 192 // Drops all packets in the NACK list up until `last_decoded_sequence_number`. 193 void DropPacketsFromNackList(uint16_t last_decoded_sequence_number); 194 195 // Gets an empty frame, creating a new frame if necessary (i.e. increases 196 // jitter buffer size). 197 VCMFrameBuffer* GetEmptyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 198 199 // Attempts to increase the size of the jitter buffer. Returns true on 200 // success, false otherwise. 201 bool TryToIncreaseJitterBufferSize() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 202 203 // Recycles oldest frames until a key frame is found. Used if jitter buffer is 204 // completely full. Returns true if a key frame was found. 205 bool RecycleFramesUntilKeyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 206 207 // Update rolling average of packets per frame. 208 void UpdateAveragePacketsPerFrame(int current_number_packets_) 209 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 210 211 // Cleans the frame list in the JB from old/empty frames. 212 // Should only be called prior to actual use. 213 void CleanUpOldOrEmptyFrames() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 214 215 // Returns true if `packet` is likely to have been retransmitted. 216 bool IsPacketRetransmitted(const VCMPacket& packet) const; 217 218 // The following three functions update the jitter estimate with the 219 // payload size, receive time and RTP timestamp of a frame. 220 void UpdateJitterEstimate(const VCMJitterSample& sample, 221 bool incomplete_frame); 222 void UpdateJitterEstimate(const VCMFrameBuffer& frame, bool incomplete_frame); 223 void UpdateJitterEstimate(int64_t latest_packet_time_ms, 224 uint32_t timestamp, 225 unsigned int frame_size, 226 bool incomplete_frame); 227 228 int NonContinuousOrIncompleteDuration() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 229 230 uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const; 231 232 // Reset frame buffer and return it to free_frames_. 233 void RecycleFrameBuffer(VCMFrameBuffer* frame) 234 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 235 236 // Empty the jitter buffer of all its data. 237 void FlushInternal() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 238 239 Clock* clock_; 240 // If we are running (have started) or not. 241 bool running_; 242 mutable Mutex mutex_; 243 // Event to signal when we have a frame ready for decoder. 244 std::unique_ptr<EventWrapper> frame_event_; 245 // Number of allocated frames. 246 int max_number_of_frames_; 247 UnorderedFrameList free_frames_ RTC_GUARDED_BY(mutex_); 248 FrameList decodable_frames_ RTC_GUARDED_BY(mutex_); 249 FrameList incomplete_frames_ RTC_GUARDED_BY(mutex_); 250 VCMDecodingState last_decoded_state_ RTC_GUARDED_BY(mutex_); 251 bool first_packet_since_reset_; 252 253 // Number of packets in a row that have been too old. 254 int num_consecutive_old_packets_; 255 // Number of packets received. 256 int num_packets_ RTC_GUARDED_BY(mutex_); 257 // Number of duplicated packets received. 258 int num_duplicated_packets_ RTC_GUARDED_BY(mutex_); 259 260 // Jitter estimation. 261 // Filter for estimating jitter. 262 JitterEstimator jitter_estimate_; 263 // Calculates network delays used for jitter calculations. 264 InterFrameDelayVariationCalculator inter_frame_delay_; 265 VCMJitterSample waiting_for_completion_; 266 267 // Holds the internal NACK list (the missing sequence numbers). 268 SequenceNumberSet missing_sequence_numbers_; 269 uint16_t latest_received_sequence_number_; 270 size_t max_nack_list_size_; 271 int max_packet_age_to_nack_; // Measured in sequence numbers. 272 int max_incomplete_time_ms_; 273 274 // Estimated rolling average of packets per frame 275 float average_packets_per_frame_; 276 // average_packets_per_frame converges fast if we have fewer than this many 277 // frames. 278 int frame_counter_; 279 }; 280 } // namespace webrtc 281 282 #endif // MODULES_VIDEO_CODING_DEPRECATED_JITTER_BUFFER_H_