media_opt_util.h (11885B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_ 12 #define MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_ 13 14 #include <math.h> 15 #include <stdlib.h> 16 17 #include <cstdint> 18 #include <memory> 19 20 #include "api/environment/environment.h" 21 #include "api/field_trials_view.h" 22 #include "rtc_base/experiments/rate_control_settings.h" 23 #include "rtc_base/numerics/exp_filter.h" 24 25 namespace webrtc { 26 namespace media_optimization { 27 28 // Number of time periods used for (max) window filter for packet loss 29 // TODO(marpan): set reasonable window size for filtered packet loss, 30 // adjustment should be based on logged/real data of loss stats/correlation. 31 constexpr int kLossPrHistorySize = 10; 32 33 // 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms 34 constexpr int kLossPrShortFilterWinMs = 1000; 35 36 // The type of filter used on the received packet loss reports. 37 enum FilterPacketLossMode { 38 kNoFilter, // No filtering on received loss. 39 kAvgFilter, // Recursive average filter. 40 kMaxFilter // Max-window filter, over the time interval of: 41 // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms. 42 }; 43 44 // Thresholds for hybrid NACK/FEC 45 // common to media optimization and the jitter buffer. 46 constexpr int64_t kLowRttNackMs = 20; 47 48 struct VCMProtectionParameters { 49 VCMProtectionParameters(); 50 51 int64_t rtt; 52 float lossPr; 53 float bitRate; 54 float packetsPerFrame; 55 float packetsPerFrameKey; 56 float frameRate; 57 float keyFrameSize; 58 uint8_t fecRateDelta; 59 uint8_t fecRateKey; 60 uint16_t codecWidth; 61 uint16_t codecHeight; 62 int numLayers; 63 }; 64 65 /******************************/ 66 /* VCMProtectionMethod class */ 67 /******************************/ 68 69 enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone }; 70 71 class VCMLossProbabilitySample { 72 public: 73 VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {} 74 75 uint8_t lossPr255; 76 int64_t timeMs; 77 }; 78 79 class VCMProtectionMethod { 80 public: 81 VCMProtectionMethod(); 82 virtual ~VCMProtectionMethod(); 83 84 // Updates the efficiency of the method using the parameters provided 85 // 86 // Input: 87 // - parameters : Parameters used to calculate efficiency 88 // 89 // Return value : True if this method is recommended in 90 // the given conditions. 91 virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0; 92 93 // Returns the protection type 94 // 95 // Return value : The protection type 96 VCMProtectionMethodEnum Type() const; 97 98 // Returns the effective packet loss for ER, required by this protection 99 // method 100 // 101 // Return value : Required effective packet loss 102 virtual uint8_t RequiredPacketLossER(); 103 104 // Extracts the FEC protection factor for Key frame, required by this 105 // protection method 106 // 107 // Return value : Required protectionFactor for Key frame 108 virtual uint8_t RequiredProtectionFactorK(); 109 110 // Extracts the FEC protection factor for Delta frame, required by this 111 // protection method 112 // 113 // Return value : Required protectionFactor for delta frame 114 virtual uint8_t RequiredProtectionFactorD(); 115 116 // Extracts whether the FEC Unequal protection (UEP) is used for Key frame. 117 // 118 // Return value : Required Unequal protection on/off state. 119 virtual bool RequiredUepProtectionK(); 120 121 // Extracts whether the the FEC Unequal protection (UEP) is used for Delta 122 // frame. 123 // 124 // Return value : Required Unequal protection on/off state. 125 virtual bool RequiredUepProtectionD(); 126 127 virtual int MaxFramesFec() const; 128 129 protected: 130 uint8_t _effectivePacketLoss; 131 uint8_t _protectionFactorK; 132 uint8_t _protectionFactorD; 133 // Estimation of residual loss after the FEC 134 float _scaleProtKey; 135 int32_t _maxPayloadSize; 136 137 bool _useUepProtectionK; 138 bool _useUepProtectionD; 139 float _corrFecCost; 140 VCMProtectionMethodEnum _type; 141 }; 142 143 class VCMNackMethod : public VCMProtectionMethod { 144 public: 145 VCMNackMethod(); 146 ~VCMNackMethod() override; 147 bool UpdateParameters(const VCMProtectionParameters* parameters) override; 148 // Get the effective packet loss 149 bool EffectivePacketLoss(const VCMProtectionParameters* parameter); 150 }; 151 152 class VCMFecMethod : public VCMProtectionMethod { 153 public: 154 explicit VCMFecMethod(const FieldTrialsView& field_trials); 155 ~VCMFecMethod() override; 156 bool UpdateParameters(const VCMProtectionParameters* parameters) override; 157 // Get the effective packet loss for ER 158 bool EffectivePacketLoss(const VCMProtectionParameters* parameters); 159 // Get the FEC protection factors 160 bool ProtectionFactor(const VCMProtectionParameters* parameters); 161 // Get the boost for key frame protection 162 uint8_t BoostCodeRateKey(uint8_t packetFrameDelta, 163 uint8_t packetFrameKey) const; 164 // Convert the rates: defined relative to total# packets or source# packets 165 uint8_t ConvertFECRate(uint8_t codeRate) const; 166 // Get the average effective recovery from FEC: for random loss model 167 float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const; 168 // Update FEC with protectionFactorD 169 void UpdateProtectionFactorD(uint8_t protectionFactorD); 170 // Update FEC with protectionFactorK 171 void UpdateProtectionFactorK(uint8_t protectionFactorK); 172 // Compute the bits per frame. Account for temporal layers when applicable. 173 int BitsPerFrame(const VCMProtectionParameters* parameters); 174 175 protected: 176 static constexpr int kUpperLimitFramesFec = 6; 177 // Thresholds values for the bytes/frame and round trip time, below which we 178 // may turn off FEC, depending on `_numLayers` and `_maxFramesFec`. 179 // Max bytes/frame for VGA, corresponds to ~140k at 25fps. 180 static constexpr int kMaxBytesPerFrameForFec = 700; 181 // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps. 182 static constexpr int kMaxBytesPerFrameForFecLow = 400; 183 // Max bytes/frame for frame size larger than VGA, ~200k at 25fps. 184 static constexpr int kMaxBytesPerFrameForFecHigh = 1000; 185 186 const RateControlSettings rate_control_settings_; 187 }; 188 189 class VCMNackFecMethod : public VCMFecMethod { 190 public: 191 VCMNackFecMethod(const FieldTrialsView& field_trials, 192 int64_t lowRttNackThresholdMs, 193 int64_t highRttNackThresholdMs); 194 ~VCMNackFecMethod() override; 195 bool UpdateParameters(const VCMProtectionParameters* parameters) override; 196 // Get the effective packet loss for ER 197 bool EffectivePacketLoss(const VCMProtectionParameters* parameters); 198 // Get the protection factors 199 bool ProtectionFactor(const VCMProtectionParameters* parameters); 200 // Get the max number of frames the FEC is allowed to be based on. 201 int MaxFramesFec() const override; 202 // Turn off the FEC based on low bitrate and other factors. 203 bool BitRateTooLowForFec(const VCMProtectionParameters* parameters); 204 205 private: 206 int ComputeMaxFramesFec(const VCMProtectionParameters* parameters); 207 208 int64_t _lowRttNackMs; 209 int64_t _highRttNackMs; 210 int _maxFramesFec; 211 }; 212 213 class VCMLossProtectionLogic { 214 public: 215 explicit VCMLossProtectionLogic(const Environment& env); 216 ~VCMLossProtectionLogic(); 217 218 // Set the protection method to be used 219 // 220 // Input: 221 // - newMethodType : New requested protection method type. If one 222 // is already set, it will be deleted and replaced 223 void SetMethod(VCMProtectionMethodEnum newMethodType); 224 225 // Update the round-trip time 226 // 227 // Input: 228 // - rtt : Round-trip time in seconds. 229 void UpdateRtt(int64_t rtt); 230 231 // Update the filtered packet loss. 232 // 233 // Input: 234 // - packetLossEnc : The reported packet loss filtered 235 // (max window or average) 236 void UpdateFilteredLossPr(uint8_t packetLossEnc); 237 238 // Update the current target bit rate. 239 // 240 // Input: 241 // - bitRate : The current target bit rate in kbits/s 242 void UpdateBitRate(float bitRate); 243 244 // Update the number of packets per frame estimate, for delta frames 245 // 246 // Input: 247 // - nPackets : Number of packets in the latest sent frame. 248 void UpdatePacketsPerFrame(float nPackets, int64_t nowMs); 249 250 // Update the number of packets per frame estimate, for key frames 251 // 252 // Input: 253 // - nPackets : umber of packets in the latest sent frame. 254 void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs); 255 256 // Update the keyFrameSize estimate 257 // 258 // Input: 259 // - keyFrameSize : The size of the latest sent key frame. 260 void UpdateKeyFrameSize(float keyFrameSize); 261 262 // Update the frame rate 263 // 264 // Input: 265 // - frameRate : The current target frame rate. 266 void UpdateFrameRate(float frameRate) { _frameRate = frameRate; } 267 268 // Update the frame size 269 // 270 // Input: 271 // - width : The codec frame width. 272 // - height : The codec frame height. 273 void UpdateFrameSize(size_t width, size_t height); 274 275 // Update the number of active layers 276 // 277 // Input: 278 // - numLayers : Number of layers used. 279 void UpdateNumLayers(int numLayers); 280 281 // The amount of packet loss to cover for with FEC. 282 // 283 // Input: 284 // - fecRateKey : Packet loss to cover for with FEC when 285 // sending key frames. 286 // - fecRateDelta : Packet loss to cover for with FEC when 287 // sending delta frames. 288 void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) { 289 _fecRateKey = fecRateKey; 290 _fecRateDelta = fecRateDelta; 291 } 292 293 // Update the protection methods with the current VCMProtectionParameters 294 // and set the requested protection settings. 295 // Return value : Returns true on update 296 bool UpdateMethod(); 297 298 // Returns the method currently selected. 299 // 300 // Return value : The protection method currently selected. 301 VCMProtectionMethod* SelectedMethod() const; 302 303 // Return the protection type of the currently selected method 304 VCMProtectionMethodEnum SelectedType() const; 305 306 // Updates the filtered loss for the average and max window packet loss, 307 // and returns the filtered loss probability in the interval [0, 255]. 308 // The returned filtered loss value depends on the parameter `filter_mode`. 309 // The input parameter `lossPr255` is the received packet loss. 310 311 // Return value : The filtered loss probability 312 uint8_t FilteredLoss(int64_t nowMs, 313 FilterPacketLossMode filter_mode, 314 uint8_t lossPr255); 315 316 void Reset(int64_t nowMs); 317 318 void Release(); 319 320 private: 321 // Sets the available loss protection methods. 322 void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now); 323 uint8_t MaxFilteredLossPr(int64_t nowMs) const; 324 325 const Environment env_; 326 std::unique_ptr<VCMProtectionMethod> _selectedMethod; 327 VCMProtectionParameters _currentParameters; 328 int64_t _rtt; 329 float _lossPr; 330 float _bitRate; 331 float _frameRate; 332 float _keyFrameSize; 333 uint8_t _fecRateKey; 334 uint8_t _fecRateDelta; 335 int64_t _lastPrUpdateT; 336 int64_t _lastPacketPerFrameUpdateT; 337 int64_t _lastPacketPerFrameUpdateTKey; 338 ExpFilter _lossPr255; 339 VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize]; 340 uint8_t _shortMaxLossPr255; 341 ExpFilter _packetsPerFrame; 342 ExpFilter _packetsPerFrameKey; 343 size_t _codecWidth; 344 size_t _codecHeight; 345 int _numLayers; 346 }; 347 348 } // namespace media_optimization 349 } // namespace webrtc 350 351 #endif // MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_