video_stream_adapter.h (11566B)
1 /* 2 * Copyright 2020 The WebRTC Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ 12 #define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ 13 14 #include <cstdint> 15 #include <optional> 16 #include <variant> 17 #include <vector> 18 19 #include "api/adaptation/resource.h" 20 #include "api/field_trials_view.h" 21 #include "api/rtp_parameters.h" 22 #include "api/scoped_refptr.h" 23 #include "api/sequence_checker.h" 24 #include "api/video/video_adaptation_counters.h" 25 #include "api/video_codecs/video_codec.h" 26 #include "call/adaptation/adaptation_constraint.h" 27 #include "call/adaptation/video_source_restrictions.h" 28 #include "call/adaptation/video_stream_input_state.h" 29 #include "call/adaptation/video_stream_input_state_provider.h" 30 #include "rtc_base/experiments/balanced_degradation_settings.h" 31 #include "rtc_base/system/no_unique_address.h" 32 #include "rtc_base/thread_annotations.h" 33 #include "video/video_stream_encoder_observer.h" 34 35 namespace webrtc { 36 37 // The listener is responsible for carrying out the reconfiguration of the video 38 // source such that the VideoSourceRestrictions are fulfilled. 39 class VideoSourceRestrictionsListener { 40 public: 41 virtual ~VideoSourceRestrictionsListener(); 42 43 // The `restrictions` are filtered by degradation preference but not the 44 // `adaptation_counters`, which are currently only reported for legacy stats 45 // calculation purposes. 46 virtual void OnVideoSourceRestrictionsUpdated( 47 VideoSourceRestrictions restrictions, 48 const VideoAdaptationCounters& adaptation_counters, 49 scoped_refptr<Resource> reason, 50 const VideoSourceRestrictions& unfiltered_restrictions) = 0; 51 }; 52 53 class VideoStreamAdapter; 54 55 extern const int kMinFrameRateFps; 56 57 VideoSourceRestrictions FilterRestrictionsByDegradationPreference( 58 VideoSourceRestrictions source_restrictions, 59 DegradationPreference degradation_preference); 60 61 int GetLowerResolutionThan(int pixel_count); 62 int GetHigherResolutionThan(int pixel_count); 63 64 // Either represents the next VideoSourceRestrictions the VideoStreamAdapter 65 // will take, or provides a Status code indicating the reason for not adapting 66 // if the adaptation is not valid. 67 class Adaptation final { 68 public: 69 enum class Status { 70 // Applying this adaptation will have an effect. All other Status codes 71 // indicate that adaptation is not possible and why. 72 kValid, 73 // Cannot adapt. The minimum or maximum adaptation has already been reached. 74 // There are no more steps to take. 75 kLimitReached, 76 // Cannot adapt. The resolution or frame rate requested by a recent 77 // adaptation has not yet been reflected in the input resolution or frame 78 // rate; adaptation is refused to avoid "double-adapting". 79 kAwaitingPreviousAdaptation, 80 // Not enough input. 81 kInsufficientInput, 82 // Adaptation disabled via degradation preference. 83 kAdaptationDisabled, 84 // Adaptation up was rejected by a VideoAdaptationConstraint. 85 kRejectedByConstraint, 86 }; 87 88 static const char* StatusToString(Status status); 89 90 Status status() const; 91 const VideoStreamInputState& input_state() const; 92 const VideoSourceRestrictions& restrictions() const; 93 const VideoAdaptationCounters& counters() const; 94 95 private: 96 friend class VideoStreamAdapter; 97 98 // Constructs with a valid adaptation. Status is kValid. 99 Adaptation(int validation_id, 100 VideoSourceRestrictions restrictions, 101 VideoAdaptationCounters counters, 102 VideoStreamInputState input_state); 103 // Constructor when adaptation is not valid. Status MUST NOT be kValid. 104 Adaptation(int validation_id, Status invalid_status); 105 106 // An Adaptation can become invalidated if the state of VideoStreamAdapter is 107 // modified before the Adaptation is applied. To guard against this, this ID 108 // has to match VideoStreamAdapter::adaptation_validation_id_ when applied. 109 // TODO(https://crbug.com/webrtc/11700): Remove the validation_id_. 110 const int validation_id_; 111 const Status status_; 112 // Input state when adaptation was made. 113 const VideoStreamInputState input_state_; 114 const VideoSourceRestrictions restrictions_; 115 const VideoAdaptationCounters counters_; 116 }; 117 118 // Owns the VideoSourceRestriction for a single stream and is responsible for 119 // adapting it up or down when told to do so. This class serves the following 120 // purposes: 121 // 1. Keep track of a stream's restrictions. 122 // 2. Provide valid ways to adapt up or down the stream's restrictions. 123 // 3. Modify the stream's restrictions in one of the valid ways. 124 class VideoStreamAdapter { 125 public: 126 VideoStreamAdapter(VideoStreamInputStateProvider* input_state_provider, 127 VideoStreamEncoderObserver* encoder_stats_observer, 128 const FieldTrialsView& field_trials); 129 ~VideoStreamAdapter(); 130 131 VideoSourceRestrictions source_restrictions() const; 132 const VideoAdaptationCounters& adaptation_counters() const; 133 void ClearRestrictions(); 134 135 void AddRestrictionsListener( 136 VideoSourceRestrictionsListener* restrictions_listener); 137 void RemoveRestrictionsListener( 138 VideoSourceRestrictionsListener* restrictions_listener); 139 void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint); 140 void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint); 141 142 // TODO(hbos): Setting the degradation preference should not clear 143 // restrictions! This is not defined in the spec and is unexpected, there is a 144 // tiny risk that people would discover and rely on this behavior. 145 void SetDegradationPreference(DegradationPreference degradation_preference); 146 147 // Returns an adaptation that we are guaranteed to be able to apply, or a 148 // status code indicating the reason why we cannot adapt. 149 Adaptation GetAdaptationUp(); 150 Adaptation GetAdaptationDown(); 151 Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, 152 const VideoSourceRestrictions& restrictions); 153 // Tries to adapt the resolution one step. This is used for initial frame 154 // dropping. Does nothing if the degradation preference is not BALANCED or 155 // MAINTAIN_FRAMERATE. In the case of BALANCED, it will try twice to reduce 156 // the resolution. If it fails twice it gives up. 157 Adaptation GetAdaptDownResolution(); 158 159 // Updates source_restrictions() the Adaptation. 160 void ApplyAdaptation(const Adaptation& adaptation, 161 scoped_refptr<Resource> resource); 162 163 struct RestrictionsWithCounters { 164 VideoSourceRestrictions restrictions; 165 VideoAdaptationCounters counters; 166 }; 167 168 static std::optional<uint32_t> GetSingleActiveLayerPixels( 169 const VideoCodec& codec); 170 171 private: 172 void BroadcastVideoRestrictionsUpdate( 173 const VideoStreamInputState& input_state, 174 const scoped_refptr<Resource>& resource); 175 176 bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state) 177 const RTC_RUN_ON(&sequence_checker_); 178 179 using RestrictionsOrState = 180 std::variant<RestrictionsWithCounters, Adaptation::Status>; 181 RestrictionsOrState GetAdaptationUpStep( 182 const VideoStreamInputState& input_state) const 183 RTC_RUN_ON(&sequence_checker_); 184 RestrictionsOrState GetAdaptationDownStep( 185 const VideoStreamInputState& input_state, 186 const RestrictionsWithCounters& current_restrictions) const 187 RTC_RUN_ON(&sequence_checker_); 188 RestrictionsOrState GetAdaptDownResolutionStepForBalanced( 189 const VideoStreamInputState& input_state) const 190 RTC_RUN_ON(&sequence_checker_); 191 RestrictionsOrState AdaptIfFpsDiffInsufficient( 192 const VideoStreamInputState& input_state, 193 const RestrictionsWithCounters& restrictions) const 194 RTC_RUN_ON(&sequence_checker_); 195 196 Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const 197 RTC_RUN_ON(&sequence_checker_); 198 Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const 199 RTC_RUN_ON(&sequence_checker_); 200 201 static RestrictionsOrState DecreaseResolution( 202 const VideoStreamInputState& input_state, 203 const RestrictionsWithCounters& current_restrictions); 204 static RestrictionsOrState IncreaseResolution( 205 const VideoStreamInputState& input_state, 206 const RestrictionsWithCounters& current_restrictions); 207 // Framerate methods are member functions because they need internal state 208 // if the degradation preference is BALANCED. 209 RestrictionsOrState DecreaseFramerate( 210 const VideoStreamInputState& input_state, 211 const RestrictionsWithCounters& current_restrictions) const 212 RTC_RUN_ON(&sequence_checker_); 213 RestrictionsOrState IncreaseFramerate( 214 const VideoStreamInputState& input_state, 215 const RestrictionsWithCounters& current_restrictions) const 216 RTC_RUN_ON(&sequence_checker_); 217 218 struct RestrictionsOrStateVisitor; 219 Adaptation RestrictionsOrStateToAdaptation( 220 RestrictionsOrState step_or_state, 221 const VideoStreamInputState& input_state) const 222 RTC_RUN_ON(&sequence_checker_); 223 224 RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; 225 // Gets the input state which is the basis of all adaptations. 226 // Thread safe. 227 VideoStreamInputStateProvider* input_state_provider_; 228 // Used to signal when min pixel limit has been reached. 229 VideoStreamEncoderObserver* const encoder_stats_observer_; 230 // Decides the next adaptation target in DegradationPreference::BALANCED. 231 const BalancedDegradationSettings balanced_settings_; 232 // To guard against applying adaptations that have become invalidated, an 233 // Adaptation that is applied has to have a matching validation ID. 234 int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_); 235 // When deciding the next target up or down, different strategies are used 236 // depending on the DegradationPreference. 237 // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference 238 DegradationPreference degradation_preference_ 239 RTC_GUARDED_BY(&sequence_checker_); 240 // Used to avoid adapting twice. Stores the resolution at the time of the last 241 // adaptation. 242 // TODO(hbos): Can we implement a more general "cooldown" mechanism of 243 // resources intead? If we already have adapted it seems like we should wait 244 // a while before adapting again, so that we are not acting on usage 245 // measurements that are made obsolete/unreliable by an "ongoing" adaptation. 246 struct AwaitingFrameSizeChange { 247 AwaitingFrameSizeChange(bool pixels_increased, int frame_size); 248 const bool pixels_increased; 249 const int frame_size_pixels; 250 }; 251 std::optional<AwaitingFrameSizeChange> awaiting_frame_size_change_ 252 RTC_GUARDED_BY(&sequence_checker_); 253 // The previous restrictions value. Starts as unrestricted. 254 VideoSourceRestrictions last_video_source_restrictions_ 255 RTC_GUARDED_BY(&sequence_checker_); 256 VideoSourceRestrictions last_filtered_restrictions_ 257 RTC_GUARDED_BY(&sequence_checker_); 258 259 std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_ 260 RTC_GUARDED_BY(&sequence_checker_); 261 std::vector<AdaptationConstraint*> adaptation_constraints_ 262 RTC_GUARDED_BY(&sequence_checker_); 263 264 RestrictionsWithCounters current_restrictions_ 265 RTC_GUARDED_BY(&sequence_checker_); 266 }; 267 268 } // namespace webrtc 269 270 #endif // CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_