video_adapter.h (7876B)
1 /* 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_BASE_VIDEO_ADAPTER_H_ 12 #define MEDIA_BASE_VIDEO_ADAPTER_H_ 13 14 #include <stdint.h> 15 16 #include <optional> 17 #include <string> 18 #include <utility> 19 20 #include "api/video/resolution.h" 21 #include "api/video/video_source_interface.h" 22 #include "common_video/framerate_controller.h" 23 #include "media/base/video_common.h" 24 #include "rtc_base/synchronization/mutex.h" 25 #include "rtc_base/system/rtc_export.h" 26 #include "rtc_base/thread_annotations.h" 27 28 namespace webrtc { 29 30 // VideoAdapter adapts an input video frame to an output frame based on the 31 // specified input and output formats. The adaptation includes dropping frames 32 // to reduce frame rate and scaling frames. 33 // VideoAdapter is thread safe. 34 class RTC_EXPORT VideoAdapter { 35 public: 36 VideoAdapter(); 37 // The source requests output frames whose width and height are divisible 38 // by `source_resolution_alignment`. 39 explicit VideoAdapter(int source_resolution_alignment); 40 virtual ~VideoAdapter(); 41 42 VideoAdapter(const VideoAdapter&) = delete; 43 VideoAdapter& operator=(const VideoAdapter&) = delete; 44 45 // Return the adapted resolution and cropping parameters given the 46 // input resolution. The input frame should first be cropped, then 47 // scaled to the final output resolution. Returns true if the frame 48 // should be adapted, and false if it should be dropped. 49 bool AdaptFrameResolution(int in_width, 50 int in_height, 51 int64_t in_timestamp_ns, 52 int* cropped_width, 53 int* cropped_height, 54 int* out_width, 55 int* out_height) RTC_LOCKS_EXCLUDED(mutex_); 56 57 // DEPRECATED. Please use OnOutputFormatRequest below. 58 // TODO(asapersson): Remove this once it is no longer used. 59 // Requests the output frame size and frame interval from 60 // `AdaptFrameResolution` to not be larger than `format`. Also, the input 61 // frame size will be cropped to match the requested aspect ratio. The 62 // requested aspect ratio is orientation agnostic and will be adjusted to 63 // maintain the input orientation, so it doesn't matter if e.g. 1280x720 or 64 // 720x1280 is requested. 65 // Note: Should be called from the source only. 66 void OnOutputFormatRequest(const std::optional<VideoFormat>& format) 67 RTC_LOCKS_EXCLUDED(mutex_); 68 69 // Requests output frame size and frame interval from `AdaptFrameResolution`. 70 // `target_aspect_ratio`: The input frame size will be cropped to match the 71 // requested aspect ratio. The aspect ratio is orientation agnostic and will 72 // be adjusted to maintain the input orientation (i.e. it doesn't matter if 73 // e.g. <1280,720> or <720,1280> is requested). 74 // `max_pixel_count`: The maximum output frame size. 75 // `max_fps`: The maximum output framerate. 76 // Note: Should be called from the source only. 77 void OnOutputFormatRequest( 78 const std::optional<std::pair<int, int>>& target_aspect_ratio, 79 const std::optional<int>& max_pixel_count, 80 const std::optional<int>& max_fps) RTC_LOCKS_EXCLUDED(mutex_); 81 82 // Same as above, but allows setting two different target aspect ratios 83 // depending on incoming frame orientation. This gives more fine-grained 84 // control and can e.g. be used to force landscape video to be cropped to 85 // portrait video. 86 void OnOutputFormatRequest( 87 const std::optional<std::pair<int, int>>& target_landscape_aspect_ratio, 88 const std::optional<int>& max_landscape_pixel_count, 89 const std::optional<std::pair<int, int>>& target_portrait_aspect_ratio, 90 const std::optional<int>& max_portrait_pixel_count, 91 const std::optional<int>& max_fps) RTC_LOCKS_EXCLUDED(mutex_); 92 93 // Requests the output frame size from `AdaptFrameResolution` to have as close 94 // as possible to `sink_wants.target_pixel_count` pixels (if set) 95 // but no more than `sink_wants.max_pixel_count`. 96 // `sink_wants.max_framerate_fps` is essentially analogous to 97 // `sink_wants.max_pixel_count`, but for framerate rather than resolution. 98 // Set `sink_wants.max_pixel_count` and/or `sink_wants.max_framerate_fps` to 99 // std::numeric_limit<int>::max() if no upper limit is desired. 100 // The sink resolution alignment requirement is given by 101 // `sink_wants.resolution_alignment`. 102 // Note: Should be called from the sink only. 103 void OnSinkWants(const VideoSinkWants& sink_wants) RTC_LOCKS_EXCLUDED(mutex_); 104 105 // Returns maximum image area, which shouldn't impose any adaptations. 106 // Can return `numeric_limits<int>::max()` if no limit is set. 107 int GetTargetPixels() const; 108 109 // Returns current frame-rate limit. 110 // Can return `numeric_limits<float>::infinity()` if no limit is set. 111 float GetMaxFramerate() const; 112 113 private: 114 // Determine if frame should be dropped based on input fps and requested fps. 115 bool DropFrame(int64_t in_timestamp_ns) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 116 117 int frames_in_ RTC_GUARDED_BY(mutex_); // Number of input frames. 118 int frames_out_ RTC_GUARDED_BY(mutex_); // Number of output frames. 119 int frames_scaled_ RTC_GUARDED_BY(mutex_); // Number of frames scaled. 120 int adaption_changes_ 121 RTC_GUARDED_BY(mutex_); // Number of changes in scale factor. 122 int previous_width_ RTC_GUARDED_BY(mutex_); // Previous adapter output width. 123 int previous_height_ 124 RTC_GUARDED_BY(mutex_); // Previous adapter output height. 125 126 // The fixed source resolution alignment requirement. 127 const int source_resolution_alignment_; 128 // The currently applied resolution alignment, as given by the requirements: 129 // - the fixed `source_resolution_alignment_`; and 130 // - the latest `sink_wants.resolution_alignment`. 131 int resolution_alignment_ RTC_GUARDED_BY(mutex_); 132 133 // Max number of pixels/fps requested via calls to OnOutputFormatRequest, 134 // OnResolutionFramerateRequest respectively. 135 // The adapted output format is the minimum of these. 136 struct OutputFormatRequest { 137 std::optional<std::pair<int, int>> target_landscape_aspect_ratio; 138 std::optional<int> max_landscape_pixel_count; 139 std::optional<std::pair<int, int>> target_portrait_aspect_ratio; 140 std::optional<int> max_portrait_pixel_count; 141 std::optional<int> max_fps; 142 143 // For logging. 144 std::string ToString() const; 145 }; 146 147 OutputFormatRequest output_format_request_ RTC_GUARDED_BY(mutex_); 148 int resolution_request_target_pixel_count_ RTC_GUARDED_BY(mutex_); 149 int resolution_request_max_pixel_count_ RTC_GUARDED_BY(mutex_); 150 int max_framerate_request_ RTC_GUARDED_BY(mutex_); 151 std::optional<Resolution> scale_resolution_down_to_ RTC_GUARDED_BY(mutex_); 152 153 // Stashed OutputFormatRequest that is used to save value of 154 // OnOutputFormatRequest in case all active encoders are using 155 // scale_resolution_down_to. I.e when all active encoders are using 156 // scale_resolution_down_to, the call to OnOutputFormatRequest is ignored 157 // and the value from scale_resolution_down_to is used instead (to scale/crop 158 // frame). This allows for an application to only use 159 // RtpEncodingParameters::request_resolution and get the same behavior as if 160 // it had used VideoAdapter::OnOutputFormatRequest. 161 std::optional<OutputFormatRequest> stashed_output_format_request_ 162 RTC_GUARDED_BY(mutex_); 163 164 FramerateController framerate_controller_ RTC_GUARDED_BY(mutex_); 165 166 // The critical section to protect the above variables. 167 mutable Mutex mutex_; 168 }; 169 170 } // namespace webrtc 171 172 173 #endif // MEDIA_BASE_VIDEO_ADAPTER_H_