video_frame.h (13951B)
1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_VIDEO_FRAME_H_ 12 #define API_VIDEO_VIDEO_FRAME_H_ 13 14 #include <stdint.h> 15 16 #include <optional> 17 #include <utility> 18 19 #include "api/rtp_packet_infos.h" 20 #include "api/scoped_refptr.h" 21 #include "api/units/time_delta.h" 22 #include "api/units/timestamp.h" 23 #include "api/video/color_space.h" 24 #include "api/video/video_frame_buffer.h" 25 #include "api/video/video_rotation.h" 26 #include "rtc_base/checks.h" 27 #include "rtc_base/system/rtc_export.h" 28 29 namespace webrtc { 30 31 class RTC_EXPORT VideoFrame { 32 public: 33 // Value used to signal that `VideoFrame::id()` is not set. 34 static constexpr uint16_t kNotSetId = 0; 35 36 struct RTC_EXPORT UpdateRect { 37 int offset_x = 0; 38 int offset_y = 0; 39 int width = 0; 40 int height = 0; 41 42 // Makes this UpdateRect a bounding box of this and other rect. 43 void Union(const UpdateRect& other); 44 45 // Makes this UpdateRect an intersection of this and other rect. 46 void Intersect(const UpdateRect& other); 47 48 // Sets everything to 0, making this UpdateRect a zero-size (empty) update. 49 void MakeEmptyUpdate(); 50 51 bool IsEmpty() const; 52 53 // Per-member equality check. Empty rectangles with different offsets would 54 // be considered different. 55 bool operator==(const UpdateRect& other) const { 56 return other.offset_x == offset_x && other.offset_y == offset_y && 57 other.width == width && other.height == height; 58 } 59 60 bool operator!=(const UpdateRect& other) const { return !(*this == other); } 61 62 // Scales update_rect given original frame dimensions. 63 // Cropping is applied first, then rect is scaled down. 64 // Update rect is snapped to 2x2 grid due to possible UV subsampling and 65 // then expanded by additional 2 pixels in each direction to accommodate any 66 // possible scaling artifacts. 67 // Note, close but not equal update_rects on original frame may result in 68 // the same scaled update rects. 69 UpdateRect ScaleWithFrame(int frame_width, 70 int frame_height, 71 int crop_x, 72 int crop_y, 73 int crop_width, 74 int crop_height, 75 int scaled_width, 76 int scaled_height) const; 77 }; 78 79 struct RTC_EXPORT ProcessingTime { 80 TimeDelta Elapsed() const { return finish - start; } 81 Timestamp start; 82 Timestamp finish; 83 }; 84 85 struct RTC_EXPORT RenderParameters { 86 bool use_low_latency_rendering = false; 87 std::optional<int32_t> max_composition_delay_in_frames; 88 89 bool operator==(const RenderParameters& other) const { 90 return other.use_low_latency_rendering == use_low_latency_rendering && 91 other.max_composition_delay_in_frames == 92 max_composition_delay_in_frames; 93 } 94 95 bool operator!=(const RenderParameters& other) const { 96 return !(*this == other); 97 } 98 }; 99 100 // Preferred way of building VideoFrame objects. 101 class RTC_EXPORT Builder { 102 public: 103 Builder(); 104 ~Builder(); 105 106 VideoFrame build(); 107 Builder& set_video_frame_buffer( 108 const scoped_refptr<VideoFrameBuffer>& buffer); 109 Builder& set_timestamp_ms(int64_t timestamp_ms); 110 Builder& set_timestamp_us(int64_t timestamp_us); 111 [[deprecated("Use set_presentation_timestamp instead")]] Builder& 112 set_capture_time_identifier( 113 const std::optional<Timestamp>& presentation_timestamp); 114 Builder& set_presentation_timestamp( 115 const std::optional<Timestamp>& presentation_timestamp); 116 Builder& set_reference_time(const std::optional<Timestamp>& reference_time); 117 Builder& set_rtp_timestamp(uint32_t rtp_timestamp); 118 // TODO(https://bugs.webrtc.org/13756): Deprecate and use set_rtp_timestamp. 119 Builder& set_timestamp_rtp(uint32_t timestamp_rtp); 120 Builder& set_ntp_time_ms(int64_t ntp_time_ms); 121 Builder& set_rotation(VideoRotation rotation); 122 Builder& set_color_space(const std::optional<ColorSpace>& color_space); 123 Builder& set_color_space(const ColorSpace* color_space); 124 Builder& set_id(uint16_t id); 125 Builder& set_update_rect(const std::optional<UpdateRect>& update_rect); 126 Builder& set_packet_infos(RtpPacketInfos packet_infos); 127 Builder& set_is_repeat_frame(bool is_repeat_frame); 128 129 private: 130 uint16_t id_ = kNotSetId; 131 scoped_refptr<VideoFrameBuffer> video_frame_buffer_; 132 int64_t timestamp_us_ = 0; 133 std::optional<Timestamp> presentation_timestamp_; 134 std::optional<Timestamp> reference_time_; 135 uint32_t timestamp_rtp_ = 0; 136 int64_t ntp_time_ms_ = 0; 137 VideoRotation rotation_ = kVideoRotation_0; 138 std::optional<ColorSpace> color_space_; 139 RenderParameters render_parameters_; 140 std::optional<UpdateRect> update_rect_; 141 RtpPacketInfos packet_infos_; 142 bool is_repeat_frame_ = false; 143 }; 144 145 // To be deprecated. Migrate all use to Builder. 146 VideoFrame(const scoped_refptr<VideoFrameBuffer>& buffer, 147 VideoRotation rotation, 148 int64_t timestamp_us); 149 VideoFrame(const scoped_refptr<VideoFrameBuffer>& buffer, 150 uint32_t timestamp_rtp, 151 int64_t render_time_ms, 152 VideoRotation rotation); 153 154 ~VideoFrame(); 155 156 // Support move and copy. 157 VideoFrame(const VideoFrame&); 158 VideoFrame(VideoFrame&&); 159 VideoFrame& operator=(const VideoFrame&); 160 VideoFrame& operator=(VideoFrame&&); 161 162 // Get frame width. 163 int width() const; 164 // Get frame height. 165 int height() const; 166 // Get frame size in pixels. 167 uint32_t size() const; 168 169 // Get frame ID. Returns `kNotSetId` if ID is not set. Not guaranteed to be 170 // transferred from the sender to the receiver, but preserved on the sender 171 // side. The id should be propagated between all frame modifications during 172 // its lifetime from capturing to sending as encoded image. It is intended to 173 // be unique over a time window of a few minutes for the peer connection to 174 // which the corresponding video stream belongs to. 175 uint16_t id() const { return id_; } 176 void set_id(uint16_t id) { id_ = id; } 177 178 // System monotonic clock, same timebase as TimeMicros(). 179 int64_t timestamp_us() const { return timestamp_us_; } 180 void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; } 181 182 // TODO(https://bugs.webrtc.org/373365537): Remove this once its usage is 183 // removed from blink. 184 const std::optional<Timestamp>& capture_time_identifier() const { 185 return presentation_timestamp_; 186 } 187 188 const std::optional<Timestamp>& presentation_timestamp() const { 189 return presentation_timestamp_; 190 } 191 void set_presentation_timestamp( 192 const std::optional<Timestamp>& presentation_timestamp) { 193 presentation_timestamp_ = presentation_timestamp; 194 } 195 196 const std::optional<Timestamp>& reference_time() const { 197 return reference_time_; 198 } 199 void set_reference_time(const std::optional<Timestamp>& reference_time) { 200 reference_time_ = reference_time; 201 } 202 203 // Set frame timestamp (90kHz). 204 void set_rtp_timestamp(uint32_t rtp_timestamp) { 205 timestamp_rtp_ = rtp_timestamp; 206 } 207 208 // Get frame timestamp (90kHz). 209 uint32_t rtp_timestamp() const { return timestamp_rtp_; } 210 211 // Set capture ntp time in milliseconds. 212 void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; } 213 214 // Get capture ntp time in milliseconds. 215 int64_t ntp_time_ms() const { return ntp_time_ms_; } 216 217 // Naming convention for Coordination of Video Orientation. Please see 218 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf 219 // 220 // "pending rotation" or "pending" = a frame that has a VideoRotation > 0. 221 // 222 // "not pending" = a frame that has a VideoRotation == 0. 223 // 224 // "apply rotation" = modify a frame from being "pending" to being "not 225 // pending" rotation (a no-op for "unrotated"). 226 // 227 VideoRotation rotation() const { return rotation_; } 228 void set_rotation(VideoRotation rotation) { rotation_ = rotation; } 229 230 // Get color space when available. 231 const std::optional<ColorSpace>& color_space() const { return color_space_; } 232 void set_color_space(const std::optional<ColorSpace>& color_space) { 233 color_space_ = color_space; 234 } 235 236 RenderParameters render_parameters() const { return render_parameters_; } 237 void set_render_parameters(const RenderParameters& render_parameters) { 238 render_parameters_ = render_parameters; 239 } 240 241 // Get render time in milliseconds. 242 int64_t render_time_ms() const; 243 244 // Return the underlying buffer. Never nullptr for a properly 245 // initialized VideoFrame. 246 scoped_refptr<VideoFrameBuffer> video_frame_buffer() const; 247 248 void set_video_frame_buffer(const scoped_refptr<VideoFrameBuffer>& buffer); 249 250 // Return true if the frame is stored in a texture. 251 bool is_texture() const { 252 return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative; 253 } 254 255 bool has_update_rect() const { return update_rect_.has_value(); } 256 257 // Returns update_rect set by the builder or set_update_rect() or whole frame 258 // rect if no update rect is available. 259 UpdateRect update_rect() const { 260 return update_rect_.value_or(UpdateRect{0, 0, width(), height()}); 261 } 262 263 // Rectangle must be within the frame dimensions. 264 void set_update_rect(const VideoFrame::UpdateRect& update_rect) { 265 RTC_DCHECK_GE(update_rect.offset_x, 0); 266 RTC_DCHECK_GE(update_rect.offset_y, 0); 267 RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width()); 268 RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height()); 269 update_rect_ = update_rect; 270 } 271 272 void clear_update_rect() { update_rect_ = std::nullopt; } 273 274 // Get information about packets used to assemble this video frame. Might be 275 // empty if the information isn't available. 276 const RtpPacketInfos& packet_infos() const { return packet_infos_; } 277 void set_packet_infos(RtpPacketInfos value) { 278 packet_infos_ = std::move(value); 279 } 280 281 const std::optional<ProcessingTime> processing_time() const { 282 return processing_time_; 283 } 284 void set_processing_time(const ProcessingTime& processing_time) { 285 processing_time_ = processing_time; 286 } 287 288 bool is_repeat_frame() const { return is_repeat_frame_; } 289 void set_is_repeat_frame(bool is_repeat_frame) { 290 is_repeat_frame_ = is_repeat_frame; 291 } 292 293 private: 294 VideoFrame(uint16_t id, 295 const scoped_refptr<VideoFrameBuffer>& buffer, 296 int64_t timestamp_us, 297 const std::optional<Timestamp>& presentation_timestamp, 298 const std::optional<Timestamp>& reference_time, 299 uint32_t timestamp_rtp, 300 int64_t ntp_time_ms, 301 VideoRotation rotation, 302 const std::optional<ColorSpace>& color_space, 303 const RenderParameters& render_parameters, 304 const std::optional<UpdateRect>& update_rect, 305 RtpPacketInfos packet_infos, 306 bool is_repeat_frame) 307 : id_(id), 308 video_frame_buffer_(buffer), 309 timestamp_rtp_(timestamp_rtp), 310 ntp_time_ms_(ntp_time_ms), 311 timestamp_us_(timestamp_us), 312 presentation_timestamp_(presentation_timestamp), 313 reference_time_(reference_time), 314 rotation_(rotation), 315 color_space_(color_space), 316 render_parameters_(render_parameters), 317 update_rect_(update_rect), 318 packet_infos_(std::move(packet_infos)), 319 is_repeat_frame_(is_repeat_frame) {} 320 321 uint16_t id_; 322 // An opaque reference counted handle that stores the pixel data. 323 scoped_refptr<VideoFrameBuffer> video_frame_buffer_; 324 uint32_t timestamp_rtp_; 325 int64_t ntp_time_ms_; 326 int64_t timestamp_us_; 327 std::optional<Timestamp> presentation_timestamp_; 328 // Contains a monotonically increasing clock time and represents the time 329 // when the frame was captured. Not all platforms provide the "true" sample 330 // capture time in |reference_time| but might instead use a somewhat delayed 331 // (by the time it took to capture the frame) version of it. 332 std::optional<Timestamp> reference_time_; 333 VideoRotation rotation_; 334 std::optional<ColorSpace> color_space_; 335 // Contains parameters that affect have the frame should be rendered. 336 RenderParameters render_parameters_; 337 // Updated since the last frame area. If present it means that the bounding 338 // box of all the changes is within the rectangular area and is close to it. 339 // If absent, it means that there's no information about the change at all and 340 // update_rect() will return a rectangle corresponding to the entire frame. 341 std::optional<UpdateRect> update_rect_; 342 // Information about packets used to assemble this video frame. This is needed 343 // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's 344 // MediaStreamTrack, in order to implement getContributingSources(). See: 345 // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources 346 RtpPacketInfos packet_infos_; 347 // Processing timestamps of the frame. For received video frames these are the 348 // timestamps when the frame is sent to the decoder and the decoded image 349 // returned from the decoder. 350 // Currently, not set for locally captured video frames. 351 std::optional<ProcessingTime> processing_time_; 352 // Indicates if this is a "repeat frame" - i.e. a copy a previous frame, 353 // inserted in order to make a video codec converge towards a stable quality 354 // in cases where a capturer is using a variable frame rate and stops 355 // producing frames when nothing has changed. 356 bool is_repeat_frame_; 357 }; 358 359 } // namespace webrtc 360 361 #endif // API_VIDEO_VIDEO_FRAME_H_