video_frame_buffer.h (12701B)
1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_VIDEO_FRAME_BUFFER_H_ 12 #define API_VIDEO_VIDEO_FRAME_BUFFER_H_ 13 14 #include <cstddef> 15 #include <cstdint> 16 #include <string> 17 18 #include "api/array_view.h" 19 #include "api/ref_count.h" 20 #include "api/scoped_refptr.h" 21 #include "rtc_base/system/rtc_export.h" 22 23 namespace webrtc { 24 25 class I420BufferInterface; 26 class I420ABufferInterface; 27 class I422BufferInterface; 28 class I444BufferInterface; 29 class I010BufferInterface; 30 class I210BufferInterface; 31 class I410BufferInterface; 32 class NV12BufferInterface; 33 34 // Base class for frame buffers of different types of pixel format and storage. 35 // The tag in type() indicates how the data is represented, and each type is 36 // implemented as a subclass. To access the pixel data, call the appropriate 37 // GetXXX() function, where XXX represents the type. There is also a function 38 // ToI420() that returns a frame buffer in I420 format, converting from the 39 // underlying representation if necessary. I420 is the most widely accepted 40 // format and serves as a fallback for video sinks that can only handle I420, 41 // e.g. the internal WebRTC software encoders. A special enum value 'kNative' is 42 // provided for external clients to implement their own frame buffer 43 // representations, e.g. as textures. The external client can produce such 44 // native frame buffers from custom video sources, and then cast it back to the 45 // correct subclass in custom video sinks. The purpose of this is to improve 46 // performance by providing an optimized path without intermediate conversions. 47 // Frame metadata such as rotation and timestamp are stored in 48 // VideoFrame, and not here. 49 class RTC_EXPORT VideoFrameBuffer : public RefCountInterface { 50 public: 51 class RTC_EXPORT PreparedFrameHandler : public webrtc::RefCountInterface { 52 public: 53 virtual void OnFramePrepared(size_t frame_identifier) = 0; 54 }; 55 56 // New frame buffer types will be added conservatively when there is an 57 // opportunity to optimize the path between some pair of video source and 58 // video sink. 59 // GENERATED_JAVA_ENUM_PACKAGE: org.webrtc 60 // GENERATED_JAVA_CLASS_NAME_OVERRIDE: VideoFrameBufferType 61 enum class Type { 62 kNative, 63 kI420, 64 kI420A, 65 kI422, 66 kI444, 67 kI010, 68 kI210, 69 kI410, 70 kNV12, 71 }; 72 73 // This function specifies in what pixel format the data is stored in. 74 virtual Type type() const = 0; 75 76 // The resolution of the frame in pixels. For formats where some planes are 77 // subsampled, this is the highest-resolution plane. 78 virtual int width() const = 0; 79 virtual int height() const = 0; 80 81 // Returns a memory-backed frame buffer in I420 format. If the pixel data is 82 // in another format, a conversion will take place. All implementations must 83 // provide a fallback to I420 for compatibility with e.g. the internal WebRTC 84 // software encoders. 85 // Conversion may fail, for example if reading the pixel data from a texture 86 // fails. If the conversion fails, nullptr is returned. 87 virtual scoped_refptr<I420BufferInterface> ToI420() = 0; 88 89 // GetI420() methods should return I420 buffer if conversion is trivial, i.e 90 // no change for binary data is needed. Otherwise these methods should return 91 // nullptr. One example of buffer with that property is 92 // WebrtcVideoFrameAdapter in Chrome - it's I420 buffer backed by a shared 93 // memory buffer. Therefore it must have type kNative. Yet, ToI420() 94 // doesn't affect binary data at all. Another example is any I420A buffer. 95 // TODO(https://crbug.com/webrtc/12021): Make this method non-virtual and 96 // behave as the other GetXXX methods below. 97 virtual const I420BufferInterface* GetI420() const; 98 99 // A format specific scale function. Default implementation works by 100 // converting to I420. But more efficient implementations may override it, 101 // especially for kNative. 102 // First, the image is cropped to `crop_width` and `crop_height` and then 103 // scaled to `scaled_width` and `scaled_height`. 104 virtual scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 105 int offset_y, 106 int crop_width, 107 int crop_height, 108 int scaled_width, 109 int scaled_height); 110 111 // Alias for common use case. 112 scoped_refptr<VideoFrameBuffer> Scale(int scaled_width, int scaled_height) { 113 return CropAndScale(0, 0, width(), height(), scaled_width, scaled_height); 114 } 115 116 // These functions should only be called if type() is of the correct type. 117 // Calling with a different type will result in a crash. 118 const I420ABufferInterface* GetI420A() const; 119 const I422BufferInterface* GetI422() const; 120 const I444BufferInterface* GetI444() const; 121 const I010BufferInterface* GetI010() const; 122 const I210BufferInterface* GetI210() const; 123 const I410BufferInterface* GetI410() const; 124 const NV12BufferInterface* GetNV12() const; 125 126 // From a kNative frame, returns a VideoFrameBuffer with a pixel format in 127 // the list of types that is in the main memory with a pixel perfect 128 // conversion for encoding with a software encoder. Returns nullptr if the 129 // frame type is not supported, mapping is not possible, or if the kNative 130 // frame has not implemented this method. Only callable if type() is kNative. 131 virtual scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer( 132 ArrayView<Type> types); 133 134 // For logging: returns a textual representation of the storage. 135 virtual std::string storage_representation() const; 136 137 // Informs the buffer about the maximum resolution for the upcoming 138 // `GetMappedBuffer()` calls. 139 // 140 virtual void PrepareMappedBufferAsync( 141 size_t width, 142 size_t height, 143 scoped_refptr<PreparedFrameHandler> handler, 144 size_t frame_identifier); 145 146 protected: 147 ~VideoFrameBuffer() override {} 148 }; 149 150 // Update when VideoFrameBuffer::Type is updated. 151 const char* VideoFrameBufferTypeToString(VideoFrameBuffer::Type type); 152 153 // This interface represents planar formats. 154 class PlanarYuvBuffer : public VideoFrameBuffer { 155 public: 156 virtual int ChromaWidth() const = 0; 157 virtual int ChromaHeight() const = 0; 158 159 // Returns the number of steps(in terms of Data*() return type) between 160 // successive rows for a given plane. 161 virtual int StrideY() const = 0; 162 virtual int StrideU() const = 0; 163 virtual int StrideV() const = 0; 164 165 protected: 166 ~PlanarYuvBuffer() override {} 167 }; 168 169 // This interface represents 8-bit color depth formats: Type::kI420, 170 // Type::kI420A, Type::kI422 and Type::kI444. 171 class PlanarYuv8Buffer : public PlanarYuvBuffer { 172 public: 173 // Returns pointer to the pixel data for a given plane. The memory is owned by 174 // the VideoFrameBuffer object and must not be freed by the caller. 175 virtual const uint8_t* DataY() const = 0; 176 virtual const uint8_t* DataU() const = 0; 177 virtual const uint8_t* DataV() const = 0; 178 179 protected: 180 ~PlanarYuv8Buffer() override {} 181 }; 182 183 class RTC_EXPORT I420BufferInterface : public PlanarYuv8Buffer { 184 public: 185 Type type() const override; 186 187 int ChromaWidth() const final; 188 int ChromaHeight() const final; 189 190 scoped_refptr<I420BufferInterface> ToI420() final; 191 const I420BufferInterface* GetI420() const final; 192 193 protected: 194 ~I420BufferInterface() override {} 195 }; 196 197 class RTC_EXPORT I420ABufferInterface : public I420BufferInterface { 198 public: 199 Type type() const final; 200 virtual const uint8_t* DataA() const = 0; 201 virtual int StrideA() const = 0; 202 203 protected: 204 ~I420ABufferInterface() override {} 205 }; 206 207 // Represents Type::kI422, 4:2:2 planar with 8 bits per pixel. 208 class I422BufferInterface : public PlanarYuv8Buffer { 209 public: 210 Type type() const final; 211 212 int ChromaWidth() const final; 213 int ChromaHeight() const final; 214 215 scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 216 int offset_y, 217 int crop_width, 218 int crop_height, 219 int scaled_width, 220 int scaled_height) override; 221 222 protected: 223 ~I422BufferInterface() override {} 224 }; 225 226 // Represents Type::kI444, 4:4:4 planar with 8 bits per pixel. 227 class I444BufferInterface : public PlanarYuv8Buffer { 228 public: 229 Type type() const final; 230 231 int ChromaWidth() const final; 232 int ChromaHeight() const final; 233 234 scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 235 int offset_y, 236 int crop_width, 237 int crop_height, 238 int scaled_width, 239 int scaled_height) override; 240 241 protected: 242 ~I444BufferInterface() override {} 243 }; 244 245 // This interface represents 8-bit to 16-bit color depth formats: Type::kI010, 246 // Type::kI210, or Type::kI410. 247 class PlanarYuv16BBuffer : public PlanarYuvBuffer { 248 public: 249 // Returns pointer to the pixel data for a given plane. The memory is owned by 250 // the VideoFrameBuffer object and must not be freed by the caller. 251 virtual const uint16_t* DataY() const = 0; 252 virtual const uint16_t* DataU() const = 0; 253 virtual const uint16_t* DataV() const = 0; 254 255 protected: 256 ~PlanarYuv16BBuffer() override {} 257 }; 258 259 // Represents Type::kI010, allocates 16 bits per pixel and fills 10 least 260 // significant bits with color information. 261 class I010BufferInterface : public PlanarYuv16BBuffer { 262 public: 263 Type type() const override; 264 265 int ChromaWidth() const final; 266 int ChromaHeight() const final; 267 268 protected: 269 ~I010BufferInterface() override {} 270 }; 271 272 // Represents Type::kI210, allocates 16 bits per pixel and fills 10 least 273 // significant bits with color information. 274 class I210BufferInterface : public PlanarYuv16BBuffer { 275 public: 276 Type type() const override; 277 278 int ChromaWidth() const final; 279 int ChromaHeight() const final; 280 281 protected: 282 ~I210BufferInterface() override {} 283 }; 284 285 // Represents Type::kI410, allocates 16 bits per pixel and fills 10 least 286 // significant bits with color information. 287 class I410BufferInterface : public PlanarYuv16BBuffer { 288 public: 289 Type type() const override; 290 291 int ChromaWidth() const final; 292 int ChromaHeight() const final; 293 294 protected: 295 ~I410BufferInterface() override {} 296 }; 297 298 class BiplanarYuvBuffer : public VideoFrameBuffer { 299 public: 300 virtual int ChromaWidth() const = 0; 301 virtual int ChromaHeight() const = 0; 302 303 // Returns the number of steps(in terms of Data*() return type) between 304 // successive rows for a given plane. 305 virtual int StrideY() const = 0; 306 virtual int StrideUV() const = 0; 307 308 protected: 309 ~BiplanarYuvBuffer() override {} 310 }; 311 312 class BiplanarYuv8Buffer : public BiplanarYuvBuffer { 313 public: 314 virtual const uint8_t* DataY() const = 0; 315 virtual const uint8_t* DataUV() const = 0; 316 317 protected: 318 ~BiplanarYuv8Buffer() override {} 319 }; 320 321 // Represents Type::kNV12. NV12 is full resolution Y and half-resolution 322 // interleved UV. 323 class RTC_EXPORT NV12BufferInterface : public BiplanarYuv8Buffer { 324 public: 325 Type type() const override; 326 327 int ChromaWidth() const final; 328 int ChromaHeight() const final; 329 330 scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 331 int offset_y, 332 int crop_width, 333 int crop_height, 334 int scaled_width, 335 int scaled_height) override; 336 337 protected: 338 ~NV12BufferInterface() override {} 339 }; 340 341 // RTC_CHECKs that common values used to calculate buffer sizes are within the 342 // range of [1..std::numeric_limits<int>::max()]. 343 // `width` and `height` must be > 0, `stride_y` must be >= `width` whereas 344 // `stride_u` and `stride_v` must be `> 0` as this is where the various yuv 345 // formats differ. 346 void CheckValidDimensions(int width, 347 int height, 348 int stride_y, 349 int stride_u, 350 int stride_v); 351 352 } // namespace webrtc 353 354 #endif // API_VIDEO_VIDEO_FRAME_BUFFER_H_