libvpx_vp9_decoder.cc (14780B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 * 10 */ 11 12 #ifdef RTC_ENABLE_VP9 13 14 #include "modules/video_coding/codecs/vp9/libvpx_vp9_decoder.h" 15 16 #include <algorithm> 17 #include <cstdint> 18 #include <cstring> 19 #include <optional> 20 21 #include "api/array_view.h" 22 #include "api/scoped_refptr.h" 23 #include "api/video/color_space.h" 24 #include "api/video/encoded_image.h" 25 #include "api/video/render_resolution.h" 26 #include "api/video/video_frame.h" 27 #include "api/video/video_frame_buffer.h" 28 #include "api/video/video_frame_type.h" 29 #include "api/video_codecs/video_decoder.h" 30 #include "common_video/include/video_frame_buffer.h" 31 #include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" 32 #include "modules/video_coding/include/video_error_codes.h" 33 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" 34 #include "rtc_base/checks.h" 35 #include "rtc_base/logging.h" 36 #include "third_party/libvpx/source/libvpx/vpx/vp8dx.h" 37 #include "third_party/libvpx/source/libvpx/vpx/vpx_decoder.h" 38 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h" 39 #include "third_party/libvpx/source/libvpx/vpx/vpx_image.h" 40 41 namespace webrtc { 42 namespace { 43 44 // Helper class for extracting VP9 colorspace. 45 ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t, 46 vpx_color_range_t range_t, 47 unsigned int bit_depth) { 48 ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kUnspecified; 49 ColorSpace::TransferID transfer = ColorSpace::TransferID::kUnspecified; 50 ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kUnspecified; 51 switch (space_t) { 52 case VPX_CS_BT_601: 53 case VPX_CS_SMPTE_170: 54 primaries = ColorSpace::PrimaryID::kSMPTE170M; 55 transfer = ColorSpace::TransferID::kSMPTE170M; 56 matrix = ColorSpace::MatrixID::kSMPTE170M; 57 break; 58 case VPX_CS_SMPTE_240: 59 primaries = ColorSpace::PrimaryID::kSMPTE240M; 60 transfer = ColorSpace::TransferID::kSMPTE240M; 61 matrix = ColorSpace::MatrixID::kSMPTE240M; 62 break; 63 case VPX_CS_BT_709: 64 primaries = ColorSpace::PrimaryID::kBT709; 65 transfer = ColorSpace::TransferID::kBT709; 66 matrix = ColorSpace::MatrixID::kBT709; 67 break; 68 case VPX_CS_BT_2020: 69 primaries = ColorSpace::PrimaryID::kBT2020; 70 switch (bit_depth) { 71 case 8: 72 transfer = ColorSpace::TransferID::kBT709; 73 break; 74 case 10: 75 transfer = ColorSpace::TransferID::kBT2020_10; 76 break; 77 default: 78 RTC_DCHECK_NOTREACHED(); 79 break; 80 } 81 matrix = ColorSpace::MatrixID::kBT2020_NCL; 82 break; 83 case VPX_CS_SRGB: 84 primaries = ColorSpace::PrimaryID::kBT709; 85 transfer = ColorSpace::TransferID::kIEC61966_2_1; 86 matrix = ColorSpace::MatrixID::kBT709; 87 break; 88 default: 89 break; 90 } 91 92 ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid; 93 switch (range_t) { 94 case VPX_CR_STUDIO_RANGE: 95 range = ColorSpace::RangeID::kLimited; 96 break; 97 case VPX_CR_FULL_RANGE: 98 range = ColorSpace::RangeID::kFull; 99 break; 100 default: 101 break; 102 } 103 return ColorSpace(primaries, transfer, matrix, range); 104 } 105 106 } // namespace 107 108 LibvpxVp9Decoder::LibvpxVp9Decoder() 109 : decode_complete_callback_(nullptr), 110 inited_(false), 111 decoder_(nullptr), 112 key_frame_required_(true) {} 113 114 LibvpxVp9Decoder::~LibvpxVp9Decoder() { 115 inited_ = true; // in order to do the actual release 116 Release(); 117 int num_buffers_in_use = libvpx_buffer_pool_.GetNumBuffersInUse(); 118 if (num_buffers_in_use > 0) { 119 // The frame buffers are reference counted and frames are exposed after 120 // decoding. There may be valid usage cases where previous frames are still 121 // referenced after ~LibvpxVp9Decoder that is not a leak. 122 RTC_LOG(LS_INFO) << num_buffers_in_use 123 << " Vp9FrameBuffers are still " 124 "referenced during ~LibvpxVp9Decoder."; 125 } 126 } 127 128 bool LibvpxVp9Decoder::Configure(const Settings& settings) { 129 if (Release() < 0) { 130 return false; 131 } 132 133 if (decoder_ == nullptr) { 134 decoder_ = new vpx_codec_ctx_t; 135 memset(decoder_, 0, sizeof(*decoder_)); 136 } 137 vpx_codec_dec_cfg_t cfg; 138 memset(&cfg, 0, sizeof(cfg)); 139 140 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 141 // We focus on webrtc fuzzing here, not libvpx itself. Use single thread for 142 // fuzzing, because: 143 // - libvpx's VP9 single thread decoder is more fuzzer friendly. It detects 144 // errors earlier than the multi-threads version. 145 // - Make peak CPU usage under control (not depending on input) 146 cfg.threads = 1; 147 #else 148 const RenderResolution& resolution = settings.max_render_resolution(); 149 if (!resolution.Valid()) { 150 // Postpone configuring number of threads until resolution is known. 151 cfg.threads = 1; 152 } else { 153 // We want to use multithreading when decoding high resolution videos. But 154 // not too many in order to avoid overhead when many stream are decoded 155 // concurrently. 156 // Set 2 thread as target for 1280x720 pixel count, and then scale up 157 // linearly from there - but cap at physical core count. 158 // For common resolutions this results in: 159 // 1 for 360p 160 // 2 for 720p 161 // 4 for 1080p 162 // 8 for 1440p 163 // 18 for 4K 164 int num_threads = std::max( 165 1, 2 * resolution.Width() * resolution.Height() / (1280 * 720)); 166 cfg.threads = std::min(settings.number_of_cores(), num_threads); 167 } 168 #endif 169 170 current_settings_ = settings; 171 172 vpx_codec_flags_t flags = 0; 173 if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) { 174 return false; 175 } 176 177 if (!libvpx_buffer_pool_.InitializeVpxUsePool(decoder_)) { 178 return false; 179 } 180 181 inited_ = true; 182 // Always start with a complete key frame. 183 key_frame_required_ = true; 184 if (std::optional<int> buffer_pool_size = settings.buffer_pool_size()) { 185 if (!libvpx_buffer_pool_.Resize(*buffer_pool_size)) { 186 return false; 187 } 188 } 189 190 vpx_codec_err_t status = 191 vpx_codec_control(decoder_, VP9D_SET_LOOP_FILTER_OPT, 1); 192 if (status != VPX_CODEC_OK) { 193 RTC_LOG(LS_ERROR) << "Failed to enable VP9D_SET_LOOP_FILTER_OPT. " 194 << vpx_codec_error(decoder_); 195 return false; 196 } 197 198 return true; 199 } 200 201 int LibvpxVp9Decoder::Decode(const EncodedImage& input_image, 202 int64_t /*render_time_ms*/) { 203 if (!inited_) { 204 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 205 } 206 if (decode_complete_callback_ == nullptr) { 207 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 208 } 209 210 if (input_image._frameType == VideoFrameType::kVideoFrameKey) { 211 std::optional<Vp9UncompressedHeader> frame_info = 212 ParseUncompressedVp9Header( 213 MakeArrayView(input_image.data(), input_image.size())); 214 if (frame_info) { 215 RenderResolution frame_resolution(frame_info->frame_width, 216 frame_info->frame_height); 217 if (frame_resolution != current_settings_.max_render_resolution()) { 218 // Resolution has changed, tear down and re-init a new decoder in 219 // order to get correct sizing. 220 Release(); 221 current_settings_.set_max_render_resolution(frame_resolution); 222 if (!Configure(current_settings_)) { 223 RTC_LOG(LS_WARNING) << "Failed to re-init decoder."; 224 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 225 } 226 } 227 } else { 228 RTC_LOG(LS_WARNING) << "Failed to parse VP9 header from key-frame."; 229 } 230 } 231 232 // Always start with a complete key frame. 233 if (key_frame_required_) { 234 if (input_image._frameType != VideoFrameType::kVideoFrameKey) 235 return WEBRTC_VIDEO_CODEC_ERROR; 236 key_frame_required_ = false; 237 } 238 vpx_codec_iter_t iter = nullptr; 239 vpx_image_t* img; 240 const uint8_t* buffer = input_image.data(); 241 if (input_image.size() == 0) { 242 buffer = nullptr; // Triggers full frame concealment. 243 } 244 // During decode libvpx may get and release buffers from 245 // `libvpx_buffer_pool_`. In practice libvpx keeps a few (~3-4) buffers alive 246 // at a time. 247 if (vpx_codec_decode(decoder_, buffer, 248 static_cast<unsigned int>(input_image.size()), nullptr, 249 VPX_DL_REALTIME)) { 250 return WEBRTC_VIDEO_CODEC_ERROR; 251 } 252 // `img->fb_priv` contains the image data, a reference counted Vp9FrameBuffer. 253 // It may be released by libvpx during future vpx_codec_decode or 254 // vpx_codec_destroy calls. 255 img = vpx_codec_get_frame(decoder_, &iter); 256 int qp; 257 vpx_codec_err_t vpx_ret = 258 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp); 259 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK); 260 int ret = ReturnFrame(img, input_image.RtpTimestamp(), qp, 261 input_image.ColorSpace()); 262 if (ret != 0) { 263 return ret; 264 } 265 return WEBRTC_VIDEO_CODEC_OK; 266 } 267 268 int LibvpxVp9Decoder::ReturnFrame(const vpx_image_t* img, 269 uint32_t timestamp, 270 int qp, 271 const ColorSpace* explicit_color_space) { 272 if (img == nullptr) { 273 // Decoder OK and nullptr image => No show frame. 274 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; 275 } 276 277 // This buffer contains all of `img`'s image data, a reference counted 278 // Vp9FrameBuffer. (libvpx is done with the buffers after a few 279 // vpx_codec_decode calls or vpx_codec_destroy). 280 scoped_refptr<Vp9FrameBufferPool::Vp9FrameBuffer> img_buffer( 281 static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv)); 282 283 // The buffer can be used directly by the VideoFrame (without copy) by 284 // using a Wrapped*Buffer. 285 scoped_refptr<VideoFrameBuffer> img_wrapped_buffer; 286 switch (img->fmt) { 287 case VPX_IMG_FMT_I420: 288 img_wrapped_buffer = WrapI420Buffer( 289 img->d_w, img->d_h, img->planes[VPX_PLANE_Y], 290 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U], 291 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V], 292 img->stride[VPX_PLANE_V], 293 // WrappedI420Buffer's mechanism for allowing the release of its 294 // frame buffer is through a callback function. This is where we 295 // should release `img_buffer`. 296 [img_buffer] {}); 297 break; 298 case VPX_IMG_FMT_I422: 299 img_wrapped_buffer = WrapI422Buffer( 300 img->d_w, img->d_h, img->planes[VPX_PLANE_Y], 301 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U], 302 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V], 303 img->stride[VPX_PLANE_V], 304 // WrappedI444Buffer's mechanism for allowing the release of its 305 // frame buffer is through a callback function. This is where we 306 // should release `img_buffer`. 307 [img_buffer] {}); 308 break; 309 case VPX_IMG_FMT_I444: 310 img_wrapped_buffer = WrapI444Buffer( 311 img->d_w, img->d_h, img->planes[VPX_PLANE_Y], 312 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U], 313 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V], 314 img->stride[VPX_PLANE_V], 315 // WrappedI444Buffer's mechanism for allowing the release of its 316 // frame buffer is through a callback function. This is where we 317 // should release `img_buffer`. 318 [img_buffer] {}); 319 break; 320 case VPX_IMG_FMT_I42016: 321 img_wrapped_buffer = WrapI010Buffer( 322 img->d_w, img->d_h, 323 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]), 324 img->stride[VPX_PLANE_Y] / 2, 325 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]), 326 img->stride[VPX_PLANE_U] / 2, 327 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]), 328 img->stride[VPX_PLANE_V] / 2, [img_buffer] {}); 329 break; 330 case VPX_IMG_FMT_I42216: 331 img_wrapped_buffer = WrapI210Buffer( 332 img->d_w, img->d_h, 333 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]), 334 img->stride[VPX_PLANE_Y] / 2, 335 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]), 336 img->stride[VPX_PLANE_U] / 2, 337 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]), 338 img->stride[VPX_PLANE_V] / 2, [img_buffer] {}); 339 break; 340 case VPX_IMG_FMT_I44416: 341 img_wrapped_buffer = WrapI410Buffer( 342 img->d_w, img->d_h, 343 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]), 344 img->stride[VPX_PLANE_Y] / 2, 345 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]), 346 img->stride[VPX_PLANE_U] / 2, 347 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]), 348 img->stride[VPX_PLANE_V] / 2, [img_buffer] {}); 349 break; 350 default: 351 RTC_LOG(LS_ERROR) << "Unsupported pixel format produced by the decoder: " 352 << static_cast<int>(img->fmt); 353 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; 354 } 355 356 auto builder = VideoFrame::Builder() 357 .set_video_frame_buffer(img_wrapped_buffer) 358 .set_rtp_timestamp(timestamp); 359 if (explicit_color_space) { 360 builder.set_color_space(*explicit_color_space); 361 } else { 362 builder.set_color_space( 363 ExtractVP9ColorSpace(img->cs, img->range, img->bit_depth)); 364 } 365 VideoFrame decoded_image = builder.build(); 366 367 decode_complete_callback_->Decoded(decoded_image, std::nullopt, qp); 368 return WEBRTC_VIDEO_CODEC_OK; 369 } 370 371 int LibvpxVp9Decoder::RegisterDecodeCompleteCallback( 372 DecodedImageCallback* callback) { 373 decode_complete_callback_ = callback; 374 return WEBRTC_VIDEO_CODEC_OK; 375 } 376 377 int LibvpxVp9Decoder::Release() { 378 int ret_val = WEBRTC_VIDEO_CODEC_OK; 379 380 if (decoder_ != nullptr) { 381 if (inited_) { 382 // When a codec is destroyed libvpx will release any buffers of 383 // `libvpx_buffer_pool_` it is currently using. 384 if (vpx_codec_destroy(decoder_)) { 385 ret_val = WEBRTC_VIDEO_CODEC_MEMORY; 386 } 387 } 388 delete decoder_; 389 decoder_ = nullptr; 390 } 391 // Releases buffers from the pool. Any buffers not in use are deleted. Buffers 392 // still referenced externally are deleted once fully released, not returning 393 // to the pool. 394 libvpx_buffer_pool_.ClearPool(); 395 inited_ = false; 396 return ret_val; 397 } 398 399 VideoDecoder::DecoderInfo LibvpxVp9Decoder::GetDecoderInfo() const { 400 DecoderInfo info; 401 info.implementation_name = "libvpx"; 402 info.is_hardware_accelerated = false; 403 return info; 404 } 405 406 const char* LibvpxVp9Decoder::ImplementationName() const { 407 return "libvpx"; 408 } 409 410 } // namespace webrtc 411 412 #endif // RTC_ENABLE_VP9