webrtc_libyuv.cc (20213B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "common_video/libyuv/include/webrtc_libyuv.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 16 #include "api/scoped_refptr.h" 17 #include "api/video/i420_buffer.h" 18 #include "api/video/video_frame.h" 19 #include "api/video/video_frame_buffer.h" 20 #include "common_video/include/video_frame_buffer.h" 21 #include "rtc_base/checks.h" 22 #include "third_party/libyuv/include/libyuv/compare.h" 23 #include "third_party/libyuv/include/libyuv/convert.h" 24 #include "third_party/libyuv/include/libyuv/convert_from.h" 25 #include "third_party/libyuv/include/libyuv/planar_functions.h" 26 #include "third_party/libyuv/include/libyuv/scale.h" 27 #include "third_party/libyuv/include/libyuv/video_common.h" 28 29 namespace webrtc { 30 31 size_t CalcBufferSize(VideoType type, int width, int height) { 32 RTC_DCHECK_GE(width, 0); 33 RTC_DCHECK_GE(height, 0); 34 switch (type) { 35 case VideoType::kI420: 36 case VideoType::kNV21: 37 case VideoType::kIYUV: 38 case VideoType::kYV12: 39 case VideoType::kNV12: { 40 int half_width = (width + 1) >> 1; 41 int half_height = (height + 1) >> 1; 42 return width * height + half_width * half_height * 2; 43 } 44 case VideoType::kARGB4444: 45 case VideoType::kRGB565: 46 case VideoType::kARGB1555: 47 case VideoType::kYUY2: 48 case VideoType::kUYVY: 49 return width * height * 2; 50 case VideoType::kRGB24: 51 case VideoType::kBGR24: 52 return width * height * 3; 53 case VideoType::kBGRA: 54 case VideoType::kARGB: 55 case VideoType::kABGR: 56 return width * height * 4; 57 case VideoType::kMJPEG: 58 case VideoType::kUnknown: 59 break; 60 } 61 RTC_DCHECK_NOTREACHED() << "Unexpected pixel format " << type; 62 return 0; 63 } 64 65 int ExtractBuffer(const scoped_refptr<I420BufferInterface>& input_frame, 66 size_t size, 67 uint8_t* buffer) { 68 RTC_DCHECK(buffer); 69 if (!input_frame) 70 return -1; 71 int width = input_frame->width(); 72 int height = input_frame->height(); 73 size_t length = CalcBufferSize(VideoType::kI420, width, height); 74 if (size < length) { 75 return -1; 76 } 77 78 int chroma_width = input_frame->ChromaWidth(); 79 int chroma_height = input_frame->ChromaHeight(); 80 81 libyuv::I420Copy(input_frame->DataY(), input_frame->StrideY(), 82 input_frame->DataU(), input_frame->StrideU(), 83 input_frame->DataV(), input_frame->StrideV(), buffer, width, 84 buffer + width * height, chroma_width, 85 buffer + width * height + chroma_width * chroma_height, 86 chroma_width, width, height); 87 88 return static_cast<int>(length); 89 } 90 91 int ExtractBuffer(const VideoFrame& input_frame, size_t size, uint8_t* buffer) { 92 return ExtractBuffer(input_frame.video_frame_buffer()->ToI420(), size, 93 buffer); 94 } 95 96 int ConvertVideoType(VideoType video_type) { 97 switch (video_type) { 98 case VideoType::kUnknown: 99 return libyuv::FOURCC_ANY; 100 case VideoType::kI420: 101 return libyuv::FOURCC_I420; 102 case VideoType::kIYUV: // same as VideoType::kYV12 103 case VideoType::kYV12: 104 return libyuv::FOURCC_YV12; 105 case VideoType::kRGB24: 106 return libyuv::FOURCC_24BG; 107 case VideoType::kBGR24: 108 return libyuv::FOURCC_RAW; 109 case VideoType::kABGR: 110 return libyuv::FOURCC_ABGR; 111 case VideoType::kRGB565: 112 return libyuv::FOURCC_RGBP; 113 case VideoType::kYUY2: 114 return libyuv::FOURCC_YUY2; 115 case VideoType::kUYVY: 116 return libyuv::FOURCC_UYVY; 117 case VideoType::kMJPEG: 118 return libyuv::FOURCC_MJPG; 119 case VideoType::kNV21: 120 return libyuv::FOURCC_NV21; 121 case VideoType::kARGB: 122 return libyuv::FOURCC_ARGB; 123 case VideoType::kBGRA: 124 return libyuv::FOURCC_BGRA; 125 case VideoType::kARGB4444: 126 return libyuv::FOURCC_R444; 127 case VideoType::kARGB1555: 128 return libyuv::FOURCC_RGBO; 129 case VideoType::kNV12: 130 return libyuv::FOURCC_NV12; 131 } 132 RTC_DCHECK_NOTREACHED() << "Unexpected pixel format " << video_type; 133 return libyuv::FOURCC_ANY; 134 } 135 136 int ConvertFromI420(const VideoFrame& src_frame, 137 VideoType dst_video_type, 138 int dst_sample_size, 139 uint8_t* dst_frame) { 140 scoped_refptr<I420BufferInterface> i420_buffer = 141 src_frame.video_frame_buffer()->ToI420(); 142 return libyuv::ConvertFromI420( 143 i420_buffer->DataY(), i420_buffer->StrideY(), i420_buffer->DataU(), 144 i420_buffer->StrideU(), i420_buffer->DataV(), i420_buffer->StrideV(), 145 dst_frame, dst_sample_size, src_frame.width(), src_frame.height(), 146 ConvertVideoType(dst_video_type)); 147 } 148 149 scoped_refptr<I420ABufferInterface> ScaleI420ABuffer( 150 const I420ABufferInterface& buffer, 151 int target_width, 152 int target_height) { 153 scoped_refptr<I420Buffer> yuv_buffer = 154 I420Buffer::Create(target_width, target_height); 155 yuv_buffer->ScaleFrom(buffer); 156 scoped_refptr<I420Buffer> axx_buffer = 157 I420Buffer::Create(target_width, target_height); 158 libyuv::ScalePlane(buffer.DataA(), buffer.StrideA(), buffer.width(), 159 buffer.height(), axx_buffer->MutableDataY(), 160 axx_buffer->StrideY(), target_width, target_height, 161 libyuv::kFilterBox); 162 scoped_refptr<I420ABufferInterface> merged_buffer = WrapI420ABuffer( 163 yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(), 164 yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(), 165 yuv_buffer->DataV(), yuv_buffer->StrideV(), axx_buffer->DataY(), 166 axx_buffer->StrideY(), 167 // To keep references alive. 168 [yuv_buffer, axx_buffer] {}); 169 return merged_buffer; 170 } 171 172 scoped_refptr<I420BufferInterface> ScaleVideoFrameBuffer( 173 const I420BufferInterface& source, 174 int dst_width, 175 int dst_height) { 176 scoped_refptr<I420Buffer> scaled_buffer = 177 I420Buffer::Create(dst_width, dst_height); 178 scaled_buffer->ScaleFrom(source); 179 return scaled_buffer; 180 } 181 182 double I420SSE(const I420BufferInterface& ref_buffer, 183 const I420BufferInterface& test_buffer) { 184 RTC_DCHECK_EQ(ref_buffer.width(), test_buffer.width()); 185 RTC_DCHECK_EQ(ref_buffer.height(), test_buffer.height()); 186 const uint64_t width = test_buffer.width(); 187 const uint64_t height = test_buffer.height(); 188 const uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( 189 ref_buffer.DataY(), ref_buffer.StrideY(), test_buffer.DataY(), 190 test_buffer.StrideY(), width, height); 191 const int width_uv = (width + 1) >> 1; 192 const int height_uv = (height + 1) >> 1; 193 const uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( 194 ref_buffer.DataU(), ref_buffer.StrideU(), test_buffer.DataU(), 195 test_buffer.StrideU(), width_uv, height_uv); 196 const uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( 197 ref_buffer.DataV(), ref_buffer.StrideV(), test_buffer.DataV(), 198 test_buffer.StrideV(), width_uv, height_uv); 199 const double samples = width * height + 2 * (width_uv * height_uv); 200 const double sse = sse_y + sse_u + sse_v; 201 return sse / (samples * 255.0 * 255.0); 202 } 203 204 // Compute PSNR for an I420A frame (all planes). Can upscale test frame. 205 double I420APSNR(const I420ABufferInterface& ref_buffer, 206 const I420ABufferInterface& test_buffer) { 207 RTC_DCHECK_GE(ref_buffer.width(), test_buffer.width()); 208 RTC_DCHECK_GE(ref_buffer.height(), test_buffer.height()); 209 if ((ref_buffer.width() != test_buffer.width()) || 210 (ref_buffer.height() != test_buffer.height())) { 211 scoped_refptr<I420ABufferInterface> scaled_buffer = 212 ScaleI420ABuffer(test_buffer, ref_buffer.width(), ref_buffer.height()); 213 return I420APSNR(ref_buffer, *scaled_buffer); 214 } 215 const int width = test_buffer.width(); 216 const int height = test_buffer.height(); 217 const uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( 218 ref_buffer.DataY(), ref_buffer.StrideY(), test_buffer.DataY(), 219 test_buffer.StrideY(), width, height); 220 const int width_uv = (width + 1) >> 1; 221 const int height_uv = (height + 1) >> 1; 222 const uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( 223 ref_buffer.DataU(), ref_buffer.StrideU(), test_buffer.DataU(), 224 test_buffer.StrideU(), width_uv, height_uv); 225 const uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( 226 ref_buffer.DataV(), ref_buffer.StrideV(), test_buffer.DataV(), 227 test_buffer.StrideV(), width_uv, height_uv); 228 const uint64_t sse_a = libyuv::ComputeSumSquareErrorPlane( 229 ref_buffer.DataA(), ref_buffer.StrideA(), test_buffer.DataA(), 230 test_buffer.StrideA(), width, height); 231 const uint64_t samples = 2 * (uint64_t)width * (uint64_t)height + 232 2 * ((uint64_t)width_uv * (uint64_t)height_uv); 233 const uint64_t sse = sse_y + sse_u + sse_v + sse_a; 234 const double psnr = libyuv::SumSquareErrorToPsnr(sse, samples); 235 return (psnr > kPerfectPSNR) ? kPerfectPSNR : psnr; 236 } 237 238 // Compute PSNR for an I420A frame (all planes) 239 double I420APSNR(const VideoFrame* ref_frame, const VideoFrame* test_frame) { 240 if (!ref_frame || !test_frame) 241 return -1; 242 RTC_DCHECK(ref_frame->video_frame_buffer()->type() == 243 VideoFrameBuffer::Type::kI420A); 244 RTC_DCHECK(test_frame->video_frame_buffer()->type() == 245 VideoFrameBuffer::Type::kI420A); 246 return I420APSNR(*ref_frame->video_frame_buffer()->GetI420A(), 247 *test_frame->video_frame_buffer()->GetI420A()); 248 } 249 250 // Compute PSNR for an I420 frame (all planes). Can upscale test frame. 251 double I420PSNR(const I420BufferInterface& ref_buffer, 252 const I420BufferInterface& test_buffer) { 253 RTC_DCHECK_GE(ref_buffer.width(), test_buffer.width()); 254 RTC_DCHECK_GE(ref_buffer.height(), test_buffer.height()); 255 if ((ref_buffer.width() != test_buffer.width()) || 256 (ref_buffer.height() != test_buffer.height())) { 257 scoped_refptr<I420Buffer> scaled_buffer = 258 I420Buffer::Create(ref_buffer.width(), ref_buffer.height()); 259 scaled_buffer->ScaleFrom(test_buffer); 260 return I420PSNR(ref_buffer, *scaled_buffer); 261 } 262 double psnr = libyuv::I420Psnr( 263 ref_buffer.DataY(), ref_buffer.StrideY(), ref_buffer.DataU(), 264 ref_buffer.StrideU(), ref_buffer.DataV(), ref_buffer.StrideV(), 265 test_buffer.DataY(), test_buffer.StrideY(), test_buffer.DataU(), 266 test_buffer.StrideU(), test_buffer.DataV(), test_buffer.StrideV(), 267 test_buffer.width(), test_buffer.height()); 268 // LibYuv sets the max psnr value to 128, we restrict it here. 269 // In case of 0 mse in one frame, 128 can skew the results significantly. 270 return (psnr > kPerfectPSNR) ? kPerfectPSNR : psnr; 271 } 272 273 // Compute PSNR for an I420 frame (all planes) 274 double I420PSNR(const VideoFrame* ref_frame, const VideoFrame* test_frame) { 275 if (!ref_frame || !test_frame) 276 return -1; 277 return I420PSNR(*ref_frame->video_frame_buffer()->ToI420(), 278 *test_frame->video_frame_buffer()->ToI420()); 279 } 280 281 double I420WeightedPSNR(const I420BufferInterface& ref_buffer, 282 const I420BufferInterface& test_buffer) { 283 RTC_DCHECK_GE(ref_buffer.width(), test_buffer.width()); 284 RTC_DCHECK_GE(ref_buffer.height(), test_buffer.height()); 285 if ((ref_buffer.width() != test_buffer.width()) || 286 (ref_buffer.height() != test_buffer.height())) { 287 scoped_refptr<I420Buffer> scaled_ref_buffer = 288 I420Buffer::Create(test_buffer.width(), test_buffer.height()); 289 scaled_ref_buffer->ScaleFrom(ref_buffer); 290 return I420WeightedPSNR(*scaled_ref_buffer, test_buffer); 291 } 292 293 // Luma. 294 int width_y = test_buffer.width(); 295 int height_y = test_buffer.height(); 296 uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( 297 ref_buffer.DataY(), ref_buffer.StrideY(), test_buffer.DataY(), 298 test_buffer.StrideY(), width_y, height_y); 299 uint64_t num_samples_y = (uint64_t)width_y * (uint64_t)height_y; 300 double psnr_y = libyuv::SumSquareErrorToPsnr(sse_y, num_samples_y); 301 302 // Chroma. 303 int width_uv = (width_y + 1) >> 1; 304 int height_uv = (height_y + 1) >> 1; 305 uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( 306 ref_buffer.DataU(), ref_buffer.StrideU(), test_buffer.DataU(), 307 test_buffer.StrideU(), width_uv, height_uv); 308 uint64_t num_samples_uv = (uint64_t)width_uv * (uint64_t)height_uv; 309 double psnr_u = libyuv::SumSquareErrorToPsnr(sse_u, num_samples_uv); 310 uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( 311 ref_buffer.DataV(), ref_buffer.StrideV(), test_buffer.DataV(), 312 test_buffer.StrideV(), width_uv, height_uv); 313 double psnr_v = libyuv::SumSquareErrorToPsnr(sse_v, num_samples_uv); 314 315 // Weights from Ohm et. al 2012. 316 double psnr_yuv = (6.0 * psnr_y + psnr_u + psnr_v) / 8.0; 317 return (psnr_yuv > kPerfectPSNR) ? kPerfectPSNR : psnr_yuv; 318 } 319 320 // Compute SSIM for an I420A frame (all planes). Can upscale test frame. 321 double I420ASSIM(const I420ABufferInterface& ref_buffer, 322 const I420ABufferInterface& test_buffer) { 323 RTC_DCHECK_GE(ref_buffer.width(), test_buffer.width()); 324 RTC_DCHECK_GE(ref_buffer.height(), test_buffer.height()); 325 if ((ref_buffer.width() != test_buffer.width()) || 326 (ref_buffer.height() != test_buffer.height())) { 327 scoped_refptr<I420ABufferInterface> scaled_buffer = 328 ScaleI420ABuffer(test_buffer, ref_buffer.width(), ref_buffer.height()); 329 return I420ASSIM(ref_buffer, *scaled_buffer); 330 } 331 const double yuv_ssim = libyuv::I420Ssim( 332 ref_buffer.DataY(), ref_buffer.StrideY(), ref_buffer.DataU(), 333 ref_buffer.StrideU(), ref_buffer.DataV(), ref_buffer.StrideV(), 334 test_buffer.DataY(), test_buffer.StrideY(), test_buffer.DataU(), 335 test_buffer.StrideU(), test_buffer.DataV(), test_buffer.StrideV(), 336 test_buffer.width(), test_buffer.height()); 337 const double a_ssim = libyuv::CalcFrameSsim( 338 ref_buffer.DataA(), ref_buffer.StrideA(), test_buffer.DataA(), 339 test_buffer.StrideA(), test_buffer.width(), test_buffer.height()); 340 return (yuv_ssim + (a_ssim * 0.8)) / 1.8; 341 } 342 343 // Compute SSIM for an I420A frame (all planes) 344 double I420ASSIM(const VideoFrame* ref_frame, const VideoFrame* test_frame) { 345 if (!ref_frame || !test_frame) 346 return -1; 347 RTC_DCHECK(ref_frame->video_frame_buffer()->type() == 348 VideoFrameBuffer::Type::kI420A); 349 RTC_DCHECK(test_frame->video_frame_buffer()->type() == 350 VideoFrameBuffer::Type::kI420A); 351 return I420ASSIM(*ref_frame->video_frame_buffer()->GetI420A(), 352 *test_frame->video_frame_buffer()->GetI420A()); 353 } 354 355 // Compute SSIM for an I420 frame (all planes). Can upscale test_buffer. 356 double I420SSIM(const I420BufferInterface& ref_buffer, 357 const I420BufferInterface& test_buffer) { 358 RTC_DCHECK_GE(ref_buffer.width(), test_buffer.width()); 359 RTC_DCHECK_GE(ref_buffer.height(), test_buffer.height()); 360 if ((ref_buffer.width() != test_buffer.width()) || 361 (ref_buffer.height() != test_buffer.height())) { 362 scoped_refptr<I420Buffer> scaled_buffer = 363 I420Buffer::Create(ref_buffer.width(), ref_buffer.height()); 364 scaled_buffer->ScaleFrom(test_buffer); 365 return I420SSIM(ref_buffer, *scaled_buffer); 366 } 367 return libyuv::I420Ssim( 368 ref_buffer.DataY(), ref_buffer.StrideY(), ref_buffer.DataU(), 369 ref_buffer.StrideU(), ref_buffer.DataV(), ref_buffer.StrideV(), 370 test_buffer.DataY(), test_buffer.StrideY(), test_buffer.DataU(), 371 test_buffer.StrideU(), test_buffer.DataV(), test_buffer.StrideV(), 372 test_buffer.width(), test_buffer.height()); 373 } 374 375 double I420SSIM(const VideoFrame* ref_frame, const VideoFrame* test_frame) { 376 if (!ref_frame || !test_frame) 377 return -1; 378 return I420SSIM(*ref_frame->video_frame_buffer()->ToI420(), 379 *test_frame->video_frame_buffer()->ToI420()); 380 } 381 382 void NV12Scale(uint8_t* tmp_buffer, 383 const uint8_t* src_y, 384 int src_stride_y, 385 const uint8_t* src_uv, 386 int src_stride_uv, 387 int src_width, 388 int src_height, 389 uint8_t* dst_y, 390 int dst_stride_y, 391 uint8_t* dst_uv, 392 int dst_stride_uv, 393 int dst_width, 394 int dst_height) { 395 const int src_chroma_width = (src_width + 1) / 2; 396 const int src_chroma_height = (src_height + 1) / 2; 397 398 if (src_width == dst_width && src_height == dst_height) { 399 // No scaling. 400 libyuv::CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, src_width, 401 src_height); 402 libyuv::CopyPlane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, 403 src_chroma_width * 2, src_chroma_height); 404 return; 405 } 406 407 // Scaling. 408 // Allocate temporary memory for spitting UV planes and scaling them. 409 const int dst_chroma_width = (dst_width + 1) / 2; 410 const int dst_chroma_height = (dst_height + 1) / 2; 411 412 uint8_t* const src_u = tmp_buffer; 413 uint8_t* const src_v = src_u + src_chroma_width * src_chroma_height; 414 uint8_t* const dst_u = src_v + src_chroma_width * src_chroma_height; 415 uint8_t* const dst_v = dst_u + dst_chroma_width * dst_chroma_height; 416 417 // Split source UV plane into separate U and V plane using the temporary data. 418 libyuv::SplitUVPlane(src_uv, src_stride_uv, src_u, src_chroma_width, src_v, 419 src_chroma_width, src_chroma_width, src_chroma_height); 420 421 // Scale the planes. 422 libyuv::I420Scale( 423 src_y, src_stride_y, src_u, src_chroma_width, src_v, src_chroma_width, 424 src_width, src_height, dst_y, dst_stride_y, dst_u, dst_chroma_width, 425 dst_v, dst_chroma_width, dst_width, dst_height, libyuv::kFilterBox); 426 427 // Merge the UV planes into the destination. 428 libyuv::MergeUVPlane(dst_u, dst_chroma_width, dst_v, dst_chroma_width, dst_uv, 429 dst_stride_uv, dst_chroma_width, dst_chroma_height); 430 } 431 432 NV12ToI420Scaler::NV12ToI420Scaler() = default; 433 NV12ToI420Scaler::~NV12ToI420Scaler() = default; 434 435 void NV12ToI420Scaler::NV12ToI420Scale(const uint8_t* src_y, 436 int src_stride_y, 437 const uint8_t* src_uv, 438 int src_stride_uv, 439 int src_width, 440 int src_height, 441 uint8_t* dst_y, 442 int dst_stride_y, 443 uint8_t* dst_u, 444 int dst_stride_u, 445 uint8_t* dst_v, 446 int dst_stride_v, 447 int dst_width, 448 int dst_height) { 449 if (src_width == dst_width && src_height == dst_height) { 450 // No scaling. 451 tmp_uv_planes_.clear(); 452 tmp_uv_planes_.shrink_to_fit(); 453 libyuv::NV12ToI420(src_y, src_stride_y, src_uv, src_stride_uv, dst_y, 454 dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v, 455 src_width, src_height); 456 return; 457 } 458 459 // Scaling. 460 // Allocate temporary memory for spitting UV planes. 461 const int src_uv_width = (src_width + 1) / 2; 462 const int src_uv_height = (src_height + 1) / 2; 463 tmp_uv_planes_.resize(src_uv_width * src_uv_height * 2); 464 tmp_uv_planes_.shrink_to_fit(); 465 466 // Split source UV plane into separate U and V plane using the temporary data. 467 uint8_t* const src_u = tmp_uv_planes_.data(); 468 uint8_t* const src_v = tmp_uv_planes_.data() + src_uv_width * src_uv_height; 469 libyuv::SplitUVPlane(src_uv, src_stride_uv, src_u, src_uv_width, src_v, 470 src_uv_width, src_uv_width, src_uv_height); 471 472 // Scale the planes into the destination. 473 libyuv::I420Scale(src_y, src_stride_y, src_u, src_uv_width, src_v, 474 src_uv_width, src_width, src_height, dst_y, dst_stride_y, 475 dst_u, dst_stride_u, dst_v, dst_stride_v, dst_width, 476 dst_height, libyuv::kFilterBox); 477 } 478 479 } // namespace webrtc