videocodec_test_libvpx.cc (18262B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <cstddef> 12 #include <cstdio> 13 #include <map> 14 #include <memory> 15 #include <utility> 16 #include <vector> 17 18 #include "api/environment/environment.h" 19 #include "api/test/create_videocodec_test_fixture.h" 20 #include "api/test/video/function_video_encoder_factory.h" 21 #include "api/test/videocodec_test_fixture.h" 22 #include "api/test/videocodec_test_stats.h" 23 #include "api/video/encoded_image.h" 24 #include "api/video/video_codec_type.h" 25 #include "api/video_codecs/sdp_video_format.h" 26 #include "media/base/media_constants.h" 27 #include "media/engine/internal_decoder_factory.h" 28 #include "media/engine/internal_encoder_factory.h" 29 #include "media/engine/simulcast_encoder_adapter.h" 30 #include "modules/video_coding/utility/vp8_header_parser.h" 31 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" 32 #include "rtc_base/checks.h" 33 #include "test/gtest.h" 34 #include "test/testsupport/file_utils.h" 35 36 namespace webrtc { 37 namespace test { 38 39 using VideoStatistics = VideoCodecTestStats::VideoStatistics; 40 41 namespace { 42 // Codec settings. 43 const int kCifWidth = 352; 44 const int kCifHeight = 288; 45 const int kNumFramesShort = 100; 46 const int kNumFramesLong = 300; 47 const size_t kBitrateRdPerfKbps[] = {100, 200, 300, 400, 500, 600, 48 700, 800, 1000, 1250, 1400, 1600, 49 1800, 2000, 2200, 2500}; 50 const size_t kNumFirstFramesToSkipAtRdPerfAnalysis = 60; 51 52 class QpFrameChecker : public VideoCodecTestFixture::EncodedFrameChecker { 53 public: 54 void CheckEncodedFrame(VideoCodecType codec, 55 const EncodedImage& encoded_frame) const override { 56 int qp; 57 if (codec == kVideoCodecVP8) { 58 EXPECT_TRUE(vp8::GetQp(encoded_frame.data(), encoded_frame.size(), &qp)); 59 } else if (codec == kVideoCodecVP9) { 60 EXPECT_TRUE(vp9::GetQp(encoded_frame.data(), encoded_frame.size(), &qp)); 61 } else { 62 RTC_DCHECK_NOTREACHED(); 63 } 64 EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP."; 65 } 66 }; 67 68 VideoCodecTestFixture::Config CreateConfig() { 69 VideoCodecTestFixture::Config config; 70 config.filename = "foreman_cif"; 71 config.filepath = ResourcePath(config.filename, "yuv"); 72 config.num_frames = kNumFramesLong; 73 config.use_single_core = true; 74 return config; 75 } 76 77 void PrintRdPerf(std::map<size_t, std::vector<VideoStatistics>> rd_stats) { 78 printf("--> Summary\n"); 79 printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n", 80 "uplink_kbps", "width", "height", "spatial_idx", "temporal_idx", 81 "target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y", 82 "psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps"); 83 for (const auto& rd_stat : rd_stats) { 84 const size_t bitrate_kbps = rd_stat.first; 85 for (const auto& layer_stat : rd_stat.second) { 86 printf( 87 "%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f " 88 "%7.2f" 89 "%13.2f %13.2f\n", 90 bitrate_kbps, layer_stat.width, layer_stat.height, 91 layer_stat.spatial_idx, layer_stat.temporal_idx, 92 layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps, 93 layer_stat.framerate_fps, layer_stat.avg_psnr, layer_stat.avg_psnr_y, 94 layer_stat.avg_psnr_u, layer_stat.avg_psnr_v, 95 layer_stat.enc_speed_fps, layer_stat.dec_speed_fps); 96 } 97 } 98 } 99 } // namespace 100 101 #if defined(RTC_ENABLE_VP9) 102 TEST(VideoCodecTestLibvpx, HighBitrateVP9) { 103 auto config = CreateConfig(); 104 config.SetCodecSettings(kVp9CodecName, 1, 1, 1, false, true, false, kCifWidth, 105 kCifHeight); 106 config.num_frames = kNumFramesShort; 107 const auto frame_checker = std::make_unique<QpFrameChecker>(); 108 config.encoded_frame_checker = frame_checker.get(); 109 auto fixture = CreateVideoCodecTestFixture(config); 110 111 std::vector<RateProfile> rate_profiles = {{500, 30, 0}}; 112 113 std::vector<RateControlThresholds> rc_thresholds = { 114 {5, 1, 0, 1, 0.3, 0.1, 0, 1}}; 115 116 std::vector<QualityThresholds> quality_thresholds = {{37, 36, 0.94, 0.92}}; 117 118 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 119 } 120 121 TEST(VideoCodecTestLibvpx, ChangeBitrateVP9) { 122 auto config = CreateConfig(); 123 config.SetCodecSettings(kVp9CodecName, 1, 1, 1, false, true, false, kCifWidth, 124 kCifHeight); 125 const auto frame_checker = std::make_unique<QpFrameChecker>(); 126 config.encoded_frame_checker = frame_checker.get(); 127 auto fixture = CreateVideoCodecTestFixture(config); 128 129 std::vector<RateProfile> rate_profiles = { 130 {200, 30, 0}, // target_kbps, input_fps, frame_num 131 {700, 30, 100}, 132 {500, 30, 200}}; 133 134 std::vector<RateControlThresholds> rc_thresholds = { 135 {5, 2, 0, 1, 0.5, 0.1, 0, 1}, 136 {15, 3, 0, 1, 0.5, 0.1, 0, 0}, 137 {11, 2, 0, 1, 0.5, 0.1, 0, 0}}; 138 139 std::vector<QualityThresholds> quality_thresholds = { 140 {34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}}; 141 142 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 143 } 144 145 TEST(VideoCodecTestLibvpx, ChangeFramerateVP9) { 146 auto config = CreateConfig(); 147 config.SetCodecSettings(kVp9CodecName, 1, 1, 1, false, true, false, kCifWidth, 148 kCifHeight); 149 const auto frame_checker = std::make_unique<QpFrameChecker>(); 150 config.encoded_frame_checker = frame_checker.get(); 151 auto fixture = CreateVideoCodecTestFixture(config); 152 153 std::vector<RateProfile> rate_profiles = { 154 {100, 24, 0}, // target_kbps, input_fps, frame_num 155 {100, 15, 100}, 156 {100, 10, 200}}; 157 158 // Framerate mismatch should be lower for lower framerate. 159 std::vector<RateControlThresholds> rc_thresholds = { 160 {10, 2, 40, 1, 0.5, 0.2, 0, 1}, 161 {8, 2, 5, 1, 0.5, 0.2, 0, 0}, 162 {5, 2, 0, 1, 0.5, 0.3, 0, 0}}; 163 164 // Quality should be higher for lower framerates for the same content. 165 std::vector<QualityThresholds> quality_thresholds = { 166 {33, 32, 0.88, 0.86}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}}; 167 168 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 169 } 170 171 TEST(VideoCodecTestLibvpx, DenoiserOnVP9) { 172 auto config = CreateConfig(); 173 config.SetCodecSettings(kVp9CodecName, 1, 1, 1, true, true, false, kCifWidth, 174 kCifHeight); 175 config.num_frames = kNumFramesShort; 176 const auto frame_checker = std::make_unique<QpFrameChecker>(); 177 config.encoded_frame_checker = frame_checker.get(); 178 auto fixture = CreateVideoCodecTestFixture(config); 179 180 std::vector<RateProfile> rate_profiles = {{500, 30, 0}}; 181 182 std::vector<RateControlThresholds> rc_thresholds = { 183 {5, 1, 0, 1, 0.3, 0.1, 0, 1}}; 184 185 std::vector<QualityThresholds> quality_thresholds = {{37.5, 36, 0.94, 0.93}}; 186 187 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 188 } 189 190 TEST(VideoCodecTestLibvpx, VeryLowBitrateVP9) { 191 auto config = CreateConfig(); 192 config.SetCodecSettings(kVp9CodecName, 1, 1, 1, false, true, true, kCifWidth, 193 kCifHeight); 194 const auto frame_checker = std::make_unique<QpFrameChecker>(); 195 config.encoded_frame_checker = frame_checker.get(); 196 auto fixture = CreateVideoCodecTestFixture(config); 197 198 std::vector<RateProfile> rate_profiles = {{50, 30, 0}}; 199 200 std::vector<RateControlThresholds> rc_thresholds = { 201 {15, 3, 75, 1, 0.5, 0.4, 2, 1}}; 202 203 std::vector<QualityThresholds> quality_thresholds = {{28, 25, 0.80, 0.65}}; 204 205 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 206 } 207 208 // TODO(marpan): Add temporal layer test for VP9, once changes are in 209 // vp9 wrapper for this. 210 211 #endif // defined(RTC_ENABLE_VP9) 212 213 TEST(VideoCodecTestLibvpx, HighBitrateVP8) { 214 auto config = CreateConfig(); 215 config.SetCodecSettings(kVp8CodecName, 1, 1, 1, true, true, false, kCifWidth, 216 kCifHeight); 217 config.num_frames = kNumFramesShort; 218 const auto frame_checker = std::make_unique<QpFrameChecker>(); 219 config.encoded_frame_checker = frame_checker.get(); 220 auto fixture = CreateVideoCodecTestFixture(config); 221 222 std::vector<RateProfile> rate_profiles = {{500, 30, 0}}; 223 224 std::vector<RateControlThresholds> rc_thresholds = { 225 {5, 1, 0, 1, 0.2, 0.1, 0, 1}}; 226 227 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) 228 std::vector<QualityThresholds> quality_thresholds = {{35, 33, 0.91, 0.89}}; 229 #else 230 std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}}; 231 #endif 232 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 233 } 234 235 TEST(VideoCodecTestLibvpx, MAYBE_ChangeBitrateVP8) { 236 auto config = CreateConfig(); 237 config.SetCodecSettings(kVp8CodecName, 1, 1, 1, true, true, false, kCifWidth, 238 kCifHeight); 239 const auto frame_checker = std::make_unique<QpFrameChecker>(); 240 config.encoded_frame_checker = frame_checker.get(); 241 auto fixture = CreateVideoCodecTestFixture(config); 242 243 std::vector<RateProfile> rate_profiles = { 244 {200, 30, 0}, // target_kbps, input_fps, frame_num 245 {800, 30, 100}, 246 {500, 30, 200}}; 247 248 std::vector<RateControlThresholds> rc_thresholds = { 249 {5, 1, 0, 1, 0.2, 0.1, 0, 1}, 250 {15.5, 1, 0, 1, 0.2, 0.1, 0, 0}, 251 {15, 1, 0, 1, 0.2, 0.1, 0, 0}}; 252 253 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) 254 std::vector<QualityThresholds> quality_thresholds = { 255 {31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}}; 256 #else 257 std::vector<QualityThresholds> quality_thresholds = { 258 {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}}; 259 #endif 260 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 261 } 262 263 TEST(VideoCodecTestLibvpx, MAYBE_ChangeFramerateVP8) { 264 auto config = CreateConfig(); 265 config.SetCodecSettings(kVp8CodecName, 1, 1, 1, true, true, false, kCifWidth, 266 kCifHeight); 267 const auto frame_checker = std::make_unique<QpFrameChecker>(); 268 config.encoded_frame_checker = frame_checker.get(); 269 auto fixture = CreateVideoCodecTestFixture(config); 270 271 std::vector<RateProfile> rate_profiles = { 272 {80, 24, 0}, // target_kbps, input_fps, frame_index_rate_update 273 {80, 15, 100}, 274 {80, 10, 200}}; 275 276 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) 277 std::vector<RateControlThresholds> rc_thresholds = { 278 {10, 2.42, 60, 1, 0.3, 0.3, 0, 1}, 279 {10, 2, 30, 1, 0.3, 0.3, 0, 0}, 280 {10, 2, 10, 1, 0.3, 0.2, 0, 0}}; 281 #else 282 std::vector<RateControlThresholds> rc_thresholds = { 283 {10, 2, 20, 1, 0.3, 0.15, 0, 1}, 284 {5, 2, 5, 1, 0.3, 0.15, 0, 0}, 285 {4, 2, 1, 1, 0.3, 0.2, 0, 0}}; 286 #endif 287 288 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) 289 std::vector<QualityThresholds> quality_thresholds = { 290 {31, 30, 0.85, 0.84}, {31.4, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}}; 291 #else 292 std::vector<QualityThresholds> quality_thresholds = { 293 {31, 30, 0.87, 0.85}, {32, 31, 0.88, 0.85}, {32, 30, 0.87, 0.82}}; 294 #endif 295 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 296 } 297 298 #if defined(WEBRTC_ANDROID) 299 #define MAYBE_TemporalLayersVP8 DISABLED_TemporalLayersVP8 300 #else 301 #define MAYBE_TemporalLayersVP8 TemporalLayersVP8 302 #endif 303 TEST(VideoCodecTestLibvpx, MAYBE_TemporalLayersVP8) { 304 auto config = CreateConfig(); 305 config.SetCodecSettings(kVp8CodecName, 1, 1, 3, true, true, false, kCifWidth, 306 kCifHeight); 307 const auto frame_checker = std::make_unique<QpFrameChecker>(); 308 config.encoded_frame_checker = frame_checker.get(); 309 auto fixture = CreateVideoCodecTestFixture(config); 310 311 std::vector<RateProfile> rate_profiles = {{200, 30, 0}, {400, 30, 150}}; 312 313 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) 314 std::vector<RateControlThresholds> rc_thresholds = { 315 {10, 1, 2.1, 1, 0.2, 0.1, 0, 1}, {12, 2, 3, 1, 0.2, 0.1, 0, 1}}; 316 #else 317 std::vector<RateControlThresholds> rc_thresholds = { 318 {5, 1, 0, 1, 0.2, 0.1, 0, 1}, {10, 2, 0, 1, 0.2, 0.1, 0, 1}}; 319 #endif 320 // Min SSIM drops because of high motion scene with complex backgound (trees). 321 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) 322 std::vector<QualityThresholds> quality_thresholds = {{31, 30, 0.85, 0.83}, 323 {31, 28, 0.85, 0.75}}; 324 #else 325 std::vector<QualityThresholds> quality_thresholds = {{32, 30, 0.88, 0.85}, 326 {33, 30, 0.89, 0.83}}; 327 #endif 328 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 329 } 330 331 #if defined(WEBRTC_ANDROID) 332 #define MAYBE_MultiresVP8 DISABLED_MultiresVP8 333 #else 334 #define MAYBE_MultiresVP8 MultiresVP8 335 #endif 336 TEST(VideoCodecTestLibvpx, MAYBE_MultiresVP8) { 337 auto config = CreateConfig(); 338 config.filename = "ConferenceMotion_1280_720_50"; 339 config.filepath = ResourcePath(config.filename, "yuv"); 340 config.num_frames = 100; 341 config.SetCodecSettings(kVp8CodecName, 3, 1, 3, true, true, false, 1280, 720); 342 const auto frame_checker = std::make_unique<QpFrameChecker>(); 343 config.encoded_frame_checker = frame_checker.get(); 344 auto fixture = CreateVideoCodecTestFixture(config); 345 346 std::vector<RateProfile> rate_profiles = {{1500, 30, 0}}; 347 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) 348 std::vector<RateControlThresholds> rc_thresholds = { 349 {4.1, 1.04, 7, 0.18, 0.14, 0.08, 0, 1}}; 350 #else 351 std::vector<RateControlThresholds> rc_thresholds = { 352 {5, 1, 5, 1, 0.3, 0.1, 0, 1}}; 353 #endif 354 std::vector<QualityThresholds> quality_thresholds = {{34, 32, 0.90, 0.88}}; 355 356 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 357 } 358 359 #if defined(WEBRTC_ANDROID) 360 #define MAYBE_SimulcastVP8 DISABLED_SimulcastVP8 361 #else 362 #define MAYBE_SimulcastVP8 SimulcastVP8 363 #endif 364 TEST(VideoCodecTestLibvpx, MAYBE_SimulcastVP8) { 365 auto config = CreateConfig(); 366 config.filename = "ConferenceMotion_1280_720_50"; 367 config.filepath = ResourcePath(config.filename, "yuv"); 368 config.num_frames = 100; 369 config.SetCodecSettings(kVp8CodecName, 3, 1, 3, true, true, false, 1280, 720); 370 const auto frame_checker = std::make_unique<QpFrameChecker>(); 371 config.encoded_frame_checker = frame_checker.get(); 372 373 InternalEncoderFactory internal_encoder_factory; 374 auto adapted_encoder_factory = std::make_unique<FunctionVideoEncoderFactory>( 375 [&](const Environment& env, const SdpVideoFormat& /* format */) { 376 return std::make_unique<SimulcastEncoderAdapter>( 377 env, &internal_encoder_factory, nullptr, SdpVideoFormat::VP8()); 378 }); 379 auto internal_decoder_factory = std::make_unique<InternalDecoderFactory>(); 380 381 auto fixture = 382 CreateVideoCodecTestFixture(config, std::move(internal_decoder_factory), 383 std::move(adapted_encoder_factory)); 384 385 std::vector<RateProfile> rate_profiles = {{1500, 30, 0}}; 386 387 std::vector<RateControlThresholds> rc_thresholds = { 388 {20, 5, 90, 1, 0.5, 0.3, 0, 1}}; 389 std::vector<QualityThresholds> quality_thresholds = {{34, 32, 0.90, 0.88}}; 390 391 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 392 } 393 394 #if defined(WEBRTC_ANDROID) 395 #define MAYBE_SvcVP9 DISABLED_SvcVP9 396 #else 397 #define MAYBE_SvcVP9 SvcVP9 398 #endif 399 TEST(VideoCodecTestLibvpx, MAYBE_SvcVP9) { 400 auto config = CreateConfig(); 401 config.filename = "ConferenceMotion_1280_720_50"; 402 config.filepath = ResourcePath(config.filename, "yuv"); 403 config.num_frames = 100; 404 config.SetCodecSettings(kVp9CodecName, 1, 3, 3, true, true, false, 1280, 720); 405 const auto frame_checker = std::make_unique<QpFrameChecker>(); 406 config.encoded_frame_checker = frame_checker.get(); 407 auto fixture = CreateVideoCodecTestFixture(config); 408 409 std::vector<RateProfile> rate_profiles = {{1500, 30, 0}}; 410 411 std::vector<RateControlThresholds> rc_thresholds = { 412 {5, 1, 5, 1, 0.3, 0.1, 0, 1}}; 413 std::vector<QualityThresholds> quality_thresholds = {{36, 34, 0.93, 0.90}}; 414 415 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); 416 } 417 418 TEST(VideoCodecTestLibvpx, DISABLED_MultiresVP8RdPerf) { 419 auto config = CreateConfig(); 420 config.filename = "FourPeople_1280x720_30"; 421 config.filepath = ResourcePath(config.filename, "yuv"); 422 config.num_frames = 300; 423 config.print_frame_level_stats = true; 424 config.SetCodecSettings(kVp8CodecName, 3, 1, 3, true, true, false, 1280, 720); 425 const auto frame_checker = std::make_unique<QpFrameChecker>(); 426 config.encoded_frame_checker = frame_checker.get(); 427 auto fixture = CreateVideoCodecTestFixture(config); 428 429 std::map<size_t, std::vector<VideoStatistics>> rd_stats; 430 for (size_t bitrate_kbps : kBitrateRdPerfKbps) { 431 std::vector<RateProfile> rate_profiles = {{bitrate_kbps, 30, 0}}; 432 433 fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr); 434 435 rd_stats[bitrate_kbps] = 436 fixture->GetStats().SliceAndCalcLayerVideoStatistic( 437 kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1); 438 } 439 440 PrintRdPerf(rd_stats); 441 } 442 443 TEST(VideoCodecTestLibvpx, DISABLED_SvcVP9RdPerf) { 444 auto config = CreateConfig(); 445 config.filename = "FourPeople_1280x720_30"; 446 config.filepath = ResourcePath(config.filename, "yuv"); 447 config.num_frames = 300; 448 config.print_frame_level_stats = true; 449 config.SetCodecSettings(kVp9CodecName, 1, 3, 3, true, true, false, 1280, 720); 450 const auto frame_checker = std::make_unique<QpFrameChecker>(); 451 config.encoded_frame_checker = frame_checker.get(); 452 auto fixture = CreateVideoCodecTestFixture(config); 453 454 std::map<size_t, std::vector<VideoStatistics>> rd_stats; 455 for (size_t bitrate_kbps : kBitrateRdPerfKbps) { 456 std::vector<RateProfile> rate_profiles = {{bitrate_kbps, 30, 0}}; 457 458 fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr); 459 460 rd_stats[bitrate_kbps] = 461 fixture->GetStats().SliceAndCalcLayerVideoStatistic( 462 kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1); 463 } 464 465 PrintRdPerf(rd_stats); 466 } 467 468 } // namespace test 469 } // namespace webrtc