error_resilience_test.cc (17475B)
1 /* 2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include "gtest/gtest.h" 13 #include "test/codec_factory.h" 14 #include "test/encode_test_driver.h" 15 #include "test/i420_video_source.h" 16 #include "test/util.h" 17 18 namespace { 19 20 const int kMaxErrorFrames = 12; 21 const int kMaxInvisibleErrorFrames = 12; 22 const int kMaxDroppableFrames = 12; 23 const int kMaxErrorResilientFrames = 12; 24 const int kMaxNoMFMVFrames = 12; 25 const int kMaxPrimRefNoneFrames = 12; 26 const int kMaxSFrames = 12; 27 const int kCpuUsed = 1; 28 29 class ErrorResilienceTestLarge 30 : public ::libaom_test::CodecTestWith2Params<libaom_test::TestMode, int>, 31 public ::libaom_test::EncoderTest { 32 protected: 33 ErrorResilienceTestLarge() 34 : EncoderTest(GET_PARAM(0)), psnr_(0.0), nframes_(0), mismatch_psnr_(0.0), 35 mismatch_nframes_(0), encoding_mode_(GET_PARAM(1)), allow_mismatch_(0), 36 enable_altref_(GET_PARAM(2)) { 37 Reset(); 38 } 39 40 ~ErrorResilienceTestLarge() override = default; 41 42 void Reset() { 43 error_nframes_ = 0; 44 invisible_error_nframes_ = 0; 45 droppable_nframes_ = 0; 46 error_resilient_nframes_ = 0; 47 nomfmv_nframes_ = 0; 48 prim_ref_none_nframes_ = 0; 49 s_nframes_ = 0; 50 } 51 52 void SetupEncoder(int bitrate, int lag) { 53 const aom_rational timebase = { 33333333, 1000000000 }; 54 cfg_.g_timebase = timebase; 55 cfg_.rc_target_bitrate = bitrate; 56 cfg_.kf_mode = AOM_KF_DISABLED; 57 cfg_.g_lag_in_frames = lag; 58 init_flags_ = AOM_CODEC_USE_PSNR; 59 } 60 61 void SetUp() override { InitializeConfig(encoding_mode_); } 62 63 void BeginPassHook(unsigned int /*pass*/) override { 64 psnr_ = 0.0; 65 nframes_ = 0; 66 decoded_nframes_ = 0; 67 mismatch_psnr_ = 0.0; 68 mismatch_nframes_ = 0; 69 } 70 71 void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override { 72 psnr_ += pkt->data.psnr.psnr[0]; 73 nframes_++; 74 } 75 76 void PreEncodeFrameHook(libaom_test::VideoSource *video, 77 libaom_test::Encoder *encoder) override { 78 if (video->frame() == 0) { 79 encoder->Control(AOME_SET_CPUUSED, kCpuUsed); 80 encoder->Control(AOME_SET_ENABLEAUTOALTREF, enable_altref_); 81 } 82 frame_flags_ &= 83 ~(AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF | 84 AOM_EFLAG_NO_REF_FRAME_MVS | AOM_EFLAG_ERROR_RESILIENT | 85 AOM_EFLAG_SET_S_FRAME | AOM_EFLAG_SET_PRIMARY_REF_NONE); 86 if (droppable_nframes_ > 0 && 87 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { 88 for (unsigned int i = 0; i < droppable_nframes_; ++i) { 89 if (droppable_frames_[i] == video->frame()) { 90 std::cout << " Encoding droppable frame: " 91 << droppable_frames_[i] << "\n"; 92 frame_flags_ |= (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | 93 AOM_EFLAG_NO_UPD_ARF); 94 break; 95 } 96 } 97 } 98 99 if (error_resilient_nframes_ > 0 && 100 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { 101 for (unsigned int i = 0; i < error_resilient_nframes_; ++i) { 102 if (error_resilient_frames_[i] == video->frame()) { 103 std::cout << " Encoding error_resilient frame: " 104 << error_resilient_frames_[i] << "\n"; 105 frame_flags_ |= AOM_EFLAG_ERROR_RESILIENT; 106 break; 107 } 108 } 109 } 110 111 if (nomfmv_nframes_ > 0 && 112 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { 113 for (unsigned int i = 0; i < nomfmv_nframes_; ++i) { 114 if (nomfmv_frames_[i] == video->frame()) { 115 std::cout << " Encoding no mfmv frame: " 116 << nomfmv_frames_[i] << "\n"; 117 frame_flags_ |= AOM_EFLAG_NO_REF_FRAME_MVS; 118 break; 119 } 120 } 121 } 122 123 if (prim_ref_none_nframes_ > 0 && 124 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { 125 for (unsigned int i = 0; i < prim_ref_none_nframes_; ++i) { 126 if (prim_ref_none_frames_[i] == video->frame()) { 127 std::cout << " Encoding no PRIMARY_REF_NONE frame: " 128 << prim_ref_none_frames_[i] << "\n"; 129 frame_flags_ |= AOM_EFLAG_SET_PRIMARY_REF_NONE; 130 break; 131 } 132 } 133 } 134 135 encoder->Control(AV1E_SET_S_FRAME_MODE, 0); 136 if (s_nframes_ > 0 && 137 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { 138 for (unsigned int i = 0; i < s_nframes_; ++i) { 139 if (s_frames_[i] == video->frame()) { 140 std::cout << " Encoding S frame: " << s_frames_[i] 141 << "\n"; 142 frame_flags_ |= AOM_EFLAG_SET_S_FRAME; 143 break; 144 } 145 } 146 } 147 } 148 149 void FramePktHook(const aom_codec_cx_pkt_t *pkt) override { 150 // Check that the encode frame flags are correctly reflected 151 // in the output frame flags. 152 const int encode_flags = pkt->data.frame.flags >> 16; 153 if ((encode_flags & (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | 154 AOM_EFLAG_NO_UPD_ARF)) == 155 (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF)) { 156 ASSERT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_DROPPABLE, 157 AOM_FRAME_IS_DROPPABLE); 158 } 159 if (encode_flags & AOM_EFLAG_SET_S_FRAME) { 160 ASSERT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_SWITCH, 161 AOM_FRAME_IS_SWITCH); 162 } 163 if (encode_flags & AOM_EFLAG_ERROR_RESILIENT) { 164 ASSERT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_ERROR_RESILIENT, 165 AOM_FRAME_IS_ERROR_RESILIENT); 166 } 167 } 168 169 double GetAveragePsnr() const { 170 if (nframes_) return psnr_ / nframes_; 171 return 0.0; 172 } 173 174 double GetAverageMismatchPsnr() const { 175 if (mismatch_nframes_) return mismatch_psnr_ / mismatch_nframes_; 176 return 0.0; 177 } 178 179 bool DoDecode() const override { 180 if (error_nframes_ > 0 && 181 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { 182 for (unsigned int i = 0; i < error_nframes_; ++i) { 183 if (error_frames_[i] == nframes_ - 1) { 184 std::cout << " Skipping decoding frame: " 185 << error_frames_[i] << "\n"; 186 return false; 187 } 188 } 189 } 190 return true; 191 } 192 193 bool DoDecodeInvisible() const override { 194 if (invisible_error_nframes_ > 0 && 195 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { 196 for (unsigned int i = 0; i < invisible_error_nframes_; ++i) { 197 if (invisible_error_frames_[i] == nframes_ - 1) { 198 std::cout << " Skipping decoding all invisible frames in " 199 "frame pkt: " 200 << invisible_error_frames_[i] << "\n"; 201 return false; 202 } 203 } 204 } 205 return true; 206 } 207 208 void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) override { 209 if (allow_mismatch_) { 210 double mismatch_psnr = compute_psnr(img1, img2); 211 mismatch_psnr_ += mismatch_psnr; 212 ++mismatch_nframes_; 213 // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n"; 214 } else { 215 ::libaom_test::EncoderTest::MismatchHook(img1, img2); 216 } 217 } 218 219 void DecompressedFrameHook(const aom_image_t &img, 220 aom_codec_pts_t pts) override { 221 (void)img; 222 (void)pts; 223 ++decoded_nframes_; 224 } 225 226 void SetErrorFrames(int num, unsigned int *list) { 227 if (num > kMaxErrorFrames) 228 num = kMaxErrorFrames; 229 else if (num < 0) 230 num = 0; 231 error_nframes_ = num; 232 for (unsigned int i = 0; i < error_nframes_; ++i) 233 error_frames_[i] = list[i]; 234 } 235 236 void SetInvisibleErrorFrames(int num, unsigned int *list) { 237 if (num > kMaxInvisibleErrorFrames) 238 num = kMaxInvisibleErrorFrames; 239 else if (num < 0) 240 num = 0; 241 invisible_error_nframes_ = num; 242 for (unsigned int i = 0; i < invisible_error_nframes_; ++i) 243 invisible_error_frames_[i] = list[i]; 244 } 245 246 void SetDroppableFrames(int num, unsigned int *list) { 247 if (num > kMaxDroppableFrames) 248 num = kMaxDroppableFrames; 249 else if (num < 0) 250 num = 0; 251 droppable_nframes_ = num; 252 for (unsigned int i = 0; i < droppable_nframes_; ++i) 253 droppable_frames_[i] = list[i]; 254 } 255 256 void SetErrorResilientFrames(int num, unsigned int *list) { 257 if (num > kMaxErrorResilientFrames) 258 num = kMaxErrorResilientFrames; 259 else if (num < 0) 260 num = 0; 261 error_resilient_nframes_ = num; 262 for (unsigned int i = 0; i < error_resilient_nframes_; ++i) 263 error_resilient_frames_[i] = list[i]; 264 } 265 266 void SetNoMFMVFrames(int num, unsigned int *list) { 267 if (num > kMaxNoMFMVFrames) 268 num = kMaxNoMFMVFrames; 269 else if (num < 0) 270 num = 0; 271 nomfmv_nframes_ = num; 272 for (unsigned int i = 0; i < nomfmv_nframes_; ++i) 273 nomfmv_frames_[i] = list[i]; 274 } 275 276 void SetPrimaryRefNoneFrames(int num, unsigned int *list) { 277 if (num > kMaxPrimRefNoneFrames) 278 num = kMaxPrimRefNoneFrames; 279 else if (num < 0) 280 num = 0; 281 prim_ref_none_nframes_ = num; 282 for (unsigned int i = 0; i < prim_ref_none_nframes_; ++i) 283 prim_ref_none_frames_[i] = list[i]; 284 } 285 286 void SetSFrames(int num, unsigned int *list) { 287 if (num > kMaxSFrames) 288 num = kMaxSFrames; 289 else if (num < 0) 290 num = 0; 291 s_nframes_ = num; 292 for (unsigned int i = 0; i < s_nframes_; ++i) s_frames_[i] = list[i]; 293 } 294 295 unsigned int GetMismatchFrames() { return mismatch_nframes_; } 296 unsigned int GetEncodedFrames() { return nframes_; } 297 unsigned int GetDecodedFrames() { return decoded_nframes_; } 298 299 void SetAllowMismatch(int allow) { allow_mismatch_ = allow; } 300 301 private: 302 double psnr_; 303 unsigned int nframes_; 304 unsigned int decoded_nframes_; 305 unsigned int error_nframes_; 306 unsigned int invisible_error_nframes_; 307 unsigned int droppable_nframes_; 308 unsigned int error_resilient_nframes_; 309 unsigned int nomfmv_nframes_; 310 unsigned int prim_ref_none_nframes_; 311 unsigned int s_nframes_; 312 double mismatch_psnr_; 313 unsigned int mismatch_nframes_; 314 unsigned int error_frames_[kMaxErrorFrames]; 315 unsigned int invisible_error_frames_[kMaxInvisibleErrorFrames]; 316 unsigned int droppable_frames_[kMaxDroppableFrames]; 317 unsigned int error_resilient_frames_[kMaxErrorResilientFrames]; 318 unsigned int nomfmv_frames_[kMaxNoMFMVFrames]; 319 unsigned int prim_ref_none_frames_[kMaxPrimRefNoneFrames]; 320 unsigned int s_frames_[kMaxSFrames]; 321 libaom_test::TestMode encoding_mode_; 322 int allow_mismatch_; 323 int enable_altref_; 324 }; 325 326 TEST_P(ErrorResilienceTestLarge, OnVersusOff) { 327 SetupEncoder(2000, 10); 328 libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 329 cfg_.g_timebase.den, cfg_.g_timebase.num, 330 0, 12); 331 332 // Global error resilient mode OFF. 333 cfg_.g_error_resilient = 0; 334 ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); 335 const double psnr_resilience_off = GetAveragePsnr(); 336 EXPECT_GT(psnr_resilience_off, 25.0); 337 338 Reset(); 339 // Error resilient mode ON for certain frames 340 unsigned int num_error_resilient_frames = 5; 341 unsigned int error_resilient_frame_list[] = { 3, 5, 6, 9, 11 }; 342 SetErrorResilientFrames(num_error_resilient_frames, 343 error_resilient_frame_list); 344 ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); 345 const double psnr_resilience_on = GetAveragePsnr(); 346 EXPECT_GT(psnr_resilience_on, 25.0); 347 348 // Test that turning on error resilient mode hurts by 10% at most. 349 if (psnr_resilience_off > 0.0) { 350 const double psnr_ratio = psnr_resilience_on / psnr_resilience_off; 351 EXPECT_GE(psnr_ratio, 0.9); 352 EXPECT_LE(psnr_ratio, 1.1); 353 } 354 } 355 356 // Check for successful decoding and no encoder/decoder mismatch 357 // if we lose (i.e., drop before decoding) a set of droppable 358 // frames (i.e., frames that don't update any reference buffers). 359 TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) { 360 if (GET_PARAM(1) == ::libaom_test::kOnePassGood && GET_PARAM(2) == 1) { 361 fprintf(stderr, "Skipping test case #1 because of bug aomedia:3002\n"); 362 return; 363 } 364 SetupEncoder(500, 10); 365 libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 366 cfg_.g_timebase.den, cfg_.g_timebase.num, 367 0, 20); 368 369 // Set an arbitrary set of error frames same as droppable frames. 370 unsigned int num_droppable_frames = 3; 371 unsigned int droppable_frame_list[] = { 5, 11, 13 }; 372 SetDroppableFrames(num_droppable_frames, droppable_frame_list); 373 SetErrorFrames(num_droppable_frames, droppable_frame_list); 374 ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); 375 // Test that no mismatches have been found 376 std::cout << " Encoded frames: " << GetEncodedFrames() << "\n"; 377 std::cout << " Decoded frames: " << GetDecodedFrames() << "\n"; 378 std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n"; 379 EXPECT_EQ(GetEncodedFrames() - GetDecodedFrames(), num_droppable_frames); 380 } 381 382 // Check for ParseAbility property of an error-resilient frame. 383 // Encode a frame in error-resilient mode (E-frame), and disallow all 384 // subsequent frames from using MFMV. If frames are dropped before the 385 // E frame, all frames starting from the E frame should be parse-able. 386 TEST_P(ErrorResilienceTestLarge, ParseAbilityTest) { 387 SetupEncoder(500, 10); 388 389 libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 390 cfg_.g_timebase.den, cfg_.g_timebase.num, 391 0, 15); 392 393 SetAllowMismatch(1); 394 395 // Note that an E-frame cannot be forced on a frame that is a 396 // show_existing_frame, or a frame that comes directly after an invisible 397 // frame. Currently, this will cause an assertion failure. 398 // Set an arbitrary error resilient (E) frame 399 unsigned int num_error_resilient_frames = 1; 400 unsigned int error_resilient_frame_list[] = { 8 }; 401 SetErrorResilientFrames(num_error_resilient_frames, 402 error_resilient_frame_list); 403 // Ensure that any invisible frames before the E frame are dropped 404 SetInvisibleErrorFrames(num_error_resilient_frames, 405 error_resilient_frame_list); 406 // Set all frames after the error resilient frame to not allow MFMV 407 unsigned int num_post_error_resilient_frames = 6; 408 unsigned int post_error_resilient_frame_list[] = { 9, 10, 11, 12, 13, 14 }; 409 SetNoMFMVFrames(num_post_error_resilient_frames, 410 post_error_resilient_frame_list); 411 412 // Set a few frames before the E frame that are lost (not decoded) 413 unsigned int num_error_frames = 5; 414 unsigned int error_frame_list[] = { 3, 4, 5, 6, 7 }; 415 SetErrorFrames(num_error_frames, error_frame_list); 416 417 ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); 418 std::cout << " Encoded frames: " << GetEncodedFrames() << "\n"; 419 std::cout << " Decoded frames: " << GetDecodedFrames() << "\n"; 420 std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n"; 421 EXPECT_EQ(GetEncodedFrames() - GetDecodedFrames(), num_error_frames); 422 // All frames following the E-frame and the E-frame are expected to have 423 // mismatches, but still be parse-able. 424 EXPECT_LE(GetMismatchFrames(), num_post_error_resilient_frames + 1); 425 } 426 427 // Check for ParseAbility property of an S frame. 428 // Encode an S-frame. If frames are dropped before the S-frame, all frames 429 // starting from the S frame should be parse-able. 430 TEST_P(ErrorResilienceTestLarge, SFrameTest) { 431 SetupEncoder(500, 10); 432 433 libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 434 cfg_.g_timebase.den, cfg_.g_timebase.num, 435 0, 15); 436 437 SetAllowMismatch(1); 438 439 // Note that an S-frame cannot be forced on a frame that is a 440 // show_existing_frame. This issue still needs to be addressed. 441 // Set an arbitrary S-frame 442 unsigned int num_s_frames = 1; 443 unsigned int s_frame_list[] = { 6 }; 444 SetSFrames(num_s_frames, s_frame_list); 445 // Ensure that any invisible frames before the S frame are dropped 446 SetInvisibleErrorFrames(num_s_frames, s_frame_list); 447 448 // Set a few frames before the S frame that are lost (not decoded) 449 unsigned int num_error_frames = 4; 450 unsigned int error_frame_list[] = { 2, 3, 4, 5 }; 451 SetErrorFrames(num_error_frames, error_frame_list); 452 453 ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); 454 std::cout << " Encoded frames: " << GetEncodedFrames() << "\n"; 455 std::cout << " Decoded frames: " << GetDecodedFrames() << "\n"; 456 std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n"; 457 EXPECT_EQ(GetEncodedFrames() - GetDecodedFrames(), num_error_frames); 458 // All frames following the S-frame and the S-frame are expected to have 459 // mismatches, but still be parse-able. 460 EXPECT_LE(GetMismatchFrames(), GetEncodedFrames() - s_frame_list[0]); 461 } 462 463 AV1_INSTANTIATE_TEST_SUITE(ErrorResilienceTestLarge, NONREALTIME_TEST_MODES, 464 ::testing::Values(0, 1)); 465 } // namespace