video_adapter_unittest.cc (57147B)
1 /* 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "media/base/video_adapter.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <limits> 16 #include <memory> 17 #include <optional> 18 #include <string> 19 #include <utility> 20 21 #include "api/field_trials.h" 22 #include "api/video/resolution.h" 23 #include "api/video/video_frame.h" 24 #include "api/video/video_source_interface.h" 25 #include "media/base/fake_frame_source.h" 26 #include "media/base/video_common.h" 27 #include "rtc_base/time_utils.h" 28 #include "test/create_test_field_trials.h" 29 #include "test/gmock.h" 30 #include "test/gtest.h" 31 32 namespace webrtc { 33 namespace { 34 constexpr int kWidth = 1280; 35 constexpr int kHeight = 720; 36 constexpr int kDefaultFps = 30; 37 38 using ::testing::_; 39 using ::testing::Eq; 40 using ::testing::Pair; 41 42 VideoSinkWants BuildSinkWants(std::optional<int> target_pixel_count, 43 int max_pixel_count, 44 int max_framerate_fps, 45 int sink_alignment = 1) { 46 VideoSinkWants wants; 47 wants.target_pixel_count = target_pixel_count; 48 wants.max_pixel_count = max_pixel_count; 49 wants.max_framerate_fps = max_framerate_fps; 50 wants.resolution_alignment = sink_alignment; 51 wants.is_active = true; 52 wants.aggregates.emplace(VideoSinkWants::Aggregates()); 53 wants.aggregates->any_active_without_requested_resolution = false; 54 return wants; 55 } 56 57 VideoSinkWants BuildSinkWants( 58 std::optional<Resolution> scale_resolution_down_to, 59 bool any_active_without_requested_resolution) { 60 VideoSinkWants wants; 61 wants.max_framerate_fps = kDefaultFps; 62 wants.resolution_alignment = 1; 63 wants.is_active = true; 64 if (scale_resolution_down_to) { 65 wants.requested_resolution.emplace(VideoSinkWants::FrameSize( 66 scale_resolution_down_to->width, scale_resolution_down_to->height)); 67 } 68 wants.aggregates.emplace(VideoSinkWants::Aggregates()); 69 wants.aggregates->any_active_without_requested_resolution = 70 any_active_without_requested_resolution; 71 return wants; 72 } 73 74 } // namespace 75 76 class VideoAdapterTest : public ::testing::Test, 77 public ::testing::WithParamInterface<bool> { 78 public: 79 VideoAdapterTest() : VideoAdapterTest("", 1) {} 80 explicit VideoAdapterTest(const std::string& field_trials, 81 int source_resolution_alignment) 82 : field_trials_(CreateTestFieldTrials(field_trials)), 83 frame_source_(std::make_unique<FakeFrameSource>( 84 kWidth, 85 kHeight, 86 VideoFormat::FpsToInterval(kDefaultFps) / kNumNanosecsPerMicrosec)), 87 adapter_(source_resolution_alignment), 88 adapter_wrapper_(std::make_unique<VideoAdapterWrapper>(&adapter_)), 89 use_new_format_request_(GetParam()) {} 90 91 protected: 92 // Wrap a VideoAdapter and collect stats. 93 class VideoAdapterWrapper { 94 public: 95 struct Stats { 96 int captured_frames = 0; 97 int dropped_frames = 0; 98 bool last_adapt_was_no_op = false; 99 100 int cropped_width = 0; 101 int cropped_height = 0; 102 int out_width = 0; 103 int out_height = 0; 104 }; 105 106 explicit VideoAdapterWrapper(VideoAdapter* adapter) 107 : video_adapter_(adapter) {} 108 109 void AdaptFrame(const VideoFrame& frame) { 110 const int in_width = frame.width(); 111 const int in_height = frame.height(); 112 int cropped_width; 113 int cropped_height; 114 int out_width; 115 int out_height; 116 if (video_adapter_->AdaptFrameResolution( 117 in_width, in_height, 118 frame.timestamp_us() * kNumNanosecsPerMicrosec, &cropped_width, 119 &cropped_height, &out_width, &out_height)) { 120 stats_.cropped_width = cropped_width; 121 stats_.cropped_height = cropped_height; 122 stats_.out_width = out_width; 123 stats_.out_height = out_height; 124 stats_.last_adapt_was_no_op = 125 (in_width == cropped_width && in_height == cropped_height && 126 in_width == out_width && in_height == out_height); 127 } else { 128 ++stats_.dropped_frames; 129 } 130 ++stats_.captured_frames; 131 } 132 133 Stats GetStats() const { return stats_; } 134 135 private: 136 VideoAdapter* video_adapter_; 137 Stats stats_; 138 }; 139 140 void VerifyAdaptedResolution(const VideoAdapterWrapper::Stats& stats, 141 int cropped_width, 142 int cropped_height, 143 int out_width, 144 int out_height) { 145 EXPECT_EQ(cropped_width, stats.cropped_width); 146 EXPECT_EQ(cropped_height, stats.cropped_height); 147 EXPECT_EQ(out_width, stats.out_width); 148 EXPECT_EQ(out_height, stats.out_height); 149 } 150 151 void OnOutputFormatRequest(int width, 152 int height, 153 const std::optional<int>& fps) { 154 if (use_new_format_request_) { 155 std::optional<std::pair<int, int>> target_aspect_ratio = 156 std::make_pair(width, height); 157 std::optional<int> max_pixel_count = width * height; 158 std::optional<int> max_fps = fps; 159 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count, 160 max_fps); 161 return; 162 } 163 adapter_.OnOutputFormatRequest( 164 VideoFormat(width, height, fps ? VideoFormat::FpsToInterval(*fps) : 0, 165 FOURCC_I420)); 166 } 167 168 // Return pair of <out resolution, cropping> 169 std::pair<Resolution, Resolution> AdaptFrameResolution(Resolution res) { 170 Resolution out; 171 Resolution cropped; 172 timestamp_ns_ += 1000000000; 173 EXPECT_TRUE(adapter_.AdaptFrameResolution( 174 res.width, res.height, timestamp_ns_, &cropped.width, &cropped.height, 175 &out.width, &out.height)); 176 return std::make_pair(out, cropped); 177 } 178 179 FieldTrials field_trials_; 180 const std::unique_ptr<FakeFrameSource> frame_source_; 181 VideoAdapter adapter_; 182 int64_t timestamp_ns_ = 0; 183 int cropped_width_; 184 int cropped_height_; 185 int out_width_; 186 int out_height_; 187 const std::unique_ptr<VideoAdapterWrapper> adapter_wrapper_; 188 const bool use_new_format_request_; 189 }; 190 191 INSTANTIATE_TEST_SUITE_P(OnOutputFormatRequests, 192 VideoAdapterTest, 193 ::testing::Values(true, false)); 194 195 // Do not adapt the frame rate or the resolution. Expect no frame drop, no 196 // cropping, and no resolution change. 197 TEST_P(VideoAdapterTest, AdaptNothing) { 198 for (int i = 0; i < 10; ++i) 199 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 200 201 // Verify no frame drop and no resolution change. 202 VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); 203 EXPECT_GE(stats.captured_frames, 10); 204 EXPECT_EQ(0, stats.dropped_frames); 205 VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); 206 EXPECT_TRUE(stats.last_adapt_was_no_op); 207 } 208 209 TEST_P(VideoAdapterTest, AdaptZeroInterval) { 210 OnOutputFormatRequest(kWidth, kHeight, std::nullopt); 211 for (int i = 0; i < 40; ++i) 212 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 213 214 // Verify no crash and that frames aren't dropped. 215 VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); 216 EXPECT_GE(stats.captured_frames, 40); 217 EXPECT_EQ(0, stats.dropped_frames); 218 VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); 219 } 220 221 // Adapt the frame rate to be half of the capture rate at the beginning. Expect 222 // the number of dropped frames to be half of the number the captured frames. 223 TEST_P(VideoAdapterTest, AdaptFramerateToHalf) { 224 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps / 2); 225 226 // Capture 10 frames and verify that every other frame is dropped. The first 227 // frame should not be dropped. 228 for (int i = 0; i < 10; ++i) 229 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 230 EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 10); 231 EXPECT_EQ(5, adapter_wrapper_->GetStats().dropped_frames); 232 } 233 234 // Adapt the frame rate to be two thirds of the capture rate at the beginning. 235 // Expect the number of dropped frames to be one thirds of the number the 236 // captured frames. 237 TEST_P(VideoAdapterTest, AdaptFramerateToTwoThirds) { 238 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps * 2 / 3); 239 240 // Capture 10 frames and verify that every third frame is dropped. The first 241 // frame should not be dropped. 242 for (int i = 0; i < 10; ++i) 243 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 244 EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 10); 245 EXPECT_EQ(3, adapter_wrapper_->GetStats().dropped_frames); 246 } 247 248 // Request frame rate twice as high as captured frame rate. Expect no frame 249 // drop. 250 TEST_P(VideoAdapterTest, AdaptFramerateHighLimit) { 251 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps * 2); 252 253 for (int i = 0; i < 10; ++i) 254 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 255 256 // Verify no frame drop. 257 EXPECT_EQ(0, adapter_wrapper_->GetStats().dropped_frames); 258 } 259 260 // Adapt the frame rate to be half of the capture rate. No resolution limit set. 261 // Expect the number of dropped frames to be half of the number the captured 262 // frames. 263 TEST_P(VideoAdapterTest, AdaptFramerateToHalfWithNoPixelLimit) { 264 adapter_.OnOutputFormatRequest(std::nullopt, std::nullopt, kDefaultFps / 2); 265 266 // Capture 10 frames and verify that every other frame is dropped. The first 267 // frame should not be dropped. 268 int expected_dropped_frames = 0; 269 for (int i = 0; i < 10; ++i) { 270 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 271 EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, i + 1); 272 if (i % 2 == 1) 273 ++expected_dropped_frames; 274 EXPECT_EQ(expected_dropped_frames, 275 adapter_wrapper_->GetStats().dropped_frames); 276 VerifyAdaptedResolution(adapter_wrapper_->GetStats(), kWidth, kHeight, 277 kWidth, kHeight); 278 } 279 } 280 281 // Adapt the frame rate to be half of the capture rate after capturing no less 282 // than 10 frames. Expect no frame dropped before adaptation and frame dropped 283 // after adaptation. 284 TEST_P(VideoAdapterTest, AdaptFramerateOntheFly) { 285 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps); 286 for (int i = 0; i < 10; ++i) 287 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 288 289 // Verify no frame drop before adaptation. 290 EXPECT_EQ(0, adapter_wrapper_->GetStats().dropped_frames); 291 292 // Adapt the frame rate. 293 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps / 2); 294 for (int i = 0; i < 20; ++i) 295 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 296 297 // Verify frame drop after adaptation. 298 EXPECT_GT(adapter_wrapper_->GetStats().dropped_frames, 0); 299 } 300 301 // Do not adapt the frame rate or the resolution. Expect no frame drop, no 302 // cropping, and no resolution change. 303 TEST_P(VideoAdapterTest, AdaptFramerateRequestMax) { 304 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 305 std::numeric_limits<int>::max(), 306 std::numeric_limits<int>::max())); 307 308 for (int i = 0; i < 10; ++i) 309 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 310 311 // Verify no frame drop and no resolution change. 312 VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); 313 EXPECT_GE(stats.captured_frames, 10); 314 EXPECT_EQ(0, stats.dropped_frames); 315 VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); 316 EXPECT_TRUE(stats.last_adapt_was_no_op); 317 } 318 319 TEST_P(VideoAdapterTest, AdaptFramerateRequestZero) { 320 adapter_.OnSinkWants( 321 BuildSinkWants(std::nullopt, std::numeric_limits<int>::max(), 0)); 322 for (int i = 0; i < 10; ++i) 323 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 324 325 // Verify no crash and that frames aren't dropped. 326 VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); 327 EXPECT_GE(stats.captured_frames, 10); 328 EXPECT_EQ(10, stats.dropped_frames); 329 } 330 331 // Adapt the frame rate to be half of the capture rate at the beginning. Expect 332 // the number of dropped frames to be half of the number the captured frames. 333 TEST_P(VideoAdapterTest, AdaptFramerateRequestHalf) { 334 adapter_.OnSinkWants(BuildSinkWants( 335 std::nullopt, std::numeric_limits<int>::max(), kDefaultFps / 2)); 336 for (int i = 0; i < 10; ++i) 337 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 338 339 // Verify no crash and that frames aren't dropped. 340 VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); 341 EXPECT_GE(stats.captured_frames, 10); 342 EXPECT_EQ(5, stats.dropped_frames); 343 VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); 344 } 345 346 // Set a very high output pixel resolution. Expect no cropping or resolution 347 // change. 348 TEST_P(VideoAdapterTest, AdaptFrameResolutionHighLimit) { 349 OnOutputFormatRequest(kWidth * 10, kHeight * 10, kDefaultFps); 350 EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, &cropped_width_, 351 &cropped_height_, &out_width_, 352 &out_height_)); 353 EXPECT_EQ(kWidth, cropped_width_); 354 EXPECT_EQ(kHeight, cropped_height_); 355 EXPECT_EQ(kWidth, out_width_); 356 EXPECT_EQ(kHeight, out_height_); 357 } 358 359 // Adapt the frame resolution to be the same as capture resolution. Expect no 360 // cropping or resolution change. 361 TEST_P(VideoAdapterTest, AdaptFrameResolutionIdentical) { 362 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps); 363 EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, &cropped_width_, 364 &cropped_height_, &out_width_, 365 &out_height_)); 366 EXPECT_EQ(kWidth, cropped_width_); 367 EXPECT_EQ(kHeight, cropped_height_); 368 EXPECT_EQ(kWidth, out_width_); 369 EXPECT_EQ(kHeight, out_height_); 370 } 371 372 // Adapt the frame resolution to be a quarter of the capture resolution. Expect 373 // no cropping, but a resolution change. 374 TEST_P(VideoAdapterTest, AdaptFrameResolutionQuarter) { 375 OnOutputFormatRequest(kWidth / 2, kHeight / 2, kDefaultFps); 376 EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, &cropped_width_, 377 &cropped_height_, &out_width_, 378 &out_height_)); 379 EXPECT_EQ(kWidth, cropped_width_); 380 EXPECT_EQ(kHeight, cropped_height_); 381 EXPECT_EQ(kWidth / 2, out_width_); 382 EXPECT_EQ(kHeight / 2, out_height_); 383 } 384 385 // Adapt the pixel resolution to 0. Expect frame drop. 386 TEST_P(VideoAdapterTest, AdaptFrameResolutionDrop) { 387 OnOutputFormatRequest(kWidth * 0, kHeight * 0, kDefaultFps); 388 EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, 389 &cropped_width_, &cropped_height_, 390 &out_width_, &out_height_)); 391 } 392 393 // Adapt the frame resolution to be a quarter of the capture resolution at the 394 // beginning. Expect no cropping but a resolution change. 395 TEST_P(VideoAdapterTest, AdaptResolution) { 396 OnOutputFormatRequest(kWidth / 2, kHeight / 2, kDefaultFps); 397 for (int i = 0; i < 10; ++i) 398 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 399 400 // Verify no frame drop, no cropping, and resolution change. 401 VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); 402 EXPECT_EQ(0, stats.dropped_frames); 403 VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth / 2, kHeight / 2); 404 } 405 406 // Adapt the frame resolution to be a quarter of the capture resolution after 407 // capturing no less than 10 frames. Expect no resolution change before 408 // adaptation and resolution change after adaptation. 409 TEST_P(VideoAdapterTest, AdaptResolutionOnTheFly) { 410 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps); 411 for (int i = 0; i < 10; ++i) 412 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 413 414 // Verify no resolution change before adaptation. 415 VerifyAdaptedResolution(adapter_wrapper_->GetStats(), kWidth, kHeight, kWidth, 416 kHeight); 417 418 // Adapt the frame resolution. 419 OnOutputFormatRequest(kWidth / 2, kHeight / 2, kDefaultFps); 420 for (int i = 0; i < 10; ++i) 421 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 422 423 // Verify resolution change after adaptation. 424 VerifyAdaptedResolution(adapter_wrapper_->GetStats(), kWidth, kHeight, 425 kWidth / 2, kHeight / 2); 426 } 427 428 // Drop all frames for resolution 0x0. 429 TEST_P(VideoAdapterTest, DropAllFrames) { 430 OnOutputFormatRequest(kWidth * 0, kHeight * 0, kDefaultFps); 431 for (int i = 0; i < 10; ++i) 432 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 433 434 // Verify all frames are dropped. 435 VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); 436 EXPECT_GE(stats.captured_frames, 10); 437 EXPECT_EQ(stats.captured_frames, stats.dropped_frames); 438 } 439 440 TEST_P(VideoAdapterTest, TestOnOutputFormatRequest) { 441 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 442 &cropped_height_, &out_width_, 443 &out_height_)); 444 EXPECT_EQ(640, cropped_width_); 445 EXPECT_EQ(400, cropped_height_); 446 EXPECT_EQ(640, out_width_); 447 EXPECT_EQ(400, out_height_); 448 449 // Format request 640x400. 450 OnOutputFormatRequest(640, 400, std::nullopt); 451 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 452 &cropped_height_, &out_width_, 453 &out_height_)); 454 EXPECT_EQ(640, cropped_width_); 455 EXPECT_EQ(400, cropped_height_); 456 EXPECT_EQ(640, out_width_); 457 EXPECT_EQ(400, out_height_); 458 459 // Request 1280x720, higher than input, but aspect 16:9. Expect cropping but 460 // no scaling. 461 OnOutputFormatRequest(1280, 720, std::nullopt); 462 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 463 &cropped_height_, &out_width_, 464 &out_height_)); 465 EXPECT_EQ(640, cropped_width_); 466 EXPECT_EQ(360, cropped_height_); 467 EXPECT_EQ(640, out_width_); 468 EXPECT_EQ(360, out_height_); 469 470 // Request 0x0. 471 OnOutputFormatRequest(0, 0, std::nullopt); 472 EXPECT_FALSE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 473 &cropped_height_, &out_width_, 474 &out_height_)); 475 476 // Request 320x200. Expect scaling, but no cropping. 477 OnOutputFormatRequest(320, 200, std::nullopt); 478 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 479 &cropped_height_, &out_width_, 480 &out_height_)); 481 EXPECT_EQ(640, cropped_width_); 482 EXPECT_EQ(400, cropped_height_); 483 EXPECT_EQ(320, out_width_); 484 EXPECT_EQ(200, out_height_); 485 486 // Request resolution close to 2/3 scale. Expect adapt down. Scaling to 2/3 487 // is not optimized and not allowed, therefore 1/2 scaling will be used 488 // instead. 489 OnOutputFormatRequest(424, 265, std::nullopt); 490 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 491 &cropped_height_, &out_width_, 492 &out_height_)); 493 EXPECT_EQ(640, cropped_width_); 494 EXPECT_EQ(400, cropped_height_); 495 EXPECT_EQ(320, out_width_); 496 EXPECT_EQ(200, out_height_); 497 498 // Request resolution of 3 / 8. Expect adapt down. 499 OnOutputFormatRequest(640 * 3 / 8, 400 * 3 / 8, std::nullopt); 500 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 501 &cropped_height_, &out_width_, 502 &out_height_)); 503 EXPECT_EQ(640, cropped_width_); 504 EXPECT_EQ(400, cropped_height_); 505 EXPECT_EQ(640 * 3 / 8, out_width_); 506 EXPECT_EQ(400 * 3 / 8, out_height_); 507 508 // Switch back up. Expect adapt. 509 OnOutputFormatRequest(320, 200, std::nullopt); 510 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 511 &cropped_height_, &out_width_, 512 &out_height_)); 513 EXPECT_EQ(640, cropped_width_); 514 EXPECT_EQ(400, cropped_height_); 515 EXPECT_EQ(320, out_width_); 516 EXPECT_EQ(200, out_height_); 517 518 // Format request 480x300. 519 OnOutputFormatRequest(480, 300, std::nullopt); 520 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 521 &cropped_height_, &out_width_, 522 &out_height_)); 523 EXPECT_EQ(640, cropped_width_); 524 EXPECT_EQ(400, cropped_height_); 525 EXPECT_EQ(480, out_width_); 526 EXPECT_EQ(300, out_height_); 527 } 528 529 TEST_P(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { 530 // Start at HD. 531 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 532 &cropped_height_, &out_width_, 533 &out_height_)); 534 EXPECT_EQ(1280, cropped_width_); 535 EXPECT_EQ(720, cropped_height_); 536 EXPECT_EQ(1280, out_width_); 537 EXPECT_EQ(720, out_height_); 538 539 // Format request for VGA. 540 OnOutputFormatRequest(640, 360, std::nullopt); 541 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 542 &cropped_height_, &out_width_, 543 &out_height_)); 544 EXPECT_EQ(1280, cropped_width_); 545 EXPECT_EQ(720, cropped_height_); 546 EXPECT_EQ(640, out_width_); 547 EXPECT_EQ(360, out_height_); 548 549 // Now, the camera reopens at VGA. 550 // Both the frame and the output format should be 640x360. 551 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, 552 &cropped_height_, &out_width_, 553 &out_height_)); 554 EXPECT_EQ(640, cropped_width_); 555 EXPECT_EQ(360, cropped_height_); 556 EXPECT_EQ(640, out_width_); 557 EXPECT_EQ(360, out_height_); 558 559 // And another view request comes in for 640x360, which should have no 560 // real impact. 561 OnOutputFormatRequest(640, 360, std::nullopt); 562 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, 563 &cropped_height_, &out_width_, 564 &out_height_)); 565 EXPECT_EQ(640, cropped_width_); 566 EXPECT_EQ(360, cropped_height_); 567 EXPECT_EQ(640, out_width_); 568 EXPECT_EQ(360, out_height_); 569 } 570 571 TEST_P(VideoAdapterTest, TestVgaWidth) { 572 // Requested output format is 640x360. 573 OnOutputFormatRequest(640, 360, std::nullopt); 574 575 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 576 &cropped_height_, &out_width_, 577 &out_height_)); 578 // Expect cropping. 579 EXPECT_EQ(640, cropped_width_); 580 EXPECT_EQ(360, cropped_height_); 581 EXPECT_EQ(640, out_width_); 582 EXPECT_EQ(360, out_height_); 583 584 // But if frames come in at 640x360, we shouldn't adapt them down. 585 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, 586 &cropped_height_, &out_width_, 587 &out_height_)); 588 EXPECT_EQ(640, cropped_width_); 589 EXPECT_EQ(360, cropped_height_); 590 EXPECT_EQ(640, out_width_); 591 EXPECT_EQ(360, out_height_); 592 593 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 594 &cropped_height_, &out_width_, 595 &out_height_)); 596 EXPECT_EQ(640, cropped_width_); 597 EXPECT_EQ(360, cropped_height_); 598 EXPECT_EQ(640, out_width_); 599 EXPECT_EQ(360, out_height_); 600 } 601 602 TEST_P(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { 603 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 604 &cropped_height_, &out_width_, 605 &out_height_)); 606 EXPECT_EQ(1280, cropped_width_); 607 EXPECT_EQ(720, cropped_height_); 608 EXPECT_EQ(1280, out_width_); 609 EXPECT_EQ(720, out_height_); 610 611 // Adapt down one step. 612 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 1280 * 720 - 1, 613 std::numeric_limits<int>::max())); 614 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 615 &cropped_height_, &out_width_, 616 &out_height_)); 617 EXPECT_EQ(1280, cropped_width_); 618 EXPECT_EQ(720, cropped_height_); 619 EXPECT_EQ(960, out_width_); 620 EXPECT_EQ(540, out_height_); 621 622 // Adapt down one step more. 623 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 960 * 540 - 1, 624 std::numeric_limits<int>::max())); 625 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 626 &cropped_height_, &out_width_, 627 &out_height_)); 628 EXPECT_EQ(1280, cropped_width_); 629 EXPECT_EQ(720, cropped_height_); 630 EXPECT_EQ(640, out_width_); 631 EXPECT_EQ(360, out_height_); 632 633 // Adapt down one step more. 634 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 640 * 360 - 1, 635 std::numeric_limits<int>::max())); 636 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 637 &cropped_height_, &out_width_, 638 &out_height_)); 639 EXPECT_EQ(1280, cropped_width_); 640 EXPECT_EQ(720, cropped_height_); 641 EXPECT_EQ(480, out_width_); 642 EXPECT_EQ(270, out_height_); 643 644 // Adapt up one step. 645 adapter_.OnSinkWants( 646 BuildSinkWants(640 * 360, 960 * 540, std::numeric_limits<int>::max())); 647 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 648 &cropped_height_, &out_width_, 649 &out_height_)); 650 EXPECT_EQ(1280, cropped_width_); 651 EXPECT_EQ(720, cropped_height_); 652 EXPECT_EQ(640, out_width_); 653 EXPECT_EQ(360, out_height_); 654 655 // Adapt up one step more. 656 adapter_.OnSinkWants( 657 BuildSinkWants(960 * 540, 1280 * 720, std::numeric_limits<int>::max())); 658 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 659 &cropped_height_, &out_width_, 660 &out_height_)); 661 EXPECT_EQ(1280, cropped_width_); 662 EXPECT_EQ(720, cropped_height_); 663 EXPECT_EQ(960, out_width_); 664 EXPECT_EQ(540, out_height_); 665 666 // Adapt up one step more. 667 adapter_.OnSinkWants( 668 BuildSinkWants(1280 * 720, 1920 * 1080, std::numeric_limits<int>::max())); 669 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 670 &cropped_height_, &out_width_, 671 &out_height_)); 672 EXPECT_EQ(1280, cropped_width_); 673 EXPECT_EQ(720, cropped_height_); 674 EXPECT_EQ(1280, out_width_); 675 EXPECT_EQ(720, out_height_); 676 } 677 678 TEST_P(VideoAdapterTest, TestOnResolutionRequestMaxZero) { 679 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 680 &cropped_height_, &out_width_, 681 &out_height_)); 682 EXPECT_EQ(1280, cropped_width_); 683 EXPECT_EQ(720, cropped_height_); 684 EXPECT_EQ(1280, out_width_); 685 EXPECT_EQ(720, out_height_); 686 687 adapter_.OnSinkWants( 688 BuildSinkWants(std::nullopt, 0, std::numeric_limits<int>::max())); 689 EXPECT_FALSE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 690 &cropped_height_, &out_width_, 691 &out_height_)); 692 } 693 694 TEST_P(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { 695 // Large step down. 696 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 640 * 360 - 1, 697 std::numeric_limits<int>::max())); 698 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 699 &cropped_height_, &out_width_, 700 &out_height_)); 701 EXPECT_EQ(1280, cropped_width_); 702 EXPECT_EQ(720, cropped_height_); 703 EXPECT_EQ(480, out_width_); 704 EXPECT_EQ(270, out_height_); 705 706 // Large step up. 707 adapter_.OnSinkWants( 708 BuildSinkWants(1280 * 720, 1920 * 1080, std::numeric_limits<int>::max())); 709 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 710 &cropped_height_, &out_width_, 711 &out_height_)); 712 EXPECT_EQ(1280, cropped_width_); 713 EXPECT_EQ(720, cropped_height_); 714 EXPECT_EQ(1280, out_width_); 715 EXPECT_EQ(720, out_height_); 716 } 717 718 TEST_P(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { 719 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 640 * 360 - 1, 720 std::numeric_limits<int>::max())); 721 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 722 &cropped_height_, &out_width_, 723 &out_height_)); 724 EXPECT_EQ(1280, cropped_width_); 725 EXPECT_EQ(720, cropped_height_); 726 EXPECT_EQ(480, out_width_); 727 EXPECT_EQ(270, out_height_); 728 729 OnOutputFormatRequest(640, 360, std::nullopt); 730 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 731 &cropped_height_, &out_width_, 732 &out_height_)); 733 EXPECT_EQ(1280, cropped_width_); 734 EXPECT_EQ(720, cropped_height_); 735 EXPECT_EQ(480, out_width_); 736 EXPECT_EQ(270, out_height_); 737 738 adapter_.OnSinkWants( 739 BuildSinkWants(std::nullopt, 960 * 720, std::numeric_limits<int>::max())); 740 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 741 &cropped_height_, &out_width_, 742 &out_height_)); 743 EXPECT_EQ(1280, cropped_width_); 744 EXPECT_EQ(720, cropped_height_); 745 EXPECT_EQ(640, out_width_); 746 EXPECT_EQ(360, out_height_); 747 } 748 749 TEST_P(VideoAdapterTest, TestOnResolutionRequestReset) { 750 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 751 &cropped_height_, &out_width_, 752 &out_height_)); 753 EXPECT_EQ(1280, cropped_width_); 754 EXPECT_EQ(720, cropped_height_); 755 EXPECT_EQ(1280, out_width_); 756 EXPECT_EQ(720, out_height_); 757 758 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 640 * 360 - 1, 759 std::numeric_limits<int>::max())); 760 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 761 &cropped_height_, &out_width_, 762 &out_height_)); 763 EXPECT_EQ(1280, cropped_width_); 764 EXPECT_EQ(720, cropped_height_); 765 EXPECT_EQ(480, out_width_); 766 EXPECT_EQ(270, out_height_); 767 768 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 769 std::numeric_limits<int>::max(), 770 std::numeric_limits<int>::max())); 771 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 772 &cropped_height_, &out_width_, 773 &out_height_)); 774 EXPECT_EQ(1280, cropped_width_); 775 EXPECT_EQ(720, cropped_height_); 776 EXPECT_EQ(1280, out_width_); 777 EXPECT_EQ(720, out_height_); 778 } 779 780 TEST_P(VideoAdapterTest, TestOnOutputFormatRequestResolutionReset) { 781 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 782 &cropped_height_, &out_width_, 783 &out_height_)); 784 EXPECT_EQ(1280, cropped_width_); 785 EXPECT_EQ(720, cropped_height_); 786 EXPECT_EQ(1280, out_width_); 787 EXPECT_EQ(720, out_height_); 788 789 adapter_.OnOutputFormatRequest(std::nullopt, 640 * 360 - 1, std::nullopt); 790 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 791 &cropped_height_, &out_width_, 792 &out_height_)); 793 EXPECT_EQ(1280, cropped_width_); 794 EXPECT_EQ(720, cropped_height_); 795 EXPECT_EQ(480, out_width_); 796 EXPECT_EQ(270, out_height_); 797 798 adapter_.OnOutputFormatRequest(std::nullopt, std::nullopt, std::nullopt); 799 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, 800 &cropped_height_, &out_width_, 801 &out_height_)); 802 EXPECT_EQ(1280, cropped_width_); 803 EXPECT_EQ(720, cropped_height_); 804 EXPECT_EQ(1280, out_width_); 805 EXPECT_EQ(720, out_height_); 806 } 807 808 TEST_P(VideoAdapterTest, TestOnOutputFormatRequestFpsReset) { 809 OnOutputFormatRequest(kWidth, kHeight, kDefaultFps / 2); 810 for (int i = 0; i < 10; ++i) 811 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 812 813 // Verify frame drop. 814 const int dropped_frames = adapter_wrapper_->GetStats().dropped_frames; 815 EXPECT_GT(dropped_frames, 0); 816 817 // Reset frame rate. 818 OnOutputFormatRequest(kWidth, kHeight, std::nullopt); 819 for (int i = 0; i < 20; ++i) 820 adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); 821 822 // Verify no frame drop after reset. 823 EXPECT_EQ(dropped_frames, adapter_wrapper_->GetStats().dropped_frames); 824 } 825 826 TEST_P(VideoAdapterTest, RequestAspectRatio) { 827 // Request aspect ratio 320/180 (16:9), smaller than input, but no resolution 828 // limit. Expect cropping but no scaling. 829 adapter_.OnOutputFormatRequest(std::make_pair(320, 180), std::nullopt, 830 std::nullopt); 831 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 832 &cropped_height_, &out_width_, 833 &out_height_)); 834 EXPECT_EQ(640, cropped_width_); 835 EXPECT_EQ(360, cropped_height_); 836 EXPECT_EQ(640, out_width_); 837 EXPECT_EQ(360, out_height_); 838 839 adapter_.OnOutputFormatRequest(std::make_pair(1280, 720), 1280 * 720 - 1, 840 std::nullopt); 841 EXPECT_TRUE(adapter_.AdaptFrameResolution(2592, 1944, 0, &cropped_width_, 842 &cropped_height_, &out_width_, 843 &out_height_)); 844 EXPECT_EQ(2592, cropped_width_); 845 EXPECT_EQ(1458, cropped_height_); 846 EXPECT_EQ(1152, out_width_); 847 EXPECT_EQ(648, out_height_); 848 } 849 850 TEST_P(VideoAdapterTest, RequestAspectRatioWithDifferentOrientation) { 851 // Request 720x1280, higher than input, but aspect 16:9. Orientation should 852 // not matter, expect cropping but no scaling. 853 OnOutputFormatRequest(720, 1280, std::nullopt); 854 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 855 &cropped_height_, &out_width_, 856 &out_height_)); 857 EXPECT_EQ(640, cropped_width_); 858 EXPECT_EQ(360, cropped_height_); 859 EXPECT_EQ(640, out_width_); 860 EXPECT_EQ(360, out_height_); 861 } 862 863 TEST_P(VideoAdapterTest, InvalidAspectRatioIgnored) { 864 // Request aspect ratio 320/0. Expect no cropping. 865 adapter_.OnOutputFormatRequest(std::make_pair(320, 0), std::nullopt, 866 std::nullopt); 867 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, 868 &cropped_height_, &out_width_, 869 &out_height_)); 870 EXPECT_EQ(640, cropped_width_); 871 EXPECT_EQ(400, cropped_height_); 872 EXPECT_EQ(640, out_width_); 873 EXPECT_EQ(400, out_height_); 874 } 875 876 TEST_P(VideoAdapterTest, TestCroppingWithResolutionRequest) { 877 // Ask for 640x360 (16:9 aspect). 878 OnOutputFormatRequest(640, 360, std::nullopt); 879 // Send 640x480 (4:3 aspect). 880 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 881 &cropped_height_, &out_width_, 882 &out_height_)); 883 // Expect cropping to 16:9 format and no scaling. 884 EXPECT_EQ(640, cropped_width_); 885 EXPECT_EQ(360, cropped_height_); 886 EXPECT_EQ(640, out_width_); 887 EXPECT_EQ(360, out_height_); 888 889 // Adapt down one step. 890 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 640 * 360 - 1, 891 std::numeric_limits<int>::max())); 892 // Expect cropping to 16:9 format and 3/4 scaling. 893 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 894 &cropped_height_, &out_width_, 895 &out_height_)); 896 EXPECT_EQ(640, cropped_width_); 897 EXPECT_EQ(360, cropped_height_); 898 EXPECT_EQ(480, out_width_); 899 EXPECT_EQ(270, out_height_); 900 901 // Adapt down one step more. 902 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 480 * 270 - 1, 903 std::numeric_limits<int>::max())); 904 // Expect cropping to 16:9 format and 1/2 scaling. 905 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 906 &cropped_height_, &out_width_, 907 &out_height_)); 908 EXPECT_EQ(640, cropped_width_); 909 EXPECT_EQ(360, cropped_height_); 910 EXPECT_EQ(320, out_width_); 911 EXPECT_EQ(180, out_height_); 912 913 // Adapt up one step. 914 adapter_.OnSinkWants( 915 BuildSinkWants(480 * 270, 640 * 360, std::numeric_limits<int>::max())); 916 // Expect cropping to 16:9 format and 3/4 scaling. 917 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 918 &cropped_height_, &out_width_, 919 &out_height_)); 920 EXPECT_EQ(640, cropped_width_); 921 EXPECT_EQ(360, cropped_height_); 922 EXPECT_EQ(480, out_width_); 923 EXPECT_EQ(270, out_height_); 924 925 // Adapt up one step more. 926 adapter_.OnSinkWants( 927 BuildSinkWants(640 * 360, 960 * 540, std::numeric_limits<int>::max())); 928 // Expect cropping to 16:9 format and no scaling. 929 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 930 &cropped_height_, &out_width_, 931 &out_height_)); 932 EXPECT_EQ(640, cropped_width_); 933 EXPECT_EQ(360, cropped_height_); 934 EXPECT_EQ(640, out_width_); 935 EXPECT_EQ(360, out_height_); 936 937 // Try to adapt up one step more. 938 adapter_.OnSinkWants( 939 BuildSinkWants(960 * 540, 1280 * 720, std::numeric_limits<int>::max())); 940 // Expect cropping to 16:9 format and no scaling. 941 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 942 &cropped_height_, &out_width_, 943 &out_height_)); 944 EXPECT_EQ(640, cropped_width_); 945 EXPECT_EQ(360, cropped_height_); 946 EXPECT_EQ(640, out_width_); 947 EXPECT_EQ(360, out_height_); 948 } 949 950 TEST_P(VideoAdapterTest, TestCroppingOddResolution) { 951 // Ask for 640x360 (16:9 aspect), with 3/16 scaling. 952 OnOutputFormatRequest(640, 360, std::nullopt); 953 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 640 * 360 * 3 / 16 * 3 / 16, 954 std::numeric_limits<int>::max())); 955 956 // Send 640x480 (4:3 aspect). 957 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, 958 &cropped_height_, &out_width_, 959 &out_height_)); 960 961 // Instead of getting the exact aspect ratio with cropped resolution 640x360, 962 // the resolution should be adjusted to get a perfect scale factor instead. 963 EXPECT_EQ(640, cropped_width_); 964 EXPECT_EQ(368, cropped_height_); 965 EXPECT_EQ(120, out_width_); 966 EXPECT_EQ(69, out_height_); 967 } 968 969 TEST_P(VideoAdapterTest, TestAdaptToVerySmallResolution) { 970 // Ask for 1920x1080 (16:9 aspect), with 1/16 scaling. 971 const int w = 1920; 972 const int h = 1080; 973 OnOutputFormatRequest(w, h, std::nullopt); 974 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, w * h * 1 / 16 * 1 / 16, 975 std::numeric_limits<int>::max())); 976 977 // Send 1920x1080 (16:9 aspect). 978 EXPECT_TRUE(adapter_.AdaptFrameResolution( 979 w, h, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_)); 980 981 // Instead of getting the exact aspect ratio with cropped resolution 1920x1080 982 // the resolution should be adjusted to get a perfect scale factor instead. 983 EXPECT_EQ(1920, cropped_width_); 984 EXPECT_EQ(1072, cropped_height_); 985 EXPECT_EQ(120, out_width_); 986 EXPECT_EQ(67, out_height_); 987 988 // Adapt back up one step to 3/32. 989 adapter_.OnSinkWants(BuildSinkWants(w * h * 3 / 32 * 3 / 32, 990 w * h * 1 / 8 * 1 / 8, 991 std::numeric_limits<int>::max())); 992 993 // Send 1920x1080 (16:9 aspect). 994 EXPECT_TRUE(adapter_.AdaptFrameResolution( 995 w, h, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_)); 996 997 EXPECT_EQ(160, out_width_); 998 EXPECT_EQ(90, out_height_); 999 } 1000 1001 TEST_P(VideoAdapterTest, AdaptFrameResolutionDropWithResolutionRequest) { 1002 OnOutputFormatRequest(0, 0, kDefaultFps); 1003 EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, 1004 &cropped_width_, &cropped_height_, 1005 &out_width_, &out_height_)); 1006 1007 adapter_.OnSinkWants(BuildSinkWants(960 * 540, 1008 std::numeric_limits<int>::max(), 1009 std::numeric_limits<int>::max())); 1010 1011 // Still expect all frames to be dropped 1012 EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, 1013 &cropped_width_, &cropped_height_, 1014 &out_width_, &out_height_)); 1015 1016 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 640 * 480 - 1, 1017 std::numeric_limits<int>::max())); 1018 1019 // Still expect all frames to be dropped 1020 EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, 1021 &cropped_width_, &cropped_height_, 1022 &out_width_, &out_height_)); 1023 } 1024 1025 // Test that we will adapt to max given a target pixel count close to max. 1026 TEST_P(VideoAdapterTest, TestAdaptToMax) { 1027 OnOutputFormatRequest(640, 360, kDefaultFps); 1028 adapter_.OnSinkWants(BuildSinkWants(640 * 360 - 1 /* target */, 1029 std::numeric_limits<int>::max(), 1030 std::numeric_limits<int>::max())); 1031 1032 EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, 1033 &cropped_height_, &out_width_, 1034 &out_height_)); 1035 EXPECT_EQ(640, out_width_); 1036 EXPECT_EQ(360, out_height_); 1037 } 1038 1039 // Test adjusting to 16:9 in landscape, and 9:16 in portrait. 1040 TEST(VideoAdapterTestMultipleOrientation, TestNormal) { 1041 VideoAdapter video_adapter; 1042 video_adapter.OnOutputFormatRequest(std::make_pair(640, 360), 640 * 360, 1043 std::make_pair(360, 640), 360 * 640, 30); 1044 1045 int cropped_width; 1046 int cropped_height; 1047 int out_width; 1048 int out_height; 1049 EXPECT_TRUE(video_adapter.AdaptFrameResolution( 1050 /* in_width= */ 640, /* in_height= */ 480, /* in_timestamp_ns= */ 0, 1051 &cropped_width, &cropped_height, &out_width, &out_height)); 1052 EXPECT_EQ(640, cropped_width); 1053 EXPECT_EQ(360, cropped_height); 1054 EXPECT_EQ(640, out_width); 1055 EXPECT_EQ(360, out_height); 1056 1057 EXPECT_TRUE(video_adapter.AdaptFrameResolution( 1058 /* in_width= */ 480, /* in_height= */ 640, 1059 /* in_timestamp_ns= */ kNumNanosecsPerSec / 30, &cropped_width, 1060 &cropped_height, &out_width, &out_height)); 1061 EXPECT_EQ(360, cropped_width); 1062 EXPECT_EQ(640, cropped_height); 1063 EXPECT_EQ(360, out_width); 1064 EXPECT_EQ(640, out_height); 1065 } 1066 1067 // Force output to be 9:16, even for landscape input. 1068 TEST(VideoAdapterTestMultipleOrientation, TestForcePortrait) { 1069 VideoAdapter video_adapter; 1070 video_adapter.OnOutputFormatRequest(std::make_pair(360, 640), 640 * 360, 1071 std::make_pair(360, 640), 360 * 640, 30); 1072 1073 int cropped_width; 1074 int cropped_height; 1075 int out_width; 1076 int out_height; 1077 EXPECT_TRUE(video_adapter.AdaptFrameResolution( 1078 /* in_width= */ 640, /* in_height= */ 480, /* in_timestamp_ns= */ 0, 1079 &cropped_width, &cropped_height, &out_width, &out_height)); 1080 EXPECT_EQ(270, cropped_width); 1081 EXPECT_EQ(480, cropped_height); 1082 EXPECT_EQ(270, out_width); 1083 EXPECT_EQ(480, out_height); 1084 1085 EXPECT_TRUE(video_adapter.AdaptFrameResolution( 1086 /* in_width= */ 480, /* in_height= */ 640, 1087 /* in_timestamp_ns= */ kNumNanosecsPerSec / 30, &cropped_width, 1088 &cropped_height, &out_width, &out_height)); 1089 EXPECT_EQ(360, cropped_width); 1090 EXPECT_EQ(640, cropped_height); 1091 EXPECT_EQ(360, out_width); 1092 EXPECT_EQ(640, out_height); 1093 } 1094 1095 TEST_P(VideoAdapterTest, AdaptResolutionInStepsFirst3_4) { 1096 OnOutputFormatRequest(kWidth, kHeight, std::nullopt); // 16:9 aspect. 1097 1098 // Scale factors: 3/4, 2/3, 3/4, 2/3, ... 1099 // Scale : 3/4, 1/2, 3/8, 1/4, 3/16, 1/8. 1100 const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160}; 1101 const int kExpectedHeights[] = {540, 360, 270, 180, 135, 90}; 1102 1103 int request_width = kWidth; 1104 int request_height = kHeight; 1105 1106 for (size_t i = 0; i < std::size(kExpectedWidths); ++i) { 1107 // Adapt down one step. 1108 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 1109 request_width * request_height - 1, 1110 std::numeric_limits<int>::max())); 1111 EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, 1112 &cropped_width_, &cropped_height_, 1113 &out_width_, &out_height_)); 1114 EXPECT_EQ(kExpectedWidths[i], out_width_); 1115 EXPECT_EQ(kExpectedHeights[i], out_height_); 1116 request_width = out_width_; 1117 request_height = out_height_; 1118 } 1119 } 1120 1121 TEST_P(VideoAdapterTest, AdaptResolutionInStepsFirst2_3) { 1122 const int kWidth1080p = 1920; 1123 const int kHeight1080p = 1080; 1124 OnOutputFormatRequest(kWidth1080p, kHeight1080p, 1125 std::nullopt); // 16:9 aspect. 1126 1127 // Scale factors: 2/3, 3/4, 2/3, 3/4, ... 1128 // Scale: 2/3, 1/2, 1/3, 1/4, 1/6, 1/8, 1/12. 1129 const int kExpectedWidths[] = {1280, 960, 640, 480, 320, 240, 160}; 1130 const int kExpectedHeights[] = {720, 540, 360, 270, 180, 135, 90}; 1131 1132 int request_width = kWidth1080p; 1133 int request_height = kHeight1080p; 1134 1135 for (size_t i = 0; i < std::size(kExpectedWidths); ++i) { 1136 // Adapt down one step. 1137 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 1138 request_width * request_height - 1, 1139 std::numeric_limits<int>::max())); 1140 EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth1080p, kHeight1080p, 0, 1141 &cropped_width_, &cropped_height_, 1142 &out_width_, &out_height_)); 1143 EXPECT_EQ(kExpectedWidths[i], out_width_); 1144 EXPECT_EQ(kExpectedHeights[i], out_height_); 1145 request_width = out_width_; 1146 request_height = out_height_; 1147 } 1148 } 1149 1150 TEST_P(VideoAdapterTest, AdaptResolutionInStepsFirst2x2_3) { 1151 const int kWidth1080p4to3 = 1440; 1152 const int kHeight1080p4to3 = 1080; 1153 OnOutputFormatRequest(kWidth1080p4to3, kHeight1080p4to3, 1154 std::nullopt); // 4:3 aspect. 1155 1156 // Scale factors: 2/3, 2/3, 3/4, 2/3, 3/4, ... 1157 // Scale : 2/3, 4/9, 1/3, 2/9, 1/6, 1/9, 1/12, 1/18, 1/24, 1/36. 1158 const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160, 120, 80, 60, 40}; 1159 const int kExpectedHeights[] = {720, 480, 360, 240, 180, 120, 90, 60, 45, 30}; 1160 1161 int request_width = kWidth1080p4to3; 1162 int request_height = kHeight1080p4to3; 1163 1164 for (size_t i = 0; i < std::size(kExpectedWidths); ++i) { 1165 // Adapt down one step. 1166 adapter_.OnSinkWants(BuildSinkWants(std::nullopt, 1167 request_width * request_height - 1, 1168 std::numeric_limits<int>::max())); 1169 EXPECT_TRUE(adapter_.AdaptFrameResolution( 1170 kWidth1080p4to3, kHeight1080p4to3, 0, &cropped_width_, &cropped_height_, 1171 &out_width_, &out_height_)); 1172 EXPECT_EQ(kExpectedWidths[i], out_width_); 1173 EXPECT_EQ(kExpectedHeights[i], out_height_); 1174 request_width = out_width_; 1175 request_height = out_height_; 1176 } 1177 } 1178 1179 TEST_P(VideoAdapterTest, AdaptResolutionWithSinkAlignment) { 1180 constexpr int kSourceWidth = 1280; 1181 constexpr int kSourceHeight = 720; 1182 constexpr int kSourceFramerate = 30; 1183 constexpr int kRequestedWidth = 480; 1184 constexpr int kRequestedHeight = 270; 1185 constexpr int kRequestedFramerate = 30; 1186 1187 OnOutputFormatRequest(kRequestedWidth, kRequestedHeight, kRequestedFramerate); 1188 1189 int frame_num = 1; 1190 for (const int sink_alignment : {2, 3, 4, 5}) { 1191 adapter_.OnSinkWants( 1192 BuildSinkWants(std::nullopt, std::numeric_limits<int>::max(), 1193 std::numeric_limits<int>::max(), sink_alignment)); 1194 EXPECT_TRUE(adapter_.AdaptFrameResolution( 1195 kSourceWidth, kSourceHeight, 1196 frame_num * kNumNanosecsPerSec / kSourceFramerate, &cropped_width_, 1197 &cropped_height_, &out_width_, &out_height_)); 1198 EXPECT_EQ(out_width_ % sink_alignment, 0); 1199 EXPECT_EQ(out_height_ % sink_alignment, 0); 1200 1201 ++frame_num; 1202 } 1203 } 1204 1205 // Verify the cases the OnOutputFormatRequest is ignored and 1206 // scale_resolution_down_to is used instead. 1207 TEST_P(VideoAdapterTest, 1208 UseScaleResolutionDownToInsteadOfOnOutputFormatRequest) { 1209 { 1210 // Both new and old API active => Use OnOutputFormatRequest 1211 OnOutputFormatRequest(640, 360, kDefaultFps); 1212 adapter_.OnSinkWants( 1213 BuildSinkWants(Resolution{.width = 960, .height = 540}, 1214 /* any_active_without_requested_resolution= */ true)); 1215 1216 EXPECT_THAT( 1217 AdaptFrameResolution(/* input frame */ {.width = 1280, .height = 720}) 1218 .first, 1219 Eq(Resolution{.width = 640, .height = 360})); 1220 } 1221 { 1222 // New API active, old API inactive, ignore OnOutputFormatRequest and use 1223 // scale_resolution_down_to. 1224 OnOutputFormatRequest(640, 360, kDefaultFps); 1225 adapter_.OnSinkWants( 1226 BuildSinkWants(Resolution{.width = 960, .height = 540}, 1227 /* any_active_without_requested_resolution= */ false)); 1228 1229 EXPECT_THAT( 1230 AdaptFrameResolution(/* input frame */ {.width = 1280, .height = 720}) 1231 .first, 1232 Eq(Resolution{.width = 960, .height = 540})); 1233 } 1234 1235 { 1236 // New API inactive, old API inactive, use OnOutputFormatRequest. 1237 OnOutputFormatRequest(640, 360, kDefaultFps); 1238 adapter_.OnSinkWants( 1239 BuildSinkWants(std::nullopt, 1240 /* any_active_without_requested_resolution= */ false)); 1241 1242 EXPECT_THAT( 1243 AdaptFrameResolution(/* input frame */ {.width = 1280, .height = 720}) 1244 .first, 1245 Eq(Resolution{.width = 640, .height = 360})); 1246 } 1247 1248 { 1249 // New API active, old API inactive, remember OnOutputFormatRequest. 1250 OnOutputFormatRequest(640, 360, kDefaultFps); 1251 adapter_.OnSinkWants( 1252 BuildSinkWants(Resolution{.width = 960, .height = 540}, 1253 /* any_active_without_requested_resolution= */ false)); 1254 1255 EXPECT_THAT( 1256 AdaptFrameResolution(/* input frame */ {.width = 1280, .height = 720}) 1257 .first, 1258 Eq(Resolution{.width = 960, .height = 540})); 1259 1260 // This is ignored since there is not any active NOT using 1261 // scale_resolution_down_to. 1262 OnOutputFormatRequest(320, 180, kDefaultFps); 1263 1264 EXPECT_THAT( 1265 AdaptFrameResolution(/* input frame */ {.width = 1280, .height = 720}) 1266 .first, 1267 Eq(Resolution{.width = 960, .height = 540})); 1268 1269 // Disable new API => fallback to last OnOutputFormatRequest. 1270 adapter_.OnSinkWants( 1271 BuildSinkWants(std::nullopt, 1272 /* any_active_without_requested_resolution= */ false)); 1273 1274 EXPECT_THAT( 1275 AdaptFrameResolution(/* input frame */ {.width = 1280, .height = 720}) 1276 .first, 1277 Eq(Resolution{.width = 320, .height = 180})); 1278 } 1279 } 1280 1281 TEST_P(VideoAdapterTest, ScaleResolutionDownToIsOrientationAgnostic) { 1282 // Request 1280x720 when frame is 720x1280. 1283 { 1284 adapter_.OnSinkWants( 1285 BuildSinkWants(Resolution{.width = 1280, .height = 720}, 1286 /* any_active_without_requested_resolution= */ false)); 1287 1288 EXPECT_THAT( 1289 AdaptFrameResolution(/* input frame */ {.width = 720, .height = 1280}) 1290 .first, 1291 Eq(Resolution{.width = 720, .height = 1280})); 1292 } 1293 // Request 720x1280 when frame is 1280x720. 1294 { 1295 adapter_.OnSinkWants( 1296 BuildSinkWants(Resolution{.width = 720, .height = 1280}, 1297 /* any_active_without_requested_resolution= */ false)); 1298 1299 EXPECT_THAT( 1300 AdaptFrameResolution(/* input frame */ {.width = 1280, .height = 720}) 1301 .first, 1302 Eq(Resolution{.width = 1280, .height = 720})); 1303 } 1304 } 1305 1306 TEST_P(VideoAdapterTest, ScaleResolutionDownToMaintainsAspectRatio) { 1307 // Request 720x720. 1308 adapter_.OnSinkWants( 1309 BuildSinkWants(Resolution{.width = 720, .height = 720}, 1310 /* any_active_without_requested_resolution= */ false)); 1311 1312 // A 1280x720 frame restricted to 720x720 produces 720x405. 1313 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, /*in_timestamp_ns=*/0, 1314 &cropped_width_, &cropped_height_, 1315 &out_width_, &out_height_)); 1316 EXPECT_EQ(out_width_, 720); 1317 EXPECT_EQ(out_height_, 405); 1318 // No cropping needed (coverage for https://crbug.com/webrtc/369865055). 1319 EXPECT_EQ(cropped_width_, 1280); 1320 EXPECT_EQ(cropped_height_, 720); 1321 } 1322 1323 class VideoAdapterWithSourceAlignmentTest : public VideoAdapterTest { 1324 protected: 1325 static constexpr int kSourceResolutionAlignment = 7; 1326 1327 VideoAdapterWithSourceAlignmentTest() 1328 : VideoAdapterTest(/*field_trials=*/"", kSourceResolutionAlignment) {} 1329 }; 1330 1331 TEST_P(VideoAdapterWithSourceAlignmentTest, AdaptResolution) { 1332 constexpr int kSourceWidth = 1280; 1333 constexpr int kSourceHeight = 720; 1334 constexpr int kRequestedWidth = 480; 1335 constexpr int kRequestedHeight = 270; 1336 constexpr int kRequestedFramerate = 30; 1337 1338 OnOutputFormatRequest(kRequestedWidth, kRequestedHeight, kRequestedFramerate); 1339 1340 EXPECT_TRUE(adapter_.AdaptFrameResolution( 1341 kSourceWidth, kSourceHeight, /*in_timestamp_ns=*/0, &cropped_width_, 1342 &cropped_height_, &out_width_, &out_height_)); 1343 EXPECT_EQ(out_width_ % kSourceResolutionAlignment, 0); 1344 EXPECT_EQ(out_height_ % kSourceResolutionAlignment, 0); 1345 } 1346 1347 TEST_P(VideoAdapterWithSourceAlignmentTest, AdaptResolutionWithSinkAlignment) { 1348 constexpr int kSourceWidth = 1280; 1349 constexpr int kSourceHeight = 720; 1350 // 7 and 8 neither divide 480 nor 270. 1351 constexpr int kRequestedWidth = 480; 1352 constexpr int kRequestedHeight = 270; 1353 constexpr int kRequestedFramerate = 30; 1354 constexpr int kSinkResolutionAlignment = 8; 1355 1356 OnOutputFormatRequest(kRequestedWidth, kRequestedHeight, kRequestedFramerate); 1357 1358 adapter_.OnSinkWants(BuildSinkWants( 1359 std::nullopt, std::numeric_limits<int>::max(), 1360 std::numeric_limits<int>::max(), kSinkResolutionAlignment)); 1361 EXPECT_TRUE(adapter_.AdaptFrameResolution( 1362 kSourceWidth, kSourceHeight, /*in_timestamp_ns=*/0, &cropped_width_, 1363 &cropped_height_, &out_width_, &out_height_)); 1364 EXPECT_EQ(out_width_ % kSourceResolutionAlignment, 0); 1365 EXPECT_EQ(out_height_ % kSourceResolutionAlignment, 0); 1366 EXPECT_EQ(out_width_ % kSinkResolutionAlignment, 0); 1367 EXPECT_EQ(out_height_ % kSinkResolutionAlignment, 0); 1368 } 1369 1370 TEST_P(VideoAdapterWithSourceAlignmentTest, 1371 ScaleResolutionDownToMaintainsAspectRatioWithAlignment) { 1372 // Request 720x720. 1373 adapter_.OnSinkWants( 1374 BuildSinkWants(Resolution{.width = 720, .height = 720}, 1375 /* any_active_without_requested_resolution= */ false)); 1376 1377 // A 1280x720 frame restricted to 720x720 produces 720x405 but this is not a 1378 // multiple of `kSourceResolutionAlignment` (= 7), the rounded up multiple of 1379 // this value that is less than the restrictions (720) is 714x406. 1380 EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, /*in_timestamp_ns=*/0, 1381 &cropped_width_, &cropped_height_, 1382 &out_width_, &out_height_)); 1383 EXPECT_EQ(out_width_, 714); 1384 EXPECT_EQ(out_height_, 406); 1385 EXPECT_EQ(out_width_ % kSourceResolutionAlignment, 0); 1386 EXPECT_EQ(out_height_ % kSourceResolutionAlignment, 0); 1387 } 1388 1389 INSTANTIATE_TEST_SUITE_P(OnOutputFormatRequests, 1390 VideoAdapterWithSourceAlignmentTest, 1391 ::testing::Values(true, false)); 1392 1393 } // namespace webrtc