libaom_av1_unittest.cc (15159B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <cstddef> 12 #include <cstdint> 13 #include <map> 14 #include <memory> 15 #include <optional> 16 #include <ostream> 17 #include <string> 18 #include <tuple> 19 #include <utility> 20 #include <vector> 21 22 #include "api/environment/environment.h" 23 #include "api/environment/environment_factory.h" 24 #include "api/transport/rtp/dependency_descriptor.h" 25 #include "api/units/data_rate.h" 26 #include "api/units/data_size.h" 27 #include "api/units/time_delta.h" 28 #include "api/video/encoded_image.h" 29 #include "api/video/video_bitrate_allocation.h" 30 #include "api/video/video_frame.h" 31 #include "api/video_codecs/scalability_mode.h" 32 #include "api/video_codecs/video_codec.h" 33 #include "api/video_codecs/video_decoder.h" 34 #include "api/video_codecs/video_encoder.h" 35 #include "modules/video_coding/codecs/av1/dav1d_decoder.h" 36 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" 37 #include "modules/video_coding/codecs/test/encoded_video_frame_producer.h" 38 #include "modules/video_coding/include/video_codec_interface.h" 39 #include "modules/video_coding/include/video_error_codes.h" 40 #include "modules/video_coding/svc/create_scalability_structure.h" 41 #include "modules/video_coding/svc/scalability_mode_util.h" 42 #include "modules/video_coding/svc/scalable_video_controller.h" 43 #include "rtc_base/checks.h" 44 #include "test/gmock.h" 45 #include "test/gtest.h" 46 47 namespace webrtc { 48 namespace { 49 50 using ::testing::ContainerEq; 51 using ::testing::Each; 52 using ::testing::ElementsAreArray; 53 using ::testing::Ge; 54 using ::testing::IsEmpty; 55 using ::testing::Not; 56 using ::testing::NotNull; 57 using ::testing::Optional; 58 using ::testing::Pointwise; 59 using ::testing::SizeIs; 60 using ::testing::Truly; 61 using ::testing::Values; 62 63 // Use small resolution for this test to make it faster. 64 constexpr int kWidth = 320; 65 constexpr int kHeight = 180; 66 constexpr int kFramerate = 30; 67 68 VideoCodec DefaultCodecSettings() { 69 VideoCodec codec_settings; 70 codec_settings.SetScalabilityMode(ScalabilityMode::kL1T1); 71 codec_settings.width = kWidth; 72 codec_settings.height = kHeight; 73 codec_settings.maxFramerate = kFramerate; 74 codec_settings.maxBitrate = 1000; 75 codec_settings.startBitrate = 1; 76 codec_settings.qpMax = 63; 77 return codec_settings; 78 } 79 VideoEncoder::Settings DefaultEncoderSettings() { 80 return VideoEncoder::Settings( 81 VideoEncoder::Capabilities(/*loss_notification=*/false), 82 /*number_of_cores=*/1, /*max_payload_size=*/1200); 83 } 84 85 class TestAv1Decoder { 86 public: 87 explicit TestAv1Decoder(const Environment& env, int decoder_id) 88 : decoder_id_(decoder_id), decoder_(CreateDav1dDecoder(env)) { 89 if (decoder_ == nullptr) { 90 ADD_FAILURE() << "Failed to create a decoder#" << decoder_id_; 91 return; 92 } 93 EXPECT_TRUE(decoder_->Configure({})); 94 EXPECT_EQ(decoder_->RegisterDecodeCompleteCallback(&callback_), 95 WEBRTC_VIDEO_CODEC_OK); 96 } 97 // This class requires pointer stability and thus not copyable nor movable. 98 TestAv1Decoder(const TestAv1Decoder&) = delete; 99 TestAv1Decoder& operator=(const TestAv1Decoder&) = delete; 100 101 void Decode(int64_t frame_id, const EncodedImage& image) { 102 ASSERT_THAT(decoder_, NotNull()); 103 int32_t error = 104 decoder_->Decode(image, /*render_time_ms=*/image.capture_time_ms_); 105 if (error != WEBRTC_VIDEO_CODEC_OK) { 106 ADD_FAILURE() << "Failed to decode frame id " << frame_id 107 << " with error code " << error << " by decoder#" 108 << decoder_id_; 109 return; 110 } 111 decoded_ids_.push_back(frame_id); 112 } 113 114 const std::vector<int64_t>& decoded_frame_ids() const { return decoded_ids_; } 115 size_t num_output_frames() const { return callback_.num_called(); } 116 117 private: 118 // Decoder callback that only counts how many times it was called. 119 // While it is tempting to replace it with a simple mock, that one requires 120 // to set expectation on number of calls in advance. Tests below unsure about 121 // expected number of calls until after calls are done. 122 class DecoderCallback : public DecodedImageCallback { 123 public: 124 size_t num_called() const { return num_called_; } 125 126 private: 127 int32_t Decoded(VideoFrame& /*decoded_image*/) override { 128 ++num_called_; 129 return 0; 130 } 131 void Decoded(VideoFrame& /*decoded_image*/, 132 std::optional<int32_t> /*decode_time_ms*/, 133 std::optional<uint8_t> /*qp*/) override { 134 ++num_called_; 135 } 136 137 int num_called_ = 0; 138 }; 139 140 const int decoder_id_; 141 std::vector<int64_t> decoded_ids_; 142 DecoderCallback callback_; 143 const std::unique_ptr<VideoDecoder> decoder_; 144 }; 145 146 TEST(LibaomAv1Test, EncodeDecode) { 147 const Environment env = CreateEnvironment(); 148 TestAv1Decoder decoder(env, /*decoder_id=*/0); 149 std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder(env); 150 VideoCodec codec_settings = DefaultCodecSettings(); 151 ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), 152 WEBRTC_VIDEO_CODEC_OK); 153 154 VideoBitrateAllocation allocation; 155 allocation.SetBitrate(0, 0, 300000); 156 encoder->SetRates(VideoEncoder::RateControlParameters( 157 allocation, codec_settings.maxFramerate)); 158 159 std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames = 160 EncodedVideoFrameProducer(*encoder).SetNumInputFrames(4).Encode(); 161 for (size_t frame_id = 0; frame_id < encoded_frames.size(); ++frame_id) { 162 decoder.Decode(static_cast<int64_t>(frame_id), 163 encoded_frames[frame_id].encoded_image); 164 } 165 166 // Check encoder produced some frames for decoder to decode. 167 ASSERT_THAT(encoded_frames, Not(IsEmpty())); 168 // Check decoder found all of them valid. 169 EXPECT_THAT(decoder.decoded_frame_ids(), SizeIs(encoded_frames.size())); 170 // Check each of them produced an output frame. 171 EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size()); 172 } 173 174 struct LayerId { 175 friend bool operator==(const LayerId& lhs, const LayerId& rhs) { 176 return std::tie(lhs.spatial_id, lhs.temporal_id) == 177 std::tie(rhs.spatial_id, rhs.temporal_id); 178 } 179 friend bool operator<(const LayerId& lhs, const LayerId& rhs) { 180 return std::tie(lhs.spatial_id, lhs.temporal_id) < 181 std::tie(rhs.spatial_id, rhs.temporal_id); 182 } 183 friend std::ostream& operator<<(std::ostream& s, const LayerId& layer) { 184 return s << "S" << layer.spatial_id << "T" << layer.temporal_id; 185 } 186 187 int spatial_id = 0; 188 int temporal_id = 0; 189 }; 190 191 struct SvcTestParam { 192 ScalabilityMode GetScalabilityMode() const { 193 std::optional<ScalabilityMode> scalability_mode = 194 ScalabilityModeFromString(name); 195 RTC_CHECK(scalability_mode.has_value()); 196 return *scalability_mode; 197 } 198 199 std::string name; 200 int num_frames_to_generate; 201 std::map<LayerId, DataRate> configured_bitrates; 202 }; 203 204 class LibaomAv1SvcTest : public ::testing::TestWithParam<SvcTestParam> {}; 205 206 TEST_P(LibaomAv1SvcTest, EncodeAndDecodeAllDecodeTargets) { 207 const SvcTestParam param = GetParam(); 208 std::unique_ptr<ScalableVideoController> svc_controller = 209 CreateScalabilityStructure(param.GetScalabilityMode()); 210 ASSERT_TRUE(svc_controller); 211 VideoBitrateAllocation allocation; 212 if (param.configured_bitrates.empty()) { 213 ScalableVideoController::StreamLayersConfig config = 214 svc_controller->StreamConfig(); 215 for (int sid = 0; sid < config.num_spatial_layers; ++sid) { 216 for (int tid = 0; tid < config.num_temporal_layers; ++tid) { 217 allocation.SetBitrate(sid, tid, 100'000); 218 } 219 } 220 } else { 221 for (const auto& kv : param.configured_bitrates) { 222 allocation.SetBitrate(kv.first.spatial_id, kv.first.temporal_id, 223 kv.second.bps()); 224 } 225 } 226 227 size_t num_decode_targets = 228 svc_controller->DependencyStructure().num_decode_targets; 229 230 const Environment env = CreateEnvironment(); 231 std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder(env); 232 VideoCodec codec_settings = DefaultCodecSettings(); 233 codec_settings.SetScalabilityMode(GetParam().GetScalabilityMode()); 234 ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), 235 WEBRTC_VIDEO_CODEC_OK); 236 encoder->SetRates(VideoEncoder::RateControlParameters( 237 allocation, codec_settings.maxFramerate)); 238 std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames = 239 EncodedVideoFrameProducer(*encoder) 240 .SetNumInputFrames(GetParam().num_frames_to_generate) 241 .SetResolution({kWidth, kHeight}) 242 .Encode(); 243 244 ASSERT_THAT( 245 encoded_frames, 246 Each(Truly([&](const EncodedVideoFrameProducer::EncodedFrame& frame) { 247 return frame.codec_specific_info.generic_frame_info && 248 frame.codec_specific_info.generic_frame_info 249 ->decode_target_indications.size() == num_decode_targets; 250 }))); 251 252 for (size_t dt = 0; dt < num_decode_targets; ++dt) { 253 TestAv1Decoder decoder(env, dt); 254 std::vector<int64_t> requested_ids; 255 for (int64_t frame_id = 0; 256 frame_id < static_cast<int64_t>(encoded_frames.size()); ++frame_id) { 257 const EncodedVideoFrameProducer::EncodedFrame& frame = 258 encoded_frames[frame_id]; 259 if (frame.codec_specific_info.generic_frame_info 260 ->decode_target_indications[dt] != 261 DecodeTargetIndication::kNotPresent) { 262 requested_ids.push_back(frame_id); 263 decoder.Decode(frame_id, frame.encoded_image); 264 } 265 EXPECT_THAT(frame.codec_specific_info.scalability_mode, 266 Optional(param.GetScalabilityMode())); 267 } 268 269 ASSERT_THAT(requested_ids, SizeIs(Ge(2u))); 270 // Check decoder found all of them valid. 271 EXPECT_THAT(decoder.decoded_frame_ids(), ContainerEq(requested_ids)) 272 << "Decoder#" << dt; 273 // Check each of them produced an output frame. 274 EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size()) 275 << "Decoder#" << dt; 276 } 277 } 278 279 MATCHER(SameLayerIdAndBitrateIsNear, "") { 280 // First check if layer id is the same. 281 return std::get<0>(arg).first == std::get<1>(arg).first && 282 // check measured bitrate is not much lower than requested. 283 std::get<0>(arg).second >= std::get<1>(arg).second * 0.75 && 284 // check measured bitrate is not much larger than requested. 285 std::get<0>(arg).second <= std::get<1>(arg).second * 1.25; 286 } 287 288 TEST_P(LibaomAv1SvcTest, SetRatesMatchMeasuredBitrate) { 289 const SvcTestParam param = GetParam(); 290 if (param.configured_bitrates.empty()) { 291 // Rates are not configured for this particular structure, skip the test. 292 return; 293 } 294 constexpr TimeDelta kDuration = TimeDelta::Seconds(5); 295 296 VideoBitrateAllocation allocation; 297 for (const auto& kv : param.configured_bitrates) { 298 allocation.SetBitrate(kv.first.spatial_id, kv.first.temporal_id, 299 kv.second.bps()); 300 } 301 302 std::unique_ptr<VideoEncoder> encoder = 303 CreateLibaomAv1Encoder(CreateEnvironment()); 304 ASSERT_TRUE(encoder); 305 VideoCodec codec_settings = DefaultCodecSettings(); 306 codec_settings.SetScalabilityMode(param.GetScalabilityMode()); 307 codec_settings.maxBitrate = allocation.get_sum_kbps(); 308 codec_settings.maxFramerate = 30; 309 ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), 310 WEBRTC_VIDEO_CODEC_OK); 311 312 encoder->SetRates(VideoEncoder::RateControlParameters( 313 allocation, codec_settings.maxFramerate)); 314 315 std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames = 316 EncodedVideoFrameProducer(*encoder) 317 .SetNumInputFrames(codec_settings.maxFramerate * kDuration.seconds()) 318 .SetResolution({codec_settings.width, codec_settings.height}) 319 .SetFramerateFps(codec_settings.maxFramerate) 320 .Encode(); 321 322 // Calculate size of each layer. 323 std::map<LayerId, DataSize> layer_size; 324 for (const auto& frame : encoded_frames) { 325 ASSERT_TRUE(frame.codec_specific_info.generic_frame_info); 326 const auto& layer = *frame.codec_specific_info.generic_frame_info; 327 LayerId layer_id = {.spatial_id = layer.spatial_id, 328 .temporal_id = layer.temporal_id}; 329 // This is almost same as 330 // layer_size[layer_id] += DataSize::Bytes(frame.encoded_image.size()); 331 // but avoids calling deleted default constructor for DataSize. 332 layer_size.emplace(layer_id, DataSize::Zero()).first->second += 333 DataSize::Bytes(frame.encoded_image.size()); 334 } 335 // Convert size of the layer into bitrate of that layer. 336 std::vector<std::pair<LayerId, DataRate>> measured_bitrates; 337 for (const auto& kv : layer_size) { 338 measured_bitrates.emplace_back(kv.first, kv.second / kDuration); 339 } 340 EXPECT_THAT(measured_bitrates, Pointwise(SameLayerIdAndBitrateIsNear(), 341 param.configured_bitrates)); 342 } 343 344 INSTANTIATE_TEST_SUITE_P( 345 Svc, 346 LibaomAv1SvcTest, 347 Values(SvcTestParam{"L1T1", /*num_frames_to_generate=*/4}, 348 SvcTestParam{"L1T2", 349 /*num_frames_to_generate=*/4, 350 /*configured_bitrates=*/ 351 {{{0, 0}, DataRate::KilobitsPerSec(60)}, 352 {{0, 1}, DataRate::KilobitsPerSec(40)}}}, 353 SvcTestParam{"L1T3", /*num_frames_to_generate=*/8}, 354 SvcTestParam{"L2T1", 355 /*num_frames_to_generate=*/3, 356 /*configured_bitrates=*/ 357 {{{0, 0}, DataRate::KilobitsPerSec(30)}, 358 {{1, 0}, DataRate::KilobitsPerSec(70)}}}, 359 SvcTestParam{"L2T1h", 360 /*num_frames_to_generate=*/3, 361 /*configured_bitrates=*/ 362 {{{0, 0}, DataRate::KilobitsPerSec(30)}, 363 {{1, 0}, DataRate::KilobitsPerSec(70)}}}, 364 SvcTestParam{"L2T1_KEY", /*num_frames_to_generate=*/3}, 365 SvcTestParam{"L3T1", /*num_frames_to_generate=*/3}, 366 SvcTestParam{"L3T3", /*num_frames_to_generate=*/8}, 367 SvcTestParam{"S2T1", /*num_frames_to_generate=*/3}, 368 // TODO: bugs.webrtc.org/15715 - Re-enable once AV1 is fixed. 369 // SvcTestParam{"S3T3", /*num_frames_to_generate=*/8}, 370 SvcTestParam{"L2T2", /*num_frames_to_generate=*/4}, 371 SvcTestParam{"L2T2_KEY", /*num_frames_to_generate=*/4}, 372 SvcTestParam{"L2T2_KEY_SHIFT", 373 /*num_frames_to_generate=*/4, 374 /*configured_bitrates=*/ 375 {{{0, 0}, DataRate::KilobitsPerSec(70)}, 376 {{0, 1}, DataRate::KilobitsPerSec(30)}, 377 {{1, 0}, DataRate::KilobitsPerSec(110)}, 378 {{1, 1}, DataRate::KilobitsPerSec(80)}}}), 379 [](const testing::TestParamInfo<SvcTestParam>& info) { 380 return info.param.name; 381 }); 382 383 } // namespace 384 } // namespace webrtc