scalability_structure_key_svc.cc (16201B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #include "modules/video_coding/svc/scalability_structure_key_svc.h" 11 12 #include <bitset> 13 #include <cstdint> 14 #include <optional> 15 #include <vector> 16 17 #include "api/transport/rtp/dependency_descriptor.h" 18 #include "api/video/video_bitrate_allocation.h" 19 #include "common_video/generic_frame_descriptor/generic_frame_info.h" 20 #include "modules/video_coding/svc/scalable_video_controller.h" 21 #include "rtc_base/checks.h" 22 23 namespace webrtc { 24 25 ScalabilityStructureKeySvc::ScalabilityStructureKeySvc(int num_spatial_layers, 26 int num_temporal_layers) 27 : num_spatial_layers_(num_spatial_layers), 28 num_temporal_layers_(num_temporal_layers), 29 active_decode_targets_( 30 (uint32_t{1} << (num_spatial_layers * num_temporal_layers)) - 1) { 31 // There is no point to use this structure without spatial scalability. 32 RTC_DCHECK_GT(num_spatial_layers, 1); 33 RTC_DCHECK_LE(num_spatial_layers, kMaxNumSpatialLayers); 34 RTC_DCHECK_LE(num_temporal_layers, kMaxNumTemporalLayers); 35 } 36 37 ScalabilityStructureKeySvc::~ScalabilityStructureKeySvc() = default; 38 39 ScalableVideoController::StreamLayersConfig 40 ScalabilityStructureKeySvc::StreamConfig() const { 41 StreamLayersConfig result; 42 result.num_spatial_layers = num_spatial_layers_; 43 result.num_temporal_layers = num_temporal_layers_; 44 result.scaling_factor_num[num_spatial_layers_ - 1] = 1; 45 result.scaling_factor_den[num_spatial_layers_ - 1] = 1; 46 for (int sid = num_spatial_layers_ - 1; sid > 0; --sid) { 47 result.scaling_factor_num[sid - 1] = 1; 48 result.scaling_factor_den[sid - 1] = 2 * result.scaling_factor_den[sid]; 49 } 50 result.uses_reference_scaling = true; 51 return result; 52 } 53 54 bool ScalabilityStructureKeySvc::TemporalLayerIsActive(int tid) const { 55 if (tid >= num_temporal_layers_) { 56 return false; 57 } 58 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 59 if (DecodeTargetIsActive(sid, tid)) { 60 return true; 61 } 62 } 63 return false; 64 } 65 66 DecodeTargetIndication ScalabilityStructureKeySvc::Dti( 67 int sid, 68 int tid, 69 const LayerFrameConfig& config) { 70 if (config.IsKeyframe() || config.Id() == kKey) { 71 RTC_DCHECK_EQ(config.TemporalId(), 0); 72 return sid < config.SpatialId() ? DecodeTargetIndication::kNotPresent 73 : DecodeTargetIndication::kSwitch; 74 } 75 76 if (sid != config.SpatialId() || tid < config.TemporalId()) { 77 return DecodeTargetIndication::kNotPresent; 78 } 79 if (tid == config.TemporalId() && tid > 0) { 80 return DecodeTargetIndication::kDiscardable; 81 } 82 return DecodeTargetIndication::kSwitch; 83 } 84 85 std::vector<ScalableVideoController::LayerFrameConfig> 86 ScalabilityStructureKeySvc::KeyframeConfig() { 87 std::vector<LayerFrameConfig> configs; 88 configs.reserve(num_spatial_layers_); 89 std::optional<int> spatial_dependency_buffer_id; 90 spatial_id_is_enabled_.reset(); 91 // Disallow temporal references cross T0 on higher temporal layers. 92 can_reference_t1_frame_for_spatial_id_.reset(); 93 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 94 if (!DecodeTargetIsActive(sid, /*tid=*/0)) { 95 continue; 96 } 97 configs.emplace_back(); 98 ScalableVideoController::LayerFrameConfig& config = configs.back(); 99 config.Id(kKey).S(sid).T(0); 100 101 if (spatial_dependency_buffer_id) { 102 config.Reference(*spatial_dependency_buffer_id); 103 } else { 104 config.Keyframe(); 105 } 106 config.Update(BufferIndex(sid, /*tid=*/0)); 107 108 spatial_id_is_enabled_.set(sid); 109 spatial_dependency_buffer_id = BufferIndex(sid, /*tid=*/0); 110 } 111 return configs; 112 } 113 114 std::vector<ScalableVideoController::LayerFrameConfig> 115 ScalabilityStructureKeySvc::T0Config() { 116 std::vector<LayerFrameConfig> configs; 117 configs.reserve(num_spatial_layers_); 118 // Disallow temporal references cross T0 on higher temporal layers. 119 can_reference_t1_frame_for_spatial_id_.reset(); 120 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 121 if (!DecodeTargetIsActive(sid, /*tid=*/0)) { 122 spatial_id_is_enabled_.reset(sid); 123 continue; 124 } 125 configs.emplace_back(); 126 configs.back().Id(kDeltaT0).S(sid).T(0).ReferenceAndUpdate( 127 BufferIndex(sid, /*tid=*/0)); 128 } 129 return configs; 130 } 131 132 std::vector<ScalableVideoController::LayerFrameConfig> 133 ScalabilityStructureKeySvc::T1Config() { 134 std::vector<LayerFrameConfig> configs; 135 configs.reserve(num_spatial_layers_); 136 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 137 if (!DecodeTargetIsActive(sid, /*tid=*/1)) { 138 continue; 139 } 140 configs.emplace_back(); 141 ScalableVideoController::LayerFrameConfig& config = configs.back(); 142 config.Id(kDeltaT1).S(sid).T(1).Reference(BufferIndex(sid, /*tid=*/0)); 143 if (num_temporal_layers_ > 2) { 144 config.Update(BufferIndex(sid, /*tid=*/1)); 145 } 146 } 147 return configs; 148 } 149 150 std::vector<ScalableVideoController::LayerFrameConfig> 151 ScalabilityStructureKeySvc::T2Config(FramePattern pattern) { 152 std::vector<LayerFrameConfig> configs; 153 configs.reserve(num_spatial_layers_); 154 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 155 if (!DecodeTargetIsActive(sid, /*tid=*/2)) { 156 continue; 157 } 158 configs.emplace_back(); 159 ScalableVideoController::LayerFrameConfig& config = configs.back(); 160 config.Id(pattern).S(sid).T(2); 161 if (can_reference_t1_frame_for_spatial_id_[sid]) { 162 config.Reference(BufferIndex(sid, /*tid=*/1)); 163 } else { 164 config.Reference(BufferIndex(sid, /*tid=*/0)); 165 } 166 } 167 return configs; 168 } 169 170 ScalabilityStructureKeySvc::FramePattern 171 ScalabilityStructureKeySvc::NextPattern(FramePattern last_pattern) const { 172 switch (last_pattern) { 173 case kNone: 174 return kKey; 175 case kDeltaT2B: 176 return kDeltaT0; 177 case kDeltaT2A: 178 if (TemporalLayerIsActive(1)) { 179 return kDeltaT1; 180 } 181 return kDeltaT0; 182 case kDeltaT1: 183 if (TemporalLayerIsActive(2)) { 184 return kDeltaT2B; 185 } 186 return kDeltaT0; 187 case kDeltaT0: 188 case kKey: 189 if (TemporalLayerIsActive(2)) { 190 return kDeltaT2A; 191 } 192 if (TemporalLayerIsActive(1)) { 193 return kDeltaT1; 194 } 195 return kDeltaT0; 196 } 197 RTC_DCHECK_NOTREACHED(); 198 return kNone; 199 } 200 201 std::vector<ScalableVideoController::LayerFrameConfig> 202 ScalabilityStructureKeySvc::NextFrameConfig(bool restart) { 203 if (active_decode_targets_.none()) { 204 last_pattern_ = kNone; 205 return {}; 206 } 207 208 if (restart) { 209 last_pattern_ = kNone; 210 } 211 212 FramePattern current_pattern = NextPattern(last_pattern_); 213 switch (current_pattern) { 214 case kKey: 215 return KeyframeConfig(); 216 case kDeltaT0: 217 return T0Config(); 218 case kDeltaT1: 219 return T1Config(); 220 case kDeltaT2A: 221 case kDeltaT2B: 222 return T2Config(current_pattern); 223 case kNone: 224 break; 225 } 226 RTC_DCHECK_NOTREACHED(); 227 return {}; 228 } 229 230 GenericFrameInfo ScalabilityStructureKeySvc::OnEncodeDone( 231 const LayerFrameConfig& config) { 232 // When encoder drops all frames for a temporal unit, it is better to reuse 233 // old temporal pattern rather than switch to next one, thus switch to next 234 // pattern defered here from the `NextFrameConfig`. 235 // In particular creating VP9 references rely on this behavior. 236 last_pattern_ = static_cast<FramePattern>(config.Id()); 237 if (config.TemporalId() == 1) { 238 can_reference_t1_frame_for_spatial_id_.set(config.SpatialId()); 239 } 240 241 GenericFrameInfo frame_info; 242 frame_info.spatial_id = config.SpatialId(); 243 frame_info.temporal_id = config.TemporalId(); 244 frame_info.encoder_buffers = config.Buffers(); 245 frame_info.decode_target_indications.reserve(num_spatial_layers_ * 246 num_temporal_layers_); 247 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 248 for (int tid = 0; tid < num_temporal_layers_; ++tid) { 249 frame_info.decode_target_indications.push_back(Dti(sid, tid, config)); 250 } 251 } 252 frame_info.part_of_chain.assign(num_spatial_layers_, false); 253 if (config.IsKeyframe() || config.Id() == kKey) { 254 RTC_DCHECK_EQ(config.TemporalId(), 0); 255 for (int sid = config.SpatialId(); sid < num_spatial_layers_; ++sid) { 256 frame_info.part_of_chain[sid] = true; 257 } 258 } else if (config.TemporalId() == 0) { 259 frame_info.part_of_chain[config.SpatialId()] = true; 260 } 261 frame_info.active_decode_targets = active_decode_targets_; 262 return frame_info; 263 } 264 265 void ScalabilityStructureKeySvc::OnRatesUpdated( 266 const VideoBitrateAllocation& bitrates) { 267 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 268 // Enable/disable spatial layers independetely. 269 bool active = bitrates.GetBitrate(sid, /*tid=*/0) > 0; 270 SetDecodeTargetIsActive(sid, /*tid=*/0, active); 271 if (!spatial_id_is_enabled_[sid] && active) { 272 // Key frame is required to reenable any spatial layer. 273 last_pattern_ = kNone; 274 } 275 276 for (int tid = 1; tid < num_temporal_layers_; ++tid) { 277 // To enable temporal layer, require bitrates for lower temporal layers. 278 active = active && bitrates.GetBitrate(sid, tid) > 0; 279 SetDecodeTargetIsActive(sid, tid, active); 280 } 281 } 282 } 283 284 ScalabilityStructureL2T1Key::~ScalabilityStructureL2T1Key() = default; 285 286 FrameDependencyStructure ScalabilityStructureL2T1Key::DependencyStructure() 287 const { 288 FrameDependencyStructure structure; 289 structure.num_decode_targets = 2; 290 structure.num_chains = 2; 291 structure.decode_target_protected_by_chain = {0, 1}; 292 structure.templates.resize(4); 293 structure.templates[0].S(0).Dtis("S-").ChainDiffs({2, 1}).FrameDiffs({2}); 294 structure.templates[1].S(0).Dtis("SS").ChainDiffs({0, 0}); 295 structure.templates[2].S(1).Dtis("-S").ChainDiffs({1, 2}).FrameDiffs({2}); 296 structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 1}).FrameDiffs({1}); 297 return structure; 298 } 299 300 ScalabilityStructureL2T2Key::~ScalabilityStructureL2T2Key() = default; 301 302 FrameDependencyStructure ScalabilityStructureL2T2Key::DependencyStructure() 303 const { 304 FrameDependencyStructure structure; 305 structure.num_decode_targets = 4; 306 structure.num_chains = 2; 307 structure.decode_target_protected_by_chain = {0, 0, 1, 1}; 308 structure.templates.resize(6); 309 auto& templates = structure.templates; 310 templates[0].S(0).T(0).Dtis("SSSS").ChainDiffs({0, 0}); 311 templates[1].S(0).T(0).Dtis("SS--").ChainDiffs({4, 3}).FrameDiffs({4}); 312 templates[2].S(0).T(1).Dtis("-D--").ChainDiffs({2, 1}).FrameDiffs({2}); 313 templates[3].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({1}); 314 templates[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 4}).FrameDiffs({4}); 315 templates[5].S(1).T(1).Dtis("---D").ChainDiffs({3, 2}).FrameDiffs({2}); 316 return structure; 317 } 318 319 ScalabilityStructureL2T3Key::~ScalabilityStructureL2T3Key() = default; 320 321 FrameDependencyStructure ScalabilityStructureL2T3Key::DependencyStructure() 322 const { 323 FrameDependencyStructure structure; 324 structure.num_decode_targets = 6; 325 structure.num_chains = 2; 326 structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1}; 327 auto& templates = structure.templates; 328 templates.resize(10); 329 templates[0].S(0).T(0).Dtis("SSSSSS").ChainDiffs({0, 0}); 330 templates[1].S(0).T(0).Dtis("SSS---").ChainDiffs({8, 7}).FrameDiffs({8}); 331 templates[2].S(0).T(1).Dtis("-DS---").ChainDiffs({4, 3}).FrameDiffs({4}); 332 templates[3].S(0).T(2).Dtis("--D---").ChainDiffs({2, 1}).FrameDiffs({2}); 333 templates[4].S(0).T(2).Dtis("--D---").ChainDiffs({6, 5}).FrameDiffs({2}); 334 templates[5].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 1}).FrameDiffs({1}); 335 templates[6].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 8}).FrameDiffs({8}); 336 templates[7].S(1).T(1).Dtis("----DS").ChainDiffs({5, 4}).FrameDiffs({4}); 337 templates[8].S(1).T(2).Dtis("-----D").ChainDiffs({3, 2}).FrameDiffs({2}); 338 templates[9].S(1).T(2).Dtis("-----D").ChainDiffs({7, 6}).FrameDiffs({2}); 339 return structure; 340 } 341 342 ScalabilityStructureL3T1Key::~ScalabilityStructureL3T1Key() = default; 343 344 FrameDependencyStructure ScalabilityStructureL3T1Key::DependencyStructure() 345 const { 346 FrameDependencyStructure structure; 347 structure.num_decode_targets = 3; 348 structure.num_chains = 3; 349 structure.decode_target_protected_by_chain = {0, 1, 2}; 350 auto& t = structure.templates; 351 t.resize(6); 352 // Templates are shown in the order frames following them appear in the 353 // stream, but in `structure.templates` array templates are sorted by 354 // (`spatial_id`, `temporal_id`) since that is a dependency descriptor 355 // requirement. 356 t[1].S(0).Dtis("SSS").ChainDiffs({0, 0, 0}); 357 t[3].S(1).Dtis("-SS").ChainDiffs({1, 1, 1}).FrameDiffs({1}); 358 t[5].S(2).Dtis("--S").ChainDiffs({2, 1, 1}).FrameDiffs({1}); 359 t[0].S(0).Dtis("S--").ChainDiffs({3, 2, 1}).FrameDiffs({3}); 360 t[2].S(1).Dtis("-S-").ChainDiffs({1, 3, 2}).FrameDiffs({3}); 361 t[4].S(2).Dtis("--S").ChainDiffs({2, 1, 3}).FrameDiffs({3}); 362 return structure; 363 } 364 365 ScalabilityStructureL3T2Key::~ScalabilityStructureL3T2Key() = default; 366 367 FrameDependencyStructure ScalabilityStructureL3T2Key::DependencyStructure() 368 const { 369 FrameDependencyStructure structure; 370 structure.num_decode_targets = 6; 371 structure.num_chains = 3; 372 structure.decode_target_protected_by_chain = {0, 0, 1, 1, 2, 2}; 373 auto& t = structure.templates; 374 t.resize(9); 375 // Templates are shown in the order frames following them appear in the 376 // stream, but in `structure.templates` array templates are sorted by 377 // (`spatial_id`, `temporal_id`) since that is a dependency descriptor 378 // requirement. 379 t[1].S(0).T(0).Dtis("SSSSSS").ChainDiffs({0, 0, 0}); 380 t[4].S(1).T(0).Dtis("--SSSS").ChainDiffs({1, 1, 1}).FrameDiffs({1}); 381 t[7].S(2).T(0).Dtis("----SS").ChainDiffs({2, 1, 1}).FrameDiffs({1}); 382 t[2].S(0).T(1).Dtis("-D----").ChainDiffs({3, 2, 1}).FrameDiffs({3}); 383 t[5].S(1).T(1).Dtis("---D--").ChainDiffs({4, 3, 2}).FrameDiffs({3}); 384 t[8].S(2).T(1).Dtis("-----D").ChainDiffs({5, 4, 3}).FrameDiffs({3}); 385 t[0].S(0).T(0).Dtis("SS----").ChainDiffs({6, 5, 4}).FrameDiffs({6}); 386 t[3].S(1).T(0).Dtis("--SS--").ChainDiffs({1, 6, 5}).FrameDiffs({6}); 387 t[6].S(2).T(0).Dtis("----SS").ChainDiffs({2, 1, 6}).FrameDiffs({6}); 388 return structure; 389 } 390 391 ScalabilityStructureL3T3Key::~ScalabilityStructureL3T3Key() = default; 392 393 FrameDependencyStructure ScalabilityStructureL3T3Key::DependencyStructure() 394 const { 395 FrameDependencyStructure structure; 396 structure.num_decode_targets = 9; 397 structure.num_chains = 3; 398 structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1, 2, 2, 2}; 399 auto& t = structure.templates; 400 t.resize(15); 401 // Templates are shown in the order frames following them appear in the 402 // stream, but in `structure.templates` array templates are sorted by 403 // (`spatial_id`, `temporal_id`) since that is a dependency descriptor 404 // requirement. Indexes are written in hex for nicer alignment. 405 t[0x0].S(0).T(0).Dtis("SSSSSSSSS").ChainDiffs({0, 0, 0}); 406 t[0x5].S(1).T(0).Dtis("---SSSSSS").ChainDiffs({1, 1, 1}).FrameDiffs({1}); 407 t[0xA].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 1}).FrameDiffs({1}); 408 t[0x3].S(0).T(2).Dtis("--D------").ChainDiffs({3, 2, 1}).FrameDiffs({3}); 409 t[0x8].S(1).T(2).Dtis("-----D---").ChainDiffs({4, 3, 2}).FrameDiffs({3}); 410 t[0xD].S(2).T(2).Dtis("--------D").ChainDiffs({5, 4, 3}).FrameDiffs({3}); 411 t[0x2].S(0).T(1).Dtis("-DS------").ChainDiffs({6, 5, 4}).FrameDiffs({6}); 412 t[0x7].S(1).T(1).Dtis("----DS---").ChainDiffs({7, 6, 5}).FrameDiffs({6}); 413 t[0xC].S(2).T(1).Dtis("-------DS").ChainDiffs({8, 7, 6}).FrameDiffs({6}); 414 t[0x4].S(0).T(2).Dtis("--D------").ChainDiffs({9, 8, 7}).FrameDiffs({3}); 415 t[0x9].S(1).T(2).Dtis("-----D---").ChainDiffs({10, 9, 8}).FrameDiffs({3}); 416 t[0xE].S(2).T(2).Dtis("--------D").ChainDiffs({11, 10, 9}).FrameDiffs({3}); 417 t[0x1].S(0).T(0).Dtis("SSS------").ChainDiffs({12, 11, 10}).FrameDiffs({12}); 418 t[0x6].S(1).T(0).Dtis("---SSS---").ChainDiffs({1, 12, 11}).FrameDiffs({12}); 419 t[0xB].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 12}).FrameDiffs({12}); 420 return structure; 421 } 422 423 } // namespace webrtc