scalability_structure_simulcast.cc (13665B)
1 /* 2 * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #include "modules/video_coding/svc/scalability_structure_simulcast.h" 11 12 #include <cstdint> 13 #include <vector> 14 15 #include "api/transport/rtp/dependency_descriptor.h" 16 #include "api/video/video_bitrate_allocation.h" 17 #include "common_video/generic_frame_descriptor/generic_frame_info.h" 18 #include "modules/video_coding/svc/scalable_video_controller.h" 19 #include "rtc_base/checks.h" 20 21 namespace webrtc { 22 namespace { 23 24 DecodeTargetIndication 25 Dti(int sid, int tid, const ScalableVideoController::LayerFrameConfig& config) { 26 if (sid != config.SpatialId() || tid < config.TemporalId()) { 27 return DecodeTargetIndication::kNotPresent; 28 } 29 if (tid == 0) { 30 RTC_DCHECK_EQ(config.TemporalId(), 0); 31 return DecodeTargetIndication::kSwitch; 32 } 33 if (tid == config.TemporalId()) { 34 return DecodeTargetIndication::kDiscardable; 35 } 36 RTC_DCHECK_GT(tid, config.TemporalId()); 37 return DecodeTargetIndication::kSwitch; 38 } 39 40 } // namespace 41 42 ScalabilityStructureSimulcast::ScalabilityStructureSimulcast( 43 int num_spatial_layers, 44 int num_temporal_layers, 45 ScalingFactor resolution_factor) 46 : num_spatial_layers_(num_spatial_layers), 47 num_temporal_layers_(num_temporal_layers), 48 resolution_factor_(resolution_factor), 49 active_decode_targets_( 50 (uint32_t{1} << (num_spatial_layers * num_temporal_layers)) - 1) { 51 RTC_DCHECK_LE(num_spatial_layers, kMaxNumSpatialLayers); 52 RTC_DCHECK_LE(num_temporal_layers, kMaxNumTemporalLayers); 53 } 54 55 ScalabilityStructureSimulcast::~ScalabilityStructureSimulcast() = default; 56 57 ScalableVideoController::StreamLayersConfig 58 ScalabilityStructureSimulcast::StreamConfig() const { 59 StreamLayersConfig result; 60 result.num_spatial_layers = num_spatial_layers_; 61 result.num_temporal_layers = num_temporal_layers_; 62 result.scaling_factor_num[num_spatial_layers_ - 1] = 1; 63 result.scaling_factor_den[num_spatial_layers_ - 1] = 1; 64 for (int sid = num_spatial_layers_ - 1; sid > 0; --sid) { 65 result.scaling_factor_num[sid - 1] = 66 resolution_factor_.num * result.scaling_factor_num[sid]; 67 result.scaling_factor_den[sid - 1] = 68 resolution_factor_.den * result.scaling_factor_den[sid]; 69 } 70 result.uses_reference_scaling = false; 71 return result; 72 } 73 74 bool ScalabilityStructureSimulcast::TemporalLayerIsActive(int tid) const { 75 if (tid >= num_temporal_layers_) { 76 return false; 77 } 78 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 79 if (DecodeTargetIsActive(sid, tid)) { 80 return true; 81 } 82 } 83 return false; 84 } 85 86 ScalabilityStructureSimulcast::FramePattern 87 ScalabilityStructureSimulcast::NextPattern() const { 88 switch (last_pattern_) { 89 case kNone: 90 case kDeltaT2B: 91 return kDeltaT0; 92 case kDeltaT2A: 93 if (TemporalLayerIsActive(1)) { 94 return kDeltaT1; 95 } 96 return kDeltaT0; 97 case kDeltaT1: 98 if (TemporalLayerIsActive(2)) { 99 return kDeltaT2B; 100 } 101 return kDeltaT0; 102 case kDeltaT0: 103 if (TemporalLayerIsActive(2)) { 104 return kDeltaT2A; 105 } 106 if (TemporalLayerIsActive(1)) { 107 return kDeltaT1; 108 } 109 return kDeltaT0; 110 } 111 RTC_DCHECK_NOTREACHED(); 112 return kDeltaT0; 113 } 114 115 std::vector<ScalableVideoController::LayerFrameConfig> 116 ScalabilityStructureSimulcast::NextFrameConfig(bool restart) { 117 std::vector<LayerFrameConfig> configs; 118 if (active_decode_targets_.none()) { 119 last_pattern_ = kNone; 120 return configs; 121 } 122 configs.reserve(num_spatial_layers_); 123 124 if (last_pattern_ == kNone || restart) { 125 can_reference_t0_frame_for_spatial_id_.reset(); 126 last_pattern_ = kNone; 127 } 128 FramePattern current_pattern = NextPattern(); 129 130 switch (current_pattern) { 131 case kDeltaT0: 132 // Disallow temporal references cross T0 on higher temporal layers. 133 can_reference_t1_frame_for_spatial_id_.reset(); 134 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 135 if (!DecodeTargetIsActive(sid, /*tid=*/0)) { 136 // Next frame from the spatial layer `sid` shouldn't depend on 137 // potentially old previous frame from the spatial layer `sid`. 138 can_reference_t0_frame_for_spatial_id_.reset(sid); 139 continue; 140 } 141 configs.emplace_back(); 142 ScalableVideoController::LayerFrameConfig& config = configs.back(); 143 config.Id(current_pattern).S(sid).T(0); 144 145 if (can_reference_t0_frame_for_spatial_id_[sid]) { 146 config.ReferenceAndUpdate(BufferIndex(sid, /*tid=*/0)); 147 } else { 148 config.Keyframe().Update(BufferIndex(sid, /*tid=*/0)); 149 } 150 can_reference_t0_frame_for_spatial_id_.set(sid); 151 } 152 break; 153 case kDeltaT1: 154 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 155 if (!DecodeTargetIsActive(sid, /*tid=*/1) || 156 !can_reference_t0_frame_for_spatial_id_[sid]) { 157 continue; 158 } 159 configs.emplace_back(); 160 ScalableVideoController::LayerFrameConfig& config = configs.back(); 161 config.Id(current_pattern) 162 .S(sid) 163 .T(1) 164 .Reference(BufferIndex(sid, /*tid=*/0)); 165 // Save frame only if there is a higher temporal layer that may need it. 166 if (num_temporal_layers_ > 2) { 167 config.Update(BufferIndex(sid, /*tid=*/1)); 168 } 169 } 170 break; 171 case kDeltaT2A: 172 case kDeltaT2B: 173 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 174 if (!DecodeTargetIsActive(sid, /*tid=*/2) || 175 !can_reference_t0_frame_for_spatial_id_[sid]) { 176 continue; 177 } 178 configs.emplace_back(); 179 ScalableVideoController::LayerFrameConfig& config = configs.back(); 180 config.Id(current_pattern).S(sid).T(2); 181 if (can_reference_t1_frame_for_spatial_id_[sid]) { 182 config.Reference(BufferIndex(sid, /*tid=*/1)); 183 } else { 184 config.Reference(BufferIndex(sid, /*tid=*/0)); 185 } 186 } 187 break; 188 case kNone: 189 RTC_DCHECK_NOTREACHED(); 190 break; 191 } 192 193 return configs; 194 } 195 196 GenericFrameInfo ScalabilityStructureSimulcast::OnEncodeDone( 197 const LayerFrameConfig& config) { 198 last_pattern_ = static_cast<FramePattern>(config.Id()); 199 if (config.TemporalId() == 1) { 200 can_reference_t1_frame_for_spatial_id_.set(config.SpatialId()); 201 } 202 GenericFrameInfo frame_info; 203 frame_info.spatial_id = config.SpatialId(); 204 frame_info.temporal_id = config.TemporalId(); 205 frame_info.encoder_buffers = config.Buffers(); 206 frame_info.decode_target_indications.reserve(num_spatial_layers_ * 207 num_temporal_layers_); 208 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 209 for (int tid = 0; tid < num_temporal_layers_; ++tid) { 210 frame_info.decode_target_indications.push_back(Dti(sid, tid, config)); 211 } 212 } 213 frame_info.part_of_chain.assign(num_spatial_layers_, false); 214 if (config.TemporalId() == 0) { 215 frame_info.part_of_chain[config.SpatialId()] = true; 216 } 217 frame_info.active_decode_targets = active_decode_targets_; 218 return frame_info; 219 } 220 221 void ScalabilityStructureSimulcast::OnRatesUpdated( 222 const VideoBitrateAllocation& bitrates) { 223 for (int sid = 0; sid < num_spatial_layers_; ++sid) { 224 // Enable/disable spatial layers independetely. 225 bool active = true; 226 for (int tid = 0; tid < num_temporal_layers_; ++tid) { 227 // To enable temporal layer, require bitrates for lower temporal layers. 228 active = active && bitrates.GetBitrate(sid, tid) > 0; 229 SetDecodeTargetIsActive(sid, tid, active); 230 } 231 } 232 } 233 234 FrameDependencyStructure ScalabilityStructureS2T1::DependencyStructure() const { 235 FrameDependencyStructure structure; 236 structure.num_decode_targets = 2; 237 structure.num_chains = 2; 238 structure.decode_target_protected_by_chain = {0, 1}; 239 structure.templates.resize(4); 240 structure.templates[0].S(0).Dtis("S-").ChainDiffs({2, 1}).FrameDiffs({2}); 241 structure.templates[1].S(0).Dtis("S-").ChainDiffs({0, 0}); 242 structure.templates[2].S(1).Dtis("-S").ChainDiffs({1, 2}).FrameDiffs({2}); 243 structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 0}); 244 return structure; 245 } 246 247 FrameDependencyStructure ScalabilityStructureS2T2::DependencyStructure() const { 248 FrameDependencyStructure structure; 249 structure.num_decode_targets = 4; 250 structure.num_chains = 2; 251 structure.decode_target_protected_by_chain = {0, 0, 1, 1}; 252 auto& t = structure.templates; 253 t.resize(6); 254 t[1].S(0).T(0).Dtis("SS--").ChainDiffs({0, 0}); 255 t[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 0}); 256 t[2].S(0).T(1).Dtis("-D--").ChainDiffs({2, 1}).FrameDiffs({2}); 257 t[5].S(1).T(1).Dtis("---D").ChainDiffs({3, 2}).FrameDiffs({2}); 258 t[0].S(0).T(0).Dtis("SS--").ChainDiffs({4, 3}).FrameDiffs({4}); 259 t[3].S(1).T(0).Dtis("--SS").ChainDiffs({1, 4}).FrameDiffs({4}); 260 return structure; 261 } 262 263 FrameDependencyStructure ScalabilityStructureS2T3::DependencyStructure() const { 264 FrameDependencyStructure structure; 265 structure.num_decode_targets = 6; 266 structure.num_chains = 2; 267 structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1}; 268 auto& t = structure.templates; 269 t.resize(10); 270 t[1].S(0).T(0).Dtis("SSS---").ChainDiffs({0, 0}); 271 t[6].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 0}); 272 t[3].S(0).T(2).Dtis("--D---").ChainDiffs({2, 1}).FrameDiffs({2}); 273 t[8].S(1).T(2).Dtis("-----D").ChainDiffs({3, 2}).FrameDiffs({2}); 274 t[2].S(0).T(1).Dtis("-DS---").ChainDiffs({4, 3}).FrameDiffs({4}); 275 t[7].S(1).T(1).Dtis("----DS").ChainDiffs({5, 4}).FrameDiffs({4}); 276 t[4].S(0).T(2).Dtis("--D---").ChainDiffs({6, 5}).FrameDiffs({2}); 277 t[9].S(1).T(2).Dtis("-----D").ChainDiffs({7, 6}).FrameDiffs({2}); 278 t[0].S(0).T(0).Dtis("SSS---").ChainDiffs({8, 7}).FrameDiffs({8}); 279 t[5].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 8}).FrameDiffs({8}); 280 return structure; 281 } 282 283 FrameDependencyStructure ScalabilityStructureS3T1::DependencyStructure() const { 284 FrameDependencyStructure structure; 285 structure.num_decode_targets = 3; 286 structure.num_chains = 3; 287 structure.decode_target_protected_by_chain = {0, 1, 2}; 288 auto& t = structure.templates; 289 t.resize(6); 290 t[1].S(0).T(0).Dtis("S--").ChainDiffs({0, 0, 0}); 291 t[3].S(1).T(0).Dtis("-S-").ChainDiffs({1, 0, 0}); 292 t[5].S(2).T(0).Dtis("--S").ChainDiffs({2, 1, 0}); 293 t[0].S(0).T(0).Dtis("S--").ChainDiffs({3, 2, 1}).FrameDiffs({3}); 294 t[2].S(1).T(0).Dtis("-S-").ChainDiffs({1, 3, 2}).FrameDiffs({3}); 295 t[4].S(2).T(0).Dtis("--S").ChainDiffs({2, 1, 3}).FrameDiffs({3}); 296 return structure; 297 } 298 299 FrameDependencyStructure ScalabilityStructureS3T2::DependencyStructure() const { 300 FrameDependencyStructure structure; 301 structure.num_decode_targets = 6; 302 structure.num_chains = 3; 303 structure.decode_target_protected_by_chain = {0, 0, 1, 1, 2, 2}; 304 auto& t = structure.templates; 305 t.resize(9); 306 // Templates are shown in the order frames following them appear in the 307 // stream, but in `structure.templates` array templates are sorted by 308 // (`spatial_id`, `temporal_id`) since that is a dependency descriptor 309 // requirement. 310 t[1].S(0).T(0).Dtis("SS----").ChainDiffs({0, 0, 0}); 311 t[4].S(1).T(0).Dtis("--SS--").ChainDiffs({1, 0, 0}); 312 t[7].S(2).T(0).Dtis("----SS").ChainDiffs({2, 1, 0}); 313 t[2].S(0).T(1).Dtis("-D----").ChainDiffs({3, 2, 1}).FrameDiffs({3}); 314 t[5].S(1).T(1).Dtis("---D--").ChainDiffs({4, 3, 2}).FrameDiffs({3}); 315 t[8].S(2).T(1).Dtis("-----D").ChainDiffs({5, 4, 3}).FrameDiffs({3}); 316 t[0].S(0).T(0).Dtis("SS----").ChainDiffs({6, 5, 4}).FrameDiffs({6}); 317 t[3].S(1).T(0).Dtis("--SS--").ChainDiffs({1, 6, 5}).FrameDiffs({6}); 318 t[6].S(2).T(0).Dtis("----SS").ChainDiffs({2, 1, 6}).FrameDiffs({6}); 319 return structure; 320 } 321 322 FrameDependencyStructure ScalabilityStructureS3T3::DependencyStructure() const { 323 FrameDependencyStructure structure; 324 structure.num_decode_targets = 9; 325 structure.num_chains = 3; 326 structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1, 2, 2, 2}; 327 auto& t = structure.templates; 328 t.resize(15); 329 // Templates are shown in the order frames following them appear in the 330 // stream, but in `structure.templates` array templates are sorted by 331 // (`spatial_id`, `temporal_id`) since that is a dependency descriptor 332 // requirement. Indexes are written in hex for nicer alignment. 333 t[0x1].S(0).T(0).Dtis("SSS------").ChainDiffs({0, 0, 0}); 334 t[0x6].S(1).T(0).Dtis("---SSS---").ChainDiffs({1, 0, 0}); 335 t[0xB].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 0}); 336 t[0x3].S(0).T(2).Dtis("--D------").ChainDiffs({3, 2, 1}).FrameDiffs({3}); 337 t[0x8].S(1).T(2).Dtis("-----D---").ChainDiffs({4, 3, 2}).FrameDiffs({3}); 338 t[0xD].S(2).T(2).Dtis("--------D").ChainDiffs({5, 4, 3}).FrameDiffs({3}); 339 t[0x2].S(0).T(1).Dtis("-DS------").ChainDiffs({6, 5, 4}).FrameDiffs({6}); 340 t[0x7].S(1).T(1).Dtis("----DS---").ChainDiffs({7, 6, 5}).FrameDiffs({6}); 341 t[0xC].S(2).T(1).Dtis("-------DS").ChainDiffs({8, 7, 6}).FrameDiffs({6}); 342 t[0x4].S(0).T(2).Dtis("--D------").ChainDiffs({9, 8, 7}).FrameDiffs({3}); 343 t[0x9].S(1).T(2).Dtis("-----D---").ChainDiffs({10, 9, 8}).FrameDiffs({3}); 344 t[0xE].S(2).T(2).Dtis("--------D").ChainDiffs({11, 10, 9}).FrameDiffs({3}); 345 t[0x0].S(0).T(0).Dtis("SSS------").ChainDiffs({12, 11, 10}).FrameDiffs({12}); 346 t[0x5].S(1).T(0).Dtis("---SSS---").ChainDiffs({1, 12, 11}).FrameDiffs({12}); 347 t[0xA].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 12}).FrameDiffs({12}); 348 return structure; 349 } 350 351 } // namespace webrtc