simulcast_to_svc_converter.cc (6963B)
1 /* 2 * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/video_coding/svc/simulcast_to_svc_converter.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <optional> 16 17 #include "api/video/encoded_image.h" 18 #include "api/video/video_bitrate_allocation.h" 19 #include "api/video_codecs/scalability_mode.h" 20 #include "api/video_codecs/video_codec.h" 21 #include "modules/video_coding/codecs/interface/common_constants.h" 22 #include "modules/video_coding/include/video_codec_interface.h" 23 #include "modules/video_coding/svc/create_scalability_structure.h" 24 #include "modules/video_coding/svc/scalability_mode_util.h" 25 #include "modules/video_coding/utility/simulcast_utility.h" 26 #include "rtc_base/checks.h" 27 28 namespace webrtc { 29 30 SimulcastToSvcConverter::SimulcastToSvcConverter(const VideoCodec& codec) { 31 config_ = codec; 32 int num_temporal_layers = 33 config_.simulcastStream[0].GetNumberOfTemporalLayers(); 34 int num_spatial_layers = config_.numberOfSimulcastStreams; 35 ScalabilityMode scalability_mode; 36 switch (num_temporal_layers) { 37 case 1: 38 scalability_mode = ScalabilityMode::kL1T1; 39 break; 40 case 2: 41 scalability_mode = ScalabilityMode::kL1T2; 42 break; 43 case 3: 44 scalability_mode = ScalabilityMode::kL1T3; 45 break; 46 default: 47 RTC_DCHECK_NOTREACHED(); 48 } 49 50 for (int i = 0; i < num_spatial_layers; ++i) { 51 config_.spatialLayers[i] = config_.simulcastStream[i]; 52 } 53 config_.simulcastStream[0] = 54 config_.simulcastStream[config_.numberOfSimulcastStreams - 1]; 55 config_.VP9()->numberOfSpatialLayers = config_.numberOfSimulcastStreams; 56 config_.VP9()->numberOfTemporalLayers = 57 config_.spatialLayers[0].numberOfTemporalLayers; 58 config_.VP9()->interLayerPred = InterLayerPredMode::kOff; 59 config_.numberOfSimulcastStreams = 1; 60 config_.UnsetScalabilityMode(); 61 62 for (int i = 0; i < num_spatial_layers; ++i) { 63 layers_.emplace_back(scalability_mode, num_temporal_layers); 64 } 65 } 66 67 VideoCodec SimulcastToSvcConverter::GetConfig() const { 68 return config_; 69 } 70 71 void SimulcastToSvcConverter::EncodeStarted(bool force_keyframe) { 72 // Check if at least one layer was encoded successfully. 73 bool some_layers_has_completed = false; 74 for (size_t i = 0; i < layers_.size(); ++i) { 75 some_layers_has_completed |= !layers_[i].awaiting_frame; 76 } 77 for (size_t i = 0; i < layers_.size(); ++i) { 78 if (layers_[i].awaiting_frame && some_layers_has_completed) { 79 // Simulcast SVC controller updates pattern on all layers, even 80 // if some layers has dropped the frame. 81 // Simulate that behavior for all controllers, not updated 82 // while rewriting frame descriptors. 83 layers_[i].video_controller->OnEncodeDone(layers_[i].layer_config); 84 } 85 layers_[i].awaiting_frame = true; 86 auto configs = layers_[i].video_controller->NextFrameConfig(force_keyframe); 87 RTC_CHECK_EQ(configs.size(), 1u); 88 layers_[i].layer_config = configs[0]; 89 } 90 } 91 92 bool SimulcastToSvcConverter::ConvertFrame(EncodedImage& encoded_image, 93 CodecSpecificInfo& codec_specific) { 94 int sid = encoded_image.SpatialIndex().value_or(0); 95 encoded_image.SetSimulcastIndex(sid); 96 encoded_image.SetSpatialIndex(std::nullopt); 97 codec_specific.end_of_picture = true; 98 if (codec_specific.scalability_mode) { 99 int num_temporal_layers = 100 ScalabilityModeToNumTemporalLayers(*codec_specific.scalability_mode); 101 RTC_DCHECK_LE(num_temporal_layers, 3); 102 if (num_temporal_layers == 1) { 103 codec_specific.scalability_mode = ScalabilityMode::kL1T1; 104 } else if (num_temporal_layers == 2) { 105 codec_specific.scalability_mode = ScalabilityMode::kL1T2; 106 } else if (num_temporal_layers == 3) { 107 codec_specific.scalability_mode = ScalabilityMode::kL1T3; 108 } 109 } 110 CodecSpecificInfoVP9& vp9_info = codec_specific.codecSpecific.VP9; 111 vp9_info.num_spatial_layers = 1; 112 vp9_info.first_active_layer = 0; 113 vp9_info.first_frame_in_picture = true; 114 if (vp9_info.ss_data_available) { 115 vp9_info.width[0] = vp9_info.width[sid]; 116 vp9_info.height[0] = vp9_info.height[sid]; 117 } 118 119 auto& video_controller = *layers_[sid].video_controller; 120 if (codec_specific.generic_frame_info) { 121 layers_[sid].awaiting_frame = false; 122 uint8_t tid = encoded_image.TemporalIndex().value_or(0); 123 auto& frame_config = layers_[sid].layer_config; 124 RTC_DCHECK_EQ(frame_config.TemporalId(), tid == kNoTemporalIdx ? 0 : tid); 125 if (frame_config.TemporalId() != (tid == kNoTemporalIdx ? 0 : tid)) { 126 return false; 127 } 128 codec_specific.generic_frame_info = 129 video_controller.OnEncodeDone(frame_config); 130 } 131 if (codec_specific.template_structure) { 132 auto resolution = codec_specific.template_structure->resolutions[sid]; 133 codec_specific.template_structure = video_controller.DependencyStructure(); 134 codec_specific.template_structure->resolutions.resize(1); 135 codec_specific.template_structure->resolutions[0] = resolution; 136 } 137 return true; 138 } 139 140 SimulcastToSvcConverter::LayerState::LayerState( 141 ScalabilityMode scalability_mode, 142 int num_temporal_layers) 143 : video_controller(CreateScalabilityStructure(scalability_mode)), 144 awaiting_frame(false) { 145 VideoBitrateAllocation dummy_bitrates; 146 for (int i = 0; i < num_temporal_layers; ++i) { 147 dummy_bitrates.SetBitrate(0, i, 10000); 148 } 149 video_controller->OnRatesUpdated(dummy_bitrates); 150 } 151 152 // static 153 bool SimulcastToSvcConverter::IsConfigSupported(const VideoCodec& codec) { 154 if (codec.numberOfSimulcastStreams <= 1 || 155 !SimulcastUtility::ValidSimulcastParameters( 156 codec, codec.numberOfSimulcastStreams)) { 157 return false; 158 } 159 // Ensure there's 4:2:1 scaling. 160 for (int i = 1; i < codec.numberOfSimulcastStreams; ++i) { 161 if (codec.simulcastStream[i].active && 162 codec.simulcastStream[i - 1].active && 163 (codec.simulcastStream[i].width != 164 codec.simulcastStream[i - 1].width * 2 || 165 codec.simulcastStream[i].height != 166 codec.simulcastStream[i - 1].height * 2)) { 167 return false; 168 } 169 } 170 int first_active_layer = -1; 171 int last_active_layer = -1; 172 int num_active_layers = 0; 173 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { 174 if (codec.simulcastStream[i].active) { 175 if (first_active_layer < 0) 176 first_active_layer = i; 177 last_active_layer = i; 178 ++num_active_layers; 179 } 180 } 181 // Active layers must form a continuous segment. Can't have holes, because 182 // most SVC encoders can't process that. 183 return num_active_layers == last_active_layer - first_active_layer + 1; 184 } 185 186 } // namespace webrtc