rtp_packetizer_h265.cc (11873B)
1 /* 2 * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/rtp_rtcp/source/rtp_packetizer_h265.h" 12 13 #include <algorithm> 14 #include <cstddef> 15 #include <cstdint> 16 #include <cstring> 17 #include <vector> 18 19 #include "api/array_view.h" 20 #include "common_video/h264/h264_common.h" 21 #include "common_video/h265/h265_common.h" 22 #include "modules/rtp_rtcp/source/byte_io.h" 23 #include "modules/rtp_rtcp/source/rtp_packet_h265_common.h" 24 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" 25 #include "rtc_base/checks.h" 26 27 namespace webrtc { 28 29 RtpPacketizerH265::RtpPacketizerH265(ArrayView<const uint8_t> payload, 30 PayloadSizeLimits limits) 31 : limits_(limits), num_packets_left_(0) { 32 for (const H264::NaluIndex& nalu : H264::FindNaluIndices(payload)) { 33 if (nalu.payload_size < 2) { 34 // Payload size has to include NALU header which is fixed 2 bytes. 35 input_fragments_.clear(); 36 return; 37 } 38 input_fragments_.push_back( 39 payload.subview(nalu.payload_start_offset, nalu.payload_size)); 40 } 41 42 if (!GeneratePackets()) { 43 // If failed to generate all the packets, discard already generated 44 // packets in case the caller would ignore return value and still try to 45 // call NextPacket(). 46 num_packets_left_ = 0; 47 while (!packets_.empty()) { 48 packets_.pop(); 49 } 50 } 51 } 52 53 RtpPacketizerH265::~RtpPacketizerH265() = default; 54 55 size_t RtpPacketizerH265::NumPackets() const { 56 return num_packets_left_; 57 } 58 59 bool RtpPacketizerH265::GeneratePackets() { 60 for (size_t i = 0; i < input_fragments_.size();) { 61 int fragment_len = input_fragments_[i].size(); 62 int single_packet_capacity = limits_.max_payload_len; 63 if (input_fragments_.size() == 1) { 64 single_packet_capacity -= limits_.single_packet_reduction_len; 65 } else if (i == 0) { 66 single_packet_capacity -= limits_.first_packet_reduction_len; 67 } else if (i + 1 == input_fragments_.size()) { 68 // Pretend that last fragment is larger instead of making last packet 69 // smaller. 70 single_packet_capacity -= limits_.last_packet_reduction_len; 71 } 72 if (fragment_len > single_packet_capacity) { 73 if (!PacketizeFu(i)) { 74 return false; 75 } 76 ++i; 77 } else { 78 i = PacketizeAp(i); 79 } 80 } 81 return true; 82 } 83 84 bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) { 85 // Fragment payload into packets (FU). 86 // Strip out the original header and leave room for the FU header. 87 ArrayView<const uint8_t> fragment = input_fragments_[fragment_index]; 88 PayloadSizeLimits limits = limits_; 89 // Refer to section 4.4.3 in RFC7798, each FU fragment will have a 2-bytes 90 // payload header and a one-byte FU header. DONL is not supported so ignore 91 // its size when calculating max_payload_len. 92 limits.max_payload_len -= 93 kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes; 94 95 // Update single/first/last packet reductions unless it is single/first/last 96 // fragment. 97 if (input_fragments_.size() != 1) { 98 // if this fragment is put into a single packet, it might still be the 99 // first or the last packet in the whole sequence of packets. 100 if (fragment_index == input_fragments_.size() - 1) { 101 limits.single_packet_reduction_len = limits_.last_packet_reduction_len; 102 } else if (fragment_index == 0) { 103 limits.single_packet_reduction_len = limits_.first_packet_reduction_len; 104 } else { 105 limits.single_packet_reduction_len = 0; 106 } 107 } 108 if (fragment_index != 0) { 109 limits.first_packet_reduction_len = 0; 110 } 111 if (fragment_index != input_fragments_.size() - 1) { 112 limits.last_packet_reduction_len = 0; 113 } 114 115 // Strip out the original header. 116 size_t payload_left = fragment.size() - kH265NalHeaderSizeBytes; 117 int offset = kH265NalHeaderSizeBytes; 118 119 std::vector<int> payload_sizes = SplitAboutEqually(payload_left, limits); 120 if (payload_sizes.empty()) { 121 return false; 122 } 123 124 for (size_t i = 0; i < payload_sizes.size(); ++i) { 125 int packet_length = payload_sizes[i]; 126 RTC_CHECK_GT(packet_length, 0); 127 uint16_t header = (fragment[0] << 8) | fragment[1]; 128 packets_.push({.source_fragment = fragment.subview(offset, packet_length), 129 .first_fragment = (i == 0), 130 .last_fragment = (i == payload_sizes.size() - 1), 131 .aggregated = false, 132 .header = header}); 133 offset += packet_length; 134 payload_left -= packet_length; 135 } 136 num_packets_left_ += payload_sizes.size(); 137 RTC_CHECK_EQ(payload_left, 0); 138 return true; 139 } 140 141 int RtpPacketizerH265::PacketizeAp(size_t fragment_index) { 142 // Aggregate fragments into one packet. 143 const bool includes_first = fragment_index == 0; 144 size_t payload_size_left = limits_.max_payload_len; 145 int aggregated_fragments = 0; 146 size_t fragment_headers_length = 0; 147 ArrayView<const uint8_t> fragment = input_fragments_[fragment_index]; 148 RTC_CHECK_GE(payload_size_left, fragment.size()); 149 ++num_packets_left_; 150 151 auto payload_size_needed = [&] { 152 size_t fragment_size = fragment.size() + fragment_headers_length; 153 bool includes_last = (fragment_index == input_fragments_.size() - 1); 154 if (includes_first && includes_last) { 155 return fragment_size + limits_.single_packet_reduction_len; 156 } else if (includes_first) { 157 return fragment_size + limits_.first_packet_reduction_len; 158 } else if (includes_last) { 159 return fragment_size + limits_.last_packet_reduction_len; 160 } else { 161 return fragment_size; 162 } 163 }; 164 165 uint16_t header = (fragment[0] << 8) | fragment[1]; 166 while (payload_size_left >= payload_size_needed()) { 167 RTC_CHECK_GT(fragment.size(), 0); 168 packets_.push({.source_fragment = fragment, 169 .first_fragment = (aggregated_fragments == 0), 170 .last_fragment = false, 171 .aggregated = true, 172 .header = header}); 173 payload_size_left -= fragment.size(); 174 payload_size_left -= fragment_headers_length; 175 176 fragment_headers_length = kH265LengthFieldSizeBytes; 177 // If we are going to try to aggregate more fragments into this packet 178 // we need to add the AP NALU header and a length field for the first 179 // NALU of this packet. 180 if (aggregated_fragments == 0) { 181 fragment_headers_length += 182 kH265PayloadHeaderSizeBytes + kH265LengthFieldSizeBytes; 183 } 184 ++aggregated_fragments; 185 186 // Next fragment. 187 ++fragment_index; 188 if (fragment_index == input_fragments_.size()) { 189 break; 190 } 191 fragment = input_fragments_[fragment_index]; 192 } 193 RTC_CHECK_GT(aggregated_fragments, 0); 194 packets_.back().last_fragment = true; 195 return fragment_index; 196 } 197 198 bool RtpPacketizerH265::NextPacket(RtpPacketToSend* rtp_packet) { 199 RTC_DCHECK(rtp_packet); 200 201 if (packets_.empty()) { 202 return false; 203 } 204 205 PacketUnit packet = packets_.front(); 206 207 if (packet.first_fragment && packet.last_fragment) { 208 // Single NAL unit packet. Do not support DONL for single NAL unit packets, 209 // DONL field is not present. 210 rtp_packet->SetPayload(packet.source_fragment); 211 packets_.pop(); 212 input_fragments_.pop_front(); 213 } else if (packet.aggregated) { 214 NextAggregatePacket(rtp_packet); 215 } else { 216 NextFragmentPacket(rtp_packet); 217 } 218 rtp_packet->SetMarker(packets_.empty()); 219 --num_packets_left_; 220 return true; 221 } 222 223 void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) { 224 size_t payload_capacity = rtp_packet->FreeCapacity(); 225 RTC_CHECK_GE(payload_capacity, kH265PayloadHeaderSizeBytes); 226 uint8_t* buffer = rtp_packet->AllocatePayload(payload_capacity); 227 RTC_CHECK(buffer); 228 PacketUnit* packet = &packets_.front(); 229 RTC_CHECK(packet->first_fragment); 230 231 /* 232 +---------------+---------------+ 233 |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| 234 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 235 |F| Type | LayerId | TID | 236 +-------------+-----------------+ 237 */ 238 // Refer to section 4.4.2 for aggregation packets and modify type to 239 // 48 in PayloadHdr for aggregate packet. Do not support DONL for aggregation 240 // packets, DONL field is not present. 241 int index = kH265PayloadHeaderSizeBytes; 242 bool is_last_fragment = packet->last_fragment; 243 244 // Refer to section 4.4.2 for aggregation packets and calculate the lowest 245 // value of LayerId and TID of all the aggregated NAL units 246 uint8_t layer_id_min = kH265MaxLayerId; 247 uint8_t temporal_id_min = kH265MaxTemporalId; 248 while (packet->aggregated) { 249 // Add NAL unit length field. 250 ArrayView<const uint8_t> fragment = packet->source_fragment; 251 uint8_t layer_id = ((fragment[0] & kH265LayerIDHMask) << 5) | 252 ((fragment[1] & kH265LayerIDLMask) >> 3); 253 layer_id_min = std::min(layer_id_min, layer_id); 254 uint8_t temporal_id = fragment[1] & kH265TIDMask; 255 temporal_id_min = std::min(temporal_id_min, temporal_id); 256 257 ByteWriter<uint16_t>::WriteBigEndian(&buffer[index], fragment.size()); 258 index += kH265LengthFieldSizeBytes; 259 // Add NAL unit. 260 memcpy(&buffer[index], fragment.data(), fragment.size()); 261 index += fragment.size(); 262 packets_.pop(); 263 input_fragments_.pop_front(); 264 if (is_last_fragment) { 265 break; 266 } 267 packet = &packets_.front(); 268 is_last_fragment = packet->last_fragment; 269 } 270 271 buffer[0] = (H265::NaluType::kAp << 1) | (layer_id_min >> 5); 272 buffer[1] = (layer_id_min << 3) | temporal_id_min; 273 RTC_CHECK(is_last_fragment); 274 rtp_packet->SetPayloadSize(index); 275 } 276 277 void RtpPacketizerH265::NextFragmentPacket(RtpPacketToSend* rtp_packet) { 278 PacketUnit* packet = &packets_.front(); 279 // NAL unit fragmented over multiple packets (FU). 280 // We do not send original NALU header, so it will be replaced by the 281 // PayloadHdr of the first packet. 282 /* 283 +---------------+---------------+ 284 |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| 285 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 286 |F| Type | LayerId | TID | 287 +-------------+-----------------+ 288 */ 289 // Refer to section section 4.4.3 for aggregation packets and modify type to 290 // 49 in PayloadHdr for aggregate packet. 291 uint8_t payload_hdr_h = 292 packet->header >> 8; // 1-bit F, 6-bit type, 1-bit layerID highest-bit 293 uint8_t payload_hdr_l = packet->header & 0xFF; 294 uint8_t layer_id_h = payload_hdr_h & kH265LayerIDHMask; 295 uint8_t fu_header = 0; 296 /* 297 +---------------+ 298 |0|1|2|3|4|5|6|7| 299 +-+-+-+-+-+-+-+-+ 300 |S|E| FuType | 301 +---------------+ 302 */ 303 // S bit indicates the start of a fragmented NAL unit. 304 // E bit indicates the end of a fragmented NAL unit. 305 // FuType must be equal to the field type value of the fragmented NAL unit. 306 fu_header |= (packet->first_fragment ? kH265SBitMask : 0); 307 fu_header |= (packet->last_fragment ? kH265EBitMask : 0); 308 uint8_t type = (payload_hdr_h & kH265TypeMask) >> 1; 309 fu_header |= type; 310 // Now update payload_hdr_h with FU type. 311 payload_hdr_h = (payload_hdr_h & kH265TypeMaskN) | 312 (H265::NaluType::kFu << 1) | layer_id_h; 313 ArrayView<const uint8_t> fragment = packet->source_fragment; 314 uint8_t* buffer = rtp_packet->AllocatePayload( 315 kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes + fragment.size()); 316 RTC_CHECK(buffer); 317 buffer[0] = payload_hdr_h; 318 buffer[1] = payload_hdr_l; 319 buffer[2] = fu_header; 320 321 // Do not support DONL for fragmentation units, DONL field is not present. 322 memcpy(buffer + kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes, 323 fragment.data(), fragment.size()); 324 if (packet->last_fragment) { 325 input_fragments_.pop_front(); 326 } 327 packets_.pop(); 328 } 329 330 } // namespace webrtc