channel_receive_frame_transformer_delegate.cc (9428B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "audio/channel_receive_frame_transformer_delegate.h" 12 13 #include <algorithm> 14 #include <cstdint> 15 #include <memory> 16 #include <optional> 17 #include <string> 18 #include <utility> 19 20 #include "api/array_view.h" 21 #include "api/frame_transformer_interface.h" 22 #include "api/rtp_headers.h" 23 #include "api/scoped_refptr.h" 24 #include "api/sequence_checker.h" 25 #include "api/task_queue/task_queue_base.h" 26 #include "api/units/time_delta.h" 27 #include "api/units/timestamp.h" 28 #include "rtc_base/buffer.h" 29 #include "rtc_base/checks.h" 30 #include "system_wrappers/include/ntp_time.h" 31 32 namespace webrtc { 33 34 class TransformableIncomingAudioFrame 35 : public TransformableAudioFrameInterface { 36 public: 37 TransformableIncomingAudioFrame(ArrayView<const uint8_t> payload, 38 const RTPHeader& header, 39 uint32_t ssrc, 40 const std::string& codec_mime_type, 41 Timestamp receive_time) 42 : TransformableAudioFrameInterface(Passkey()), 43 payload_(payload.data(), payload.size()), 44 header_(header), 45 ssrc_(ssrc), 46 codec_mime_type_(codec_mime_type), 47 receive_time_(receive_time) {} 48 ~TransformableIncomingAudioFrame() override = default; 49 ArrayView<const uint8_t> GetData() const override { return payload_; } 50 51 void SetData(ArrayView<const uint8_t> data) override { 52 payload_.SetData(data.data(), data.size()); 53 } 54 55 void SetRTPTimestamp(uint32_t timestamp) override { 56 header_.timestamp = timestamp; 57 } 58 59 uint8_t GetPayloadType() const override { return header_.payloadType; } 60 uint32_t GetSsrc() const override { return ssrc_; } 61 uint32_t GetTimestamp() const override { return header_.timestamp; } 62 ArrayView<const uint32_t> GetContributingSources() const override { 63 return ArrayView<const uint32_t>(header_.arrOfCSRCs, header_.numCSRCs); 64 } 65 Direction GetDirection() const override { return Direction::kReceiver; } 66 67 std::string GetMimeType() const override { return codec_mime_type_; } 68 const std::optional<uint16_t> SequenceNumber() const override { 69 return header_.sequenceNumber; 70 } 71 72 std::optional<uint64_t> AbsoluteCaptureTimestamp() const override { 73 // This could be extracted from received header extensions + extrapolation, 74 // if required in future, eg for being able to re-send received frames. 75 return std::nullopt; 76 } 77 const RTPHeader& Header() const { return header_; } 78 79 FrameType Type() const override { 80 if (!header_.extension.audio_level()) { 81 // Audio level extension not set. 82 return FrameType::kAudioFrameCN; 83 } 84 return header_.extension.audio_level()->voice_activity() 85 ? FrameType::kAudioFrameSpeech 86 : FrameType::kAudioFrameCN; 87 } 88 89 std::optional<uint8_t> AudioLevel() const override { 90 if (header_.extension.audio_level()) { 91 return header_.extension.audio_level()->level(); 92 } 93 return std::nullopt; 94 } 95 96 bool CanSetAudioLevel() const override { return true; } 97 98 void SetAudioLevel(std::optional<uint8_t> audio_level_dbov) override { 99 header_.extension.set_audio_level( 100 audio_level_dbov.has_value() 101 ? std::make_optional(webrtc::AudioLevel( 102 /*voice_activity=*/true, 103 std::min(*audio_level_dbov, static_cast<uint8_t>(127u)))) 104 : std::nullopt); 105 } 106 107 std::optional<Timestamp> ReceiveTime() const override { 108 return receive_time_ == Timestamp::MinusInfinity() 109 ? std::nullopt 110 : std::optional<Timestamp>(receive_time_); 111 } 112 113 std::optional<Timestamp> CaptureTime() const override { 114 if (header_.extension.absolute_capture_time) { 115 return Timestamp::Micros(UQ32x32ToInt64Us( 116 header_.extension.absolute_capture_time->absolute_capture_timestamp)); 117 } 118 return std::nullopt; 119 } 120 121 std::optional<TimeDelta> SenderCaptureTimeOffset() const override { 122 if (header_.extension.absolute_capture_time && 123 header_.extension.absolute_capture_time 124 ->estimated_capture_clock_offset) { 125 return TimeDelta::Micros( 126 Q32x32ToInt64Us(*header_.extension.absolute_capture_time 127 ->estimated_capture_clock_offset)); 128 } 129 return std::nullopt; 130 } 131 132 private: 133 Buffer payload_; 134 RTPHeader header_; 135 uint32_t ssrc_; 136 std::string codec_mime_type_; 137 Timestamp receive_time_; 138 }; 139 140 ChannelReceiveFrameTransformerDelegate::ChannelReceiveFrameTransformerDelegate( 141 ReceiveFrameCallback receive_frame_callback, 142 scoped_refptr<FrameTransformerInterface> frame_transformer, 143 TaskQueueBase* channel_receive_thread) 144 : receive_frame_callback_(receive_frame_callback), 145 frame_transformer_(std::move(frame_transformer)), 146 channel_receive_thread_(channel_receive_thread) {} 147 148 void ChannelReceiveFrameTransformerDelegate::Init() { 149 RTC_DCHECK_RUN_ON(&sequence_checker_); 150 frame_transformer_->RegisterTransformedFrameCallback( 151 scoped_refptr<TransformedFrameCallback>(this)); 152 } 153 154 void ChannelReceiveFrameTransformerDelegate::Reset() { 155 RTC_DCHECK_RUN_ON(&sequence_checker_); 156 frame_transformer_->UnregisterTransformedFrameCallback(); 157 frame_transformer_ = nullptr; 158 receive_frame_callback_ = ReceiveFrameCallback(); 159 } 160 161 void ChannelReceiveFrameTransformerDelegate::Transform( 162 ArrayView<const uint8_t> packet, 163 const RTPHeader& header, 164 uint32_t ssrc, 165 const std::string& codec_mime_type, 166 Timestamp receive_time) { 167 RTC_DCHECK_RUN_ON(&sequence_checker_); 168 if (short_circuit_) { 169 receive_frame_callback_(packet, header, receive_time); 170 } else { 171 frame_transformer_->Transform( 172 std::make_unique<TransformableIncomingAudioFrame>( 173 packet, header, ssrc, codec_mime_type, receive_time)); 174 } 175 } 176 177 void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame( 178 std::unique_ptr<TransformableFrameInterface> frame) { 179 scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate(this); 180 channel_receive_thread_->PostTask( 181 [delegate = std::move(delegate), frame = std::move(frame)]() mutable { 182 delegate->ReceiveFrame(std::move(frame)); 183 }); 184 } 185 186 void ChannelReceiveFrameTransformerDelegate::StartShortCircuiting() { 187 scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate(this); 188 channel_receive_thread_->PostTask([delegate = std::move(delegate)]() mutable { 189 RTC_DCHECK_RUN_ON(&delegate->sequence_checker_); 190 delegate->short_circuit_ = true; 191 }); 192 } 193 194 void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( 195 std::unique_ptr<TransformableFrameInterface> frame) const { 196 RTC_DCHECK_RUN_ON(&sequence_checker_); 197 if (!receive_frame_callback_) 198 return; 199 200 auto* transformed_frame = 201 static_cast<TransformableAudioFrameInterface*>(frame.get()); 202 Timestamp receive_time = 203 transformed_frame->ReceiveTime().value_or(Timestamp::MinusInfinity()); 204 RTPHeader header; 205 if (frame->GetDirection() == 206 TransformableFrameInterface::Direction::kSender) { 207 header.payloadType = transformed_frame->GetPayloadType(); 208 header.timestamp = transformed_frame->GetTimestamp(); 209 header.ssrc = transformed_frame->GetSsrc(); 210 if (transformed_frame->AbsoluteCaptureTimestamp().has_value()) { 211 header.extension.absolute_capture_time = AbsoluteCaptureTime(); 212 header.extension.absolute_capture_time->absolute_capture_timestamp = 213 transformed_frame->AbsoluteCaptureTimestamp().value(); 214 } 215 if (transformed_frame->AudioLevel().has_value()) { 216 // TODO(crbug.com/webrtc/419746427): Add support for voice activity in 217 // TransformableAudioFrameInterface. 218 header.extension.set_audio_level(AudioLevel( 219 /*voice_activity=*/true, *transformed_frame->AudioLevel())); 220 } 221 } else { 222 auto* transformed_incoming_frame = 223 static_cast<TransformableIncomingAudioFrame*>(frame.get()); 224 header = transformed_incoming_frame->Header(); 225 } 226 227 // TODO(crbug.com/1464860): Take an explicit struct with the required 228 // information rather than the RTPHeader to make it easier to 229 // construct the required information when injecting transformed frames not 230 // originally from this receiver. 231 receive_frame_callback_(frame->GetData(), header, receive_time); 232 } 233 234 scoped_refptr<FrameTransformerInterface> 235 ChannelReceiveFrameTransformerDelegate::FrameTransformer() { 236 RTC_DCHECK_RUN_ON(&sequence_checker_); 237 return frame_transformer_; 238 } 239 240 std::unique_ptr<TransformableAudioFrameInterface> CloneReceiverAudioFrame( 241 TransformableAudioFrameInterface* original) { 242 RTC_CHECK(original->GetDirection() == 243 TransformableFrameInterface::Direction::kReceiver); 244 245 auto* original_incoming_frame = 246 static_cast<TransformableIncomingAudioFrame*>(original); 247 return std::make_unique<TransformableIncomingAudioFrame>( 248 original->GetData(), original_incoming_frame->Header(), 249 original->GetSsrc(), original->GetMimeType(), 250 original->ReceiveTime().value_or(Timestamp::MinusInfinity())); 251 } 252 } // namespace webrtc