channel_send_frame_transformer_delegate.cc (9581B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "audio/channel_send_frame_transformer_delegate.h" 12 13 #include <cstddef> 14 #include <cstdint> 15 #include <memory> 16 #include <optional> 17 #include <string> 18 #include <utility> 19 #include <vector> 20 21 #include "api/array_view.h" 22 #include "api/frame_transformer_interface.h" 23 #include "api/scoped_refptr.h" 24 #include "api/sequence_checker.h" 25 #include "api/task_queue/task_queue_base.h" 26 #include "api/units/time_delta.h" 27 #include "api/units/timestamp.h" 28 #include "modules/audio_coding/include/audio_coding_module_typedefs.h" 29 #include "rtc_base/buffer.h" 30 #include "rtc_base/checks.h" 31 #include "rtc_base/synchronization/mutex.h" 32 33 namespace webrtc { 34 namespace { 35 36 using IfaceFrameType = TransformableAudioFrameInterface::FrameType; 37 38 IfaceFrameType InternalFrameTypeToInterfaceFrameType( 39 const AudioFrameType frame_type) { 40 switch (frame_type) { 41 case AudioFrameType::kEmptyFrame: 42 return IfaceFrameType::kEmptyFrame; 43 case AudioFrameType::kAudioFrameSpeech: 44 return IfaceFrameType::kAudioFrameSpeech; 45 case AudioFrameType::kAudioFrameCN: 46 return IfaceFrameType::kAudioFrameCN; 47 } 48 RTC_DCHECK_NOTREACHED(); 49 return IfaceFrameType::kEmptyFrame; 50 } 51 52 AudioFrameType InterfaceFrameTypeToInternalFrameType( 53 const IfaceFrameType frame_type) { 54 switch (frame_type) { 55 case IfaceFrameType::kEmptyFrame: 56 return AudioFrameType::kEmptyFrame; 57 case IfaceFrameType::kAudioFrameSpeech: 58 return AudioFrameType::kAudioFrameSpeech; 59 case IfaceFrameType::kAudioFrameCN: 60 return AudioFrameType::kAudioFrameCN; 61 } 62 RTC_DCHECK_NOTREACHED(); 63 return AudioFrameType::kEmptyFrame; 64 } 65 } // namespace 66 67 class TransformableOutgoingAudioFrame 68 : public TransformableAudioFrameInterface { 69 public: 70 TransformableOutgoingAudioFrame( 71 AudioFrameType frame_type, 72 uint8_t payload_type, 73 uint32_t rtp_timestamp_with_offset, 74 const uint8_t* payload_data, 75 size_t payload_size, 76 std::optional<uint64_t> absolute_capture_timestamp_ms, 77 uint32_t ssrc, 78 std::vector<uint32_t> csrcs, 79 const std::string& codec_mime_type, 80 std::optional<uint16_t> sequence_number, 81 std::optional<uint8_t> audio_level_dbov) 82 : TransformableAudioFrameInterface(Passkey()), 83 frame_type_(frame_type), 84 payload_type_(payload_type), 85 rtp_timestamp_with_offset_(rtp_timestamp_with_offset), 86 payload_(payload_data, payload_size), 87 absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms), 88 ssrc_(ssrc), 89 csrcs_(std::move(csrcs)), 90 codec_mime_type_(codec_mime_type), 91 sequence_number_(sequence_number), 92 audio_level_dbov_(audio_level_dbov) {} 93 ~TransformableOutgoingAudioFrame() override = default; 94 ArrayView<const uint8_t> GetData() const override { return payload_; } 95 void SetData(ArrayView<const uint8_t> data) override { 96 payload_.SetData(data.data(), data.size()); 97 } 98 uint32_t GetTimestamp() const override { return rtp_timestamp_with_offset_; } 99 uint32_t GetSsrc() const override { return ssrc_; } 100 101 IfaceFrameType Type() const override { 102 return InternalFrameTypeToInterfaceFrameType(frame_type_); 103 } 104 105 uint8_t GetPayloadType() const override { return payload_type_; } 106 bool CanSetPayloadType() const override { return true; } 107 void SetPayloadType(uint8_t payload_type) override { 108 payload_type_ = payload_type; 109 } 110 Direction GetDirection() const override { return Direction::kSender; } 111 std::string GetMimeType() const override { return codec_mime_type_; } 112 113 ArrayView<const uint32_t> GetContributingSources() const override { 114 return csrcs_; 115 } 116 117 const std::optional<uint16_t> SequenceNumber() const override { 118 return sequence_number_; 119 } 120 121 void SetRTPTimestamp(uint32_t rtp_timestamp_with_offset) override { 122 rtp_timestamp_with_offset_ = rtp_timestamp_with_offset; 123 } 124 125 std::optional<uint64_t> AbsoluteCaptureTimestamp() const override { 126 return absolute_capture_timestamp_ms_; 127 } 128 129 std::optional<uint8_t> AudioLevel() const override { 130 return audio_level_dbov_; 131 } 132 133 bool CanSetAudioLevel() const override { return true; } 134 void SetAudioLevel(std::optional<uint8_t> audio_level_dbov) override { 135 if (audio_level_dbov.has_value() && audio_level_dbov > 127u) { 136 audio_level_dbov = 127u; 137 } 138 audio_level_dbov_ = audio_level_dbov; 139 } 140 141 std::optional<Timestamp> ReceiveTime() const override { return std::nullopt; } 142 143 std::optional<Timestamp> CaptureTime() const override { 144 return absolute_capture_timestamp_ms_ 145 ? std::make_optional( 146 Timestamp::Millis(*absolute_capture_timestamp_ms_)) 147 : std::nullopt; 148 } 149 bool CanSetCaptureTime() const override { return true; } 150 void SetCaptureTime(std::optional<Timestamp> capture_time) override { 151 absolute_capture_timestamp_ms_ = 152 capture_time ? std::make_optional(capture_time->ms()) : std::nullopt; 153 } 154 std::optional<TimeDelta> SenderCaptureTimeOffset() const override { 155 return std::nullopt; 156 } 157 158 private: 159 AudioFrameType frame_type_; 160 uint8_t payload_type_; 161 uint32_t rtp_timestamp_with_offset_; 162 Buffer payload_; 163 std::optional<uint64_t> absolute_capture_timestamp_ms_; 164 uint32_t ssrc_; 165 std::vector<uint32_t> csrcs_; 166 std::string codec_mime_type_; 167 std::optional<uint16_t> sequence_number_; 168 std::optional<uint8_t> audio_level_dbov_; 169 }; 170 171 ChannelSendFrameTransformerDelegate::ChannelSendFrameTransformerDelegate( 172 SendFrameCallback send_frame_callback, 173 scoped_refptr<FrameTransformerInterface> frame_transformer, 174 TaskQueueBase* encoder_queue) 175 : send_frame_callback_(send_frame_callback), 176 frame_transformer_(std::move(frame_transformer)), 177 encoder_queue_(encoder_queue) {} 178 179 void ChannelSendFrameTransformerDelegate::Init() { 180 frame_transformer_->RegisterTransformedFrameCallback( 181 scoped_refptr<TransformedFrameCallback>(this)); 182 } 183 184 void ChannelSendFrameTransformerDelegate::Reset() { 185 frame_transformer_->UnregisterTransformedFrameCallback(); 186 frame_transformer_ = nullptr; 187 188 MutexLock lock(&send_lock_); 189 send_frame_callback_ = SendFrameCallback(); 190 } 191 192 void ChannelSendFrameTransformerDelegate::Transform( 193 AudioFrameType frame_type, 194 uint8_t payload_type, 195 uint32_t rtp_timestamp, 196 const uint8_t* payload_data, 197 size_t payload_size, 198 int64_t absolute_capture_timestamp_ms, 199 uint32_t ssrc, 200 const std::string& codec_mime_type, 201 std::optional<uint8_t> audio_level_dbov, 202 const std::vector<uint32_t>& csrcs) { 203 { 204 MutexLock lock(&send_lock_); 205 if (short_circuit_) { 206 send_frame_callback_(frame_type, payload_type, rtp_timestamp, 207 ArrayView<const uint8_t>(payload_data, payload_size), 208 absolute_capture_timestamp_ms, csrcs, 209 audio_level_dbov); 210 return; 211 } 212 } 213 frame_transformer_->Transform( 214 std::make_unique<TransformableOutgoingAudioFrame>( 215 frame_type, payload_type, rtp_timestamp, payload_data, payload_size, 216 absolute_capture_timestamp_ms, ssrc, csrcs, codec_mime_type, 217 /*sequence_number=*/std::nullopt, audio_level_dbov)); 218 } 219 220 void ChannelSendFrameTransformerDelegate::OnTransformedFrame( 221 std::unique_ptr<TransformableFrameInterface> frame) { 222 MutexLock lock(&send_lock_); 223 if (!send_frame_callback_) 224 return; 225 scoped_refptr<ChannelSendFrameTransformerDelegate> delegate(this); 226 encoder_queue_->PostTask( 227 [delegate = std::move(delegate), frame = std::move(frame)]() mutable { 228 delegate->SendFrame(std::move(frame)); 229 }); 230 } 231 232 void ChannelSendFrameTransformerDelegate::StartShortCircuiting() { 233 MutexLock lock(&send_lock_); 234 short_circuit_ = true; 235 } 236 237 void ChannelSendFrameTransformerDelegate::SendFrame( 238 std::unique_ptr<TransformableFrameInterface> frame) const { 239 MutexLock lock(&send_lock_); 240 RTC_DCHECK_RUN_ON(encoder_queue_); 241 if (!send_frame_callback_) 242 return; 243 auto* transformed_frame = 244 static_cast<TransformableAudioFrameInterface*>(frame.get()); 245 send_frame_callback_( 246 InterfaceFrameTypeToInternalFrameType(transformed_frame->Type()), 247 transformed_frame->GetPayloadType(), transformed_frame->GetTimestamp(), 248 transformed_frame->GetData(), 249 transformed_frame->AbsoluteCaptureTimestamp() 250 ? *transformed_frame->AbsoluteCaptureTimestamp() 251 : 0, 252 transformed_frame->GetContributingSources(), 253 transformed_frame->AudioLevel()); 254 } 255 256 std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame( 257 TransformableAudioFrameInterface* original) { 258 std::vector<uint32_t> csrcs; 259 csrcs.assign(original->GetContributingSources().begin(), 260 original->GetContributingSources().end()); 261 return std::make_unique<TransformableOutgoingAudioFrame>( 262 InterfaceFrameTypeToInternalFrameType(original->Type()), 263 original->GetPayloadType(), original->GetTimestamp(), 264 original->GetData().data(), original->GetData().size(), 265 original->AbsoluteCaptureTimestamp(), original->GetSsrc(), 266 std::move(csrcs), original->GetMimeType(), original->SequenceNumber(), 267 original->AudioLevel()); 268 } 269 270 } // namespace webrtc