frame_transformer_interface.h (8576B)
1 /* 2 * Copyright 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_FRAME_TRANSFORMER_INTERFACE_H_ 12 #define API_FRAME_TRANSFORMER_INTERFACE_H_ 13 14 #include <cstdint> 15 #include <memory> 16 #include <optional> 17 #include <string> 18 19 #include "api/array_view.h" 20 #include "api/ref_count.h" 21 #include "api/scoped_refptr.h" 22 #include "api/units/time_delta.h" 23 #include "api/units/timestamp.h" 24 #include "api/video/video_frame_metadata.h" 25 #include "rtc_base/checks.h" 26 #include "rtc_base/system/rtc_export.h" 27 28 namespace webrtc { 29 30 // Owns the frame payload data. 31 class TransformableFrameInterface { 32 public: 33 // Only a known list of internal implementations of transformable frames are 34 // permitted to allow internal downcasting. This is enforced via the 35 // internally-constructable Passkey. 36 // TODO: bugs.webrtc.org/339815768 - Remove this passkey once the 37 // downcasts are removed. 38 class Passkey; 39 RTC_EXPORT explicit TransformableFrameInterface(Passkey); 40 41 TransformableFrameInterface(TransformableFrameInterface&&) = default; 42 TransformableFrameInterface& operator=(TransformableFrameInterface&&) = 43 default; 44 45 virtual ~TransformableFrameInterface() = default; 46 47 // Returns the frame payload data. The data is valid until the next non-const 48 // method call. 49 virtual ArrayView<const uint8_t> GetData() const = 0; 50 51 // Copies `data` into the owned frame payload data. 52 virtual void SetData(ArrayView<const uint8_t> data) = 0; 53 54 virtual uint8_t GetPayloadType() const = 0; 55 virtual bool CanSetPayloadType() const { return false; } 56 virtual void SetPayloadType(uint8_t payload_type) { RTC_DCHECK_NOTREACHED(); } 57 virtual uint32_t GetSsrc() const = 0; 58 virtual uint32_t GetTimestamp() const = 0; 59 virtual void SetRTPTimestamp(uint32_t timestamp) = 0; 60 61 // TODO(https://bugs.webrtc.org/373365537): Remove this once its usage is 62 // removed from blink. 63 [[deprecated( 64 "Use GetPresentationTimestamp instead")]] virtual std::optional<Timestamp> 65 GetCaptureTimeIdentifier() const { 66 return std::nullopt; 67 } 68 69 // TODO(https://bugs.webrtc.org/14878): Change this to pure virtual after it 70 // is implemented everywhere. 71 virtual std::optional<Timestamp> GetPresentationTimestamp() const { 72 return std::nullopt; 73 } 74 75 enum class Direction { 76 kUnknown, 77 kReceiver, 78 kSender, 79 }; 80 // TODO(crbug.com/1250638): Remove this distinction between receiver and 81 // sender frames to allow received frames to be directly re-transmitted on 82 // other PeerConnectionss. 83 virtual Direction GetDirection() const { return Direction::kUnknown; } 84 virtual std::string GetMimeType() const = 0; 85 86 // Timestamp at which the packet has been first seen on the network interface. 87 // Only defined for received frames. 88 virtual std::optional<Timestamp> ReceiveTime() const = 0; 89 90 // Timestamp at which the frame was captured in the capturer system. 91 // For receiver frames, the timestamp is expressed in the capturer system's 92 // clock relative to the NTP epoch (January 1st 1970 00:00 UTC) and is 93 // available only if the absolute capture timestamp header extension is 94 // enabled. 95 // For sender frames, the timestamp is expressed relative to the local 96 // system clock's default epoch. 97 virtual std::optional<Timestamp> CaptureTime() const = 0; 98 virtual bool CanSetCaptureTime() const { return false; } 99 virtual void SetCaptureTime(std::optional<Timestamp> capture_time) { 100 RTC_DCHECK_NOTREACHED(); 101 } 102 103 // Offset between the sender system's clock and the capturer system's clock. 104 // Can be used to express the capture time in the local system's clock as 105 // long as the local system can determine the offset between its local clock 106 // and the sender system's clock. 107 // Accessible only if the absolute capture timestamp header extension is 108 // enabled. 109 virtual std::optional<TimeDelta> SenderCaptureTimeOffset() const = 0; 110 }; 111 112 class TransformableVideoFrameInterface : public TransformableFrameInterface { 113 public: 114 RTC_EXPORT explicit TransformableVideoFrameInterface(Passkey passkey); 115 virtual ~TransformableVideoFrameInterface() = default; 116 virtual bool IsKeyFrame() const = 0; 117 virtual std::optional<std::string> Rid() const { return std::nullopt; } 118 virtual VideoFrameMetadata Metadata() const = 0; 119 virtual void SetMetadata(const VideoFrameMetadata&) = 0; 120 }; 121 122 // Extends the TransformableFrameInterface to expose audio-specific information. 123 class TransformableAudioFrameInterface : public TransformableFrameInterface { 124 public: 125 RTC_EXPORT explicit TransformableAudioFrameInterface(Passkey passkey); 126 virtual ~TransformableAudioFrameInterface() = default; 127 128 virtual ArrayView<const uint32_t> GetContributingSources() const = 0; 129 130 virtual const std::optional<uint16_t> SequenceNumber() const = 0; 131 132 // TODO(crbug.com/391114797): Delete this function. 133 virtual std::optional<uint64_t> AbsoluteCaptureTimestamp() const = 0; 134 135 enum class FrameType { kEmptyFrame, kAudioFrameSpeech, kAudioFrameCN }; 136 137 // TODO(crbug.com/1456628): Change this to pure virtual after it 138 // is implemented everywhere. 139 virtual FrameType Type() const { return FrameType::kEmptyFrame; } 140 141 // Audio level in -dBov. Values range from 0 to 127, representing 0 to -127 142 // dBov. 127 represents digital silence. Only present on remote frames if 143 // the audio level header extension was included. 144 virtual std::optional<uint8_t> AudioLevel() const = 0; 145 virtual bool CanSetAudioLevel() const { return false; } 146 virtual void SetAudioLevel(std::optional<uint8_t> audio_level_dbov) { 147 RTC_DCHECK_NOTREACHED(); 148 } 149 }; 150 151 // Objects implement this interface to be notified with the transformed frame. 152 class TransformedFrameCallback : public RefCountInterface { 153 public: 154 virtual void OnTransformedFrame( 155 std::unique_ptr<TransformableFrameInterface> frame) = 0; 156 157 // Request to no longer be called on each frame, instead having frames be 158 // sent directly to OnTransformedFrame without additional work. 159 // TODO(crbug.com/1502781): Make pure virtual once all mocks have 160 // implementations. 161 virtual void StartShortCircuiting() {} 162 163 protected: 164 ~TransformedFrameCallback() override = default; 165 }; 166 167 // Transforms encoded frames. The transformed frame is sent in a callback using 168 // the TransformedFrameCallback interface (see above). 169 class FrameTransformerInterface : public RefCountInterface { 170 public: 171 // Transforms `frame` using the implementing class' processing logic. 172 virtual void Transform( 173 std::unique_ptr<TransformableFrameInterface> transformable_frame) = 0; 174 175 virtual void RegisterTransformedFrameCallback( 176 scoped_refptr<TransformedFrameCallback>) {} 177 virtual void RegisterTransformedFrameSinkCallback( 178 scoped_refptr<TransformedFrameCallback>, 179 uint32_t /* ssrc */) {} 180 virtual void UnregisterTransformedFrameCallback() {} 181 virtual void UnregisterTransformedFrameSinkCallback(uint32_t /* ssrc */) {} 182 183 protected: 184 ~FrameTransformerInterface() override = default; 185 }; 186 187 // An interface implemented by classes that can host a transform. 188 // Currently this is implemented by the RTCRtpSender and RTCRtpReceiver. 189 class FrameTransformerHost { 190 public: 191 virtual ~FrameTransformerHost() {} 192 virtual void SetFrameTransformer( 193 scoped_refptr<FrameTransformerInterface> frame_transformer) = 0; 194 // TODO: bugs.webrtc.org/15929 - To be added: 195 // virtual AddIncomingMediaType(RtpCodec codec) = 0; 196 // virtual AddOutgoingMediaType(RtpCodec codec) = 0; 197 }; 198 199 //------------------------------------------------------------------------------ 200 // Implementation details follow 201 //------------------------------------------------------------------------------ 202 class TransformableFrameInterface::Passkey { 203 public: 204 ~Passkey() = default; 205 206 private: 207 // Explicit list of allowed internal implmentations of 208 // TransformableFrameInterface. 209 friend class TransformableOutgoingAudioFrame; 210 friend class TransformableIncomingAudioFrame; 211 friend class TransformableVideoSenderFrame; 212 friend class TransformableVideoReceiverFrame; 213 214 friend class MockTransformableFrame; 215 friend class MockTransformableAudioFrame; 216 friend class MockTransformableVideoFrame; 217 Passkey() = default; 218 }; 219 220 } // namespace webrtc 221 222 #endif // API_FRAME_TRANSFORMER_INTERFACE_H_