videoprocessor.h (10722B)
1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ 12 #define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ 13 14 #include <stddef.h> 15 #include <stdint.h> 16 17 #include <map> 18 #include <memory> 19 #include <optional> 20 #include <utility> 21 #include <vector> 22 23 #include "api/environment/environment.h" 24 #include "api/scoped_refptr.h" 25 #include "api/sequence_checker.h" 26 #include "api/task_queue/task_queue_base.h" 27 #include "api/test/videocodec_test_fixture.h" 28 #include "api/test/videocodec_test_stats.h" 29 #include "api/video/encoded_image.h" 30 #include "api/video/i420_buffer.h" 31 #include "api/video/video_bitrate_allocator.h" 32 #include "api/video/video_codec_type.h" 33 #include "api/video/video_frame.h" 34 #include "api/video/video_frame_buffer.h" 35 #include "api/video_codecs/video_decoder.h" 36 #include "api/video_codecs/video_encoder.h" 37 #include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h" 38 #include "modules/video_coding/include/video_codec_interface.h" 39 #include "modules/video_coding/utility/ivf_file_writer.h" 40 #include "rtc_base/checks.h" 41 #include "rtc_base/system/no_unique_address.h" 42 #include "rtc_base/thread_annotations.h" 43 #include "test/testsupport/frame_reader.h" 44 #include "test/testsupport/frame_writer.h" 45 46 namespace webrtc { 47 namespace test { 48 49 // Handles encoding/decoding of video using the VideoEncoder/VideoDecoder 50 // interfaces. This is done in a sequential manner in order to be able to 51 // measure times properly. 52 // The class processes a frame at the time for the configured input file. 53 // It maintains state of where in the source input file the processing is at. 54 // TODO(webrtc:14852): Deprecated in favor VideoCodecTester. 55 class VideoProcessor { 56 public: 57 using VideoDecoderList = std::vector<std::unique_ptr<VideoDecoder>>; 58 using LayerKey = std::pair<int /* spatial_idx */, int /* temporal_idx */>; 59 using IvfFileWriterMap = std::map<LayerKey, std::unique_ptr<IvfFileWriter>>; 60 // TODO(brandtr): Consider changing FrameWriterList to be a FrameWriterMap, 61 // to be able to save different TLs separately. 62 using FrameWriterList = std::vector<std::unique_ptr<FrameWriter>>; 63 using FrameStatistics = VideoCodecTestStats::FrameStatistics; 64 65 VideoProcessor(const Environment& env, 66 VideoEncoder* encoder, 67 VideoDecoderList* decoders, 68 FrameReader* input_frame_reader, 69 const VideoCodecTestFixture::Config& config, 70 VideoCodecTestStatsImpl* stats, 71 IvfFileWriterMap* encoded_frame_writers, 72 FrameWriterList* decoded_frame_writers); 73 ~VideoProcessor(); 74 75 VideoProcessor(const VideoProcessor&) = delete; 76 VideoProcessor& operator=(const VideoProcessor&) = delete; 77 78 // Reads a frame and sends it to the encoder. When the encode callback 79 // is received, the encoded frame is buffered. After encoding is finished 80 // buffered frame is sent to decoder. Quality evaluation is done in 81 // the decode callback. 82 void ProcessFrame(); 83 84 // Updates the encoder with target rates. Must be called at least once. 85 void SetRates(size_t bitrate_kbps, double framerate_fps); 86 87 // Signals processor to finalize frame processing and handle possible tail 88 // drops. If not called expelicitly, this will be called in dtor. It is 89 // unexpected to get ProcessFrame() or SetRates() calls after Finalize(). 90 void Finalize(); 91 92 private: 93 class VideoProcessorEncodeCompleteCallback 94 : public webrtc::EncodedImageCallback { 95 public: 96 explicit VideoProcessorEncodeCompleteCallback( 97 VideoProcessor* video_processor) 98 : video_processor_(video_processor), 99 task_queue_(TaskQueueBase::Current()) { 100 RTC_DCHECK(video_processor_); 101 RTC_DCHECK(task_queue_); 102 } 103 104 Result OnEncodedImage( 105 const webrtc::EncodedImage& encoded_image, 106 const webrtc::CodecSpecificInfo* codec_specific_info) override { 107 RTC_CHECK(codec_specific_info); 108 109 // Post the callback to the right task queue, if needed. 110 if (!task_queue_->IsCurrent()) { 111 VideoProcessor* video_processor = video_processor_; 112 task_queue_->PostTask([video_processor, encoded_image, 113 codec_specific_info = *codec_specific_info] { 114 video_processor->FrameEncoded(encoded_image, codec_specific_info); 115 }); 116 return Result(Result::OK, 0); 117 } 118 119 video_processor_->FrameEncoded(encoded_image, *codec_specific_info); 120 return Result(Result::OK, 0); 121 } 122 123 private: 124 VideoProcessor* const video_processor_; 125 TaskQueueBase* const task_queue_; 126 }; 127 128 class VideoProcessorDecodeCompleteCallback 129 : public webrtc::DecodedImageCallback { 130 public: 131 explicit VideoProcessorDecodeCompleteCallback( 132 VideoProcessor* video_processor, 133 size_t simulcast_svc_idx) 134 : video_processor_(video_processor), 135 simulcast_svc_idx_(simulcast_svc_idx), 136 task_queue_(TaskQueueBase::Current()) { 137 RTC_DCHECK(video_processor_); 138 RTC_DCHECK(task_queue_); 139 } 140 141 int32_t Decoded(webrtc::VideoFrame& image) override; 142 143 int32_t Decoded(webrtc::VideoFrame& image, 144 int64_t /* decode_time_ms */) override { 145 return Decoded(image); 146 } 147 148 void Decoded(webrtc::VideoFrame& image, 149 std::optional<int32_t> /* decode_time_ms */, 150 std::optional<uint8_t> /* qp */) override { 151 Decoded(image); 152 } 153 154 private: 155 VideoProcessor* const video_processor_; 156 const size_t simulcast_svc_idx_; 157 TaskQueueBase* const task_queue_; 158 }; 159 160 // Invoked by the callback adapter when a frame has completed encoding. 161 void FrameEncoded(const webrtc::EncodedImage& encoded_image, 162 const webrtc::CodecSpecificInfo& codec_specific); 163 164 // Invoked by the callback adapter when a frame has completed decoding. 165 void FrameDecoded(const webrtc::VideoFrame& image, size_t simulcast_svc_idx); 166 167 void DecodeFrame(const EncodedImage& encoded_image, size_t simulcast_svc_idx); 168 169 // In order to supply the SVC decoders with super frames containing all 170 // lower layer frames, we merge and store the layer frames in this method. 171 const webrtc::EncodedImage* BuildAndStoreSuperframe( 172 const EncodedImage& encoded_image, 173 VideoCodecType codec, 174 size_t frame_number, 175 size_t simulcast_svc_idx, 176 bool inter_layer_predicted) RTC_RUN_ON(sequence_checker_); 177 178 void CalcFrameQuality(const I420BufferInterface& decoded_frame, 179 FrameStatistics* frame_stat); 180 181 void WriteDecodedFrame(const I420BufferInterface& decoded_frame, 182 FrameWriter& frame_writer); 183 184 void HandleTailDrops(); 185 186 // Test config. 187 const VideoCodecTestFixture::Config config_; 188 const size_t num_simulcast_or_spatial_layers_; 189 const bool analyze_frame_quality_; 190 191 // Frame statistics. 192 VideoCodecTestStatsImpl* const stats_; 193 194 // Codecs. 195 webrtc::VideoEncoder* const encoder_; 196 VideoDecoderList* const decoders_; 197 const std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_; 198 199 // Target bitrate and framerate per frame. 200 std::map<size_t, RateProfile> target_rates_ RTC_GUARDED_BY(sequence_checker_); 201 202 // Adapters for the codec callbacks. 203 VideoProcessorEncodeCompleteCallback encode_callback_; 204 // Assign separate callback object to each decoder. This allows us to identify 205 // decoded layer in frame decode callback. 206 // simulcast_svc_idx -> decode callback. 207 std::vector<std::unique_ptr<VideoProcessorDecodeCompleteCallback>> 208 decode_callback_; 209 210 // Each call to ProcessFrame() will read one frame from `input_frame_reader_`. 211 FrameReader* const input_frame_reader_; 212 213 // Input frames are used as reference for frame quality evaluations. 214 // Async codecs might queue frames. To handle that we keep input frame 215 // and release it after corresponding coded frame is decoded and quality 216 // measurement is done. 217 // frame_number -> frame. 218 std::map<size_t, VideoFrame> input_frames_ RTC_GUARDED_BY(sequence_checker_); 219 220 // Encoder delivers coded frame layer-by-layer. We store coded frames and 221 // then, after all layers are encoded, decode them. Such separation of 222 // frame processing on superframe level simplifies encoding/decoding time 223 // measurement. 224 // simulcast_svc_idx -> merged SVC encoded frame. 225 std::vector<EncodedImage> merged_encoded_frames_ 226 RTC_GUARDED_BY(sequence_checker_); 227 228 // These (optional) file writers are used to persistently store the encoded 229 // and decoded bitstreams. Each frame writer is enabled by being non-null. 230 IvfFileWriterMap* const encoded_frame_writers_; 231 FrameWriterList* const decoded_frame_writers_; 232 233 // Metadata for inputed/encoded/decoded frames. Used for frame identification, 234 // frame drop detection, etc. We assume that encoded/decoded frames are 235 // ordered within each simulcast/spatial layer, but we do not make any 236 // assumptions of frame ordering between layers. 237 size_t last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_); 238 size_t last_inputed_timestamp_ RTC_GUARDED_BY(sequence_checker_); 239 // simulcast_svc_idx -> encode status. 240 std::vector<bool> first_encoded_frame_ RTC_GUARDED_BY(sequence_checker_); 241 // simulcast_svc_idx -> frame_number. 242 std::vector<size_t> last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); 243 // simulcast_svc_idx -> decode status. 244 std::vector<bool> first_decoded_frame_ RTC_GUARDED_BY(sequence_checker_); 245 // simulcast_svc_idx -> frame_number. 246 std::vector<size_t> last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); 247 // simulcast_svc_idx -> buffer. 248 std::vector<scoped_refptr<I420Buffer>> last_decoded_frame_buffer_ 249 RTC_GUARDED_BY(sequence_checker_); 250 251 // Time spent in frame encode callback. It is accumulated for layers and 252 // reset when frame encode starts. When next layer is encoded post-encode time 253 // is substracted from measured encode time. Thus we get pure encode time. 254 int64_t post_encode_time_ns_ RTC_GUARDED_BY(sequence_checker_); 255 256 // Indicates whether Finalize() was called or not. 257 bool is_finalized_ RTC_GUARDED_BY(sequence_checker_); 258 259 // This class must be operated on a TaskQueue. 260 RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; 261 }; 262 263 } // namespace test 264 } // namespace webrtc 265 266 #endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_