video_broadcaster.cc (8276B)
1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "media/base/video_broadcaster.h" 12 13 #include <algorithm> 14 #include <numeric> 15 #include <optional> 16 #include <vector> 17 18 #include "api/scoped_refptr.h" 19 #include "api/video/i420_buffer.h" 20 #include "api/video/video_frame.h" 21 #include "api/video/video_frame_buffer.h" 22 #include "api/video/video_rotation.h" 23 #include "api/video/video_sink_interface.h" 24 #include "api/video/video_source_interface.h" 25 #include "api/video_track_source_constraints.h" 26 #include "media/base/video_source_base.h" 27 #include "rtc_base/checks.h" 28 #include "rtc_base/logging.h" 29 #include "rtc_base/synchronization/mutex.h" 30 31 namespace webrtc { 32 33 VideoBroadcaster::VideoBroadcaster() = default; 34 VideoBroadcaster::~VideoBroadcaster() = default; 35 36 void VideoBroadcaster::AddOrUpdateSink(VideoSinkInterface<VideoFrame>* sink, 37 const VideoSinkWants& wants) { 38 RTC_DCHECK(sink != nullptr); 39 MutexLock lock(&sinks_and_wants_lock_); 40 if (!FindSinkPair(sink)) { 41 // `Sink` is a new sink, which didn't receive previous frame. 42 previous_frame_sent_to_all_sinks_ = false; 43 44 if (last_constraints_.has_value()) { 45 RTC_LOG(LS_INFO) << __func__ << " forwarding stored constraints min_fps " 46 << last_constraints_->min_fps.value_or(-1) << " max_fps " 47 << last_constraints_->max_fps.value_or(-1); 48 sink->OnConstraintsChanged(*last_constraints_); 49 } 50 } 51 VideoSourceBase::AddOrUpdateSink(sink, wants); 52 UpdateWants(); 53 } 54 55 void VideoBroadcaster::RemoveSink(VideoSinkInterface<VideoFrame>* sink) { 56 RTC_DCHECK(sink != nullptr); 57 MutexLock lock(&sinks_and_wants_lock_); 58 VideoSourceBase::RemoveSink(sink); 59 UpdateWants(); 60 } 61 62 bool VideoBroadcaster::frame_wanted() const { 63 MutexLock lock(&sinks_and_wants_lock_); 64 return !sink_pairs().empty(); 65 } 66 67 VideoSinkWants VideoBroadcaster::wants() const { 68 MutexLock lock(&sinks_and_wants_lock_); 69 return current_wants_; 70 } 71 72 void VideoBroadcaster::OnFrame(const VideoFrame& frame) { 73 MutexLock lock(&sinks_and_wants_lock_); 74 bool current_frame_was_discarded = false; 75 for (auto& sink_pair : sink_pairs()) { 76 if (sink_pair.wants.rotation_applied && 77 frame.rotation() != kVideoRotation_0) { 78 // Calls to OnFrame are not synchronized with changes to the sink wants. 79 // When rotation_applied is set to true, one or a few frames may get here 80 // with rotation still pending. Protect sinks that don't expect any 81 // pending rotation. 82 RTC_LOG(LS_VERBOSE) << "Discarding frame with unexpected rotation."; 83 sink_pair.sink->OnDiscardedFrame(); 84 current_frame_was_discarded = true; 85 continue; 86 } 87 if (sink_pair.wants.black_frames) { 88 VideoFrame black_frame = VideoFrame::Builder() 89 .set_video_frame_buffer(GetBlackFrameBuffer( 90 frame.width(), frame.height())) 91 .set_rotation(frame.rotation()) 92 .set_timestamp_us(frame.timestamp_us()) 93 .set_id(frame.id()) 94 .build(); 95 sink_pair.sink->OnFrame(black_frame); 96 } else if (!previous_frame_sent_to_all_sinks_ && frame.has_update_rect()) { 97 // Since last frame was not sent to some sinks, no reliable update 98 // information is available, so we need to clear the update rect. 99 VideoFrame copy = frame; 100 copy.clear_update_rect(); 101 sink_pair.sink->OnFrame(copy); 102 } else { 103 sink_pair.sink->OnFrame(frame); 104 } 105 } 106 previous_frame_sent_to_all_sinks_ = !current_frame_was_discarded; 107 } 108 109 void VideoBroadcaster::OnDiscardedFrame() { 110 MutexLock lock(&sinks_and_wants_lock_); 111 for (auto& sink_pair : sink_pairs()) { 112 sink_pair.sink->OnDiscardedFrame(); 113 } 114 } 115 116 void VideoBroadcaster::ProcessConstraints( 117 const VideoTrackSourceConstraints& constraints) { 118 MutexLock lock(&sinks_and_wants_lock_); 119 RTC_LOG(LS_INFO) << __func__ << " min_fps " 120 << constraints.min_fps.value_or(-1) << " max_fps " 121 << constraints.max_fps.value_or(-1) << " broadcasting to " 122 << sink_pairs().size() << " sinks."; 123 last_constraints_ = constraints; 124 for (auto& sink_pair : sink_pairs()) 125 sink_pair.sink->OnConstraintsChanged(constraints); 126 } 127 128 void VideoBroadcaster::UpdateWants() { 129 VideoSinkWants wants; 130 wants.rotation_applied = false; 131 wants.resolution_alignment = 1; 132 wants.aggregates.emplace(VideoSinkWants::Aggregates()); 133 wants.is_active = false; 134 135 // TODO(webrtc:14451) : I think it makes sense to always 136 // "ignore" encoders that are not active. But that would 137 // probably require a controlled roll out with a field trials? 138 // To play it safe, only ignore inactive encoders is there is an 139 // active encoder using the new api (scale_resolution_down_to), 140 // this means that there is only a behavioural change when using new 141 // api. 142 bool ignore_inactive_encoders_old_api = false; 143 for (auto& sink : sink_pairs()) { 144 if (sink.wants.is_active && sink.wants.requested_resolution.has_value()) { 145 ignore_inactive_encoders_old_api = true; 146 break; 147 } 148 } 149 150 for (auto& sink : sink_pairs()) { 151 if (!sink.wants.is_active && 152 (sink.wants.requested_resolution || ignore_inactive_encoders_old_api)) { 153 continue; 154 } 155 // wants.rotation_applied == ANY(sink.wants.rotation_applied) 156 if (sink.wants.rotation_applied) { 157 wants.rotation_applied = true; 158 } 159 // wants.max_pixel_count == MIN(sink.wants.max_pixel_count) 160 if (sink.wants.max_pixel_count < wants.max_pixel_count) { 161 wants.max_pixel_count = sink.wants.max_pixel_count; 162 } 163 // Select the minimum requested target_pixel_count, if any, of all sinks so 164 // that we don't over utilize the resources for any one. 165 // TODO(sprang): Consider using the median instead, since the limit can be 166 // expressed by max_pixel_count. 167 if (sink.wants.target_pixel_count && 168 (!wants.target_pixel_count || 169 (*sink.wants.target_pixel_count < *wants.target_pixel_count))) { 170 wants.target_pixel_count = sink.wants.target_pixel_count; 171 } 172 // Select the minimum for the requested max framerates. 173 if (sink.wants.max_framerate_fps < wants.max_framerate_fps) { 174 wants.max_framerate_fps = sink.wants.max_framerate_fps; 175 } 176 wants.resolution_alignment = 177 std::lcm(wants.resolution_alignment, sink.wants.resolution_alignment); 178 179 // Pick MAX(requested_resolution) since the actual can be downscaled in 180 // encoder instead. 181 if (sink.wants.requested_resolution) { 182 if (!wants.requested_resolution) { 183 wants.requested_resolution = sink.wants.requested_resolution; 184 } else { 185 wants.requested_resolution->width = 186 std::max(wants.requested_resolution->width, 187 sink.wants.requested_resolution->width); 188 wants.requested_resolution->height = 189 std::max(wants.requested_resolution->height, 190 sink.wants.requested_resolution->height); 191 } 192 } else if (sink.wants.is_active) { 193 wants.aggregates->any_active_without_requested_resolution = true; 194 } 195 196 wants.is_active |= sink.wants.is_active; 197 } 198 199 if (wants.target_pixel_count && 200 *wants.target_pixel_count >= wants.max_pixel_count) { 201 wants.target_pixel_count.emplace(wants.max_pixel_count); 202 } 203 current_wants_ = wants; 204 } 205 206 const scoped_refptr<VideoFrameBuffer>& VideoBroadcaster::GetBlackFrameBuffer( 207 int width, 208 int height) { 209 if (!black_frame_buffer_ || black_frame_buffer_->width() != width || 210 black_frame_buffer_->height() != height) { 211 scoped_refptr<I420Buffer> buffer = I420Buffer::Create(width, height); 212 I420Buffer::SetBlack(buffer.get()); 213 black_frame_buffer_ = buffer; 214 } 215 216 return black_frame_buffer_; 217 } 218 219 } // namespace webrtc