shared_screencast_stream.cc (37797B)
1 /* 2 * Copyright 2022 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/desktop_capture/linux/wayland/shared_screencast_stream.h" 12 13 #include <fcntl.h> 14 #include <libdrm/drm_fourcc.h> 15 #include <pipewire/pipewire.h> 16 #include <spa/buffer/buffer.h> 17 #include <spa/buffer/meta.h> 18 #include <spa/param/format.h> 19 #include <spa/param/param.h> 20 #include <spa/param/video/format-utils.h> 21 #include <spa/param/video/raw.h> 22 #include <spa/pod/builder.h> 23 #include <spa/pod/iter.h> 24 #include <spa/pod/vararg.h> 25 #include <spa/support/loop.h> 26 #include <spa/utils/defs.h> 27 #include <spa/utils/hook.h> 28 #include <spa/utils/type.h> 29 #include <sys/mman.h> 30 #include <sys/types.h> 31 32 #include <algorithm> 33 #include <cerrno> 34 #include <cstdint> 35 #include <cstring> 36 #include <memory> 37 #include <optional> 38 #include <utility> 39 #include <vector> 40 41 #include "api/scoped_refptr.h" 42 #include "modules/desktop_capture/desktop_capture_types.h" 43 #include "modules/desktop_capture/desktop_capturer.h" 44 #include "modules/desktop_capture/desktop_frame.h" 45 #include "modules/desktop_capture/desktop_geometry.h" 46 #include "modules/desktop_capture/desktop_region.h" 47 #include "modules/desktop_capture/linux/wayland/egl_dmabuf.h" 48 #include "modules/desktop_capture/linux/wayland/screencast_stream_utils.h" 49 #include "modules/desktop_capture/mouse_cursor.h" 50 #include "modules/desktop_capture/screen_capture_frame_queue.h" 51 #include "modules/desktop_capture/shared_desktop_frame.h" 52 #include "modules/portal/pipewire_utils.h" 53 #include "rtc_base/checks.h" 54 #include "rtc_base/logging.h" 55 #include "rtc_base/sanitizer.h" 56 #include "rtc_base/synchronization/mutex.h" 57 #include "rtc_base/thread_annotations.h" 58 #include "rtc_base/time_utils.h" 59 60 // Wrapper for gfxVars::UseDMABuf() as we can't include gfxVars here. 61 // We don't want to use dmabuf of known broken systems. 62 // See FEATURE_DMABUF for details. 63 namespace mozilla::gfx { 64 bool IsDMABufEnabled(); 65 } 66 67 namespace webrtc { 68 69 const int kBytesPerPixel = 4; 70 const int kVideoDamageRegionCount = 16; 71 72 constexpr int kCursorBpp = 4; 73 constexpr int CursorMetaSize(int w, int h) { 74 return (sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + 75 w * h * kCursorBpp); 76 } 77 78 constexpr PipeWireVersion kDmaBufModifierMinVersion = {.major = 0, 79 .minor = 3, 80 .micro = 33}; 81 constexpr PipeWireVersion kDropSingleModifierMinVersion = {.major = 0, 82 .minor = 3, 83 .micro = 40}; 84 85 class SharedScreenCastStreamPrivate { 86 public: 87 SharedScreenCastStreamPrivate(); 88 ~SharedScreenCastStreamPrivate(); 89 90 bool StartScreenCastStream(uint32_t stream_node_id, 91 int fd, 92 uint32_t width = 0, 93 uint32_t height = 0, 94 bool is_cursor_embedded = false, 95 DesktopCapturer::Callback* callback = nullptr); 96 void UpdateScreenCastStreamResolution(uint32_t width, uint32_t height); 97 void UpdateScreenCastStreamFrameRate(uint32_t frame_rate); 98 void SetUseDamageRegion(bool use_damage_region) { 99 use_damage_region_ = use_damage_region; 100 } 101 void SetObserver(SharedScreenCastStream::Observer* observer) { 102 observer_ = observer; 103 } 104 void StopScreenCastStream(); 105 std::unique_ptr<SharedDesktopFrame> CaptureFrame(); 106 std::unique_ptr<MouseCursor> CaptureCursor(); 107 DesktopVector CaptureCursorPosition(); 108 109 private: 110 // Stops the streams and cleans up any in-use elements. 111 void StopAndCleanupStream(); 112 113 SharedScreenCastStream::Observer* observer_ = nullptr; 114 115 // Track damage region updates that were reported since the last time 116 // frame was captured 117 DesktopRegion damage_region_ RTC_GUARDED_BY(&latest_frame_lock_); 118 119 uint32_t pw_stream_node_id_ = 0; 120 121 DesktopSize stream_size_ = {}; 122 DesktopSize frame_size_; 123 124 Mutex queue_lock_; 125 ScreenCaptureFrameQueue<SharedDesktopFrame> queue_ 126 RTC_GUARDED_BY(&queue_lock_); 127 Mutex latest_frame_lock_ RTC_ACQUIRED_AFTER(queue_lock_); 128 SharedDesktopFrame* latest_available_frame_ 129 RTC_GUARDED_BY(&latest_frame_lock_) = nullptr; 130 std::unique_ptr<MouseCursor> mouse_cursor_; 131 DesktopVector mouse_cursor_position_ = DesktopVector(-1, -1); 132 133 int64_t modifier_; 134 std::unique_ptr<EglDmaBuf> egl_dmabuf_; 135 // List of modifiers we query as supported by the graphics card/driver 136 std::vector<uint64_t> modifiers_; 137 138 // PipeWire types 139 struct pw_context* pw_context_ = nullptr; 140 struct pw_core* pw_core_ = nullptr; 141 struct pw_stream* pw_stream_ = nullptr; 142 struct pw_thread_loop* pw_main_loop_ = nullptr; 143 struct spa_source* renegotiate_ = nullptr; 144 145 spa_hook spa_core_listener_; 146 spa_hook spa_stream_listener_; 147 148 // A number used to verify all previous methods and the resulting 149 // events have been handled. 150 int server_version_sync_ = 0; 151 // Version of the running PipeWire server we communicate with 152 PipeWireVersion pw_server_version_; 153 // Version of the library used to run our code 154 PipeWireVersion pw_client_version_; 155 156 // Resolution parameters. 157 uint32_t width_ = 0; 158 uint32_t height_ = 0; 159 // Frame rate. 160 uint32_t frame_rate_ = 60; 161 162 bool use_damage_region_ = true; 163 164 // Specifies whether the pipewire stream has been initialized with a request 165 // to embed cursor into the captured frames. 166 bool is_cursor_embedded_ = false; 167 168 // event handlers 169 pw_core_events pw_core_events_ = {}; 170 pw_stream_events pw_stream_events_ = {}; 171 172 struct spa_video_info_raw spa_video_format_; 173 174 void ProcessBuffer(pw_buffer* buffer); 175 bool ProcessMemFDBuffer(pw_buffer* buffer, 176 DesktopFrame& frame, 177 const DesktopVector& offset); 178 bool ProcessDMABuffer(pw_buffer* buffer, 179 DesktopFrame& frame, 180 const DesktopVector& offset); 181 void ConvertRGBxToBGRx(uint8_t* frame, uint32_t size); 182 void UpdateFrameUpdatedRegions(const spa_buffer* spa_buffer, 183 DesktopFrame& frame); 184 185 // PipeWire callbacks 186 static void OnCoreError(void* data, 187 uint32_t id, 188 int seq, 189 int res, 190 const char* message); 191 static void OnCoreDone(void* user_data, uint32_t id, int seq); 192 static void OnCoreInfo(void* user_data, const pw_core_info* info); 193 static void OnStreamParamChanged(void* data, 194 uint32_t id, 195 const struct spa_pod* format); 196 static void OnStreamStateChanged(void* data, 197 pw_stream_state old_state, 198 pw_stream_state state, 199 const char* error_message); 200 static void OnStreamProcess(void* data); 201 // This will be invoked in case we fail to process DMA-BUF PW buffer using 202 // negotiated stream parameters (modifier). We will drop the modifier we 203 // failed to use and try to use a different one or fallback to shared memory 204 // buffers. 205 static void OnRenegotiateFormat(void* data, uint64_t); 206 207 DesktopCapturer::Callback* callback_; 208 }; 209 210 void SharedScreenCastStreamPrivate::OnCoreError(void* data, 211 uint32_t id, 212 int seq, 213 int res, 214 const char* message) { 215 SharedScreenCastStreamPrivate* stream = 216 static_cast<SharedScreenCastStreamPrivate*>(data); 217 RTC_DCHECK(stream); 218 219 RTC_LOG(LS_ERROR) << "PipeWire remote error: " << message; 220 pw_thread_loop_signal(stream->pw_main_loop_, false); 221 } 222 223 void SharedScreenCastStreamPrivate::OnCoreInfo(void* data, 224 const pw_core_info* info) { 225 SharedScreenCastStreamPrivate* stream = 226 static_cast<SharedScreenCastStreamPrivate*>(data); 227 RTC_DCHECK(stream); 228 229 stream->pw_server_version_ = PipeWireVersion::Parse(info->version); 230 } 231 232 void SharedScreenCastStreamPrivate::OnCoreDone(void* data, 233 uint32_t id, 234 int seq) { 235 const SharedScreenCastStreamPrivate* stream = 236 static_cast<SharedScreenCastStreamPrivate*>(data); 237 RTC_DCHECK(stream); 238 239 if (id == PW_ID_CORE && stream->server_version_sync_ == seq) { 240 pw_thread_loop_signal(stream->pw_main_loop_, false); 241 } 242 } 243 244 // static 245 void SharedScreenCastStreamPrivate::OnStreamStateChanged( 246 void* data, 247 pw_stream_state old_state, 248 pw_stream_state state, 249 const char* error_message) { 250 SharedScreenCastStreamPrivate* that = 251 static_cast<SharedScreenCastStreamPrivate*>(data); 252 RTC_DCHECK(that); 253 254 switch (state) { 255 case PW_STREAM_STATE_ERROR: 256 RTC_LOG(LS_ERROR) << "PipeWire stream state error: " << error_message; 257 break; 258 case PW_STREAM_STATE_PAUSED: 259 if (that->observer_ && old_state != PW_STREAM_STATE_STREAMING) { 260 that->observer_->OnStreamConfigured(); 261 } 262 break; 263 case PW_STREAM_STATE_STREAMING: 264 case PW_STREAM_STATE_UNCONNECTED: 265 case PW_STREAM_STATE_CONNECTING: 266 break; 267 } 268 } 269 270 // static 271 void SharedScreenCastStreamPrivate::OnStreamParamChanged( 272 void* data, 273 uint32_t id, 274 const struct spa_pod* format) { 275 SharedScreenCastStreamPrivate* that = 276 static_cast<SharedScreenCastStreamPrivate*>(data); 277 RTC_DCHECK(that); 278 279 RTC_LOG(LS_INFO) << "PipeWire stream format changed."; 280 if (!format || id != SPA_PARAM_Format) { 281 return; 282 } 283 284 spa_format_video_raw_parse(format, &that->spa_video_format_); 285 286 if (that->observer_ && that->spa_video_format_.max_framerate.denom) { 287 that->observer_->OnFrameRateChanged( 288 that->spa_video_format_.max_framerate.num / 289 that->spa_video_format_.max_framerate.denom); 290 } 291 292 auto width = that->spa_video_format_.size.width; 293 auto height = that->spa_video_format_.size.height; 294 auto stride = SPA_ROUND_UP_N(width * kBytesPerPixel, 4); 295 auto size = height * stride; 296 297 that->stream_size_ = DesktopSize(width, height); 298 299 uint8_t buffer[2048] = {}; 300 auto builder = spa_pod_builder{.data = buffer, .size = sizeof(buffer)}; 301 302 // Setup buffers and meta header for new format. 303 304 // When SPA_FORMAT_VIDEO_modifier is present we can use DMA-BUFs as 305 // the server announces support for it. 306 // See https://github.com/PipeWire/pipewire/blob/master/doc/dma-buf.dox 307 const bool has_modifier = 308 spa_pod_find_prop(format, nullptr, SPA_FORMAT_VIDEO_modifier); 309 that->modifier_ = 310 has_modifier ? that->spa_video_format_.modifier : DRM_FORMAT_MOD_INVALID; 311 std::vector<const spa_pod*> params; 312 const int buffer_types = has_modifier && mozilla::gfx::IsDMABufEnabled() 313 ? (1 << SPA_DATA_DmaBuf) | (1 << SPA_DATA_MemFd) 314 : (1 << SPA_DATA_MemFd); 315 316 params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( 317 &builder, SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, 318 SPA_PARAM_BUFFERS_size, SPA_POD_Int(size), SPA_PARAM_BUFFERS_stride, 319 SPA_POD_Int(stride), SPA_PARAM_BUFFERS_buffers, 320 SPA_POD_CHOICE_RANGE_Int(8, 1, 32), SPA_PARAM_BUFFERS_dataType, 321 SPA_POD_CHOICE_FLAGS_Int(buffer_types)))); 322 params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( 323 &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, 324 SPA_POD_Id(SPA_META_Header), SPA_PARAM_META_size, 325 SPA_POD_Int(sizeof(struct spa_meta_header))))); 326 params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( 327 &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, 328 SPA_POD_Id(SPA_META_VideoCrop), SPA_PARAM_META_size, 329 SPA_POD_Int(sizeof(struct spa_meta_region))))); 330 params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( 331 &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, 332 SPA_POD_Id(SPA_META_Cursor), SPA_PARAM_META_size, 333 SPA_POD_CHOICE_RANGE_Int(CursorMetaSize(64, 64), CursorMetaSize(1, 1), 334 CursorMetaSize(384, 384))))); 335 params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( 336 &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, 337 SPA_POD_Id(SPA_META_VideoDamage), SPA_PARAM_META_size, 338 SPA_POD_CHOICE_RANGE_Int( 339 sizeof(struct spa_meta_region) * kVideoDamageRegionCount, 340 sizeof(struct spa_meta_region) * 1, 341 sizeof(struct spa_meta_region) * kVideoDamageRegionCount)))); 342 343 pw_stream_update_params(that->pw_stream_, params.data(), params.size()); 344 } 345 346 // static 347 void SharedScreenCastStreamPrivate::OnStreamProcess(void* data) { 348 SharedScreenCastStreamPrivate* that = 349 static_cast<SharedScreenCastStreamPrivate*>(data); 350 RTC_DCHECK(that); 351 352 struct pw_buffer* next_buffer; 353 struct pw_buffer* buffer = nullptr; 354 355 next_buffer = pw_stream_dequeue_buffer(that->pw_stream_); 356 while (next_buffer) { 357 buffer = next_buffer; 358 next_buffer = pw_stream_dequeue_buffer(that->pw_stream_); 359 360 if (next_buffer) { 361 pw_stream_queue_buffer(that->pw_stream_, buffer); 362 } 363 } 364 365 if (!buffer) { 366 return; 367 } 368 369 struct spa_meta_header* header = 370 static_cast<spa_meta_header*>(spa_buffer_find_meta_data( 371 buffer->buffer, SPA_META_Header, sizeof(*header))); 372 if (header && (header->flags & SPA_META_HEADER_FLAG_CORRUPTED)) { 373 RTC_LOG(LS_INFO) << "Dropping corrupted buffer"; 374 if (that->observer_) { 375 that->observer_->OnBufferCorruptedMetadata(); 376 } 377 // Queue buffer for reuse; it will not be processed further. 378 pw_stream_queue_buffer(that->pw_stream_, buffer); 379 return; 380 } 381 382 that->ProcessBuffer(buffer); 383 384 pw_stream_queue_buffer(that->pw_stream_, buffer); 385 } 386 387 void SharedScreenCastStreamPrivate::OnRenegotiateFormat(void* data, uint64_t) { 388 SharedScreenCastStreamPrivate* that = 389 static_cast<SharedScreenCastStreamPrivate*>(data); 390 RTC_DCHECK(that); 391 392 { 393 PipeWireThreadLoopLock thread_loop_lock(that->pw_main_loop_); 394 395 uint8_t buffer[4096] = {}; 396 397 spa_pod_builder builder = 398 spa_pod_builder{.data = buffer, .size = sizeof(buffer)}; 399 400 std::vector<const spa_pod*> params; 401 struct spa_rectangle resolution = 402 SPA_RECTANGLE(that->width_, that->height_); 403 struct spa_fraction frame_rate = SPA_FRACTION(that->frame_rate_, 1); 404 405 for (uint32_t format : {SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_RGBA, 406 SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBx}) { 407 if (!that->modifiers_.empty()) { 408 params.push_back( 409 BuildFormat(&builder, format, that->modifiers_, 410 that->width_ && that->height_ ? &resolution : nullptr, 411 &frame_rate)); 412 } 413 params.push_back(BuildFormat( 414 &builder, format, /*modifiers=*/{}, 415 that->width_ && that->height_ ? &resolution : nullptr, &frame_rate)); 416 } 417 418 pw_stream_update_params(that->pw_stream_, params.data(), params.size()); 419 } 420 } 421 422 SharedScreenCastStreamPrivate::SharedScreenCastStreamPrivate() {} 423 424 SharedScreenCastStreamPrivate::~SharedScreenCastStreamPrivate() { 425 StopAndCleanupStream(); 426 } 427 428 RTC_NO_SANITIZE("cfi-icall") 429 bool SharedScreenCastStreamPrivate::StartScreenCastStream( 430 uint32_t stream_node_id, 431 int fd, 432 uint32_t width, 433 uint32_t height, 434 bool is_cursor_embedded, 435 DesktopCapturer::Callback* callback) { 436 width_ = width; 437 height_ = height; 438 callback_ = callback; 439 is_cursor_embedded_ = is_cursor_embedded; 440 if (!InitializePipeWire()) { 441 RTC_LOG(LS_ERROR) << "Unable to open PipeWire library"; 442 return false; 443 } 444 if (mozilla::gfx::IsDMABufEnabled()) { 445 egl_dmabuf_ = std::make_unique<EglDmaBuf>(); 446 } 447 448 pw_stream_node_id_ = stream_node_id; 449 450 pw_init(/*argc=*/nullptr, /*argc=*/nullptr); 451 452 pw_main_loop_ = pw_thread_loop_new("pipewire-main-loop", nullptr); 453 454 pw_context_ = 455 pw_context_new(pw_thread_loop_get_loop(pw_main_loop_), nullptr, 0); 456 if (!pw_context_) { 457 RTC_LOG(LS_ERROR) << "Failed to create PipeWire context"; 458 return false; 459 } 460 461 if (pw_thread_loop_start(pw_main_loop_) < 0) { 462 RTC_LOG(LS_ERROR) << "Failed to start main PipeWire loop"; 463 return false; 464 } 465 466 pw_client_version_ = PipeWireVersion::Parse(pw_get_library_version()); 467 468 // Initialize event handlers, remote end and stream-related. 469 pw_core_events_.version = PW_VERSION_CORE_EVENTS; 470 pw_core_events_.info = &OnCoreInfo; 471 pw_core_events_.done = &OnCoreDone; 472 pw_core_events_.error = &OnCoreError; 473 474 pw_stream_events_.version = PW_VERSION_STREAM_EVENTS; 475 pw_stream_events_.state_changed = &OnStreamStateChanged; 476 pw_stream_events_.param_changed = &OnStreamParamChanged; 477 pw_stream_events_.process = &OnStreamProcess; 478 479 { 480 PipeWireThreadLoopLock thread_loop_lock(pw_main_loop_); 481 482 if (fd != kInvalidPipeWireFd) { 483 pw_core_ = pw_context_connect_fd( 484 pw_context_, fcntl(fd, F_DUPFD_CLOEXEC, 0), nullptr, 0); 485 } else { 486 pw_core_ = pw_context_connect(pw_context_, nullptr, 0); 487 } 488 489 if (!pw_core_) { 490 RTC_LOG(LS_ERROR) << "Failed to connect PipeWire context"; 491 return false; 492 } 493 494 pw_core_add_listener(pw_core_, &spa_core_listener_, &pw_core_events_, this); 495 496 // Add an event that can be later invoked by pw_loop_signal_event() 497 renegotiate_ = pw_loop_add_event(pw_thread_loop_get_loop(pw_main_loop_), 498 OnRenegotiateFormat, this); 499 500 server_version_sync_ = 501 pw_core_sync(pw_core_, PW_ID_CORE, server_version_sync_); 502 503 pw_thread_loop_wait(pw_main_loop_); 504 505 pw_properties* reuseProps = 506 pw_properties_new_string("pipewire.client.reuse=1"); 507 pw_stream_ = pw_stream_new(pw_core_, "webrtc-consume-stream", reuseProps); 508 509 if (!pw_stream_) { 510 RTC_LOG(LS_ERROR) << "Failed to create PipeWire stream"; 511 return false; 512 } 513 514 pw_stream_add_listener(pw_stream_, &spa_stream_listener_, 515 &pw_stream_events_, this); 516 uint8_t buffer[4096] = {}; 517 518 spa_pod_builder builder = 519 spa_pod_builder{.data = buffer, .size = sizeof(buffer)}; 520 521 std::vector<const spa_pod*> params; 522 const bool has_required_pw_client_version = 523 pw_client_version_ >= kDmaBufModifierMinVersion; 524 const bool has_required_pw_server_version = 525 pw_server_version_ >= kDmaBufModifierMinVersion; 526 struct spa_rectangle resolution; 527 bool set_resolution = false; 528 if (width && height) { 529 resolution = SPA_RECTANGLE(width, height); 530 set_resolution = true; 531 } 532 struct spa_fraction default_frame_rate = SPA_FRACTION(frame_rate_, 1); 533 for (uint32_t format : {SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_RGBA, 534 SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBx}) { 535 // Modifiers can be used with PipeWire >= 0.3.33 536 if (egl_dmabuf_ && 537 has_required_pw_client_version && has_required_pw_server_version) { 538 modifiers_ = egl_dmabuf_->QueryDmaBufModifiers(format); 539 540 if (!modifiers_.empty()) { 541 params.push_back(BuildFormat(&builder, format, modifiers_, 542 set_resolution ? &resolution : nullptr, 543 &default_frame_rate)); 544 } 545 } 546 547 params.push_back(BuildFormat(&builder, format, /*modifiers=*/{}, 548 set_resolution ? &resolution : nullptr, 549 &default_frame_rate)); 550 } 551 552 if (pw_stream_connect(pw_stream_, PW_DIRECTION_INPUT, pw_stream_node_id_, 553 PW_STREAM_FLAG_AUTOCONNECT, params.data(), 554 params.size()) != 0) { 555 RTC_LOG(LS_ERROR) << "Could not connect receiving stream."; 556 return false; 557 } 558 559 RTC_LOG(LS_INFO) << "PipeWire remote opened."; 560 } 561 return true; 562 } 563 564 RTC_NO_SANITIZE("cfi-icall") 565 void SharedScreenCastStreamPrivate::UpdateScreenCastStreamResolution( 566 uint32_t width, 567 uint32_t height) { 568 if (!width || !height) { 569 RTC_LOG(LS_WARNING) << "Bad resolution specified: " << width << "x" 570 << height; 571 return; 572 } 573 if (!pw_main_loop_) { 574 RTC_LOG(LS_WARNING) << "No main pipewire loop, ignoring resolution change"; 575 return; 576 } 577 if (!renegotiate_) { 578 RTC_LOG(LS_WARNING) << "Can not renegotiate stream params, ignoring " 579 << "resolution change"; 580 return; 581 } 582 if (width_ != width || height_ != height) { 583 width_ = width; 584 height_ = height; 585 pw_loop_signal_event(pw_thread_loop_get_loop(pw_main_loop_), renegotiate_); 586 } 587 } 588 589 RTC_NO_SANITIZE("cfi-icall") 590 void SharedScreenCastStreamPrivate::UpdateScreenCastStreamFrameRate( 591 uint32_t frame_rate) { 592 if (!pw_main_loop_) { 593 RTC_LOG(LS_WARNING) << "No main pipewire loop, ignoring frame rate change"; 594 return; 595 } 596 if (!renegotiate_) { 597 RTC_LOG(LS_WARNING) << "Can not renegotiate stream params, ignoring " 598 << "frame rate change"; 599 return; 600 } 601 if (frame_rate_ != frame_rate) { 602 frame_rate_ = frame_rate; 603 pw_loop_signal_event(pw_thread_loop_get_loop(pw_main_loop_), renegotiate_); 604 } 605 } 606 607 void SharedScreenCastStreamPrivate::StopScreenCastStream() { 608 StopAndCleanupStream(); 609 } 610 611 void SharedScreenCastStreamPrivate::StopAndCleanupStream() { 612 // We get buffers on the PipeWire thread, but this is called from the capturer 613 // thread, so we need to wait on and stop the pipewire thread before we 614 // disconnect the stream so that we can guarantee we aren't in the middle of 615 // processing a new frame. 616 617 // Even if we *do* somehow have the other objects without a pipewire thread, 618 // destroying them without a thread causes a crash. 619 if (!pw_main_loop_) 620 return; 621 622 // While we can stop the thread now, we cannot destroy it until we've cleaned 623 // up the other members. 624 pw_thread_loop_stop(pw_main_loop_); 625 626 if (pw_stream_) { 627 pw_stream_disconnect(pw_stream_); 628 pw_stream_destroy(pw_stream_); 629 pw_stream_ = nullptr; 630 631 { 632 MutexLock lock(&queue_lock_); 633 queue_.Reset(); 634 } 635 { 636 MutexLock latest_frame_lock(&latest_frame_lock_); 637 latest_available_frame_ = nullptr; 638 } 639 } 640 641 if (pw_core_) { 642 pw_core_disconnect(pw_core_); 643 pw_core_ = nullptr; 644 } 645 646 if (pw_context_) { 647 pw_context_destroy(pw_context_); 648 pw_context_ = nullptr; 649 } 650 651 pw_thread_loop_destroy(pw_main_loop_); 652 pw_main_loop_ = nullptr; 653 } 654 655 std::unique_ptr<SharedDesktopFrame> 656 SharedScreenCastStreamPrivate::CaptureFrame() { 657 MutexLock latest_frame_lock(&latest_frame_lock_); 658 659 if (!pw_stream_ || !latest_available_frame_) { 660 return std::unique_ptr<SharedDesktopFrame>{}; 661 } 662 663 std::unique_ptr<SharedDesktopFrame> frame = latest_available_frame_->Share(); 664 if (use_damage_region_) { 665 frame->mutable_updated_region()->Swap(&damage_region_); 666 damage_region_.Clear(); 667 } 668 669 return frame; 670 } 671 672 std::unique_ptr<MouseCursor> SharedScreenCastStreamPrivate::CaptureCursor() { 673 if (!mouse_cursor_) { 674 return nullptr; 675 } 676 677 return std::move(mouse_cursor_); 678 } 679 680 DesktopVector SharedScreenCastStreamPrivate::CaptureCursorPosition() { 681 return mouse_cursor_position_; 682 } 683 684 void SharedScreenCastStreamPrivate::UpdateFrameUpdatedRegions( 685 const spa_buffer* spa_buffer, 686 DesktopFrame& frame) { 687 latest_frame_lock_.AssertHeld(); 688 689 if (!use_damage_region_) { 690 frame.mutable_updated_region()->SetRect( 691 DesktopRect::MakeSize(frame.size())); 692 return; 693 } 694 695 const struct spa_meta* video_damage = static_cast<struct spa_meta*>( 696 spa_buffer_find_meta(spa_buffer, SPA_META_VideoDamage)); 697 if (!video_damage) { 698 damage_region_.SetRect(DesktopRect::MakeSize(frame.size())); 699 return; 700 } 701 702 frame.mutable_updated_region()->Clear(); 703 spa_meta_region* meta_region; 704 spa_meta_for_each(meta_region, video_damage) { 705 // Skip empty regions 706 if (meta_region->region.size.width == 0 || 707 meta_region->region.size.height == 0) { 708 continue; 709 } 710 711 damage_region_.AddRect(DesktopRect::MakeXYWH( 712 meta_region->region.position.x, meta_region->region.position.y, 713 meta_region->region.size.width, meta_region->region.size.height)); 714 } 715 } 716 717 RTC_NO_SANITIZE("cfi-icall") 718 void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) { 719 int64_t capture_start_time_nanos = TimeNanos(); 720 if (callback_) { 721 callback_->OnFrameCaptureStart(); 722 } 723 724 spa_buffer* spa_buffer = buffer->buffer; 725 726 // Try to update the mouse cursor first, because it can be the only 727 // information carried by the buffer 728 { 729 const struct spa_meta_cursor* cursor = 730 static_cast<struct spa_meta_cursor*>(spa_buffer_find_meta_data( 731 spa_buffer, SPA_META_Cursor, sizeof(*cursor))); 732 733 if (cursor) { 734 if (spa_meta_cursor_is_valid(cursor)) { 735 struct spa_meta_bitmap* bitmap = nullptr; 736 737 if (cursor->bitmap_offset) 738 bitmap = 739 SPA_MEMBER(cursor, cursor->bitmap_offset, struct spa_meta_bitmap); 740 741 if (bitmap && bitmap->size.width > 0 && bitmap->size.height > 0) { 742 const uint8_t* bitmap_data = 743 SPA_MEMBER(bitmap, bitmap->offset, uint8_t); 744 // TODO(bugs.webrtc.org/436974448): Convert `spa_video_format` to 745 // `FourCC`. 746 BasicDesktopFrame* mouse_frame = new BasicDesktopFrame( 747 DesktopSize(bitmap->size.width, bitmap->size.height), 748 FOURCC_ARGB); 749 mouse_frame->CopyPixelsFrom( 750 bitmap_data, bitmap->stride, 751 DesktopRect::MakeWH(bitmap->size.width, bitmap->size.height)); 752 mouse_cursor_ = std::make_unique<MouseCursor>( 753 mouse_frame, DesktopVector(cursor->hotspot.x, cursor->hotspot.y)); 754 755 if (observer_) { 756 observer_->OnCursorShapeChanged(); 757 } 758 } 759 mouse_cursor_position_.set(cursor->position.x, cursor->position.y); 760 761 if (observer_) { 762 observer_->OnCursorPositionChanged(); 763 } 764 } else { 765 // Indicate an invalid cursor 766 mouse_cursor_position_.set(-1, -1); 767 } 768 } 769 } 770 771 if (spa_buffer->datas[0].chunk->flags & SPA_CHUNK_FLAG_CORRUPTED) { 772 RTC_LOG(LS_INFO) << "Dropping buffer with corrupted or missing data"; 773 if (observer_) { 774 observer_->OnBufferCorruptedData(); 775 } 776 return; 777 } 778 779 if (spa_buffer->datas[0].type == SPA_DATA_MemFd && 780 spa_buffer->datas[0].chunk->size == 0) { 781 RTC_LOG(LS_INFO) << "Dropping buffer with empty data"; 782 if (observer_) { 783 observer_->OnEmptyBuffer(); 784 } 785 return; 786 } 787 788 // Use SPA_META_VideoCrop metadata to get the frame size. KDE and GNOME do 789 // handle screen/window sharing differently. KDE/KWin doesn't use 790 // SPA_META_VideoCrop metadata and when sharing a window, it always sets 791 // stream size to size of the window. With that we just allocate the 792 // DesktopFrame using the size of the stream itself. GNOME/Mutter 793 // always sets stream size to the size of the whole screen, even when sharing 794 // a window. To get the real window size we have to use SPA_META_VideoCrop 795 // metadata. This gives us the size we need in order to allocate the 796 // DesktopFrame. 797 798 struct spa_meta_region* videocrop_metadata = 799 static_cast<struct spa_meta_region*>(spa_buffer_find_meta_data( 800 spa_buffer, SPA_META_VideoCrop, sizeof(*videocrop_metadata))); 801 802 // Video size from metadata is bigger than an actual video stream size. 803 // The metadata are wrong or we should up-scale the video...in both cases 804 // just quit now. 805 if (videocrop_metadata && 806 (videocrop_metadata->region.size.width > 807 static_cast<uint32_t>(stream_size_.width()) || 808 videocrop_metadata->region.size.height > 809 static_cast<uint32_t>(stream_size_.height()))) { 810 RTC_LOG(LS_ERROR) << "Stream metadata sizes are wrong!"; 811 812 if (observer_) { 813 observer_->OnFailedToProcessBuffer(); 814 } 815 816 return; 817 } 818 819 // Use SPA_META_VideoCrop metadata to get the DesktopFrame size in case 820 // a windows is shared and it represents just a small portion of the 821 // stream itself. This will be for example used in case of GNOME (Mutter) 822 // where the stream will have the size of the screen itself, but we care 823 // only about smaller portion representing the window inside. 824 bool videocrop_metadata_use = false; 825 const struct spa_rectangle* videocrop_metadata_size = 826 videocrop_metadata ? &videocrop_metadata->region.size : nullptr; 827 828 if (videocrop_metadata_size && videocrop_metadata_size->width != 0 && 829 videocrop_metadata_size->height != 0 && 830 (static_cast<int>(videocrop_metadata_size->width) < 831 stream_size_.width() || 832 static_cast<int>(videocrop_metadata_size->height) < 833 stream_size_.height())) { 834 videocrop_metadata_use = true; 835 } 836 837 if (videocrop_metadata_use) { 838 frame_size_ = DesktopSize(videocrop_metadata_size->width, 839 videocrop_metadata_size->height); 840 } else { 841 frame_size_ = stream_size_; 842 } 843 844 // Get the position of the video crop within the stream. Just double-check 845 // that the position doesn't exceed the size of the stream itself. 846 // NOTE: Currently it looks there is no implementation using this. 847 uint32_t y_offset = 848 videocrop_metadata_use && 849 (videocrop_metadata->region.position.y + frame_size_.height() <= 850 stream_size_.height()) 851 ? videocrop_metadata->region.position.y 852 : 0; 853 uint32_t x_offset = 854 videocrop_metadata_use && 855 (videocrop_metadata->region.position.x + frame_size_.width() <= 856 stream_size_.width()) 857 ? videocrop_metadata->region.position.x 858 : 0; 859 DesktopVector offset = DesktopVector(x_offset, y_offset); 860 861 MutexLock lock(&queue_lock_); 862 863 queue_.MoveToNextFrame(); 864 if (queue_.current_frame() && queue_.current_frame()->IsShared()) { 865 RTC_DLOG(LS_WARNING) << "Overwriting frame that is still shared"; 866 867 if (observer_) { 868 observer_->OnFailedToProcessBuffer(); 869 } 870 } 871 872 if (!queue_.current_frame() || 873 !queue_.current_frame()->size().equals(frame_size_)) { 874 std::unique_ptr<DesktopFrame> frame(new BasicDesktopFrame( 875 DesktopSize(frame_size_.width(), frame_size_.height()), FOURCC_ARGB)); 876 queue_.ReplaceCurrentFrame(SharedDesktopFrame::Wrap(std::move(frame))); 877 } 878 879 bool bufferProcessed = false; 880 if (spa_buffer->datas[0].type == SPA_DATA_MemFd) { 881 bufferProcessed = 882 ProcessMemFDBuffer(buffer, *queue_.current_frame(), offset); 883 } else if (spa_buffer->datas[0].type == SPA_DATA_DmaBuf) { 884 bufferProcessed = ProcessDMABuffer(buffer, *queue_.current_frame(), offset); 885 } 886 887 if (!bufferProcessed) { 888 if (observer_) { 889 observer_->OnFailedToProcessBuffer(); 890 } 891 MutexLock latest_frame_lock(&latest_frame_lock_); 892 latest_available_frame_ = nullptr; 893 return; 894 } 895 896 // TODO(bugs.webrtc.org/436974448): Remove this conversion when arbitrary 897 // pixel formats are supported. 898 if (spa_video_format_.format == SPA_VIDEO_FORMAT_RGBx || 899 spa_video_format_.format == SPA_VIDEO_FORMAT_RGBA) { 900 uint8_t* tmp_src = queue_.current_frame()->data(); 901 for (int i = 0; i < frame_size_.height(); ++i) { 902 // If both sides decided to go with the RGBx format we need to convert 903 // it to BGRx to match color format expected by WebRTC. 904 ConvertRGBxToBGRx(tmp_src, queue_.current_frame()->stride()); 905 tmp_src += queue_.current_frame()->stride(); 906 } 907 } 908 909 if (observer_) { 910 observer_->OnDesktopFrameChanged(); 911 } 912 913 std::unique_ptr<SharedDesktopFrame> frame; 914 { 915 MutexLock latest_frame_lock(&latest_frame_lock_); 916 917 UpdateFrameUpdatedRegions(spa_buffer, *queue_.current_frame()); 918 queue_.current_frame()->set_may_contain_cursor(is_cursor_embedded_); 919 920 latest_available_frame_ = queue_.current_frame(); 921 922 if (!callback_) { 923 return; 924 } 925 926 frame = latest_available_frame_->Share(); 927 frame->set_capturer_id(DesktopCapturerId::kWaylandCapturerLinux); 928 frame->set_capture_time_ms((TimeNanos() - capture_start_time_nanos) / 929 kNumNanosecsPerMillisec); 930 if (use_damage_region_) { 931 frame->mutable_updated_region()->Swap(&damage_region_); 932 damage_region_.Clear(); 933 } 934 } 935 936 if (callback_) { 937 callback_->OnCaptureResult(DesktopCapturer::Result::SUCCESS, 938 std::move(frame)); 939 } 940 } 941 942 RTC_NO_SANITIZE("cfi-icall") 943 bool SharedScreenCastStreamPrivate::ProcessMemFDBuffer( 944 pw_buffer* buffer, 945 DesktopFrame& frame, 946 const DesktopVector& offset) { 947 spa_buffer* spa_buffer = buffer->buffer; 948 ScopedBuf map; 949 uint8_t* src = nullptr; 950 951 map.initialize( 952 static_cast<uint8_t*>( 953 mmap(nullptr, 954 spa_buffer->datas[0].maxsize + spa_buffer->datas[0].mapoffset, 955 PROT_READ, MAP_PRIVATE, spa_buffer->datas[0].fd, 0)), 956 spa_buffer->datas[0].maxsize + spa_buffer->datas[0].mapoffset, 957 spa_buffer->datas[0].fd); 958 959 if (!map) { 960 RTC_LOG(LS_ERROR) << "Failed to mmap the memory: " << std::strerror(errno); 961 return false; 962 } 963 964 src = SPA_MEMBER(map.get(), spa_buffer->datas[0].mapoffset, uint8_t); 965 966 uint32_t buffer_stride = spa_buffer->datas[0].chunk->stride; 967 uint32_t src_stride = buffer_stride; 968 969 uint8_t* updated_src = 970 src + (src_stride * offset.y()) + (kBytesPerPixel * offset.x()); 971 972 frame.CopyPixelsFrom( 973 updated_src, (src_stride - (kBytesPerPixel * offset.x())), 974 DesktopRect::MakeWH(frame.size().width(), frame.size().height())); 975 976 return true; 977 } 978 979 RTC_NO_SANITIZE("cfi-icall") 980 bool SharedScreenCastStreamPrivate::ProcessDMABuffer( 981 pw_buffer* buffer, 982 DesktopFrame& frame, 983 const DesktopVector& offset) { 984 spa_buffer* spa_buffer = buffer->buffer; 985 986 const uint n_planes = spa_buffer->n_datas; 987 988 if (!n_planes || !egl_dmabuf_) { 989 return false; 990 } 991 992 std::vector<EglDmaBuf::PlaneData> plane_datas; 993 for (uint32_t i = 0; i < n_planes; ++i) { 994 EglDmaBuf::PlaneData data = { 995 .fd = static_cast<int32_t>(spa_buffer->datas[i].fd), 996 .stride = static_cast<uint32_t>(spa_buffer->datas[i].chunk->stride), 997 .offset = static_cast<uint32_t>(spa_buffer->datas[i].chunk->offset)}; 998 plane_datas.push_back(data); 999 } 1000 1001 const bool imported = egl_dmabuf_->ImageFromDmaBuf( 1002 stream_size_, spa_video_format_.format, plane_datas, modifier_, offset, 1003 frame.size(), frame.data()); 1004 if (!imported) { 1005 RTC_LOG(LS_ERROR) << "Dropping DMA-BUF modifier: " << modifier_ 1006 << " and trying to renegotiate stream parameters"; 1007 1008 if (pw_server_version_ >= kDropSingleModifierMinVersion) { 1009 std::erase(modifiers_, modifier_); 1010 } else { 1011 modifiers_.clear(); 1012 } 1013 1014 pw_loop_signal_event(pw_thread_loop_get_loop(pw_main_loop_), renegotiate_); 1015 return false; 1016 } 1017 1018 return true; 1019 } 1020 1021 void SharedScreenCastStreamPrivate::ConvertRGBxToBGRx(uint8_t* frame, 1022 uint32_t size) { 1023 for (uint32_t i = 0; i < size; i += 4) { 1024 uint8_t tempR = frame[i]; 1025 uint8_t tempB = frame[i + 2]; 1026 frame[i] = tempB; 1027 frame[i + 2] = tempR; 1028 } 1029 } 1030 1031 SharedScreenCastStream::SharedScreenCastStream() 1032 : private_(std::make_unique<SharedScreenCastStreamPrivate>()) {} 1033 1034 SharedScreenCastStream::~SharedScreenCastStream() {} 1035 1036 webrtc::scoped_refptr<SharedScreenCastStream> 1037 SharedScreenCastStream::CreateDefault() { 1038 // Explicit new, to access non-public constructor. 1039 return scoped_refptr<SharedScreenCastStream>(new SharedScreenCastStream()); 1040 } 1041 1042 bool SharedScreenCastStream::StartScreenCastStream(uint32_t stream_node_id) { 1043 return private_->StartScreenCastStream(stream_node_id, kInvalidPipeWireFd); 1044 } 1045 1046 bool SharedScreenCastStream::StartScreenCastStream( 1047 uint32_t stream_node_id, 1048 int fd, 1049 uint32_t width, 1050 uint32_t height, 1051 bool is_cursor_embedded, 1052 DesktopCapturer::Callback* callback) { 1053 return private_->StartScreenCastStream(stream_node_id, fd, width, height, 1054 is_cursor_embedded, callback); 1055 } 1056 1057 void SharedScreenCastStream::UpdateScreenCastStreamResolution(uint32_t width, 1058 uint32_t height) { 1059 private_->UpdateScreenCastStreamResolution(width, height); 1060 } 1061 1062 void SharedScreenCastStream::UpdateScreenCastStreamFrameRate( 1063 uint32_t frame_rate) { 1064 private_->UpdateScreenCastStreamFrameRate(frame_rate); 1065 } 1066 1067 void SharedScreenCastStream::SetUseDamageRegion(bool use_damage_region) { 1068 private_->SetUseDamageRegion(use_damage_region); 1069 } 1070 1071 void SharedScreenCastStream::SetObserver( 1072 SharedScreenCastStream::Observer* observer) { 1073 private_->SetObserver(observer); 1074 } 1075 1076 void SharedScreenCastStream::StopScreenCastStream() { 1077 private_->StopScreenCastStream(); 1078 } 1079 1080 std::unique_ptr<SharedDesktopFrame> SharedScreenCastStream::CaptureFrame() { 1081 return private_->CaptureFrame(); 1082 } 1083 1084 std::unique_ptr<MouseCursor> SharedScreenCastStream::CaptureCursor() { 1085 return private_->CaptureCursor(); 1086 } 1087 1088 std::optional<DesktopVector> SharedScreenCastStream::CaptureCursorPosition() { 1089 DesktopVector position = private_->CaptureCursorPosition(); 1090 1091 // Consider only (x >= 0 and y >= 0) a valid position 1092 if (position.x() < 0 || position.y() < 0) { 1093 return std::nullopt; 1094 } 1095 1096 return position; 1097 } 1098 1099 } // namespace webrtc