video_capture_pipewire.cc (17667B)
1 /* 2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/video_capture/linux/video_capture_pipewire.h" 12 13 #include <pipewire/pipewire.h> 14 #include <spa/buffer/buffer.h> 15 #include <spa/buffer/meta.h> 16 #include <spa/param/format-utils.h> 17 #include <spa/param/format.h> 18 #include <spa/param/param.h> 19 #include <spa/param/video/format-utils.h> 20 #include <spa/param/video/raw.h> 21 #include <spa/pod/builder.h> 22 #include <spa/pod/iter.h> 23 #include <spa/pod/vararg.h> 24 #include <spa/utils/defs.h> 25 #include <spa/utils/result.h> 26 #include <spa/utils/type.h> 27 #include <sys/mman.h> 28 29 #include <algorithm> 30 #include <cerrno> 31 #include <cstddef> 32 #include <cstdint> 33 #include <cstring> 34 #include <new> 35 #include <vector> 36 37 #include "api/sequence_checker.h" 38 #include "api/video/video_rotation.h" 39 #include "common_video/libyuv/include/webrtc_libyuv.h" 40 #include "modules/portal/pipewire_utils.h" 41 #include "modules/video_capture/linux/pipewire_session.h" 42 #include "modules/video_capture/video_capture_defines.h" 43 #include "modules/video_capture/video_capture_impl.h" 44 #include "modules/video_capture/video_capture_options.h" 45 #include "rtc_base/checks.h" 46 #include "rtc_base/logging.h" 47 #include "rtc_base/race_checker.h" 48 #include "rtc_base/sanitizer.h" 49 #include "rtc_base/synchronization/mutex.h" 50 #include "system_wrappers/include/clock.h" 51 52 namespace webrtc { 53 namespace videocapturemodule { 54 55 struct { 56 uint32_t spa_format; 57 VideoType video_type; 58 } constexpr kSupportedFormats[] = { 59 {.spa_format = SPA_VIDEO_FORMAT_I420, .video_type = VideoType::kI420}, 60 {.spa_format = SPA_VIDEO_FORMAT_NV12, .video_type = VideoType::kNV12}, 61 {.spa_format = SPA_VIDEO_FORMAT_YUY2, .video_type = VideoType::kYUY2}, 62 {.spa_format = SPA_VIDEO_FORMAT_UYVY, .video_type = VideoType::kUYVY}, 63 // PipeWire is big-endian for the formats, while libyuv is little-endian 64 // This means that BGRA == ARGB, RGBA == ABGR and similar 65 // This follows mapping in libcamera PipeWire plugin: 66 // https://gitlab.freedesktop.org/pipewire/pipewire/-/blob/master/spa/plugins/libcamera/libcamera-utils.cpp 67 {.spa_format = SPA_VIDEO_FORMAT_BGRA, .video_type = VideoType::kARGB}, 68 {.spa_format = SPA_VIDEO_FORMAT_RGBA, .video_type = VideoType::kABGR}, 69 {.spa_format = SPA_VIDEO_FORMAT_ARGB, .video_type = VideoType::kBGRA}, 70 {.spa_format = SPA_VIDEO_FORMAT_RGB, .video_type = VideoType::kBGR24}, 71 {.spa_format = SPA_VIDEO_FORMAT_BGR, .video_type = VideoType::kRGB24}, 72 {.spa_format = SPA_VIDEO_FORMAT_RGB16, .video_type = VideoType::kRGB565}, 73 }; 74 75 VideoType VideoCaptureModulePipeWire::PipeWireRawFormatToVideoType( 76 uint32_t spa_format) { 77 for (const auto& spa_and_pixel_format : kSupportedFormats) { 78 if (spa_and_pixel_format.spa_format == spa_format) 79 return spa_and_pixel_format.video_type; 80 } 81 RTC_LOG(LS_WARNING) << "Unsupported pixel format: " << spa_format; 82 return VideoType::kUnknown; 83 } 84 85 uint32_t VideoCaptureModulePipeWire::VideoTypeToPipeWireRawFormat( 86 VideoType type) { 87 for (const auto& spa_and_pixel_format : kSupportedFormats) { 88 if (spa_and_pixel_format.video_type == type) 89 return spa_and_pixel_format.spa_format; 90 } 91 RTC_LOG(LS_WARNING) << "Unsupported video type: " << static_cast<int>(type); 92 return SPA_VIDEO_FORMAT_UNKNOWN; 93 } 94 95 VideoCaptureModulePipeWire::VideoCaptureModulePipeWire( 96 Clock* clock, 97 VideoCaptureOptions* options) 98 : VideoCaptureImpl(clock), 99 session_(options->pipewire_session()), 100 initialized_(false), 101 started_(false) {} 102 103 VideoCaptureModulePipeWire::~VideoCaptureModulePipeWire() { 104 RTC_DCHECK_RUN_ON(&api_checker_); 105 106 StopCapture(); 107 } 108 109 int32_t VideoCaptureModulePipeWire::Init(const char* deviceUniqueId) { 110 RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); 111 RTC_DCHECK_RUN_ON(&api_checker_); 112 113 auto node = 114 std::find_if(session_->nodes_.begin(), session_->nodes_.end(), 115 [deviceUniqueId](const PipeWireNode::PipeWireNodePtr& node) { 116 return node->unique_id() == deviceUniqueId; 117 }); 118 if (node == session_->nodes_.end()) 119 return -1; 120 121 node_id_ = (*node)->id(); 122 123 const int len = strlen(deviceUniqueId); 124 _deviceUniqueId = new (std::nothrow) char[len + 1]; 125 memcpy(_deviceUniqueId, deviceUniqueId, len + 1); 126 127 return 0; 128 } 129 130 static spa_pod* BuildFormat(spa_pod_builder* builder, 131 VideoType video_type, 132 uint32_t width, 133 uint32_t height, 134 float frame_rate) { 135 spa_pod_frame frame; 136 137 const uint32_t media_subtype = video_type == VideoType::kMJPEG 138 ? SPA_MEDIA_SUBTYPE_mjpg 139 : SPA_MEDIA_SUBTYPE_raw; 140 141 spa_pod_builder_push_object(builder, &frame, SPA_TYPE_OBJECT_Format, 142 SPA_PARAM_EnumFormat); 143 spa_pod_builder_add(builder, SPA_FORMAT_mediaType, 144 SPA_POD_Id(SPA_MEDIA_TYPE_video), SPA_FORMAT_mediaSubtype, 145 SPA_POD_Id(media_subtype), 0); 146 147 if (media_subtype == SPA_MEDIA_SUBTYPE_raw) { 148 const uint32_t format = 149 VideoCaptureModulePipeWire::VideoTypeToPipeWireRawFormat(video_type); 150 RTC_CHECK(format != SPA_VIDEO_FORMAT_UNKNOWN); 151 spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(format), 152 0); 153 } 154 155 spa_rectangle resolution = spa_rectangle{.width = width, .height = height}; 156 spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_size, 157 SPA_POD_Rectangle(&resolution), 0); 158 159 // Framerate can be also set to 0 to be unspecified 160 if (frame_rate) { 161 spa_fraction framerate = 162 spa_fraction{.num = static_cast<uint32_t>(frame_rate), .denom = 1}; 163 spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_framerate, 164 SPA_POD_Fraction(&framerate), 0); 165 } else { 166 // Default to some reasonable values 167 spa_fraction preferred_frame_rate = 168 spa_fraction{.num = static_cast<uint32_t>(30), .denom = 1}; 169 spa_fraction min_frame_rate = spa_fraction{.num = 1, .denom = 1}; 170 spa_fraction max_frame_rate = spa_fraction{.num = 30, .denom = 1}; 171 spa_pod_builder_add( 172 builder, SPA_FORMAT_VIDEO_framerate, 173 SPA_POD_CHOICE_RANGE_Fraction(&preferred_frame_rate, &min_frame_rate, 174 &max_frame_rate), 175 0); 176 } 177 178 return static_cast<spa_pod*>(spa_pod_builder_pop(builder, &frame)); 179 } 180 181 RTC_NO_SANITIZE("cfi-icall") 182 int32_t VideoCaptureModulePipeWire::StartCapture( 183 const VideoCaptureCapability& capability) { 184 RTC_DCHECK_RUN_ON(&api_checker_); 185 186 if (initialized_) { 187 if (capability == _requestedCapability) { 188 return 0; 189 } else { 190 StopCapture(); 191 } 192 } 193 194 uint8_t buffer[1024] = {}; 195 196 // We don't want members above to be guarded by capture_checker_ as 197 // it's meant to be for members that are accessed on the API thread 198 // only when we are not capturing. The code above can be called many 199 // times while sharing instance of VideoCapturePipeWire between 200 // websites and therefore it would not follow the requirements of this 201 // checker. 202 RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); 203 PipeWireThreadLoopLock thread_loop_lock(session_->pw_main_loop_); 204 205 RTC_LOG(LS_VERBOSE) << "Creating new PipeWire stream for node " << node_id_; 206 207 pw_properties* reuse_props = 208 pw_properties_new_string("pipewire.client.reuse=1"); 209 stream_ = pw_stream_new(session_->pw_core_, "camera-stream", reuse_props); 210 211 if (!stream_) { 212 RTC_LOG(LS_ERROR) << "Failed to create camera stream!"; 213 return -1; 214 } 215 216 static const pw_stream_events stream_events{ 217 .version = PW_VERSION_STREAM_EVENTS, 218 .state_changed = &OnStreamStateChanged, 219 .param_changed = &OnStreamParamChanged, 220 .process = &OnStreamProcess, 221 }; 222 223 pw_stream_add_listener(stream_, &stream_listener_, &stream_events, this); 224 225 spa_pod_builder builder = 226 spa_pod_builder{.data = buffer, .size = sizeof(buffer)}; 227 std::vector<const spa_pod*> params; 228 uint32_t width = capability.width; 229 uint32_t height = capability.height; 230 uint32_t frame_rate = capability.maxFPS; 231 VideoType video_type = capability.videoType; 232 233 params.push_back( 234 BuildFormat(&builder, video_type, width, height, frame_rate)); 235 236 int res = pw_stream_connect( 237 stream_, PW_DIRECTION_INPUT, node_id_, 238 static_cast<enum pw_stream_flags>(PW_STREAM_FLAG_AUTOCONNECT | 239 PW_STREAM_FLAG_DONT_RECONNECT), 240 params.data(), params.size()); 241 if (res != 0) { 242 RTC_LOG(LS_ERROR) << "Could not connect to camera stream: " 243 << spa_strerror(res); 244 return -1; 245 } 246 247 _requestedCapability = capability; 248 initialized_ = true; 249 250 return 0; 251 } 252 253 int32_t VideoCaptureModulePipeWire::StopCapture() { 254 RTC_DCHECK_RUN_ON(&api_checker_); 255 256 PipeWireThreadLoopLock thread_loop_lock(session_->pw_main_loop_); 257 // PipeWireSession is guarded by API checker so just make sure we do 258 // race detection when the PipeWire loop is locked/stopped to not run 259 // any callback at this point. 260 RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); 261 if (stream_) { 262 pw_stream_destroy(stream_); 263 stream_ = nullptr; 264 } 265 266 _requestedCapability = VideoCaptureCapability(); 267 return 0; 268 } 269 270 bool VideoCaptureModulePipeWire::CaptureStarted() { 271 RTC_DCHECK_RUN_ON(&api_checker_); 272 MutexLock lock(&api_lock_); 273 274 return started_; 275 } 276 277 int32_t VideoCaptureModulePipeWire::CaptureSettings( 278 VideoCaptureCapability& settings) { 279 RTC_DCHECK_RUN_ON(&api_checker_); 280 281 settings = _requestedCapability; 282 283 return 0; 284 } 285 286 void VideoCaptureModulePipeWire::OnStreamParamChanged( 287 void* data, 288 uint32_t id, 289 const struct spa_pod* format) { 290 VideoCaptureModulePipeWire* that = 291 static_cast<VideoCaptureModulePipeWire*>(data); 292 RTC_DCHECK(that); 293 RTC_CHECK_RUNS_SERIALIZED(&that->capture_checker_); 294 295 if (format && id == SPA_PARAM_Format) 296 that->OnFormatChanged(format); 297 } 298 299 RTC_NO_SANITIZE("cfi-icall") 300 void VideoCaptureModulePipeWire::OnFormatChanged(const struct spa_pod* format) { 301 RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); 302 303 uint32_t media_type, media_subtype; 304 305 if (spa_format_parse(format, &media_type, &media_subtype) < 0) { 306 RTC_LOG(LS_ERROR) << "Failed to parse video format."; 307 return; 308 } 309 310 switch (media_subtype) { 311 case SPA_MEDIA_SUBTYPE_raw: { 312 struct spa_video_info_raw f; 313 spa_format_video_raw_parse(format, &f); 314 configured_capability_.width = f.size.width; 315 configured_capability_.height = f.size.height; 316 configured_capability_.videoType = PipeWireRawFormatToVideoType(f.format); 317 configured_capability_.maxFPS = f.framerate.num / f.framerate.denom; 318 break; 319 } 320 case SPA_MEDIA_SUBTYPE_mjpg: { 321 struct spa_video_info_mjpg f; 322 spa_format_video_mjpg_parse(format, &f); 323 configured_capability_.width = f.size.width; 324 configured_capability_.height = f.size.height; 325 configured_capability_.videoType = VideoType::kMJPEG; 326 configured_capability_.maxFPS = f.framerate.num / f.framerate.denom; 327 break; 328 } 329 default: 330 configured_capability_.videoType = VideoType::kUnknown; 331 } 332 333 if (configured_capability_.videoType == VideoType::kUnknown) { 334 RTC_LOG(LS_ERROR) << "Unsupported video format."; 335 return; 336 } 337 338 RTC_LOG(LS_VERBOSE) << "Configured capture format = " 339 << static_cast<int>(configured_capability_.videoType); 340 341 uint8_t buffer[1024] = {}; 342 auto builder = spa_pod_builder{.data = buffer, .size = sizeof(buffer)}; 343 344 // Setup buffers and meta header for new format. 345 std::vector<const spa_pod*> params; 346 spa_pod_frame frame; 347 spa_pod_builder_push_object(&builder, &frame, SPA_TYPE_OBJECT_ParamBuffers, 348 SPA_PARAM_Buffers); 349 350 if (media_subtype == SPA_MEDIA_SUBTYPE_raw) { 351 // Enforce stride without padding. 352 size_t stride; 353 switch (configured_capability_.videoType) { 354 case VideoType::kI420: 355 case VideoType::kNV12: 356 stride = configured_capability_.width; 357 break; 358 case VideoType::kYUY2: 359 case VideoType::kUYVY: 360 case VideoType::kRGB565: 361 stride = configured_capability_.width * 2; 362 break; 363 case VideoType::kRGB24: 364 case VideoType::kBGR24: 365 stride = configured_capability_.width * 3; 366 break; 367 case VideoType::kARGB: 368 case VideoType::kABGR: 369 case VideoType::kBGRA: 370 stride = configured_capability_.width * 4; 371 break; 372 default: 373 RTC_LOG(LS_ERROR) << "Unsupported video format."; 374 return; 375 } 376 spa_pod_builder_add(&builder, SPA_PARAM_BUFFERS_stride, SPA_POD_Int(stride), 377 0); 378 } 379 380 const int buffer_types = 381 (1 << SPA_DATA_DmaBuf) | (1 << SPA_DATA_MemFd) | (1 << SPA_DATA_MemPtr); 382 spa_pod_builder_add( 383 &builder, SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(8, 1, 32), 384 SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(buffer_types), 0); 385 params.push_back( 386 static_cast<spa_pod*>(spa_pod_builder_pop(&builder, &frame))); 387 388 params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( 389 &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, 390 SPA_POD_Id(SPA_META_Header), SPA_PARAM_META_size, 391 SPA_POD_Int(sizeof(struct spa_meta_header))))); 392 params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( 393 &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, 394 SPA_POD_Id(SPA_META_VideoTransform), SPA_PARAM_META_size, 395 SPA_POD_Int(sizeof(struct spa_meta_videotransform))))); 396 pw_stream_update_params(stream_, params.data(), params.size()); 397 } 398 399 void VideoCaptureModulePipeWire::OnStreamStateChanged( 400 void* data, 401 pw_stream_state old_state, 402 pw_stream_state state, 403 const char* error_message) { 404 VideoCaptureModulePipeWire* that = 405 static_cast<VideoCaptureModulePipeWire*>(data); 406 RTC_DCHECK(that); 407 408 MutexLock lock(&that->api_lock_); 409 switch (state) { 410 case PW_STREAM_STATE_STREAMING: 411 that->started_ = true; 412 break; 413 case PW_STREAM_STATE_ERROR: 414 RTC_LOG(LS_ERROR) << "PipeWire stream state error: " << error_message; 415 [[fallthrough]]; 416 case PW_STREAM_STATE_PAUSED: 417 case PW_STREAM_STATE_UNCONNECTED: 418 case PW_STREAM_STATE_CONNECTING: 419 that->started_ = false; 420 break; 421 } 422 RTC_LOG(LS_VERBOSE) << "PipeWire stream state change: " 423 << pw_stream_state_as_string(old_state) << " -> " 424 << pw_stream_state_as_string(state); 425 } 426 427 void VideoCaptureModulePipeWire::OnStreamProcess(void* data) { 428 VideoCaptureModulePipeWire* that = 429 static_cast<VideoCaptureModulePipeWire*>(data); 430 RTC_DCHECK(that); 431 RTC_CHECK_RUNS_SERIALIZED(&that->capture_checker_); 432 that->ProcessBuffers(); 433 } 434 435 static VideoRotation VideorotationFromPipeWireTransform(uint32_t transform) { 436 switch (transform) { 437 case SPA_META_TRANSFORMATION_90: 438 return kVideoRotation_90; 439 case SPA_META_TRANSFORMATION_180: 440 return kVideoRotation_180; 441 case SPA_META_TRANSFORMATION_270: 442 return kVideoRotation_270; 443 default: 444 return kVideoRotation_0; 445 } 446 } 447 448 RTC_NO_SANITIZE("cfi-icall") 449 void VideoCaptureModulePipeWire::ProcessBuffers() { 450 RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); 451 452 while (pw_buffer* buffer = pw_stream_dequeue_buffer(stream_)) { 453 spa_buffer* spaBuffer = buffer->buffer; 454 struct spa_meta_header* h; 455 h = static_cast<struct spa_meta_header*>( 456 spa_buffer_find_meta_data(spaBuffer, SPA_META_Header, sizeof(*h))); 457 458 struct spa_meta_videotransform* videotransform; 459 videotransform = 460 static_cast<struct spa_meta_videotransform*>(spa_buffer_find_meta_data( 461 spaBuffer, SPA_META_VideoTransform, sizeof(*videotransform))); 462 if (videotransform) { 463 VideoRotation rotation = 464 VideorotationFromPipeWireTransform(videotransform->transform); 465 SetCaptureRotation(rotation); 466 SetApplyRotation(rotation != kVideoRotation_0); 467 } 468 469 if (h->flags & SPA_META_HEADER_FLAG_CORRUPTED) { 470 RTC_LOG(LS_INFO) << "Dropping corruped frame."; 471 pw_stream_queue_buffer(stream_, buffer); 472 continue; 473 } 474 475 if (spaBuffer->datas[0].type == SPA_DATA_DmaBuf || 476 spaBuffer->datas[0].type == SPA_DATA_MemFd) { 477 ScopedBuf frame; 478 frame.initialize( 479 static_cast<uint8_t*>( 480 mmap(nullptr, spaBuffer->datas[0].maxsize, PROT_READ, MAP_SHARED, 481 spaBuffer->datas[0].fd, spaBuffer->datas[0].mapoffset)), 482 spaBuffer->datas[0].maxsize, spaBuffer->datas[0].fd, 483 spaBuffer->datas[0].type == SPA_DATA_DmaBuf); 484 485 if (!frame) { 486 RTC_LOG(LS_ERROR) << "Failed to mmap the memory: " 487 << std::strerror(errno); 488 return; 489 } 490 491 IncomingFrame( 492 SPA_MEMBER(frame.get(), spaBuffer->datas[0].mapoffset, uint8_t), 493 spaBuffer->datas[0].chunk->size, configured_capability_); 494 } else { // SPA_DATA_MemPtr 495 IncomingFrame(static_cast<uint8_t*>(spaBuffer->datas[0].data), 496 spaBuffer->datas[0].chunk->size, configured_capability_); 497 } 498 499 pw_stream_queue_buffer(stream_, buffer); 500 } 501 } 502 503 } // namespace videocapturemodule 504 } // namespace webrtc