CamerasParent.cpp (60057B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set sw=2 ts=8 et ft=cpp : */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 5 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "CamerasParent.h" 8 9 #include <algorithm> 10 11 #include "CamerasTypes.h" 12 #include "MediaEngineSource.h" 13 #include "PerformanceRecorder.h" 14 #include "VideoEngine.h" 15 #include "VideoFrameUtils.h" 16 #include "api/video/video_frame_buffer.h" 17 #include "common/browser_logging/WebRtcLog.h" 18 #include "common_video/libyuv/include/webrtc_libyuv.h" 19 #include "mozilla/Assertions.h" 20 #include "mozilla/Logging.h" 21 #include "mozilla/Preferences.h" 22 #include "mozilla/ProfilerMarkers.h" 23 #include "mozilla/Services.h" 24 #include "mozilla/StaticPrefs_media.h" 25 #include "mozilla/dom/CanonicalBrowsingContext.h" 26 #include "mozilla/dom/WindowGlobalParent.h" 27 #include "mozilla/ipc/BackgroundParent.h" 28 #include "mozilla/ipc/PBackgroundParent.h" 29 #include "mozilla/media/MediaUtils.h" 30 #include "nsIPermissionManager.h" 31 #include "nsIThread.h" 32 #include "nsNetUtil.h" 33 #include "nsThreadUtils.h" 34 #include "video_engine/desktop_capture_impl.h" 35 #include "video_engine/video_capture_factory.h" 36 37 #if defined(_WIN32) 38 # include <process.h> 39 # define getpid() _getpid() 40 #endif 41 42 mozilla::LazyLogModule gCamerasParentLog("CamerasParent"); 43 #define LOG(...) \ 44 MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, (__VA_ARGS__)) 45 #define LOG_FUNCTION() \ 46 MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, \ 47 ("CamerasParent(%p)::%s", this, __func__)) 48 #define LOG_VERBOSE(...) \ 49 MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Verbose, (__VA_ARGS__)) 50 #define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug) 51 52 namespace mozilla { 53 using dom::VideoResizeModeEnum; 54 using media::ShutdownBlockingTicket; 55 namespace camera { 56 57 uint32_t ResolutionFeasibilityDistance(int32_t candidate, int32_t requested) { 58 // The purpose of this function is to find a smallest resolution 59 // which is larger than all requested capabilities. 60 // Then we can use down-scaling to fulfill each request. 61 62 MOZ_DIAGNOSTIC_ASSERT(candidate >= 0, "Candidate unexpectedly negative"); 63 MOZ_DIAGNOSTIC_ASSERT(requested >= 0, "Requested unexpectedly negative"); 64 65 if (candidate == 0) { 66 // Treat width|height capability of 0 as "can do any". 67 // This allows for orthogonal capabilities that are not in discrete steps. 68 return 0; 69 } 70 71 uint32_t distance = 72 std::abs(candidate - requested) * 1000 / std::max(candidate, requested); 73 if (candidate >= requested) { 74 // This is a good case, the candidate covers the requested resolution. 75 return distance; 76 } 77 78 // This is a bad case, the candidate is lower than the requested resolution. 79 // This is penalized with an added weight of 10000. 80 return 10000 + distance; 81 } 82 83 uint32_t FeasibilityDistance(int32_t candidate, int32_t requested) { 84 MOZ_DIAGNOSTIC_ASSERT(candidate >= 0, "Candidate unexpectedly negative"); 85 MOZ_DIAGNOSTIC_ASSERT(requested >= 0, "Requested unexpectedly negative"); 86 87 if (candidate == 0) { 88 // Treat maxFPS capability of 0 as "can do any". 89 // This allows for orthogonal capabilities that are not in discrete steps. 90 return 0; 91 } 92 93 return std::abs(candidate - requested) * 1000 / 94 std::max(candidate, requested); 95 } 96 97 class CamerasParent::VideoEngineArray 98 : public media::Refcountable<nsTArray<RefPtr<VideoEngine>>> {}; 99 100 // Singleton video engines. The sEngines RefPtr is IPC background thread only 101 // and outlives the CamerasParent instances. The array elements are video 102 // capture thread only. 103 using VideoEngineArray = CamerasParent::VideoEngineArray; 104 static StaticRefPtr<VideoEngineArray> sEngines; 105 // Number of CamerasParents instances in the current process for which 106 // mVideoCaptureThread has been set. IPC background thread only. 107 static int32_t sNumCamerasParents = 0; 108 // Video processing thread - where webrtc.org capturer code runs. Outlives the 109 // CamerasParent instances. IPC background thread only. 110 static StaticRefPtr<nsIThread> sVideoCaptureThread; 111 // Main VideoCaptureFactory used to create and manage all capture related 112 // objects. Created on IPC background thread, destroyed on main thread on 113 // shutdown. Outlives the CamerasParent instances. 114 static StaticRefPtr<VideoCaptureFactory> sVideoCaptureFactory; 115 // All live capturers across all CamerasParent instances. The array and its 116 // members are only modified on the video capture thread. The outermost refcount 117 // is IPC background thread only. 118 static StaticRefPtr< 119 media::Refcountable<nsTArray<std::unique_ptr<AggregateCapturer>>>> 120 sCapturers; 121 122 static void ClearCameraDeviceInfo() { 123 ipc::AssertIsOnBackgroundThread(); 124 if (sVideoCaptureThread) { 125 MOZ_ASSERT(sEngines); 126 MOZ_ALWAYS_SUCCEEDS(sVideoCaptureThread->Dispatch( 127 NS_NewRunnableFunction(__func__, [engines = RefPtr(sEngines.get())] { 128 if (VideoEngine* engine = engines->ElementAt(CameraEngine)) { 129 engine->ClearVideoCaptureDeviceInfo(); 130 } 131 }))); 132 } 133 } 134 135 static inline nsCString FakeCameraPref() { 136 return nsDependentCString( 137 StaticPrefs::GetPrefName_media_getusermedia_camera_fake_force()); 138 } 139 140 static void OnPrefChange(const char* aPref, void* aClosure) { 141 MOZ_ASSERT(NS_IsMainThread()); 142 MOZ_ASSERT(FakeCameraPref() == aPref); 143 if (!sVideoCaptureFactory) { 144 return; 145 } 146 nsCOMPtr<nsISerialEventTarget> backgroundTarget = 147 ipc::BackgroundParent::GetBackgroundThread(); 148 if (!backgroundTarget) { 149 return; 150 } 151 MOZ_ALWAYS_SUCCEEDS(backgroundTarget->Dispatch(NS_NewRunnableFunction( 152 "CamerasParent::OnPrefChange", &ClearCameraDeviceInfo))); 153 } 154 155 static VideoCaptureFactory* EnsureVideoCaptureFactory() { 156 ipc::AssertIsOnBackgroundThread(); 157 158 if (sVideoCaptureFactory) { 159 return sVideoCaptureFactory; 160 } 161 162 sVideoCaptureFactory = MakeRefPtr<VideoCaptureFactory>(); 163 NS_DispatchToMainThread( 164 NS_NewRunnableFunction("CamerasParent::EnsureVideoCaptureFactory", []() { 165 Preferences::RegisterCallback(&OnPrefChange, FakeCameraPref()); 166 RunOnShutdown([] { 167 sVideoCaptureFactory = nullptr; 168 Preferences::UnregisterCallback(&OnPrefChange, FakeCameraPref()); 169 }); 170 })); 171 return sVideoCaptureFactory; 172 } 173 174 static already_AddRefed<nsISerialEventTarget> 175 MakeAndAddRefVideoCaptureThreadAndSingletons() { 176 ipc::AssertIsOnBackgroundThread(); 177 178 MOZ_ASSERT_IF(sVideoCaptureThread, sNumCamerasParents > 0); 179 MOZ_ASSERT_IF(!sVideoCaptureThread, sNumCamerasParents == 0); 180 181 if (!sVideoCaptureThread) { 182 LOG("Spinning up WebRTC Cameras Thread"); 183 nsIThreadManager::ThreadCreationOptions options; 184 #ifdef XP_WIN 185 // Windows desktop capture needs a UI thread 186 options.isUiThread = true; 187 #endif 188 nsCOMPtr<nsIThread> videoCaptureThread; 189 if (NS_FAILED(NS_NewNamedThread("VideoCapture", 190 getter_AddRefs(videoCaptureThread), nullptr, 191 options))) { 192 return nullptr; 193 } 194 sVideoCaptureThread = videoCaptureThread.forget(); 195 196 sEngines = MakeRefPtr<VideoEngineArray>(); 197 sEngines->AppendElements(CaptureEngine::MaxEngine); 198 199 sCapturers = MakeRefPtr< 200 media::Refcountable<nsTArray<std::unique_ptr<AggregateCapturer>>>>(); 201 } 202 203 ++sNumCamerasParents; 204 return do_AddRef(sVideoCaptureThread); 205 } 206 207 static void ReleaseVideoCaptureThreadAndSingletons() { 208 ipc::AssertIsOnBackgroundThread(); 209 210 if (--sNumCamerasParents > 0) { 211 // Other CamerasParent instances are using the singleton classes. 212 return; 213 } 214 215 MOZ_ASSERT(sNumCamerasParents == 0, "Double release!"); 216 217 // No other CamerasParent instances alive. Clean up. 218 LOG("Shutting down VideoEngines and the VideoCapture thread"); 219 MOZ_ALWAYS_SUCCEEDS(sVideoCaptureThread->Dispatch(NS_NewRunnableFunction( 220 __func__, [engines = RefPtr(sEngines.forget()), 221 capturers = RefPtr(sCapturers.forget())] { 222 MOZ_ASSERT(capturers->IsEmpty(), "No capturers expected on shutdown"); 223 for (RefPtr<VideoEngine>& engine : *engines) { 224 if (engine) { 225 VideoEngine::Delete(engine); 226 engine = nullptr; 227 } 228 } 229 }))); 230 231 MOZ_ALWAYS_SUCCEEDS(RefPtr(sVideoCaptureThread.forget())->AsyncShutdown()); 232 } 233 234 // 3 threads are involved in this code: 235 // - the main thread for some setups, and occassionally for video capture setup 236 // calls that don't work correctly elsewhere. 237 // - the IPC thread on which PBackground is running and which receives and 238 // sends messages 239 // - a thread which will execute the actual (possibly slow) camera access 240 // called "VideoCapture". On Windows this is a thread with an event loop 241 // suitable for UI access. 242 243 void CamerasParent::OnDeviceChange() { 244 LOG_FUNCTION(); 245 246 mPBackgroundEventTarget->Dispatch( 247 NS_NewRunnableFunction(__func__, [this, self = RefPtr(this)]() { 248 if (IsShuttingDown()) { 249 LOG("OnDeviceChanged failure: parent shutting down."); 250 return; 251 } 252 (void)SendDeviceChange(); 253 })); 254 }; 255 256 class DeliverFrameRunnable : public mozilla::Runnable { 257 public: 258 // Constructor for when no ShmemBuffer (of the right size) was available, so 259 // keep the frame around until we can allocate one on PBackground (in Run). 260 DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine, 261 int aCaptureId, nsTArray<int>&& aStreamIds, 262 const TrackingId& aTrackingId, 263 const webrtc::VideoFrame& aFrame, 264 const VideoFrameProperties& aProperties) 265 : Runnable("camera::DeliverFrameRunnable"), 266 mParent(aParent), 267 mCapEngine(aEngine), 268 mCaptureId(aCaptureId), 269 mStreamIds(std::move(aStreamIds)), 270 mTrackingId(aTrackingId), 271 mBuffer(aFrame), 272 mProperties(aProperties) {} 273 274 DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine, 275 int aCaptureId, nsTArray<int>&& aStreamIds, 276 const TrackingId& aTrackingId, ShmemBuffer aBuffer, 277 VideoFrameProperties& aProperties) 278 : Runnable("camera::DeliverFrameRunnable"), 279 mParent(aParent), 280 mCapEngine(aEngine), 281 mCaptureId(aCaptureId), 282 mStreamIds(std::move(aStreamIds)), 283 mTrackingId(aTrackingId), 284 mBuffer(std::move(aBuffer)), 285 mProperties(aProperties) {} 286 287 NS_IMETHOD Run() override { 288 // runs on BackgroundEventTarget 289 MOZ_ASSERT(GetCurrentSerialEventTarget() == 290 mParent->mPBackgroundEventTarget); 291 if (mParent->IsShuttingDown()) { 292 // Communication channel is being torn down 293 return NS_OK; 294 } 295 mParent->DeliverFrameOverIPC(mCapEngine, mCaptureId, mStreamIds, 296 mTrackingId, std::move(mBuffer), mProperties); 297 return NS_OK; 298 } 299 300 private: 301 const RefPtr<CamerasParent> mParent; 302 const CaptureEngine mCapEngine; 303 const int mCaptureId; 304 const nsTArray<int> mStreamIds; 305 const TrackingId mTrackingId; 306 Variant<ShmemBuffer, webrtc::VideoFrame> mBuffer; 307 const VideoFrameProperties mProperties; 308 }; 309 310 int CamerasParent::DeliverFrameOverIPC( 311 CaptureEngine aCapEngine, int aCaptureId, const Span<const int>& aStreamIds, 312 const TrackingId& aTrackingId, 313 Variant<ShmemBuffer, webrtc::VideoFrame>&& aBuffer, 314 const VideoFrameProperties& aProps) { 315 // No ShmemBuffers were available, so construct one now of the right size 316 // and copy into it. That is an extra copy, but we expect this to be 317 // the exceptional case, because we just assured the next call *will* have a 318 // buffer of the right size. 319 if (!aBuffer.is<ShmemBuffer>()) { 320 // Get a shared memory buffer from the pool, at least size big 321 ShmemBuffer shMemBuff; 322 { 323 auto guard = mShmemPools.Lock(); 324 auto it = guard->find(aCaptureId); 325 if (it != guard->end()) { 326 auto& [_, pool] = *it; 327 shMemBuff = pool.Get(this, aProps.bufferSize()); 328 } 329 } 330 331 if (!shMemBuff.Valid()) { 332 LOG("No usable Video shmem in DeliverFrame (out of buffers?)"); 333 // We can skip this frame if we run out of buffers, it's not a real error. 334 return 0; 335 } 336 337 PerformanceRecorder<CopyVideoStage> rec( 338 "CamerasParent::AltBufferToShmem"_ns, aTrackingId, aProps.width(), 339 aProps.height()); 340 VideoFrameUtils::CopyVideoFrameBuffers(shMemBuff, 341 aBuffer.as<webrtc::VideoFrame>()); 342 rec.Record(); 343 344 if (!SendDeliverFrame(aCaptureId, aStreamIds, std::move(shMemBuff.Get()), 345 aProps)) { 346 return -1; 347 } 348 } else { 349 MOZ_ASSERT(aBuffer.as<ShmemBuffer>().Valid()); 350 // ShmemBuffer was available, we're all good. A single copy happened 351 // in the original webrtc callback. 352 if (!SendDeliverFrame(aCaptureId, aStreamIds, 353 std::move(aBuffer.as<ShmemBuffer>().Get()), aProps)) { 354 return -1; 355 } 356 } 357 358 return 0; 359 } 360 361 bool CamerasParent::IsWindowCapturing(uint64_t aWindowId, 362 const nsACString& aUniqueId) const { 363 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 364 for (const auto& capturer : *mCapturers) { 365 if (capturer->mUniqueId != aUniqueId) { 366 continue; 367 } 368 auto streamsGuard = capturer->mStreams.ConstLock(); 369 for (const auto& stream : *streamsGuard) { 370 if (stream->mWindowId == aWindowId) { 371 return true; 372 } 373 } 374 } 375 return false; 376 } 377 378 ShmemBuffer CamerasParent::GetBuffer(int aCaptureId, size_t aSize) { 379 auto guard = mShmemPools.Lock(); 380 auto it = guard->find(aCaptureId); 381 if (it == guard->end()) { 382 return ShmemBuffer(); 383 } 384 auto& [_, pool] = *it; 385 return pool.GetIfAvailable(aSize); 386 } 387 388 /*static*/ 389 std::unique_ptr<AggregateCapturer> AggregateCapturer::Create( 390 nsISerialEventTarget* aVideoCaptureThread, CaptureEngine aCapEng, 391 VideoEngine* aEngine, const nsCString& aUniqueId, uint64_t aWindowId, 392 nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities, 393 CamerasParent* aParent) { 394 MOZ_ASSERT(aVideoCaptureThread->IsOnCurrentThread()); 395 int captureId = aEngine->CreateVideoCapture(aUniqueId.get(), aWindowId); 396 if (captureId < 0) { 397 return nullptr; 398 } 399 auto capturer = WrapUnique( 400 new AggregateCapturer(aVideoCaptureThread, aCapEng, aEngine, aUniqueId, 401 captureId, std::move(aCapabilities))); 402 capturer->AddStream(aParent, captureId, aWindowId); 403 aEngine->WithEntry(captureId, [&](VideoEngine::CaptureEntry& aEntry) -> void { 404 aEntry.VideoCapture()->SetTrackingId(capturer->mTrackingId.mUniqueInProcId); 405 aEntry.VideoCapture()->RegisterCaptureDataCallback(capturer.get()); 406 if (auto* event = aEntry.CaptureEndedEvent()) { 407 capturer->mCaptureEndedListener = 408 event->Connect(aVideoCaptureThread, capturer.get(), 409 &AggregateCapturer::OnCaptureEnded); 410 } 411 }); 412 return capturer; 413 } 414 415 AggregateCapturer::AggregateCapturer( 416 nsISerialEventTarget* aVideoCaptureThread, CaptureEngine aCapEng, 417 VideoEngine* aEngine, const nsCString& aUniqueId, int aCaptureId, 418 nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities) 419 : mVideoCaptureThread(aVideoCaptureThread), 420 mCapEngine(aCapEng), 421 mEngine(aEngine), 422 mUniqueId(aUniqueId), 423 mCaptureId(aCaptureId), 424 mTrackingId(CaptureEngineToTrackingSourceStr(aCapEng), mCaptureId), 425 mCapabilities(std::move(aCapabilities)), 426 mStreams("CallbackHelper::mStreams") { 427 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 428 } 429 430 AggregateCapturer::~AggregateCapturer() { 431 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 432 #ifdef DEBUG 433 { 434 auto streamsGuard = mStreams.Lock(); 435 MOZ_ASSERT(streamsGuard->IsEmpty()); 436 } 437 #endif 438 mCaptureEndedListener.DisconnectIfExists(); 439 mEngine->WithEntry(mCaptureId, [&](VideoEngine::CaptureEntry& aEntry) { 440 if (auto* cap = aEntry.VideoCapture().get()) { 441 cap->DeRegisterCaptureDataCallback(this); 442 cap->StopCaptureIfAllClientsClose(); 443 } 444 }); 445 MOZ_ALWAYS_FALSE(mEngine->ReleaseVideoCapture(mCaptureId)); 446 } 447 448 void AggregateCapturer::AddStream(CamerasParent* aParent, int aStreamId, 449 uint64_t aWindowId) { 450 auto streamsGuard = mStreams.Lock(); 451 #ifdef DEBUG 452 for (const auto& stream : *streamsGuard) { 453 MOZ_ASSERT(stream->mId != aStreamId); 454 } 455 #endif 456 streamsGuard->AppendElement( 457 new Stream{.mParent = aParent, .mId = aStreamId, .mWindowId = aWindowId}); 458 } 459 460 auto AggregateCapturer::RemoveStream(int aStreamId) -> RemoveStreamResult { 461 auto streamsGuard = mStreams.Lock(); 462 size_t idx = streamsGuard->IndexOf( 463 aStreamId, 0, 464 [](const auto& aElem, const auto& aId) { return aElem->mId - aId; }); 465 if (idx == streamsGuard->NoIndex) { 466 return {.mNumRemainingStreams = 0, .mNumRemainingStreamsForParent = 0}; 467 } 468 CamerasParent* parent = streamsGuard->ElementAt(idx)->mParent; 469 streamsGuard->RemoveElementAt(idx); 470 size_t remainingForParent = 0; 471 for (const auto& s : *streamsGuard) { 472 if (s->mParent == parent) { 473 remainingForParent += 1; 474 } 475 } 476 return {.mNumRemainingStreams = streamsGuard->Length(), 477 .mNumRemainingStreamsForParent = remainingForParent}; 478 } 479 480 auto AggregateCapturer::RemoveStreamsFor(CamerasParent* aParent) 481 -> RemoveStreamResult { 482 auto streamsGuard = mStreams.Lock(); 483 streamsGuard->RemoveElementsBy( 484 [&](const auto& aElem) { return aElem->mParent == aParent; }); 485 return {.mNumRemainingStreams = streamsGuard->Length(), 486 .mNumRemainingStreamsForParent = 0}; 487 } 488 489 Maybe<int> AggregateCapturer::CaptureIdFor(int aStreamId) { 490 auto streamsGuard = mStreams.Lock(); 491 for (auto& stream : *streamsGuard) { 492 if (stream->mId == aStreamId) { 493 return Some(mCaptureId); 494 } 495 } 496 return Nothing(); 497 } 498 499 void AggregateCapturer::SetConfigurationFor( 500 int aStreamId, const webrtc::VideoCaptureCapability& aCapability, 501 const NormalizedConstraints& aConstraints, 502 const dom::VideoResizeModeEnum& aResizeMode, bool aStarted) { 503 auto streamsGuard = mStreams.Lock(); 504 for (auto& stream : *streamsGuard) { 505 if (stream->mId == aStreamId) { 506 stream->mConfiguration = { 507 .mCapability = aCapability, 508 .mConstraints = aConstraints, 509 .mResizeMode = aResizeMode, 510 }; 511 stream->mStarted = aStarted; 512 break; 513 } 514 } 515 } 516 517 webrtc::VideoCaptureCapability AggregateCapturer::CombinedCapability() { 518 Maybe<webrtc::VideoCaptureCapability> combinedCap; 519 CamerasParent* someParent{}; 520 const auto streamsGuard = mStreams.ConstLock(); 521 for (const auto& stream : *streamsGuard) { 522 if (!stream->mStarted) { 523 continue; 524 } 525 if (!someParent) { 526 someParent = stream->mParent; 527 } 528 const auto& cap = stream->mConfiguration.mCapability; 529 if (!combinedCap) { 530 combinedCap = Some(cap); 531 continue; 532 } 533 auto combinedRes = combinedCap->width * combinedCap->height; 534 combinedCap->maxFPS = std::max(combinedCap->maxFPS, cap.maxFPS); 535 if (mCapEngine == CaptureEngine::CameraEngine) { 536 auto newCombinedRes = cap.width * cap.height; 537 if (newCombinedRes > combinedRes) { 538 combinedCap->videoType = cap.videoType; 539 } 540 combinedCap->width = std::max(combinedCap->width, cap.width); 541 combinedCap->height = std::max(combinedCap->height, cap.height); 542 } 543 } 544 if (mCapEngine == CameraEngine) { 545 const webrtc::VideoCaptureCapability* minDistanceCapability{}; 546 uint64_t minDistance = UINT64_MAX; 547 548 for (const auto& candidateCapability : mCapabilities) { 549 if (candidateCapability.videoType != combinedCap->videoType) { 550 continue; 551 } 552 // The first priority is finding a suitable resolution. 553 // So here we raise the weight of width and height 554 uint64_t distance = 555 uint64_t(ResolutionFeasibilityDistance(candidateCapability.width, 556 combinedCap->width)) + 557 uint64_t(ResolutionFeasibilityDistance(candidateCapability.height, 558 combinedCap->height)) + 559 uint64_t(FeasibilityDistance(candidateCapability.maxFPS, 560 combinedCap->maxFPS)); 561 if (distance < minDistance) { 562 minDistanceCapability = &candidateCapability; 563 minDistance = distance; 564 } 565 } 566 if (minDistanceCapability) { 567 combinedCap = Some(*minDistanceCapability); 568 } 569 } 570 return combinedCap.extract(); 571 } 572 573 void AggregateCapturer::OnCaptureEnded() { 574 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 575 std::multimap<CamerasParent*, int> parentsAndIds; 576 { 577 auto streamsGuard = mStreams.Lock(); 578 for (const auto& stream : *streamsGuard) { 579 parentsAndIds.insert({stream->mParent, stream->mId}); 580 } 581 } 582 583 for (auto it = parentsAndIds.begin(); it != parentsAndIds.end();) { 584 const auto& parent = it->first; 585 auto nextParentIt = parentsAndIds.upper_bound(parent); 586 AutoTArray<int, 4> ids; 587 while (it != nextParentIt) { 588 const auto& [_, id] = *it; 589 ids.AppendElement(id); 590 ++it; 591 } 592 nsIEventTarget* target = parent->GetBackgroundEventTarget(); 593 MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( 594 __func__, [parent = RefPtr(parent), ids = std::move(ids)] { 595 (void)parent->SendCaptureEnded(ids); 596 }))); 597 } 598 } 599 600 void AggregateCapturer::OnFrame(const webrtc::VideoFrame& aVideoFrame) { 601 std::multimap<CamerasParent*, int> parentsAndIds; 602 { 603 // Proactively drop frames that would not get processed anyway. 604 auto streamsGuard = mStreams.Lock(); 605 606 for (auto& stream : *streamsGuard) { 607 auto& c = stream->mConfiguration; 608 const double maxFramerate = static_cast<double>( 609 c.mCapability.maxFPS > 0 ? c.mCapability.maxFPS : 120); 610 const double desiredFramerate = 611 c.mResizeMode == VideoResizeModeEnum::Crop_and_scale 612 ? c.mConstraints.mFrameRate.Get(maxFramerate) 613 : maxFramerate; 614 const double targetFramerate = std::clamp(desiredFramerate, 0.01, 120.); 615 616 // Allow 5% higher fps than configured as frame time sampling is timing 617 // dependent. 618 const auto minInterval = 619 media::TimeUnit(1000, static_cast<int64_t>(1050 * targetFramerate)); 620 const auto frameTime = 621 media::TimeUnit::FromMicroseconds(aVideoFrame.timestamp_us()); 622 const auto frameInterval = frameTime - stream->mLastFrameTime; 623 if (frameInterval < minInterval) { 624 continue; 625 } 626 stream->mLastFrameTime = frameTime; 627 LOG_VERBOSE("CamerasParent::%s parent=%p, id=%d.", __func__, 628 stream->mParent, stream->mId); 629 parentsAndIds.insert({stream->mParent, stream->mId}); 630 } 631 } 632 633 if (profiler_thread_is_being_profiled_for_markers()) { 634 PROFILER_MARKER_UNTYPED( 635 nsPrintfCString("CaptureVideoFrame %dx%d %s %s", aVideoFrame.width(), 636 aVideoFrame.height(), 637 webrtc::VideoFrameBufferTypeToString( 638 aVideoFrame.video_frame_buffer()->type()), 639 mTrackingId.ToString().get()), 640 MEDIA_RT); 641 } 642 643 // Get frame properties 644 camera::VideoFrameProperties properties; 645 VideoFrameUtils::InitFrameBufferProperties(aVideoFrame, properties); 646 647 for (auto it = parentsAndIds.begin(); it != parentsAndIds.end();) { 648 const auto& parent = it->first; 649 auto nextParentIt = parentsAndIds.upper_bound(parent); 650 AutoTArray<int, 4> ids; 651 while (it != nextParentIt) { 652 const auto& [_, id] = *it; 653 ids.AppendElement(id); 654 ++it; 655 } 656 657 LOG_VERBOSE("CamerasParent(%p)::%s", parent, __func__); 658 RefPtr<DeliverFrameRunnable> runnable = nullptr; 659 // Get a shared memory buffer to copy the frame data into 660 ShmemBuffer shMemBuffer = 661 parent->GetBuffer(mCaptureId, properties.bufferSize()); 662 if (!shMemBuffer.Valid()) { 663 // Either we ran out of buffers or they're not the right size yet 664 LOG("Correctly sized Video shmem not available in DeliverFrame"); 665 // We will do the copy into a(n extra) temporary buffer inside 666 // the DeliverFrameRunnable constructor. 667 } else { 668 // Shared memory buffers of the right size are available, do the copy 669 // here. 670 PerformanceRecorder<CopyVideoStage> rec( 671 "CamerasParent::VideoFrameToShmem"_ns, mTrackingId, 672 aVideoFrame.width(), aVideoFrame.height()); 673 VideoFrameUtils::CopyVideoFrameBuffers( 674 shMemBuffer.GetBytes(), properties.bufferSize(), aVideoFrame); 675 rec.Record(); 676 runnable = new DeliverFrameRunnable(parent, mCapEngine, mCaptureId, 677 std::move(ids), mTrackingId, 678 std::move(shMemBuffer), properties); 679 } 680 if (!runnable) { 681 runnable = new DeliverFrameRunnable(parent, mCapEngine, mCaptureId, 682 std::move(ids), mTrackingId, 683 aVideoFrame, properties); 684 } 685 nsIEventTarget* target = parent->GetBackgroundEventTarget(); 686 target->Dispatch(runnable, NS_DISPATCH_NORMAL); 687 } 688 } 689 690 ipc::IPCResult CamerasParent::RecvReleaseFrame(const int& aCaptureId, 691 ipc::Shmem&& aShmem) { 692 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 693 694 auto guard = mShmemPools.Lock(); 695 auto it = guard->find(aCaptureId); 696 if (it == guard->end()) { 697 MOZ_ASSERT_UNREACHABLE( 698 "Releasing shmem but pool is already gone. Shmem must have been " 699 "deallocated."); 700 return IPC_FAIL(this, "Shmem was already deallocated"); 701 } 702 auto& [_, pool] = *it; 703 pool.Put(ShmemBuffer(aShmem)); 704 return IPC_OK(); 705 } 706 707 void CamerasParent::CloseEngines() { 708 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 709 LOG_FUNCTION(); 710 711 // Stop the capturers. 712 for (const auto& capturer : Reversed(*mCapturers)) { 713 auto removed = capturer->RemoveStreamsFor(this); 714 if (removed.mNumRemainingStreams == 0) { 715 auto capEngine = capturer->mCapEngine; 716 auto captureId = capturer->mCaptureId; 717 size_t idx = mCapturers->LastIndexOf(capturer); 718 mCapturers->RemoveElementAtUnsafe(idx); 719 LOG("Forcing shutdown of engine %d, capturer %d", capEngine, captureId); 720 } 721 } 722 723 mDeviceChangeEventListener.DisconnectIfExists(); 724 mDeviceChangeEventListenerConnected = false; 725 } 726 727 std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> 728 CamerasParent::GetDeviceInfo(int aEngine) { 729 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 730 LOG_VERBOSE("CamerasParent(%p)::%s", this, __func__); 731 732 auto* engine = EnsureInitialized(aEngine); 733 if (!engine) { 734 return nullptr; 735 } 736 auto info = engine->GetOrCreateVideoCaptureDeviceInfo(); 737 738 if (!mDeviceChangeEventListenerConnected && aEngine == CameraEngine) { 739 mDeviceChangeEventListener = engine->DeviceChangeEvent().Connect( 740 mVideoCaptureThread, this, &CamerasParent::OnDeviceChange); 741 mDeviceChangeEventListenerConnected = true; 742 } 743 744 return info; 745 } 746 747 VideoEngine* CamerasParent::EnsureInitialized(int aEngine) { 748 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 749 LOG_VERBOSE("CamerasParent(%p)::%s", this, __func__); 750 CaptureEngine capEngine = static_cast<CaptureEngine>(aEngine); 751 752 if (VideoEngine* engine = mEngines->ElementAt(capEngine); engine) { 753 return engine; 754 } 755 756 CaptureDeviceType captureDeviceType = CaptureDeviceType::Camera; 757 switch (capEngine) { 758 case ScreenEngine: 759 captureDeviceType = CaptureDeviceType::Screen; 760 break; 761 case BrowserEngine: 762 captureDeviceType = CaptureDeviceType::Browser; 763 break; 764 case WinEngine: 765 captureDeviceType = CaptureDeviceType::Window; 766 break; 767 case CameraEngine: 768 captureDeviceType = CaptureDeviceType::Camera; 769 break; 770 default: 771 LOG("Invalid webrtc Video engine"); 772 return nullptr; 773 } 774 775 RefPtr<VideoEngine> engine = 776 VideoEngine::Create(captureDeviceType, mVideoCaptureFactory); 777 if (!engine) { 778 LOG("VideoEngine::Create failed"); 779 return nullptr; 780 } 781 782 return mEngines->ElementAt(capEngine) = std::move(engine); 783 } 784 785 // Dispatch the runnable to do the camera operation on the 786 // specific Cameras thread, preventing us from blocking, and 787 // chain a runnable to send back the result on the IPC thread. 788 // It would be nice to get rid of the code duplication here, 789 // perhaps via Promises. 790 ipc::IPCResult CamerasParent::RecvNumberOfCaptureDevices( 791 const CaptureEngine& aCapEngine) { 792 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 793 MOZ_ASSERT(!mDestroyed); 794 795 LOG_FUNCTION(); 796 LOG("CaptureEngine=%d", aCapEngine); 797 798 using Promise = MozPromise<int, bool, true>; 799 InvokeAsync(mVideoCaptureThread, __func__, 800 [this, self = RefPtr(this), aCapEngine] { 801 int num = -1; 802 if (auto devInfo = GetDeviceInfo(aCapEngine)) { 803 num = static_cast<int>(devInfo->NumberOfDevices()); 804 } 805 806 return Promise::CreateAndResolve( 807 num, "CamerasParent::RecvNumberOfCaptureDevices"); 808 }) 809 ->Then( 810 mPBackgroundEventTarget, __func__, 811 [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { 812 int nrDevices = aValue.ResolveValue(); 813 814 if (mDestroyed) { 815 LOG("RecvNumberOfCaptureDevices failure: child not alive"); 816 return; 817 } 818 819 if (nrDevices < 0) { 820 LOG("RecvNumberOfCaptureDevices couldn't find devices"); 821 (void)SendReplyFailure(); 822 return; 823 } 824 825 LOG("RecvNumberOfCaptureDevices: %d", nrDevices); 826 (void)SendReplyNumberOfCaptureDevices(nrDevices); 827 }); 828 return IPC_OK(); 829 } 830 831 ipc::IPCResult CamerasParent::RecvEnsureInitialized( 832 const CaptureEngine& aCapEngine) { 833 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 834 MOZ_ASSERT(!mDestroyed); 835 836 LOG_FUNCTION(); 837 838 using Promise = MozPromise<bool, bool, true>; 839 InvokeAsync(mVideoCaptureThread, __func__, 840 [this, self = RefPtr(this), aCapEngine] { 841 return Promise::CreateAndResolve( 842 EnsureInitialized(aCapEngine), 843 "CamerasParent::RecvEnsureInitialized"); 844 }) 845 ->Then( 846 mPBackgroundEventTarget, __func__, 847 [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { 848 bool result = aValue.ResolveValue(); 849 850 if (mDestroyed) { 851 LOG("RecvEnsureInitialized: child not alive"); 852 return; 853 } 854 855 if (!result) { 856 LOG("RecvEnsureInitialized failed"); 857 (void)SendReplyFailure(); 858 return; 859 } 860 861 LOG("RecvEnsureInitialized succeeded"); 862 (void)SendReplySuccess(); 863 }); 864 return IPC_OK(); 865 } 866 867 ipc::IPCResult CamerasParent::RecvNumberOfCapabilities( 868 const CaptureEngine& aCapEngine, const nsACString& aUniqueId) { 869 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 870 MOZ_ASSERT(!mDestroyed); 871 872 LOG_FUNCTION(); 873 LOG("Getting caps for %s", PromiseFlatCString(aUniqueId).get()); 874 875 using Promise = MozPromise<int, bool, true>; 876 InvokeAsync( 877 mVideoCaptureThread, __func__, 878 [this, self = RefPtr(this), id = nsCString(aUniqueId), aCapEngine]() { 879 int num = -1; 880 if (auto devInfo = GetDeviceInfo(aCapEngine)) { 881 num = devInfo->NumberOfCapabilities(id.get()); 882 } 883 return Promise::CreateAndResolve( 884 num, "CamerasParent::RecvNumberOfCapabilities"); 885 }) 886 ->Then( 887 mPBackgroundEventTarget, __func__, 888 [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { 889 int aNrCapabilities = aValue.ResolveValue(); 890 891 if (mDestroyed) { 892 LOG("RecvNumberOfCapabilities: child not alive"); 893 return; 894 } 895 896 if (aNrCapabilities < 0) { 897 LOG("RecvNumberOfCapabilities couldn't find capabilities"); 898 (void)SendReplyFailure(); 899 return; 900 } 901 902 LOG("RecvNumberOfCapabilities: %d", aNrCapabilities); 903 (void)SendReplyNumberOfCapabilities(aNrCapabilities); 904 }); 905 return IPC_OK(); 906 } 907 908 ipc::IPCResult CamerasParent::RecvGetCaptureCapability( 909 const CaptureEngine& aCapEngine, const nsACString& aUniqueId, 910 const int& aIndex) { 911 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 912 MOZ_ASSERT(!mDestroyed); 913 914 LOG_FUNCTION(); 915 LOG("RecvGetCaptureCapability: %s %d", PromiseFlatCString(aUniqueId).get(), 916 aIndex); 917 918 using Promise = MozPromise<webrtc::VideoCaptureCapability, int, true>; 919 InvokeAsync(mVideoCaptureThread, __func__, 920 [this, self = RefPtr(this), id = nsCString(aUniqueId), aCapEngine, 921 aIndex] { 922 nsTArray<webrtc::VideoCaptureCapability> const* capabilities = 923 EnsureCapabilitiesPopulated(aCapEngine, id); 924 webrtc::VideoCaptureCapability webrtcCaps; 925 if (!capabilities) { 926 return Promise::CreateAndReject( 927 -1, "CamerasParent::RecvGetCaptureCapability"); 928 } 929 if (aIndex < 0 || 930 static_cast<size_t>(aIndex) >= capabilities->Length()) { 931 return Promise::CreateAndReject( 932 -2, "CamerasParent::RecvGetCaptureCapability"); 933 } 934 return Promise::CreateAndResolve( 935 capabilities->ElementAt(aIndex), 936 "CamerasParent::RecvGetCaptureCapability"); 937 }) 938 ->Then( 939 mPBackgroundEventTarget, __func__, 940 [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { 941 if (mDestroyed) { 942 LOG("RecvGetCaptureCapability: child not alive"); 943 return; 944 } 945 946 if (aValue.IsReject()) { 947 LOG("RecvGetCaptureCapability: reply failure"); 948 (void)SendReplyFailure(); 949 return; 950 } 951 952 auto webrtcCaps = aValue.ResolveValue(); 953 VideoCaptureCapability capCap( 954 webrtcCaps.width, webrtcCaps.height, webrtcCaps.maxFPS, 955 static_cast<int>(webrtcCaps.videoType), webrtcCaps.interlaced); 956 LOG("Capability: %u %u %u %d %d", webrtcCaps.width, 957 webrtcCaps.height, webrtcCaps.maxFPS, 958 static_cast<int>(webrtcCaps.videoType), webrtcCaps.interlaced); 959 (void)SendReplyGetCaptureCapability(capCap); 960 }); 961 return IPC_OK(); 962 } 963 964 ipc::IPCResult CamerasParent::RecvGetCaptureDevice( 965 const CaptureEngine& aCapEngine, const int& aDeviceIndex) { 966 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 967 MOZ_ASSERT(!mDestroyed); 968 969 LOG_FUNCTION(); 970 971 using Data = std::tuple<nsCString, nsCString, pid_t, int>; 972 using Promise = MozPromise<Data, bool, true>; 973 InvokeAsync(mVideoCaptureThread, __func__, 974 [this, self = RefPtr(this), aCapEngine, aDeviceIndex] { 975 char deviceName[MediaEngineSource::kMaxDeviceNameLength]; 976 char deviceUniqueId[MediaEngineSource::kMaxUniqueIdLength]; 977 nsCString name; 978 nsCString uniqueId; 979 pid_t devicePid = 0; 980 int error = -1; 981 if (auto devInfo = GetDeviceInfo(aCapEngine)) { 982 error = devInfo->GetDeviceName( 983 aDeviceIndex, deviceName, sizeof(deviceName), 984 deviceUniqueId, sizeof(deviceUniqueId), nullptr, 0, 985 &devicePid); 986 } 987 988 if (error == 0) { 989 name.Assign(deviceName); 990 uniqueId.Assign(deviceUniqueId); 991 } 992 993 return Promise::CreateAndResolve( 994 std::make_tuple(std::move(name), std::move(uniqueId), 995 devicePid, error), 996 "CamerasParent::RecvGetCaptureDevice"); 997 }) 998 ->Then( 999 mPBackgroundEventTarget, __func__, 1000 [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { 1001 const auto& [name, uniqueId, devicePid, error] = 1002 aValue.ResolveValue(); 1003 if (mDestroyed) { 1004 return; 1005 } 1006 if (error != 0) { 1007 LOG("GetCaptureDevice failed: %d", error); 1008 (void)SendReplyFailure(); 1009 return; 1010 } 1011 bool scary = (devicePid == getpid()); 1012 1013 LOG("Returning %s name %s id (pid = %d)%s", name.get(), 1014 uniqueId.get(), devicePid, (scary ? " (scary)" : "")); 1015 (void)SendReplyGetCaptureDevice(name, uniqueId, scary); 1016 }); 1017 return IPC_OK(); 1018 } 1019 1020 // Find out whether the given window with id has permission to use the 1021 // camera. If the permission is not persistent, we'll make it a one-shot by 1022 // removing the (session) permission. 1023 static bool HasCameraPermission(const uint64_t& aWindowId) { 1024 MOZ_ASSERT(NS_IsMainThread()); 1025 1026 RefPtr<dom::WindowGlobalParent> window = 1027 dom::WindowGlobalParent::GetByInnerWindowId(aWindowId); 1028 if (!window) { 1029 // Could not find window by id 1030 return false; 1031 } 1032 1033 // when we delegate permission from first party, we should use the top level 1034 // window 1035 RefPtr<dom::BrowsingContext> topBC = window->BrowsingContext()->Top(); 1036 window = topBC->Canonical()->GetCurrentWindowGlobal(); 1037 1038 // Return false if the window is not the currently-active window for its 1039 // BrowsingContext. 1040 if (!window || !window->IsCurrentGlobal()) { 1041 return false; 1042 } 1043 1044 nsIPrincipal* principal = window->DocumentPrincipal(); 1045 if (principal->GetIsNullPrincipal()) { 1046 return false; 1047 } 1048 1049 if (principal->IsSystemPrincipal()) { 1050 return true; 1051 } 1052 1053 MOZ_ASSERT(principal->GetIsContentPrincipal()); 1054 1055 nsresult rv; 1056 // Name used with nsIPermissionManager 1057 static const nsLiteralCString cameraPermission = "MediaManagerVideo"_ns; 1058 nsCOMPtr<nsIPermissionManager> mgr = 1059 do_GetService(NS_PERMISSIONMANAGER_CONTRACTID, &rv); 1060 if (NS_WARN_IF(NS_FAILED(rv))) { 1061 return false; 1062 } 1063 1064 uint32_t video = nsIPermissionManager::UNKNOWN_ACTION; 1065 rv = mgr->TestExactPermissionFromPrincipal(principal, cameraPermission, 1066 &video); 1067 if (NS_WARN_IF(NS_FAILED(rv))) { 1068 return false; 1069 } 1070 1071 bool allowed = (video == nsIPermissionManager::ALLOW_ACTION); 1072 1073 // Session permissions are removed after one use. 1074 if (allowed) { 1075 mgr->RemoveFromPrincipal(principal, cameraPermission); 1076 } 1077 1078 return allowed; 1079 } 1080 1081 ipc::IPCResult CamerasParent::RecvAllocateCapture( 1082 const CaptureEngine& aCapEngine, const nsACString& aUniqueIdUTF8, 1083 const uint64_t& aWindowID) { 1084 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 1085 MOZ_ASSERT(!mDestroyed); 1086 1087 LOG("CamerasParent(%p)::%s: Verifying permissions", this, __func__); 1088 1089 using Promise1 = MozPromise<bool, bool, true>; 1090 using Promise2 = MozPromise<Maybe<int>, bool, true>; 1091 InvokeAsync( 1092 GetMainThreadSerialEventTarget(), __func__, 1093 [aWindowID] { 1094 // Verify whether the claimed origin has received permission 1095 // to use the camera, either persistently or this session (one 1096 // shot). 1097 bool allowed = HasCameraPermission(aWindowID); 1098 if (!allowed && Preferences::GetBool( 1099 "media.navigator.permission.disabled", false)) { 1100 // Developer preference for turning off permission check. 1101 allowed = true; 1102 LOG("No permission but checks are disabled"); 1103 } 1104 if (!allowed) { 1105 LOG("No camera permission for this origin"); 1106 } 1107 return Promise1::CreateAndResolve(allowed, 1108 "CamerasParent::RecvAllocateCapture"); 1109 }) 1110 ->Then( 1111 mVideoCaptureThread, __func__, 1112 [this, self = RefPtr(this), aCapEngine, aWindowID, 1113 unique_id = nsCString(aUniqueIdUTF8)]( 1114 Promise1::ResolveOrRejectValue&& aValue) { 1115 VideoEngine* engine = EnsureInitialized(aCapEngine); 1116 if (!engine) { 1117 return Promise2::CreateAndResolve( 1118 Nothing(), "CamerasParent::RecvAllocateCapture no engine"); 1119 } 1120 bool allowed = aValue.ResolveValue(); 1121 if (!allowed && IsWindowCapturing(aWindowID, unique_id)) { 1122 allowed = true; 1123 LOG("No permission but window is already capturing this device"); 1124 } 1125 if (!allowed) { 1126 return Promise2::CreateAndResolve( 1127 Nothing(), "CamerasParent::RecvAllocateCapture"); 1128 } 1129 1130 nsTArray<webrtc::VideoCaptureCapability> capabilities; 1131 if (const auto* caps = 1132 EnsureCapabilitiesPopulated(aCapEngine, unique_id)) { 1133 capabilities.AppendElements(*caps); 1134 } 1135 1136 auto created = GetOrCreateCapturer(aCapEngine, aWindowID, unique_id, 1137 std::move(capabilities)); 1138 return Promise2::CreateAndResolve( 1139 created.mCapturer ? Some(created.mStreamId) : Nothing(), 1140 "CamerasParent::RecvAllocateCapture"); 1141 }) 1142 ->Then( 1143 mPBackgroundEventTarget, __func__, 1144 [this, self = RefPtr(this)](Promise2::ResolveOrRejectValue&& aValue) { 1145 const Maybe<int> captureId = aValue.ResolveValue(); 1146 if (mDestroyed) { 1147 LOG("RecvAllocateCapture: child not alive"); 1148 return; 1149 } 1150 1151 if (!captureId) { 1152 (void)SendReplyFailure(); 1153 LOG("RecvAllocateCapture: failed to create capturer"); 1154 return; 1155 } 1156 1157 LOG("Allocated device nr %d", *captureId); 1158 (void)SendReplyAllocateCapture(*captureId); 1159 }); 1160 return IPC_OK(); 1161 } 1162 1163 ipc::IPCResult CamerasParent::RecvReleaseCapture( 1164 const CaptureEngine& aCapEngine, const int& aStreamId) { 1165 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 1166 MOZ_ASSERT(!mDestroyed); 1167 1168 LOG_FUNCTION(); 1169 LOG("RecvReleaseCapture stream nr %d", aStreamId); 1170 1171 using Promise = MozPromise<int, bool, true>; 1172 InvokeAsync(mVideoCaptureThread, __func__, 1173 [this, self = RefPtr(this), aCapEngine, aStreamId] { 1174 return Promise::CreateAndResolve( 1175 ReleaseStream(aCapEngine, aStreamId), 1176 "CamerasParent::RecvReleaseCapture"); 1177 }) 1178 ->Then(mPBackgroundEventTarget, __func__, 1179 [this, self = RefPtr(this), 1180 aStreamId](Promise::ResolveOrRejectValue&& aValue) { 1181 int error = aValue.ResolveValue(); 1182 1183 if (mDestroyed) { 1184 LOG("RecvReleaseCapture: child not alive"); 1185 return; 1186 } 1187 1188 if (error != 0) { 1189 (void)SendReplyFailure(); 1190 LOG("RecvReleaseCapture: Failed to free stream nr %d", 1191 aStreamId); 1192 return; 1193 } 1194 1195 (void)SendReplySuccess(); 1196 LOG("Freed stream nr %d", aStreamId); 1197 }); 1198 return IPC_OK(); 1199 } 1200 1201 ipc::IPCResult CamerasParent::RecvStartCapture( 1202 const CaptureEngine& aCapEngine, const int& aStreamId, 1203 const VideoCaptureCapability& aIpcCaps, 1204 const NormalizedConstraints& aConstraints, 1205 const dom::VideoResizeModeEnum& aResizeMode) { 1206 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 1207 MOZ_ASSERT(!mDestroyed); 1208 1209 LOG_FUNCTION(); 1210 1211 using Promise = MozPromise<int, bool, true>; 1212 InvokeAsync( 1213 mVideoCaptureThread, __func__, 1214 [this, self = RefPtr(this), aCapEngine, aStreamId, aIpcCaps, aConstraints, 1215 aResizeMode] { 1216 LOG_FUNCTION(); 1217 1218 if (!EnsureInitialized(aCapEngine)) { 1219 return Promise::CreateAndResolve(-1, 1220 "CamerasParent::RecvStartCapture"); 1221 } 1222 1223 AggregateCapturer* cbh = GetCapturer(aCapEngine, aStreamId); 1224 if (!cbh) { 1225 return Promise::CreateAndResolve(-1, 1226 "CamerasParent::RecvStartCapture"); 1227 } 1228 1229 int error = -1; 1230 mEngines->ElementAt(aCapEngine) 1231 ->WithEntry(cbh->mCaptureId, [&](VideoEngine::CaptureEntry& cap) { 1232 webrtc::VideoCaptureCapability capability; 1233 capability.width = aIpcCaps.width(); 1234 capability.height = aIpcCaps.height(); 1235 capability.maxFPS = aIpcCaps.maxFPS(); 1236 capability.videoType = 1237 static_cast<webrtc::VideoType>(aIpcCaps.videoType()); 1238 capability.interlaced = aIpcCaps.interlaced(); 1239 1240 if (cbh) { 1241 cbh->SetConfigurationFor(aStreamId, capability, aConstraints, 1242 aResizeMode, /*aStarted=*/true); 1243 error = 1244 cap.VideoCapture()->StartCapture(cbh->CombinedCapability()); 1245 if (error) { 1246 cbh->SetConfigurationFor(aStreamId, capability, aConstraints, 1247 aResizeMode, /*aStarted=*/false); 1248 } 1249 } 1250 }); 1251 1252 return Promise::CreateAndResolve(error, 1253 "CamerasParent::RecvStartCapture"); 1254 }) 1255 ->Then( 1256 mPBackgroundEventTarget, __func__, 1257 [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { 1258 int error = aValue.ResolveValue(); 1259 1260 if (mDestroyed) { 1261 LOG("RecvStartCapture failure: child is not alive"); 1262 return; 1263 } 1264 1265 if (error != 0) { 1266 LOG("RecvStartCapture failure: StartCapture failed"); 1267 (void)SendReplyFailure(); 1268 return; 1269 } 1270 1271 (void)SendReplySuccess(); 1272 }); 1273 return IPC_OK(); 1274 } 1275 1276 ipc::IPCResult CamerasParent::RecvFocusOnSelectedSource( 1277 const CaptureEngine& aCapEngine, const int& aStreamId) { 1278 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 1279 MOZ_ASSERT(!mDestroyed); 1280 1281 LOG_FUNCTION(); 1282 1283 using Promise = MozPromise<bool, bool, true>; 1284 InvokeAsync(mVideoCaptureThread, __func__, 1285 [this, self = RefPtr(this), aCapEngine, aStreamId] { 1286 bool result = false; 1287 auto* capturer = GetCapturer(aCapEngine, aStreamId); 1288 if (!capturer) { 1289 return Promise::CreateAndResolve( 1290 result, "CamerasParent::RecvFocusOnSelectedSource"); 1291 } 1292 if (auto* engine = EnsureInitialized(aCapEngine)) { 1293 engine->WithEntry( 1294 capturer->mCaptureId, 1295 [&](VideoEngine::CaptureEntry& cap) { 1296 if (cap.VideoCapture()) { 1297 result = cap.VideoCapture()->FocusOnSelectedSource(); 1298 } 1299 }); 1300 } 1301 return Promise::CreateAndResolve( 1302 result, "CamerasParent::RecvFocusOnSelectedSource"); 1303 }) 1304 ->Then( 1305 mPBackgroundEventTarget, __func__, 1306 [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { 1307 bool result = aValue.ResolveValue(); 1308 if (mDestroyed) { 1309 LOG("RecvFocusOnSelectedSource failure: child is not alive"); 1310 return; 1311 } 1312 1313 if (!result) { 1314 (void)SendReplyFailure(); 1315 LOG("RecvFocusOnSelectedSource failure."); 1316 return; 1317 } 1318 1319 (void)SendReplySuccess(); 1320 }); 1321 return IPC_OK(); 1322 } 1323 1324 auto CamerasParent::GetOrCreateCapturer( 1325 CaptureEngine aEngine, uint64_t aWindowId, const nsCString& aUniqueId, 1326 nsTArray<webrtc::VideoCaptureCapability>&& aCapabilities) 1327 -> GetOrCreateCapturerResult { 1328 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 1329 VideoEngine* engine = EnsureInitialized(aEngine); 1330 const auto ensureShmemPool = [&](int aCaptureId) { 1331 auto guard = mShmemPools.Lock(); 1332 constexpr size_t kMaxShmemBuffers = 1; 1333 guard->try_emplace(aCaptureId, kMaxShmemBuffers); 1334 }; 1335 for (auto& capturer : *mCapturers) { 1336 if (capturer->mCapEngine != aEngine) { 1337 continue; 1338 } 1339 if (capturer->mUniqueId.Equals(aUniqueId)) { 1340 int streamId = engine->GenerateId(); 1341 ensureShmemPool(capturer->mCaptureId); 1342 capturer->AddStream(this, streamId, aWindowId); 1343 return {.mCapturer = capturer.get(), .mStreamId = streamId}; 1344 } 1345 } 1346 std::unique_ptr aggregate = 1347 AggregateCapturer::Create(mVideoCaptureThread, aEngine, engine, aUniqueId, 1348 aWindowId, std::move(aCapabilities), this); 1349 if (!aggregate) { 1350 return {}; 1351 } 1352 NotNull capturer = mCapturers->AppendElement(std::move(aggregate)); 1353 ensureShmemPool(capturer->get()->mCaptureId); 1354 return {.mCapturer = capturer->get(), 1355 .mStreamId = capturer->get()->mCaptureId}; 1356 } 1357 1358 AggregateCapturer* CamerasParent::GetCapturer(CaptureEngine aEngine, 1359 int aStreamId) { 1360 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 1361 for (auto& capturer : *mCapturers) { 1362 if (capturer->mCapEngine != aEngine) { 1363 continue; 1364 } 1365 Maybe captureId = capturer->CaptureIdFor(aStreamId); 1366 if (captureId) { 1367 return capturer.get(); 1368 } 1369 } 1370 return nullptr; 1371 } 1372 1373 int CamerasParent::ReleaseStream(CaptureEngine aEngine, int aStreamId) { 1374 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 1375 auto* capturer = GetCapturer(aEngine, aStreamId); 1376 if (!capturer) { 1377 return -1; 1378 } 1379 auto removed = capturer->RemoveStream(aStreamId); 1380 if (removed.mNumRemainingStreams == 0) { 1381 mCapturers->RemoveElement(capturer); 1382 } 1383 return 0; 1384 } 1385 1386 nsTArray<webrtc::VideoCaptureCapability> const* 1387 CamerasParent::EnsureCapabilitiesPopulated(CaptureEngine aEngine, 1388 const nsCString& aUniqueId) { 1389 MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); 1390 if (auto iter = mAllCandidateCapabilities.find(aUniqueId); 1391 iter != mAllCandidateCapabilities.end()) { 1392 return &iter->second; 1393 } 1394 auto devInfo = GetDeviceInfo(aEngine); 1395 if (!devInfo) { 1396 return nullptr; 1397 } 1398 const int num = devInfo->NumberOfCapabilities(aUniqueId.get()); 1399 if (num <= 0) { 1400 return nullptr; 1401 } 1402 nsTArray<webrtc::VideoCaptureCapability> capabilities(num); 1403 for (int i = 0; i < num; ++i) { 1404 webrtc::VideoCaptureCapability capability; 1405 if (devInfo->GetCapability(aUniqueId.get(), i, capability)) { 1406 return nullptr; 1407 } 1408 capabilities.AppendElement(capability); 1409 } 1410 const auto& [iter, _] = 1411 mAllCandidateCapabilities.emplace(aUniqueId, std::move(capabilities)); 1412 return &iter->second; 1413 } 1414 1415 ipc::IPCResult CamerasParent::RecvStopCapture(const CaptureEngine& aCapEngine, 1416 const int& aStreamId) { 1417 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 1418 MOZ_ASSERT(!mDestroyed); 1419 1420 LOG_FUNCTION(); 1421 1422 nsresult rv = mVideoCaptureThread->Dispatch(NS_NewRunnableFunction( 1423 __func__, [this, self = RefPtr(this), aCapEngine, aStreamId] { 1424 auto* capturer = GetCapturer(aCapEngine, aStreamId); 1425 if (capturer) { 1426 capturer->SetConfigurationFor( 1427 aStreamId, webrtc::VideoCaptureCapability{}, 1428 NormalizedConstraints{}, dom::VideoResizeModeEnum::None, 1429 /*aStarted=*/false); 1430 } 1431 })); 1432 1433 if (mDestroyed) { 1434 if (NS_FAILED(rv)) { 1435 return IPC_FAIL_NO_REASON(this); 1436 } 1437 } else { 1438 if (NS_SUCCEEDED(rv)) { 1439 if (!SendReplySuccess()) { 1440 return IPC_FAIL_NO_REASON(this); 1441 } 1442 } else { 1443 if (!SendReplyFailure()) { 1444 return IPC_FAIL_NO_REASON(this); 1445 } 1446 } 1447 } 1448 return IPC_OK(); 1449 } 1450 1451 void CamerasParent::ActorDestroy(ActorDestroyReason aWhy) { 1452 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 1453 LOG_FUNCTION(); 1454 1455 // Release shared memory now, it's our last chance 1456 { 1457 auto guard = mShmemPools.Lock(); 1458 for (auto& [captureId, pool] : *guard) { 1459 pool.Cleanup(this); 1460 } 1461 } 1462 // We don't want to receive callbacks or anything if we can't 1463 // forward them anymore anyway. 1464 mDestroyed = true; 1465 // We don't need to listen for shutdown any longer. Disconnect the request. 1466 // This breaks the reference cycle between CamerasParent and the shutdown 1467 // promise's Then handler. 1468 mShutdownRequest.DisconnectIfExists(); 1469 1470 if (mVideoCaptureThread) { 1471 // Shut down the WebRTC stack, on the video capture thread. 1472 MOZ_ALWAYS_SUCCEEDS(mVideoCaptureThread->Dispatch( 1473 NewRunnableMethod(__func__, this, &CamerasParent::CloseEngines))); 1474 } 1475 } 1476 1477 void CamerasParent::OnShutdown() { 1478 ipc::AssertIsOnBackgroundThread(); 1479 LOG("CamerasParent(%p) ShutdownEvent", this); 1480 mShutdownRequest.Complete(); 1481 (void)Send__delete__(this); 1482 } 1483 1484 CamerasParent::CamerasParent() 1485 : mShutdownBlocker(ShutdownBlockingTicket::Create( 1486 u"CamerasParent"_ns, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), 1487 __LINE__)), 1488 mVideoCaptureThread(mShutdownBlocker 1489 ? MakeAndAddRefVideoCaptureThreadAndSingletons() 1490 : nullptr), 1491 mEngines(sEngines), 1492 mCapturers(sCapturers), 1493 mVideoCaptureFactory(EnsureVideoCaptureFactory()), 1494 mShmemPools("CamerasParent::mShmemPools"), 1495 mPBackgroundEventTarget(GetCurrentSerialEventTarget()), 1496 mDestroyed(false) { 1497 MOZ_ASSERT(mPBackgroundEventTarget != nullptr, 1498 "GetCurrentThreadEventTarget failed"); 1499 LOG("CamerasParent: %p", this); 1500 1501 // Don't dispatch from the constructor a runnable that may toggle the 1502 // reference count, because the IPC thread does not get a reference until 1503 // after the constructor returns. 1504 } 1505 1506 /* static */ 1507 auto CamerasParent::RequestCameraAccess(bool aAllowPermissionRequest) 1508 -> RefPtr<CameraAccessRequestPromise> { 1509 ipc::AssertIsOnBackgroundThread(); 1510 1511 // Special case for PipeWire where we at this point just need to make sure 1512 // we have information about camera availabilty through the camera portal 1513 if (!aAllowPermissionRequest) { 1514 return EnsureVideoCaptureFactory()->UpdateCameraAvailability()->Then( 1515 GetCurrentSerialEventTarget(), 1516 "CamerasParent::RequestCameraAccess update camera availability", 1517 [](const VideoCaptureFactory::UpdateCameraAvailabilityPromise:: 1518 ResolveOrRejectValue& aValue) { 1519 LOG("Camera availability updated to %s", 1520 aValue.IsResolve() 1521 ? aValue.ResolveValue() == 1522 VideoCaptureFactory::CameraAvailability::Available 1523 ? "available" 1524 : "not available" 1525 : "still unknown"); 1526 return CameraAccessRequestPromise::CreateAndResolve( 1527 CamerasAccessStatus::RequestRequired, 1528 "CamerasParent::RequestCameraAccess camera availability updated"); 1529 }); 1530 } 1531 1532 static StaticRefPtr<CameraAccessRequestPromise> sCameraAccessRequestPromise; 1533 if (!sCameraAccessRequestPromise) { 1534 sCameraAccessRequestPromise = RefPtr<CameraAccessRequestPromise>( 1535 EnsureVideoCaptureFactory()->InitCameraBackend()->Then( 1536 GetCurrentSerialEventTarget(), 1537 "CamerasParent::RequestCameraAccess camera backend init handler", 1538 [](nsresult aRv) mutable { 1539 MOZ_ASSERT(NS_SUCCEEDED(aRv)); 1540 ClearCameraDeviceInfo(); 1541 return CameraAccessRequestPromise::CreateAndResolve( 1542 CamerasAccessStatus::Granted, 1543 "CamerasParent::RequestCameraAccess camera backend init " 1544 "resolve"); 1545 }, 1546 [](nsresult aRv) mutable { 1547 MOZ_ASSERT(NS_FAILED(aRv)); 1548 return CameraAccessRequestPromise::CreateAndResolve( 1549 aRv == NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR 1550 ? CamerasAccessStatus::Rejected 1551 : CamerasAccessStatus::Error, 1552 "CamerasParent::RequestCameraAccess camera backend init " 1553 "reject"); 1554 })); 1555 static nsresult clearingRv = NS_DispatchToMainThread(NS_NewRunnableFunction( 1556 __func__, [] { ClearOnShutdown(&sCameraAccessRequestPromise); })); 1557 (void)clearingRv; 1558 } 1559 1560 // If camera acess is granted, all is jolly. But we need to handle rejection. 1561 return sCameraAccessRequestPromise->Then( 1562 GetCurrentSerialEventTarget(), 1563 "CamerasParent::CameraAccessRequestPromise rejection handler", 1564 [](CamerasAccessStatus aStatus) { 1565 return CameraAccessRequestPromise::CreateAndResolve( 1566 aStatus, "CamerasParent::RequestCameraAccess resolve"); 1567 }, 1568 [promise = RefPtr(sCameraAccessRequestPromise.get()), 1569 aAllowPermissionRequest](void_t aRv) { 1570 if (promise == sCameraAccessRequestPromise) { 1571 sCameraAccessRequestPromise = nullptr; 1572 return CameraAccessRequestPromise::CreateAndResolve( 1573 CamerasAccessStatus::Error, 1574 "CamerasParent::RequestCameraAccess reject"); 1575 } 1576 return CamerasParent::RequestCameraAccess(aAllowPermissionRequest); 1577 }); 1578 } 1579 1580 // RecvPCamerasConstructor() is used because IPC messages, for 1581 // Send__delete__(), cannot be sent from AllocPCamerasParent(). 1582 ipc::IPCResult CamerasParent::RecvPCamerasConstructor() { 1583 MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); 1584 1585 // AsyncShutdown barriers are available only for ShutdownPhases as late as 1586 // XPCOMWillShutdown. The IPC background thread shuts down during 1587 // XPCOMShutdownThreads, so actors may be created when AsyncShutdown barriers 1588 // are no longer available. Should shutdown be past XPCOMWillShutdown we end 1589 // up with a null mShutdownBlocker. 1590 1591 if (!mShutdownBlocker) { 1592 LOG("CamerasParent(%p) Got no ShutdownBlockingTicket. We are already in " 1593 "shutdown. Deleting.", 1594 this); 1595 return Send__delete__(this) ? IPC_OK() : IPC_FAIL(this, "Failed to send"); 1596 } 1597 1598 if (!mVideoCaptureThread) { 1599 return Send__delete__(this) ? IPC_OK() : IPC_FAIL(this, "Failed to send"); 1600 } 1601 1602 NS_DispatchToMainThread( 1603 NS_NewRunnableFunction(__func__, [this, self = RefPtr(this)] { 1604 mLogHandle = new nsMainThreadPtrHolder<WebrtcLogSinkHandle>( 1605 "CamerasParent::mLogHandle", EnsureWebrtcLogging()); 1606 })); 1607 1608 MOZ_ASSERT(mEngines); 1609 1610 mShutdownBlocker->ShutdownPromise() 1611 ->Then(mPBackgroundEventTarget, "CamerasParent OnShutdown", 1612 [this, self = RefPtr(this)]( 1613 const ShutdownPromise::ResolveOrRejectValue& aValue) { 1614 MOZ_ASSERT(aValue.IsResolve(), 1615 "ShutdownBlockingTicket must have been destroyed " 1616 "without us disconnecting the shutdown request"); 1617 OnShutdown(); 1618 }) 1619 ->Track(mShutdownRequest); 1620 1621 return IPC_OK(); 1622 } 1623 1624 CamerasParent::~CamerasParent() { 1625 ipc::AssertIsOnBackgroundThread(); 1626 LOG_FUNCTION(); 1627 1628 if (!mVideoCaptureThread) { 1629 // No video engines or video capture thread to shutdown here. 1630 return; 1631 } 1632 1633 MOZ_ASSERT(mShutdownBlocker, 1634 "A ShutdownBlocker is a prerequisite for mVideoCaptureThread"); 1635 1636 ReleaseVideoCaptureThreadAndSingletons(); 1637 } 1638 1639 already_AddRefed<CamerasParent> CamerasParent::Create() { 1640 ipc::AssertIsOnBackgroundThread(); 1641 return MakeAndAddRef<CamerasParent>(); 1642 } 1643 1644 } // namespace camera 1645 } // namespace mozilla 1646 1647 #undef LOG 1648 #undef LOG_FUNCTION 1649 #undef LOG_VERBOSE 1650 #undef LOG_ENABLED