VRManager.cpp (48721B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "VRManager.h" 8 9 #include "GeckoProfiler.h" 10 #include "VRManagerParent.h" 11 #include "VRShMem.h" 12 #include "VRThread.h" 13 #include "gfxVR.h" 14 #include "mozilla/ClearOnShutdown.h" 15 #include "mozilla/dom/VRDisplay.h" 16 #include "mozilla/dom/GamepadEventTypes.h" 17 #include "mozilla/layers/TextureHost.h" 18 #include "mozilla/layers/CompositorThread.h" 19 #include "mozilla/Preferences.h" 20 #include "mozilla/Services.h" 21 #include "mozilla/StaticPrefs_dom.h" 22 #include "nsIObserverService.h" 23 24 #include "gfxVR.h" 25 #include <cstring> 26 27 #include "ipc/VRLayerParent.h" 28 #if !defined(MOZ_WIDGET_ANDROID) 29 # include "VRServiceHost.h" 30 #endif 31 32 #ifdef XP_WIN 33 # include "CompositorD3D11.h" 34 # include "TextureD3D11.h" 35 # include <d3d11.h> 36 # include "gfxWindowsPlatform.h" 37 # include "mozilla/gfx/DeviceManagerDx.h" 38 #elif defined(XP_MACOSX) 39 # include "mozilla/gfx/MacIOSurface.h" 40 # include <errno.h> 41 #elif defined(MOZ_WIDGET_ANDROID) 42 # include <string.h> 43 # include <pthread.h> 44 # include "GeckoVRManager.h" 45 # include "mozilla/java/GeckoSurfaceTextureWrappers.h" 46 # include "mozilla/layers/CompositorThread.h" 47 #endif // defined(MOZ_WIDGET_ANDROID) 48 49 using namespace mozilla; 50 using namespace mozilla::gfx; 51 using namespace mozilla::layers; 52 using namespace mozilla::gl; 53 54 using mozilla::dom::GamepadHandle; 55 56 namespace mozilla::gfx { 57 58 /** 59 * When VR content is active, we run the tasks at 1ms 60 * intervals, enabling multiple events to be processed 61 * per frame, such as haptic feedback pulses. 62 */ 63 const uint32_t kVRActiveTaskInterval = 1; // milliseconds 64 65 /** 66 * When VR content is inactive, we run the tasks at 100ms 67 * intervals, enabling VR display enumeration and 68 * presentation startup to be relatively responsive 69 * while not consuming unnecessary resources. 70 */ 71 const uint32_t kVRIdleTaskInterval = 100; // milliseconds 72 73 /** 74 * Max frame duration before the watchdog submits a new one. 75 * Probably we can get rid of this when we enforce that SubmitFrame can only be 76 * called in a VRDisplay loop. 77 */ 78 const double kVRMaxFrameSubmitDuration = 4000.0f; // milliseconds 79 80 static StaticRefPtr<VRManager> sVRManagerSingleton; 81 82 static bool ValidVRManagerProcess() { 83 return XRE_IsParentProcess() || XRE_IsGPUProcess(); 84 } 85 86 /* static */ 87 VRManager* VRManager::Get() { 88 MOZ_ASSERT(sVRManagerSingleton != nullptr); 89 MOZ_ASSERT(ValidVRManagerProcess()); 90 91 return sVRManagerSingleton; 92 } 93 94 /* static */ 95 VRManager* VRManager::MaybeGet() { 96 MOZ_ASSERT(ValidVRManagerProcess()); 97 98 return sVRManagerSingleton; 99 } 100 101 Atomic<uint32_t> VRManager::sDisplayBase(0); 102 103 /* static */ 104 uint32_t VRManager::AllocateDisplayID() { return ++sDisplayBase; } 105 106 /*static*/ 107 void VRManager::ManagerInit() { 108 MOZ_ASSERT(NS_IsMainThread()); 109 110 if (!ValidVRManagerProcess()) { 111 return; 112 } 113 114 // Enable gamepad extensions while VR is enabled. 115 // Preference only can be set at the Parent process. 116 if (StaticPrefs::dom_vr_enabled() && XRE_IsParentProcess()) { 117 Preferences::SetBool("dom.gamepad.extensions.enabled", true); 118 } 119 120 if (sVRManagerSingleton == nullptr) { 121 sVRManagerSingleton = new VRManager(); 122 ClearOnShutdown(&sVRManagerSingleton); 123 } 124 } 125 126 VRManager::VRManager() 127 : mState(VRManagerState::Disabled), 128 mAccumulator100ms(0.0f), 129 mRuntimeDetectionRequested(false), 130 mRuntimeDetectionCompleted(false), 131 mEnumerationRequested(false), 132 mEnumerationCompleted(false), 133 mVRDisplaysRequested(false), 134 mVRDisplaysRequestedNonFocus(false), 135 mVRControllersRequested(false), 136 mFrameStarted(false), 137 mTaskInterval(0), 138 mCurrentSubmitTaskMonitor("CurrentSubmitTaskMonitor"), 139 mCurrentSubmitTask(nullptr), 140 mLastSubmittedFrameId(0), 141 mLastStartedFrame(0), 142 mRuntimeSupportFlags(VRDisplayCapabilityFlags::Cap_None), 143 mAppPaused(false), 144 mShmem(nullptr), 145 mHapticPulseRemaining{}, 146 mDisplayInfo{}, 147 mLastUpdateDisplayInfo{}, 148 mBrowserState{}, 149 mLastSensorState{} { 150 MOZ_ASSERT(sVRManagerSingleton == nullptr); 151 MOZ_ASSERT(NS_IsMainThread()); 152 MOZ_ASSERT(ValidVRManagerProcess()); 153 154 #if !defined(MOZ_WIDGET_ANDROID) 155 // XRE_IsGPUProcess() is helping us to check some platforms like 156 // Win 7 try which are not using GPU process but VR process is enabled. 157 mVRProcessEnabled = 158 StaticPrefs::dom_vr_process_enabled_AtStartup() && XRE_IsGPUProcess(); 159 VRServiceHost::Init(mVRProcessEnabled); 160 mServiceHost = VRServiceHost::Get(); 161 // We must shutdown before VRServiceHost, which is cleared 162 // on ShutdownPhase::XPCOMShutdownFinal, potentially before VRManager. 163 // We hold a reference to VRServiceHost to ensure it stays 164 // alive until we have shut down. 165 #else 166 // For Android, there is no VRProcess available and no VR service is 167 // created, so default to false. 168 mVRProcessEnabled = false; 169 #endif // !defined(MOZ_WIDGET_ANDROID) 170 171 nsCOMPtr<nsIObserverService> service = services::GetObserverService(); 172 if (service) { 173 service->AddObserver(this, "application-background", false); 174 service->AddObserver(this, "application-foreground", false); 175 } 176 } 177 178 void VRManager::OpenShmem() { 179 if (mShmem == nullptr) { 180 mShmem = new VRShMem(nullptr, true /*aRequiresMutex*/); 181 182 #if !defined(MOZ_WIDGET_ANDROID) 183 mShmem->CreateShMem(mVRProcessEnabled /*aCreateOnSharedMemory*/); 184 // The VR Service accesses all hardware from a separate process 185 // and replaces the other VRManager when enabled. 186 // If the VR process is not enabled, create an in-process VRService. 187 if (!mVRProcessEnabled) { 188 // If the VR process is disabled, attempt to create a 189 // VR service within the current process 190 mServiceHost->CreateService(mShmem->GetExternalShmem()); 191 return; 192 } 193 #else 194 mShmem->CreateShMemForAndroid(); 195 #endif 196 } else { 197 mShmem->ClearShMem(); 198 } 199 200 // Reset local information for new connection 201 mDisplayInfo.Clear(); 202 mLastUpdateDisplayInfo.Clear(); 203 mFrameStarted = false; 204 mBrowserState.Clear(); 205 mLastSensorState.Clear(); 206 mEnumerationCompleted = false; 207 mDisplayInfo.mGroupMask = kVRGroupContent; 208 } 209 210 void VRManager::CloseShmem() { 211 if (mShmem != nullptr) { 212 mShmem->CloseShMem(); 213 delete mShmem; 214 mShmem = nullptr; 215 } 216 } 217 218 VRManager::~VRManager() { 219 MOZ_ASSERT(NS_IsMainThread()); 220 MOZ_ASSERT(mState == VRManagerState::Disabled); 221 222 nsCOMPtr<nsIObserverService> service = services::GetObserverService(); 223 if (service) { 224 service->RemoveObserver(this, "application-background"); 225 service->RemoveObserver(this, "application-foreground"); 226 } 227 228 #if !defined(MOZ_WIDGET_ANDROID) 229 mServiceHost->Shutdown(); 230 #endif 231 CloseShmem(); 232 } 233 234 void VRManager::AddLayer(VRLayerParent* aLayer) { 235 mLayers.AppendElement(aLayer); 236 mDisplayInfo.mPresentingGroups |= aLayer->GetGroup(); 237 if (mLayers.Length() == 1) { 238 StartPresentation(); 239 } 240 241 // Ensure that the content process receives the change immediately 242 if (mState != VRManagerState::Enumeration && 243 mState != VRManagerState::RuntimeDetection) { 244 DispatchVRDisplayInfoUpdate(); 245 } 246 } 247 248 void VRManager::RemoveLayer(VRLayerParent* aLayer) { 249 mLayers.RemoveElement(aLayer); 250 if (mLayers.Length() == 0) { 251 StopPresentation(); 252 } 253 mDisplayInfo.mPresentingGroups = 0; 254 for (auto layer : mLayers) { 255 mDisplayInfo.mPresentingGroups |= layer->GetGroup(); 256 } 257 258 // Ensure that the content process receives the change immediately 259 if (mState != VRManagerState::Enumeration && 260 mState != VRManagerState::RuntimeDetection) { 261 DispatchVRDisplayInfoUpdate(); 262 } 263 } 264 265 void VRManager::AddVRManagerParent(VRManagerParent* aVRManagerParent) { 266 mVRManagerParents.Insert(aVRManagerParent); 267 } 268 269 void VRManager::RemoveVRManagerParent(VRManagerParent* aVRManagerParent) { 270 mVRManagerParents.Remove(aVRManagerParent); 271 if (mVRManagerParents.IsEmpty()) { 272 Destroy(); 273 } 274 } 275 276 void VRManager::UpdateRequestedDevices() { 277 bool bHaveEventListener = false; 278 bool bHaveEventListenerNonFocus = false; 279 bool bHaveControllerListener = false; 280 281 for (VRManagerParent* vmp : mVRManagerParents) { 282 bHaveEventListener |= vmp->HaveEventListener() && vmp->GetVRActiveStatus(); 283 bHaveEventListenerNonFocus |= 284 vmp->HaveEventListener() && !vmp->GetVRActiveStatus(); 285 bHaveControllerListener |= vmp->HaveControllerListener(); 286 } 287 288 mVRDisplaysRequested = bHaveEventListener; 289 mVRDisplaysRequestedNonFocus = bHaveEventListenerNonFocus; 290 // We only currently allow controllers to be used when 291 // also activating a VR display 292 mVRControllersRequested = mVRDisplaysRequested && bHaveControllerListener; 293 } 294 295 /** 296 * VRManager::NotifyVsync must be called on every 2d vsync (usually at 60hz). 297 * This must be called even when no WebVR site is active. 298 * If we don't have a 2d display attached to the system, we can call this 299 * at the VR display's native refresh rate. 300 **/ 301 void VRManager::NotifyVsync(const TimeStamp& aVsyncTimestamp) { 302 if (mState != VRManagerState::Active) { 303 return; 304 } 305 /** 306 * If the display isn't presenting, refresh the sensors and trigger 307 * VRDisplay.requestAnimationFrame at the normal 2d display refresh rate. 308 */ 309 if (mDisplayInfo.mPresentingGroups == 0) { 310 StartFrame(); 311 } 312 } 313 314 void VRManager::StartTasks() { 315 if (!mTaskTimer) { 316 mTaskInterval = GetOptimalTaskInterval(); 317 mTaskTimer = NS_NewTimer(); 318 mTaskTimer->SetTarget(CompositorThread()); 319 mTaskTimer->InitWithNamedFuncCallback( 320 TaskTimerCallback, this, mTaskInterval, 321 nsITimer::TYPE_REPEATING_PRECISE_CAN_SKIP, 322 "VRManager::TaskTimerCallback"_ns); 323 } 324 } 325 326 void VRManager::StopTasks() { 327 if (mTaskTimer) { 328 mTaskTimer->Cancel(); 329 mTaskTimer = nullptr; 330 } 331 } 332 333 /*static*/ 334 void VRManager::TaskTimerCallback(nsITimer* aTimer, void* aClosure) { 335 /** 336 * It is safe to use the pointer passed in aClosure to reference the 337 * VRManager object as the timer is canceled in VRManager::Destroy. 338 * VRManager::Destroy set mState to VRManagerState::Disabled, which 339 * is asserted in the VRManager destructor, guaranteeing that this 340 * functions runs if and only if the VRManager object is valid. 341 */ 342 VRManager* self = static_cast<VRManager*>(aClosure); 343 self->RunTasks(); 344 345 if (self->mAppPaused) { 346 // When the apps goes the background (e.g. Android) we should stop the 347 // tasks. 348 self->StopTasks(); 349 self->mState = VRManagerState::Idle; 350 } 351 } 352 353 void VRManager::RunTasks() { 354 // Will be called once every 1ms when a VR presentation 355 // is active or once per vsync when a VR presentation is 356 // not active. 357 358 if (mState == VRManagerState::Disabled) { 359 // We may have been destroyed but still have messages 360 // in the queue from mTaskTimer. Bail out to avoid 361 // running them. 362 return; 363 } 364 365 TimeStamp now = TimeStamp::Now(); 366 double lastTickMs = mAccumulator100ms; 367 double deltaTime = 0.0f; 368 if (!mLastTickTime.IsNull()) { 369 deltaTime = (now - mLastTickTime).ToMilliseconds(); 370 } 371 mAccumulator100ms += deltaTime; 372 mLastTickTime = now; 373 374 if (deltaTime > 0.0f && floor(mAccumulator100ms) != floor(lastTickMs)) { 375 // Even if more than 1 ms has passed, we will only 376 // execute Run1msTasks() once. 377 Run1msTasks(deltaTime); 378 } 379 380 if (floor(mAccumulator100ms * 0.1f) != floor(lastTickMs * 0.1f)) { 381 // Even if more than 10 ms has passed, we will only 382 // execute Run10msTasks() once. 383 Run10msTasks(); 384 } 385 386 if (mAccumulator100ms >= 100.0f) { 387 // Even if more than 100 ms has passed, we will only 388 // execute Run100msTasks() once. 389 Run100msTasks(); 390 mAccumulator100ms = fmod(mAccumulator100ms, 100.0f); 391 } 392 393 uint32_t optimalTaskInterval = GetOptimalTaskInterval(); 394 if (mTaskTimer && optimalTaskInterval != mTaskInterval) { 395 mTaskTimer->SetDelay(optimalTaskInterval); 396 mTaskInterval = optimalTaskInterval; 397 } 398 } 399 400 uint32_t VRManager::GetOptimalTaskInterval() { 401 /** 402 * When either VR content is detected or VR hardware 403 * has already been activated, we schedule tasks more 404 * frequently. 405 */ 406 bool wantGranularTasks = mVRDisplaysRequested || mVRControllersRequested || 407 mDisplayInfo.mDisplayID != 0; 408 if (wantGranularTasks) { 409 return kVRActiveTaskInterval; 410 } 411 412 return kVRIdleTaskInterval; 413 } 414 415 /** 416 * Run1msTasks() is guaranteed not to be 417 * called more than once within 1ms. 418 * When VR is not active, this will be 419 * called once per VSync if it wasn't 420 * called within the last 1ms. 421 */ 422 void VRManager::Run1msTasks(double aDeltaTime) { UpdateHaptics(aDeltaTime); } 423 424 /** 425 * Run10msTasks() is guaranteed not to be 426 * called more than once within 10ms. 427 * When VR is not active, this will be 428 * called once per VSync if it wasn't 429 * called within the last 10ms. 430 */ 431 void VRManager::Run10msTasks() { 432 UpdateRequestedDevices(); 433 CheckWatchDog(); 434 ExpireNavigationTransition(); 435 PullState(); 436 PushState(); 437 } 438 439 /** 440 * Run100msTasks() is guaranteed not to be 441 * called more than once within 100ms. 442 * When VR is not active, this will be 443 * called once per VSync if it wasn't 444 * called within the last 100ms. 445 */ 446 void VRManager::Run100msTasks() { 447 // We must continually refresh the VR display enumeration to check 448 // for events that we must fire such as Window.onvrdisplayconnect 449 // Note that enumeration itself may activate display hardware, such 450 // as Oculus, so we only do this when we know we are displaying content 451 // that is looking for VR displays. 452 #if !defined(MOZ_WIDGET_ANDROID) 453 mServiceHost->Refresh(); 454 CheckForPuppetCompletion(); 455 #endif 456 ProcessManagerState(); 457 } 458 459 void VRManager::CheckForInactiveTimeout() { 460 // Shut down the VR devices when not in use 461 if (mVRDisplaysRequested || mVRDisplaysRequestedNonFocus || 462 mVRControllersRequested || mEnumerationRequested || 463 mRuntimeDetectionRequested || mState == VRManagerState::Enumeration || 464 mState == VRManagerState::RuntimeDetection) { 465 // We are using a VR device, keep it alive 466 mLastActiveTime = TimeStamp::Now(); 467 } else if (mLastActiveTime.IsNull()) { 468 Shutdown(); 469 } else { 470 TimeDuration duration = TimeStamp::Now() - mLastActiveTime; 471 if (duration.ToMilliseconds() > StaticPrefs::dom_vr_inactive_timeout()) { 472 Shutdown(); 473 // We must not throttle the next enumeration request 474 // after an idle timeout, as it may result in the 475 // user needing to refresh the browser to detect 476 // VR hardware when leaving and returning to a VR 477 // site. 478 mLastDisplayEnumerationTime = TimeStamp(); 479 } 480 } 481 } 482 483 void VRManager::CheckForShutdown() { 484 // Check for remote end shutdown 485 if (mDisplayInfo.mDisplayState.shutdown) { 486 Shutdown(); 487 } 488 } 489 490 #if !defined(MOZ_WIDGET_ANDROID) 491 void VRManager::CheckForPuppetCompletion() { 492 // Notify content process about completion of puppet test resets 493 if (mState != VRManagerState::Active) { 494 for (const auto& key : mManagerParentsWaitingForPuppetReset) { 495 (void)key->SendNotifyPuppetResetComplete(); 496 } 497 mManagerParentsWaitingForPuppetReset.Clear(); 498 } 499 // Notify content process about completion of puppet test scripts 500 if (mManagerParentRunningPuppet) { 501 mServiceHost->CheckForPuppetCompletion(); 502 } 503 } 504 505 void VRManager::NotifyPuppetComplete() { 506 // Notify content process about completion of puppet test scripts 507 if (mManagerParentRunningPuppet) { 508 (void)mManagerParentRunningPuppet->SendNotifyPuppetCommandBufferCompleted( 509 true); 510 mManagerParentRunningPuppet = nullptr; 511 } 512 } 513 514 #endif // !defined(MOZ_WIDGET_ANDROID) 515 516 void VRManager::StartFrame() { 517 if (mState != VRManagerState::Active) { 518 return; 519 } 520 AUTO_PROFILER_MARKER("GetSensorState", OTHER); 521 522 /** 523 * Do not start more VR frames until the last submitted frame is already 524 * processed, or the last has stalled for more than 525 * kVRMaxFrameSubmitDuration milliseconds. 526 */ 527 TimeStamp now = TimeStamp::Now(); 528 const TimeStamp lastFrameStart = 529 mLastFrameStart[mDisplayInfo.mFrameId % kVRMaxLatencyFrames]; 530 const bool isPresenting = mLastUpdateDisplayInfo.GetPresentingGroups() != 0; 531 double duration = 532 lastFrameStart.IsNull() ? 0.0 : (now - lastFrameStart).ToMilliseconds(); 533 if (isPresenting && mLastStartedFrame > 0 && 534 mDisplayInfo.mDisplayState.lastSubmittedFrameId < mLastStartedFrame && 535 duration < kVRMaxFrameSubmitDuration) { 536 return; 537 } 538 539 mDisplayInfo.mFrameId++; 540 size_t bufferIndex = mDisplayInfo.mFrameId % kVRMaxLatencyFrames; 541 mDisplayInfo.mLastSensorState[bufferIndex] = mLastSensorState; 542 mLastFrameStart[bufferIndex] = now; 543 mFrameStarted = true; 544 mLastStartedFrame = mDisplayInfo.mFrameId; 545 546 DispatchVRDisplayInfoUpdate(); 547 } 548 549 void VRManager::DetectRuntimes() { 550 if (mState == VRManagerState::RuntimeDetection) { 551 // Runtime detection has already been started. 552 // This additional request will also receive the 553 // result from the first request. 554 return; 555 } 556 557 // Detect XR runtimes to determine if they are 558 // capable of supporting VR or AR sessions, while 559 // avoiding activating any XR devices or persistent 560 // background software. 561 if (mRuntimeDetectionCompleted) { 562 // We have already detected runtimes, so we can 563 // immediately respond with the same results. 564 // This will require the user to restart the browser 565 // after installing or removing an XR device 566 // runtime. 567 DispatchRuntimeCapabilitiesUpdate(); 568 return; 569 } 570 mRuntimeDetectionRequested = true; 571 ProcessManagerState(); 572 } 573 574 void VRManager::EnumerateDevices() { 575 if (mState == VRManagerState::Enumeration || 576 (mRuntimeDetectionCompleted && 577 (mVRDisplaysRequested || mEnumerationRequested))) { 578 // Enumeration has already been started. 579 // This additional request will also receive the 580 // result from the first request. 581 return; 582 } 583 // Activate XR runtimes and enumerate XR devices. 584 mEnumerationRequested = true; 585 ProcessManagerState(); 586 } 587 588 void VRManager::ProcessManagerState() { 589 switch (mState) { 590 case VRManagerState::Disabled: 591 ProcessManagerState_Disabled(); 592 break; 593 case VRManagerState::Idle: 594 ProcessManagerState_Idle(); 595 break; 596 case VRManagerState::RuntimeDetection: 597 ProcessManagerState_DetectRuntimes(); 598 break; 599 case VRManagerState::Enumeration: 600 ProcessManagerState_Enumeration(); 601 break; 602 case VRManagerState::Active: 603 ProcessManagerState_Active(); 604 break; 605 case VRManagerState::Stopping: 606 ProcessManagerState_Stopping(); 607 break; 608 } 609 CheckForInactiveTimeout(); 610 CheckForShutdown(); 611 } 612 613 void VRManager::ProcessManagerState_Disabled() { 614 MOZ_ASSERT(mState == VRManagerState::Disabled); 615 616 if (!StaticPrefs::dom_vr_enabled() && !StaticPrefs::dom_vr_webxr_enabled()) { 617 return; 618 } 619 620 if (mRuntimeDetectionRequested || mEnumerationRequested || 621 mVRDisplaysRequested) { 622 StartTasks(); 623 mState = VRManagerState::Idle; 624 } 625 } 626 627 void VRManager::ProcessManagerState_Stopping() { 628 MOZ_ASSERT(mState == VRManagerState::Stopping); 629 PullState(); 630 /** 631 * In the case of Desktop, the VRService shuts itself down. 632 * Before it's finished stopping, it sets a flag in the ShMem 633 * to let VRManager know that it's done. VRManager watches for 634 * this flag and transitions out of the VRManagerState::Stopping 635 * state to VRManagerState::Idle. 636 */ 637 #if defined(MOZ_WIDGET_ANDROID) 638 // On Android, the VR service never actually shuts 639 // down or requests VRManager to stop. 640 Shutdown(); 641 #endif // defined(MOZ_WIDGET_ANDROID) 642 } 643 644 void VRManager::ProcessManagerState_Idle_StartEnumeration() { 645 MOZ_ASSERT(mState == VRManagerState::Idle); 646 647 if (!mEarliestRestartTime.IsNull() && 648 mEarliestRestartTime > TimeStamp::Now()) { 649 // When the VR Service shuts down it informs us of how long we 650 // must wait until we can re-start it. 651 // We must wait until mEarliestRestartTime before attempting 652 // to enumerate again. 653 return; 654 } 655 656 /** 657 * Throttle the rate of enumeration to the interval set in 658 * VRDisplayEnumerateInterval 659 */ 660 if (!mLastDisplayEnumerationTime.IsNull()) { 661 TimeDuration duration = TimeStamp::Now() - mLastDisplayEnumerationTime; 662 if (duration.ToMilliseconds() < 663 StaticPrefs::dom_vr_display_enumerate_interval()) { 664 return; 665 } 666 } 667 668 /** 669 * If we get this far, don't try again until 670 * the VRDisplayEnumerateInterval elapses 671 */ 672 mLastDisplayEnumerationTime = TimeStamp::Now(); 673 674 OpenShmem(); 675 676 mEnumerationRequested = false; 677 // We must block until enumeration has completed in order 678 // to signal that the WebVR promise should be resolved at the 679 // right time. 680 #if defined(MOZ_WIDGET_ANDROID) 681 // In Android, we need to make sure calling 682 // GeckoVRManager::SetExternalContext() from an external VR service 683 // before doing enumeration. 684 if (!mShmem->GetExternalShmem()) { 685 mShmem->CreateShMemForAndroid(); 686 } 687 if (mShmem->GetExternalShmem()) { 688 mState = VRManagerState::Enumeration; 689 } else { 690 // Not connected to shmem, so no devices to enumerate. 691 mDisplayInfo.Clear(); 692 DispatchVRDisplayInfoUpdate(); 693 } 694 #else 695 696 PushState(); 697 698 /** 699 * We must start the VR Service thread 700 * and VR Process before enumeration. 701 * We don't want to start this until we will 702 * actualy enumerate, to avoid continuously 703 * re-launching the thread/process when 704 * no hardware is found or a VR software update 705 * is in progress 706 */ 707 mServiceHost->StartService(); 708 mState = VRManagerState::Enumeration; 709 #endif // MOZ_WIDGET_ANDROID 710 } 711 712 void VRManager::ProcessManagerState_Idle_StartRuntimeDetection() { 713 MOZ_ASSERT(mState == VRManagerState::Idle); 714 715 OpenShmem(); 716 mBrowserState.detectRuntimesOnly = true; 717 mRuntimeDetectionRequested = false; 718 719 // We must block until enumeration has completed in order 720 // to signal that the WebVR promise should be resolved at the 721 // right time. 722 #if defined(MOZ_WIDGET_ANDROID) 723 // In Android, we need to make sure calling 724 // GeckoVRManager::SetExternalContext() from an external VR service 725 // before doing enumeration. 726 if (!mShmem->GetExternalShmem()) { 727 mShmem->CreateShMemForAndroid(); 728 } 729 if (mShmem->GetExternalShmem()) { 730 mState = VRManagerState::RuntimeDetection; 731 } else { 732 // Not connected to shmem, so no runtimes to detect. 733 mRuntimeSupportFlags = VRDisplayCapabilityFlags::Cap_None; 734 mRuntimeDetectionCompleted = true; 735 DispatchRuntimeCapabilitiesUpdate(); 736 } 737 #else 738 739 PushState(); 740 741 /** 742 * We must start the VR Service thread 743 * and VR Process before enumeration. 744 * We don't want to start this until we will 745 * actualy enumerate, to avoid continuously 746 * re-launching the thread/process when 747 * no hardware is found or a VR software update 748 * is in progress 749 */ 750 mServiceHost->StartService(); 751 mState = VRManagerState::RuntimeDetection; 752 #endif // MOZ_WIDGET_ANDROID 753 } 754 755 void VRManager::ProcessManagerState_Idle() { 756 MOZ_ASSERT(mState == VRManagerState::Idle); 757 758 if (!mRuntimeDetectionCompleted) { 759 // Check if we should start detecting runtimes 760 // We must alwasy detect runtimes before doing anything 761 // else with the VR process. 762 // This will happen only once per browser startup. 763 if (mRuntimeDetectionRequested || mEnumerationRequested) { 764 ProcessManagerState_Idle_StartRuntimeDetection(); 765 } 766 return; 767 } 768 769 // Check if we should start activating enumerating XR hardware 770 if (mRuntimeDetectionCompleted && 771 (mVRDisplaysRequested || mEnumerationRequested)) { 772 ProcessManagerState_Idle_StartEnumeration(); 773 } 774 } 775 776 void VRManager::ProcessManagerState_DetectRuntimes() { 777 MOZ_ASSERT(mState == VRManagerState::RuntimeDetection); 778 MOZ_ASSERT(mShmem != nullptr); 779 780 PullState(); 781 if (mEnumerationCompleted) { 782 /** 783 * When mBrowserState.detectRuntimesOnly is set, the 784 * VRService and VR process will shut themselves down 785 * automatically after detecting runtimes. 786 * mEnumerationCompleted is also used in this case, 787 * but to mean "enumeration of runtimes" not 788 * "enumeration of VR devices". 789 * 790 * We set mState to `VRManagerState::Stopping` 791 * to make sure that we don't try to do anything 792 * else with the active VRService until it has stopped. 793 * We must start another one when an XR session will be 794 * requested. 795 * 796 * This logic is optimized for the WebXR design, but still 797 * works for WebVR so it can continue to function until 798 * deprecated and removed. 799 */ 800 mState = VRManagerState::Stopping; 801 mRuntimeSupportFlags = mDisplayInfo.mDisplayState.capabilityFlags & 802 (VRDisplayCapabilityFlags::Cap_ImmersiveVR | 803 VRDisplayCapabilityFlags::Cap_ImmersiveAR | 804 VRDisplayCapabilityFlags::Cap_Inline); 805 mRuntimeDetectionCompleted = true; 806 DispatchRuntimeCapabilitiesUpdate(); 807 } 808 } 809 810 void VRManager::ProcessManagerState_Enumeration() { 811 MOZ_ASSERT(mState == VRManagerState::Enumeration); 812 MOZ_ASSERT(mShmem != nullptr); 813 814 PullState(); 815 if (mEnumerationCompleted) { 816 if (mDisplayInfo.mDisplayState.isConnected) { 817 mDisplayInfo.mDisplayID = VRManager::AllocateDisplayID(); 818 mState = VRManagerState::Active; 819 } else { 820 mDisplayInfo.Clear(); 821 mState = VRManagerState::Stopping; 822 } 823 DispatchVRDisplayInfoUpdate(); 824 } 825 } 826 827 void VRManager::ProcessManagerState_Active() { 828 MOZ_ASSERT(mState == VRManagerState::Active); 829 830 if (mDisplayInfo != mLastUpdateDisplayInfo) { 831 // While the display is active, send continuous updates 832 DispatchVRDisplayInfoUpdate(); 833 } 834 } 835 836 void VRManager::DispatchVRDisplayInfoUpdate() { 837 for (VRManagerParent* vmp : mVRManagerParents) { 838 (void)vmp->SendUpdateDisplayInfo(mDisplayInfo); 839 } 840 mLastUpdateDisplayInfo = mDisplayInfo; 841 } 842 843 void VRManager::DispatchRuntimeCapabilitiesUpdate() { 844 VRDisplayCapabilityFlags flags = mRuntimeSupportFlags; 845 if (StaticPrefs::dom_vr_always_support_vr()) { 846 flags |= VRDisplayCapabilityFlags::Cap_ImmersiveVR; 847 } 848 849 if (StaticPrefs::dom_vr_always_support_ar()) { 850 flags |= VRDisplayCapabilityFlags::Cap_ImmersiveAR; 851 } 852 853 for (VRManagerParent* vmp : mVRManagerParents) { 854 (void)vmp->SendUpdateRuntimeCapabilities(flags); 855 } 856 } 857 858 void VRManager::StopAllHaptics() { 859 if (mState != VRManagerState::Active) { 860 return; 861 } 862 for (size_t i = 0; i < std::size(mBrowserState.hapticState); i++) { 863 ClearHapticSlot(i); 864 } 865 PushState(); 866 } 867 868 void VRManager::VibrateHaptic(GamepadHandle aGamepadHandle, 869 uint32_t aHapticIndex, double aIntensity, 870 double aDuration, 871 const VRManagerPromise& aPromise) 872 873 { 874 if (mState != VRManagerState::Active) { 875 return; 876 } 877 // VRDisplayClient::FireGamepadEvents() assigns a controller ID with 878 // ranges based on displayID. We must translate this to the indexes 879 // understood by VRDisplayExternal. 880 uint32_t controllerBaseIndex = 881 kVRControllerMaxCount * mDisplayInfo.mDisplayID; 882 uint32_t controllerIndex = aGamepadHandle.GetValue() - controllerBaseIndex; 883 884 TimeStamp now = TimeStamp::Now(); 885 size_t bestSlotIndex = 0; 886 // Default to an empty slot, or the slot holding the oldest haptic pulse 887 for (size_t i = 0; i < std::size(mBrowserState.hapticState); i++) { 888 const VRHapticState& state = mBrowserState.hapticState[i]; 889 if (state.inputFrameID == 0) { 890 // Unused slot, use it 891 bestSlotIndex = i; 892 break; 893 } 894 if (mHapticPulseRemaining[i] < mHapticPulseRemaining[bestSlotIndex]) { 895 // If no empty slots are available, fall back to overriding 896 // the pulse which is ending soonest. 897 bestSlotIndex = i; 898 } 899 } 900 // Override the last pulse on the same actuator if present. 901 for (size_t i = 0; i < std::size(mBrowserState.hapticState); i++) { 902 const VRHapticState& state = mBrowserState.hapticState[i]; 903 if (state.inputFrameID == 0) { 904 // This is an empty slot -- no match 905 continue; 906 } 907 if (state.controllerIndex == controllerIndex && 908 state.hapticIndex == aHapticIndex) { 909 // Found pulse on same actuator -- let's override it. 910 bestSlotIndex = i; 911 } 912 } 913 ClearHapticSlot(bestSlotIndex); 914 915 // Populate the selected slot with new haptic state 916 size_t bufferIndex = mDisplayInfo.mFrameId % kVRMaxLatencyFrames; 917 VRHapticState& bestSlot = mBrowserState.hapticState[bestSlotIndex]; 918 bestSlot.inputFrameID = 919 mDisplayInfo.mLastSensorState[bufferIndex].inputFrameID; 920 bestSlot.controllerIndex = controllerIndex; 921 bestSlot.hapticIndex = aHapticIndex; 922 bestSlot.pulseStart = (float)(now - mLastFrameStart[bufferIndex]).ToSeconds(); 923 bestSlot.pulseDuration = 924 (float)aDuration * 0.001f; // Convert from ms to seconds 925 bestSlot.pulseIntensity = (float)aIntensity; 926 927 mHapticPulseRemaining[bestSlotIndex] = aDuration; 928 MOZ_ASSERT(bestSlotIndex <= mHapticPromises.Length()); 929 if (bestSlotIndex == mHapticPromises.Length()) { 930 mHapticPromises.AppendElement( 931 UniquePtr<VRManagerPromise>(new VRManagerPromise(aPromise))); 932 } else { 933 mHapticPromises[bestSlotIndex] = 934 UniquePtr<VRManagerPromise>(new VRManagerPromise(aPromise)); 935 } 936 PushState(); 937 } 938 939 void VRManager::StopVibrateHaptic(GamepadHandle aGamepadHandle) { 940 if (mState != VRManagerState::Active) { 941 return; 942 } 943 // VRDisplayClient::FireGamepadEvents() assigns a controller ID with 944 // ranges based on displayID. We must translate this to the indexes 945 // understood by VRDisplayExternal. 946 uint32_t controllerBaseIndex = 947 kVRControllerMaxCount * mDisplayInfo.mDisplayID; 948 uint32_t controllerIndex = aGamepadHandle.GetValue() - controllerBaseIndex; 949 950 for (size_t i = 0; i < std::size(mBrowserState.hapticState); i++) { 951 VRHapticState& state = mBrowserState.hapticState[i]; 952 if (state.controllerIndex == controllerIndex) { 953 memset(&state, 0, sizeof(VRHapticState)); 954 } 955 } 956 PushState(); 957 } 958 959 void VRManager::NotifyVibrateHapticCompleted(const VRManagerPromise& aPromise) { 960 aPromise.mParent->SendReplyGamepadVibrateHaptic(aPromise.mPromiseID); 961 } 962 963 void VRManager::StartVRNavigation(const uint32_t& aDisplayID) { 964 if (mState != VRManagerState::Active) { 965 return; 966 } 967 /** 968 * We only support a single VRSession with a single VR display at a 969 * time; however, due to the asynchronous nature of the API, it's possible 970 * that the previously used VR display was a different one than the one now 971 * allocated. We catch these cases to avoid automatically activating the new 972 * VR displays. This situation is expected to be very rare and possibly never 973 * seen. Perhaps further simplification could be made in the content process 974 * code which passes around displayID's that may no longer be needed. 975 **/ 976 if (mDisplayInfo.GetDisplayID() != aDisplayID) { 977 return; 978 } 979 mBrowserState.navigationTransitionActive = true; 980 mVRNavigationTransitionEnd = TimeStamp(); 981 PushState(); 982 } 983 984 void VRManager::StopVRNavigation(const uint32_t& aDisplayID, 985 const TimeDuration& aTimeout) { 986 if (mState != VRManagerState::Active) { 987 return; 988 } 989 if (mDisplayInfo.GetDisplayID() != aDisplayID) { 990 return; 991 } 992 if (aTimeout.ToMilliseconds() <= 0) { 993 mBrowserState.navigationTransitionActive = false; 994 mVRNavigationTransitionEnd = TimeStamp(); 995 PushState(); 996 } 997 mVRNavigationTransitionEnd = TimeStamp::Now() + aTimeout; 998 } 999 1000 #if !defined(MOZ_WIDGET_ANDROID) 1001 1002 bool VRManager::RunPuppet(const nsTArray<uint64_t>& aBuffer, 1003 VRManagerParent* aManagerParent) { 1004 if (!StaticPrefs::dom_vr_puppet_enabled()) { 1005 // Sanity check to ensure that a compromised content process 1006 // can't use this to escalate permissions. 1007 return false; 1008 } 1009 if (mManagerParentRunningPuppet != nullptr) { 1010 // Only one parent may run a puppet at a time 1011 return false; 1012 } 1013 mManagerParentRunningPuppet = aManagerParent; 1014 mServiceHost->PuppetSubmit(aBuffer); 1015 return true; 1016 } 1017 1018 void VRManager::ResetPuppet(VRManagerParent* aManagerParent) { 1019 if (!StaticPrefs::dom_vr_puppet_enabled()) { 1020 return; 1021 } 1022 1023 mManagerParentsWaitingForPuppetReset.Insert(aManagerParent); 1024 if (mManagerParentRunningPuppet != nullptr) { 1025 (void)mManagerParentRunningPuppet->SendNotifyPuppetCommandBufferCompleted( 1026 false); 1027 mManagerParentRunningPuppet = nullptr; 1028 } 1029 mServiceHost->PuppetReset(); 1030 // In the event that we are shut down, the task timer won't be running 1031 // to trigger CheckForPuppetCompletion. 1032 // In this case, CheckForPuppetCompletion() would immediately resolve 1033 // the promises for mManagerParentsWaitingForPuppetReset. 1034 // We can simply call it once here to handle that case. 1035 CheckForPuppetCompletion(); 1036 } 1037 1038 #endif // !defined(MOZ_WIDGET_ANDROID) 1039 1040 void VRManager::PullState( 1041 const std::function<bool()>& aWaitCondition /* = nullptr */) { 1042 if (mShmem != nullptr) { 1043 mShmem->PullSystemState(mDisplayInfo.mDisplayState, mLastSensorState, 1044 &mDisplayInfo.mControllerState, 1045 mEnumerationCompleted, aWaitCondition); 1046 } 1047 } 1048 1049 void VRManager::PushState(bool aNotifyCond) { 1050 if (mShmem != nullptr) { 1051 mShmem->PushBrowserState(mBrowserState, aNotifyCond); 1052 } 1053 } 1054 1055 void VRManager::Destroy() { 1056 if (mState == VRManagerState::Disabled) { 1057 return; 1058 } 1059 Shutdown(); 1060 StopTasks(); 1061 mState = VRManagerState::Disabled; 1062 } 1063 1064 void VRManager::Shutdown() { 1065 if (mState == VRManagerState::Disabled || mState == VRManagerState::Idle) { 1066 return; 1067 } 1068 1069 if (mDisplayInfo.mDisplayState.shutdown) { 1070 // Shutdown was requested by VR Service, so we must throttle 1071 // as requested by the VR Service 1072 TimeStamp now = TimeStamp::Now(); 1073 mEarliestRestartTime = 1074 now + TimeDuration::FromMilliseconds( 1075 (double)mDisplayInfo.mDisplayState.minRestartInterval); 1076 } 1077 1078 StopAllHaptics(); 1079 StopPresentation(); 1080 CancelCurrentSubmitTask(); 1081 ShutdownSubmitThread(); 1082 1083 mDisplayInfo.Clear(); 1084 mEnumerationCompleted = false; 1085 1086 if (mState == VRManagerState::RuntimeDetection) { 1087 /** 1088 * We have failed to detect runtimes before shutting down. 1089 * Ensure that promises are resolved 1090 * 1091 * This call to DispatchRuntimeCapabilitiesUpdate will only 1092 * happen when we have failed to detect runtimes. In that case, 1093 * mRuntimeSupportFlags will be 0 and send the correct message 1094 * to the content process. 1095 * 1096 * When we are successful, we store the result in mRuntimeSupportFlags 1097 * and never try again unless the browser is restarted. mRuntimeSupportFlags 1098 * is never reset back to 0 in that case but we will never re-enter the 1099 * VRManagerState::RuntimeDetection state and hit this code path again. 1100 */ 1101 DispatchRuntimeCapabilitiesUpdate(); 1102 } 1103 1104 if (mState == VRManagerState::Enumeration) { 1105 // We have failed to enumerate VR devices before shutting down. 1106 // Ensure that promises are resolved 1107 DispatchVRDisplayInfoUpdate(); 1108 } 1109 1110 #if !defined(MOZ_WIDGET_ANDROID) 1111 mServiceHost->StopService(); 1112 #endif 1113 mState = VRManagerState::Idle; 1114 1115 // We will close Shmem in the DTOR to avoid 1116 // mSubmitThread is still running but its shmem 1117 // has been released. 1118 } 1119 1120 void VRManager::ShutdownVRManagerParents() { 1121 // Close removes the CanvasParent from the set so take a copy first. 1122 const auto parents = ToTArray<nsTArray<VRManagerParent*>>(mVRManagerParents); 1123 for (RefPtr<VRManagerParent> vrManagerParent : parents) { 1124 vrManagerParent->Close(); 1125 } 1126 1127 MOZ_DIAGNOSTIC_ASSERT(mVRManagerParents.IsEmpty(), 1128 "Closing should have cleared all entries."); 1129 } 1130 1131 void VRManager::CheckWatchDog() { 1132 /** 1133 * We will trigger a new frame immediately after a successful frame 1134 * texture submission. If content fails to call VRDisplay.submitFrame 1135 * after dom.vr.display.rafMaxDuration milliseconds has elapsed since the 1136 * last VRDisplay.requestAnimationFrame, we act as a "watchdog" and 1137 * kick-off a new VRDisplay.requestAnimationFrame to avoid a render loop 1138 * stall and to give content a chance to recover. 1139 * 1140 * If the lower level VR platform API's are rejecting submitted frames, 1141 * such as when the Oculus "Health and Safety Warning" is displayed, 1142 * we will not kick off the next frame immediately after 1143 * VRDisplay.submitFrame as it would result in an unthrottled render loop 1144 * that would free run at potentially extreme frame rates. To ensure that 1145 * content has a chance to resume its presentation when the frames are 1146 * accepted once again, we rely on this "watchdog" to act as a VR refresh 1147 * driver cycling at a rate defined by dom.vr.display.rafMaxDuration. 1148 * 1149 * This number must be larger than the slowest expected frame time during 1150 * normal VR presentation, but small enough not to break content that 1151 * makes assumptions of reasonably minimal VSync rate. 1152 * 1153 * The slowest expected refresh rate for a VR display currently is an 1154 * Oculus CV1 when ASW (Asynchronous Space Warp) is enabled, at 45hz. 1155 * A dom.vr.display.rafMaxDuration value of 50 milliseconds results in a 1156 * 20hz rate, which avoids inadvertent triggering of the watchdog during 1157 * Oculus ASW even if every second frame is dropped. 1158 */ 1159 if (mState != VRManagerState::Active) { 1160 return; 1161 } 1162 bool bShouldStartFrame = false; 1163 1164 // If content fails to call VRDisplay.submitFrame, we must eventually 1165 // time-out and trigger a new frame. 1166 TimeStamp lastFrameStart = 1167 mLastFrameStart[mDisplayInfo.mFrameId % kVRMaxLatencyFrames]; 1168 if (lastFrameStart.IsNull()) { 1169 bShouldStartFrame = true; 1170 } else { 1171 TimeDuration duration = TimeStamp::Now() - lastFrameStart; 1172 if (duration.ToMilliseconds() > 1173 StaticPrefs::dom_vr_display_rafMaxDuration()) { 1174 bShouldStartFrame = true; 1175 } 1176 } 1177 1178 if (bShouldStartFrame) { 1179 StartFrame(); 1180 } 1181 } 1182 1183 void VRManager::ExpireNavigationTransition() { 1184 if (mState != VRManagerState::Active) { 1185 return; 1186 } 1187 if (!mVRNavigationTransitionEnd.IsNull() && 1188 TimeStamp::Now() > mVRNavigationTransitionEnd) { 1189 mBrowserState.navigationTransitionActive = false; 1190 } 1191 } 1192 1193 void VRManager::UpdateHaptics(double aDeltaTime) { 1194 if (mState != VRManagerState::Active) { 1195 return; 1196 } 1197 bool bNeedPush = false; 1198 // Check for any haptic pulses that have ended and clear them 1199 for (size_t i = 0; i < std::size(mBrowserState.hapticState); i++) { 1200 const VRHapticState& state = mBrowserState.hapticState[i]; 1201 if (state.inputFrameID == 0) { 1202 // Nothing in this slot 1203 continue; 1204 } 1205 mHapticPulseRemaining[i] -= aDeltaTime; 1206 if (mHapticPulseRemaining[i] <= 0.0f) { 1207 // The pulse has finished 1208 ClearHapticSlot(i); 1209 bNeedPush = true; 1210 } 1211 } 1212 if (bNeedPush) { 1213 PushState(); 1214 } 1215 } 1216 1217 void VRManager::ClearHapticSlot(size_t aSlot) { 1218 MOZ_ASSERT(aSlot < std::size(mBrowserState.hapticState)); 1219 memset(&mBrowserState.hapticState[aSlot], 0, sizeof(VRHapticState)); 1220 mHapticPulseRemaining[aSlot] = 0.0f; 1221 if (aSlot < mHapticPromises.Length() && mHapticPromises[aSlot]) { 1222 NotifyVibrateHapticCompleted(*(mHapticPromises[aSlot])); 1223 mHapticPromises[aSlot] = nullptr; 1224 } 1225 } 1226 1227 void VRManager::ShutdownSubmitThread() { 1228 if (mSubmitThread) { 1229 mSubmitThread->Shutdown(); 1230 mSubmitThread = nullptr; 1231 } 1232 } 1233 1234 void VRManager::StartPresentation() { 1235 if (mState != VRManagerState::Active) { 1236 return; 1237 } 1238 if (mBrowserState.presentationActive) { 1239 return; 1240 } 1241 1242 // Indicate that we are ready to start immersive mode 1243 mBrowserState.presentationActive = true; 1244 mBrowserState.layerState[0].type = VRLayerType::LayerType_Stereo_Immersive; 1245 PushState(); 1246 1247 mDisplayInfo.mDisplayState.lastSubmittedFrameId = 0; 1248 mLastSubmittedFrameId = 0; 1249 mLastStartedFrame = 0; 1250 } 1251 1252 void VRManager::StopPresentation() { 1253 if (mState != VRManagerState::Active) { 1254 return; 1255 } 1256 if (!mBrowserState.presentationActive) { 1257 return; 1258 } 1259 1260 // Indicate that we have stopped immersive mode 1261 mBrowserState.presentationActive = false; 1262 memset(mBrowserState.layerState, 0, 1263 sizeof(VRLayerState) * std::size(mBrowserState.layerState)); 1264 1265 PushState(true); 1266 } 1267 1268 bool VRManager::IsPresenting() { 1269 if (mShmem) { 1270 return mDisplayInfo.mPresentingGroups != 0; 1271 } 1272 return false; 1273 } 1274 1275 void VRManager::SetGroupMask(uint32_t aGroupMask) { 1276 if (mState != VRManagerState::Active) { 1277 return; 1278 } 1279 mDisplayInfo.mGroupMask = aGroupMask; 1280 } 1281 1282 void VRManager::SubmitFrame(VRLayerParent* aLayer, 1283 const layers::SurfaceDescriptor& aTexture, 1284 uint64_t aFrameId, const gfx::Rect& aLeftEyeRect, 1285 const gfx::Rect& aRightEyeRect) { 1286 if (mState != VRManagerState::Active) { 1287 return; 1288 } 1289 MonitorAutoLock lock(mCurrentSubmitTaskMonitor); 1290 if ((mDisplayInfo.mGroupMask & aLayer->GetGroup()) == 0) { 1291 // Suppress layers hidden by the group mask 1292 return; 1293 } 1294 1295 // Ensure that we only accept the first SubmitFrame call per RAF cycle. 1296 if (!mFrameStarted || aFrameId != mDisplayInfo.mFrameId) { 1297 return; 1298 } 1299 1300 /** 1301 * Do not queue more submit frames until the last submitted frame is 1302 * already processed and the new WebGL texture is ready. 1303 */ 1304 if (mLastSubmittedFrameId > 0 && 1305 mLastSubmittedFrameId != 1306 mDisplayInfo.mDisplayState.lastSubmittedFrameId) { 1307 mLastStartedFrame = 0; 1308 return; 1309 } 1310 1311 mLastSubmittedFrameId = aFrameId; 1312 1313 mFrameStarted = false; 1314 1315 RefPtr<CancelableRunnable> task = NewCancelableRunnableMethod< 1316 StoreCopyPassByConstLRef<layers::SurfaceDescriptor>, uint64_t, 1317 StoreCopyPassByConstLRef<gfx::Rect>, StoreCopyPassByConstLRef<gfx::Rect>>( 1318 "gfx::VRManager::SubmitFrameInternal", this, 1319 &VRManager::SubmitFrameInternal, aTexture, aFrameId, aLeftEyeRect, 1320 aRightEyeRect); 1321 1322 if (!mCurrentSubmitTask) { 1323 mCurrentSubmitTask = task; 1324 #if !defined(MOZ_WIDGET_ANDROID) 1325 if (!mSubmitThread) { 1326 mSubmitThread = new VRThread("VR_SubmitFrame"_ns); 1327 } 1328 mSubmitThread->Start(); 1329 mSubmitThread->PostTask(task.forget()); 1330 #else 1331 CompositorThread()->Dispatch(task.forget()); 1332 #endif // defined(MOZ_WIDGET_ANDROID) 1333 } 1334 } 1335 1336 bool VRManager::SubmitFrame(const layers::SurfaceDescriptor& aTexture, 1337 uint64_t aFrameId, const gfx::Rect& aLeftEyeRect, 1338 const gfx::Rect& aRightEyeRect) { 1339 if (mState != VRManagerState::Active) { 1340 return false; 1341 } 1342 #if defined(XP_WIN) || defined(XP_MACOSX) || defined(MOZ_WIDGET_ANDROID) 1343 MOZ_ASSERT(mBrowserState.layerState[0].type == 1344 VRLayerType::LayerType_Stereo_Immersive); 1345 VRLayer_Stereo_Immersive& layer = 1346 mBrowserState.layerState[0].layer_stereo_immersive; 1347 1348 switch (aTexture.type()) { 1349 # if defined(XP_WIN) 1350 case SurfaceDescriptor::TSurfaceDescriptorD3D10: { 1351 const SurfaceDescriptorD3D10& surf = 1352 aTexture.get_SurfaceDescriptorD3D10(); 1353 auto handle = surf.handle()->ClonePlatformHandle(); 1354 layer.textureType = 1355 VRLayerTextureType::LayerTextureType_D3D10SurfaceDescriptor; 1356 layer.textureHandle = (void*)handle.release(); 1357 layer.textureSize.width = surf.size().width; 1358 layer.textureSize.height = surf.size().height; 1359 } break; 1360 # elif defined(XP_MACOSX) 1361 case SurfaceDescriptor::TSurfaceDescriptorMacIOSurface: { 1362 // MacIOSurface ptr can't be fetched or used at different threads. 1363 // Both of fetching and using this MacIOSurface are at the VRService 1364 // thread. 1365 const auto& desc = aTexture.get_SurfaceDescriptorMacIOSurface(); 1366 layer.textureType = VRLayerTextureType::LayerTextureType_MacIOSurface; 1367 layer.textureHandle = desc.surfaceId(); 1368 RefPtr<MacIOSurface> surf = MacIOSurface::LookupSurface( 1369 desc.surfaceId(), !desc.isOpaque(), desc.yUVColorSpace()); 1370 if (surf) { 1371 layer.textureSize.width = surf->GetDevicePixelWidth(); 1372 layer.textureSize.height = surf->GetDevicePixelHeight(); 1373 } 1374 } break; 1375 # elif defined(MOZ_WIDGET_ANDROID) 1376 case SurfaceDescriptor::TSurfaceTextureDescriptor: { 1377 const SurfaceTextureDescriptor& desc = 1378 aTexture.get_SurfaceTextureDescriptor(); 1379 java::GeckoSurfaceTexture::LocalRef surfaceTexture = 1380 java::GeckoSurfaceTexture::Lookup(desc.handle()); 1381 if (!surfaceTexture) { 1382 NS_WARNING("VRManager::SubmitFrame failed to get a SurfaceTexture"); 1383 return false; 1384 } 1385 layer.textureType = 1386 VRLayerTextureType::LayerTextureType_GeckoSurfaceTexture; 1387 layer.textureHandle = desc.handle(); 1388 layer.textureSize.width = desc.size().width; 1389 layer.textureSize.height = desc.size().height; 1390 } break; 1391 # endif 1392 default: { 1393 MOZ_ASSERT(false); 1394 return false; 1395 } 1396 } 1397 1398 layer.frameId = aFrameId; 1399 layer.inputFrameId = 1400 mDisplayInfo.mLastSensorState[mDisplayInfo.mFrameId % kVRMaxLatencyFrames] 1401 .inputFrameID; 1402 1403 layer.leftEyeRect.x = aLeftEyeRect.x; 1404 layer.leftEyeRect.y = aLeftEyeRect.y; 1405 layer.leftEyeRect.width = aLeftEyeRect.width; 1406 layer.leftEyeRect.height = aLeftEyeRect.height; 1407 layer.rightEyeRect.x = aRightEyeRect.x; 1408 layer.rightEyeRect.y = aRightEyeRect.y; 1409 layer.rightEyeRect.width = aRightEyeRect.width; 1410 layer.rightEyeRect.height = aRightEyeRect.height; 1411 1412 PushState(true); 1413 1414 PullState([&]() { 1415 return (mDisplayInfo.mDisplayState.lastSubmittedFrameId >= aFrameId) || 1416 mDisplayInfo.mDisplayState.suppressFrames || 1417 !mDisplayInfo.mDisplayState.isConnected; 1418 }); 1419 1420 if (mDisplayInfo.mDisplayState.suppressFrames || 1421 !mDisplayInfo.mDisplayState.isConnected) { 1422 // External implementation wants to supress frames, service has shut 1423 // down or hardware has been disconnected. 1424 return false; 1425 } 1426 1427 return mDisplayInfo.mDisplayState.lastSubmittedFrameSuccessful; 1428 #else 1429 MOZ_ASSERT(false); // Not implmented for this platform 1430 return false; 1431 #endif 1432 } 1433 1434 void VRManager::SubmitFrameInternal(const layers::SurfaceDescriptor& aTexture, 1435 uint64_t aFrameId, 1436 const gfx::Rect& aLeftEyeRect, 1437 const gfx::Rect& aRightEyeRect) { 1438 #if !defined(MOZ_WIDGET_ANDROID) 1439 MOZ_ASSERT(mSubmitThread->GetThread() == NS_GetCurrentThread()); 1440 #endif // !defined(MOZ_WIDGET_ANDROID) 1441 AUTO_PROFILER_MARKER("SubmitFrameAtVRDisplayExternal", OTHER); 1442 1443 { // scope lock 1444 MonitorAutoLock lock(mCurrentSubmitTaskMonitor); 1445 1446 if (!SubmitFrame(aTexture, aFrameId, aLeftEyeRect, aRightEyeRect)) { 1447 mCurrentSubmitTask = nullptr; 1448 return; 1449 } 1450 mCurrentSubmitTask = nullptr; 1451 } 1452 1453 #if defined(XP_WIN) || defined(XP_MACOSX) 1454 1455 /** 1456 * Trigger the next VSync immediately after we are successfully 1457 * submitting frames. As SubmitFrame is responsible for throttling 1458 * the render loop, if we don't successfully call it, we shouldn't trigger 1459 * StartFrame immediately, as it will run unbounded. 1460 * If StartFrame is not called here due to SubmitFrame failing, the 1461 * fallback "watchdog" code in VRManager::NotifyVSync() will cause 1462 * frames to continue at a lower refresh rate until frame submission 1463 * succeeds again. 1464 */ 1465 CompositorThread()->Dispatch(NewRunnableMethod("gfx::VRManager::StartFrame", 1466 this, &VRManager::StartFrame)); 1467 #elif defined(MOZ_WIDGET_ANDROID) 1468 // We are already in the CompositorThreadHolder event loop on Android. 1469 StartFrame(); 1470 #endif 1471 } 1472 1473 void VRManager::CancelCurrentSubmitTask() { 1474 MonitorAutoLock lock(mCurrentSubmitTaskMonitor); 1475 if (mCurrentSubmitTask) { 1476 mCurrentSubmitTask->Cancel(); 1477 mCurrentSubmitTask = nullptr; 1478 } 1479 } 1480 1481 //----------------------------------------------------------------------------- 1482 // VRManager::nsIObserver 1483 //----------------------------------------------------------------------------- 1484 1485 NS_IMETHODIMP 1486 VRManager::Observe(nsISupports* subject, const char* topic, 1487 const char16_t* data) { 1488 if (!StaticPrefs::dom_vr_enabled() && !StaticPrefs::dom_vr_webxr_enabled()) { 1489 return NS_OK; 1490 } 1491 1492 if (!strcmp(topic, "application-background")) { 1493 // StopTasks() is called later in the timer thread based on this flag to 1494 // avoid threading issues. 1495 mAppPaused = true; 1496 } else if (!strcmp(topic, "application-foreground") && mAppPaused) { 1497 mAppPaused = false; 1498 // When the apps goes the foreground (e.g. Android) we should restart the 1499 // tasks. 1500 StartTasks(); 1501 } 1502 return NS_OK; 1503 } 1504 1505 NS_IMPL_ISUPPORTS(VRManager, nsIObserver) 1506 1507 } // namespace mozilla::gfx