wgc_capture_session.cc (31346B)
1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <dispatcherqueue.h> 12 #include <windows.graphics.capture.interop.h> 13 #include <windows.graphics.directx.direct3d11.interop.h> 14 #include <windows.graphics.h> 15 #include <wrl/client.h> 16 #include <wrl/event.h> 17 18 #include <algorithm> 19 #include <cstdint> 20 #include <cstring> 21 #include <memory> 22 #include <utility> 23 24 #include "api/make_ref_counted.h" 25 #include "api/sequence_checker.h" 26 #include "api/units/time_delta.h" 27 #include "modules/desktop_capture/desktop_capture_options.h" 28 #include "modules/desktop_capture/desktop_frame.h" 29 #include "modules/desktop_capture/desktop_geometry.h" 30 #include "modules/desktop_capture/shared_desktop_frame.h" 31 #include "modules/desktop_capture/win/screen_capture_utils.h" 32 #include "modules/desktop_capture/win/wgc_capture_session.h" 33 #include "rtc_base/checks.h" 34 #include "rtc_base/logging.h" 35 #include "rtc_base/thread.h" 36 #include "rtc_base/time_utils.h" 37 #include "rtc_base/win/create_direct3d_device.h" 38 #include "rtc_base/win/get_activation_factory.h" 39 #include "rtc_base/win/windows_version.h" 40 #include "system_wrappers/include/metrics.h" 41 42 using Microsoft::WRL::ComPtr; 43 namespace WGC = ABI::Windows::Graphics::Capture; 44 45 namespace webrtc { 46 namespace { 47 48 // We must use a BGRA pixel format that has 4 bytes per pixel, as required by 49 // the DesktopFrame interface. 50 constexpr auto kPixelFormat = ABI::Windows::Graphics::DirectX:: 51 DirectXPixelFormat::DirectXPixelFormat_B8G8R8A8UIntNormalized; 52 53 // We must wait a little longer for the first frame to avoid failing the 54 // capture when there is a longer startup time. 55 constexpr int kFirstFrameTimeoutMs = 5000; 56 57 // These values are persisted to logs. Entries should not be renumbered and 58 // numeric values should never be reused. 59 enum class StartCaptureResult { 60 kSuccess = 0, 61 kSourceClosed = 1, 62 kAddClosedFailed = 2, 63 kDxgiDeviceCastFailed = 3, 64 kD3dDelayLoadFailed = 4, 65 kD3dDeviceCreationFailed = 5, 66 kFramePoolActivationFailed = 6, 67 kFramePoolCastFailed = 7, 68 // kGetItemSizeFailed = 8, (deprecated) 69 kCreateFramePoolFailed = 9, 70 kCreateCaptureSessionFailed = 10, 71 kStartCaptureFailed = 11, 72 kMaxValue = kStartCaptureFailed 73 }; 74 75 // These values are persisted to logs. Entries should not be renumbered and 76 // numeric values should never be reused. 77 enum class GetFrameResult { 78 kSuccess = 0, 79 kItemClosed = 1, 80 kTryGetNextFrameFailed = 2, 81 kFrameDropped = 3, 82 kGetSurfaceFailed = 4, 83 kDxgiInterfaceAccessFailed = 5, 84 kTexture2dCastFailed = 6, 85 kCreateMappedTextureFailed = 7, 86 kMapFrameFailed = 8, 87 kGetContentSizeFailed = 9, 88 kResizeMappedTextureFailed = 10, 89 kRecreateFramePoolFailed = 11, 90 kFramePoolEmpty = 12, 91 kWaitForFirstFrameFailed = 13, 92 kMaxValue = kWaitForFirstFrameFailed 93 }; 94 95 enum class WaitForFirstFrameResult { 96 kSuccess = 0, 97 kTryGetNextFrameFailed = 1, 98 kAddFrameArrivedCallbackFailed = 2, 99 kWaitingTimedOut = 3, 100 kRemoveFrameArrivedCallbackFailed = 4, 101 kMaxValue = kRemoveFrameArrivedCallbackFailed 102 }; 103 104 void RecordStartCaptureResult(StartCaptureResult error) { 105 RTC_HISTOGRAM_ENUMERATION( 106 "WebRTC.DesktopCapture.Win.WgcCaptureSessionStartResult", 107 static_cast<int>(error), static_cast<int>(StartCaptureResult::kMaxValue)); 108 } 109 110 void RecordGetFrameResult(GetFrameResult error) { 111 RTC_HISTOGRAM_ENUMERATION( 112 "WebRTC.DesktopCapture.Win.WgcCaptureSessionGetFrameResult", 113 static_cast<int>(error), static_cast<int>(GetFrameResult::kMaxValue)); 114 } 115 116 void RecordGetFirstFrameTime(int64_t elapsed_time_ms) { 117 RTC_HISTOGRAM_COUNTS( 118 "WebRTC.DesktopCapture.Win.WgcCaptureSessionTimeToFirstFrame", 119 elapsed_time_ms, /*min=*/1, /*max=*/5000, /*bucket_count=*/100); 120 } 121 122 void RecordWaitForFirstFrameResult(WaitForFirstFrameResult error) { 123 RTC_HISTOGRAM_ENUMERATION( 124 "WebRTC.DesktopCapture.Win.WgcCaptureSessionWaitForFirstFrameResult", 125 static_cast<int>(error), 126 static_cast<int>(WaitForFirstFrameResult::kMaxValue)); 127 } 128 129 bool SizeHasChanged(ABI::Windows::Graphics::SizeInt32 size_new, 130 ABI::Windows::Graphics::SizeInt32 size_old) { 131 return (size_new.Height != size_old.Height || 132 size_new.Width != size_old.Width); 133 } 134 135 bool DoesWgcSkipStaticFrames() { 136 return (rtc_win::GetVersion() >= rtc_win::Version::VERSION_WIN11_24H2); 137 } 138 139 } // namespace 140 141 WgcCaptureSession::RefCountedEvent::RefCountedEvent(bool manual_reset, 142 bool initially_signaled) 143 : Event(manual_reset, initially_signaled) {} 144 145 WgcCaptureSession::RefCountedEvent::~RefCountedEvent() = default; 146 147 WgcCaptureSession::AgileFrameArrivedHandler::AgileFrameArrivedHandler( 148 scoped_refptr<RefCountedEvent> event) 149 : frame_arrived_event_(event) {} 150 151 IFACEMETHODIMP WgcCaptureSession::AgileFrameArrivedHandler::Invoke( 152 ABI::Windows::Graphics::Capture::IDirect3D11CaptureFramePool* sender, 153 IInspectable* args) { 154 frame_arrived_event_->Set(); 155 return S_OK; 156 } 157 158 WgcCaptureSession::WgcCaptureSession(intptr_t source_id, 159 ComPtr<ID3D11Device> d3d11_device, 160 ComPtr<WGC::IGraphicsCaptureItem> item, 161 ABI::Windows::Graphics::SizeInt32 size) 162 : d3d11_device_(std::move(d3d11_device)), 163 item_(std::move(item)), 164 size_(size), 165 source_id_(source_id) { 166 is_window_source_ = ::IsWindow(reinterpret_cast<HWND>(source_id_)); 167 } 168 169 WgcCaptureSession::~WgcCaptureSession() { 170 RemoveEventHandlers(); 171 } 172 173 HRESULT WgcCaptureSession::StartCapture(const DesktopCaptureOptions& options) { 174 RTC_DCHECK_RUN_ON(&sequence_checker_); 175 RTC_DCHECK(!is_capture_started_); 176 177 if (item_closed_) { 178 RTC_LOG(LS_ERROR) << "The target source has been closed."; 179 RecordStartCaptureResult(StartCaptureResult::kSourceClosed); 180 return E_ABORT; 181 } 182 183 RTC_DCHECK(d3d11_device_); 184 RTC_DCHECK(item_); 185 186 // Listen for the Closed event, to detect if the source we are capturing is 187 // closed (e.g. application window is closed or monitor is disconnected). If 188 // it is, we should abort the capture. 189 item_closed_token_ = std::make_unique<EventRegistrationToken>(); 190 auto closed_handler = 191 Microsoft::WRL::Callback<ABI::Windows::Foundation::ITypedEventHandler< 192 WGC::GraphicsCaptureItem*, IInspectable*>>( 193 this, &WgcCaptureSession::OnItemClosed); 194 HRESULT hr = 195 item_->add_Closed(closed_handler.Get(), item_closed_token_.get()); 196 if (FAILED(hr)) { 197 RecordStartCaptureResult(StartCaptureResult::kAddClosedFailed); 198 return hr; 199 } 200 201 ComPtr<IDXGIDevice> dxgi_device; 202 hr = d3d11_device_->QueryInterface(IID_PPV_ARGS(&dxgi_device)); 203 if (FAILED(hr)) { 204 RecordStartCaptureResult(StartCaptureResult::kDxgiDeviceCastFailed); 205 return hr; 206 } 207 208 if (!ResolveCoreWinRTDirect3DDelayload()) { 209 RecordStartCaptureResult(StartCaptureResult::kD3dDelayLoadFailed); 210 return E_FAIL; 211 } 212 213 hr = CreateDirect3DDeviceFromDXGIDevice(dxgi_device.Get(), &direct3d_device_); 214 if (FAILED(hr)) { 215 RecordStartCaptureResult(StartCaptureResult::kD3dDeviceCreationFailed); 216 return hr; 217 } 218 219 ComPtr<WGC::IDirect3D11CaptureFramePoolStatics> frame_pool_statics; 220 hr = GetActivationFactory< 221 WGC::IDirect3D11CaptureFramePoolStatics, 222 RuntimeClass_Windows_Graphics_Capture_Direct3D11CaptureFramePool>( 223 &frame_pool_statics); 224 if (FAILED(hr)) { 225 RecordStartCaptureResult(StartCaptureResult::kFramePoolActivationFailed); 226 return hr; 227 } 228 229 // Cast to FramePoolStatics2 so we can use CreateFreeThreaded and avoid the 230 // need to have a DispatcherQueue. Sometimes, the time to obtain the first 231 // frame ever in a stream can take longer. To avoid timeouts, 232 // CreateFreeThreaded is needed so that the frame processing done by WGC can 233 // happen on a different thread while the main thread is waiting for it. 234 ComPtr<WGC::IDirect3D11CaptureFramePoolStatics2> frame_pool_statics2; 235 hr = frame_pool_statics->QueryInterface(IID_PPV_ARGS(&frame_pool_statics2)); 236 if (FAILED(hr)) { 237 RecordStartCaptureResult(StartCaptureResult::kFramePoolCastFailed); 238 return hr; 239 } 240 241 hr = frame_pool_statics2->CreateFreeThreaded( 242 direct3d_device_.Get(), kPixelFormat, kNumBuffers, size_, &frame_pool_); 243 if (FAILED(hr)) { 244 RecordStartCaptureResult(StartCaptureResult::kCreateFramePoolFailed); 245 return hr; 246 } 247 248 hr = frame_pool_->CreateCaptureSession(item_.Get(), &session_); 249 if (FAILED(hr)) { 250 RecordStartCaptureResult(StartCaptureResult::kCreateCaptureSessionFailed); 251 return hr; 252 } 253 254 if (!options.prefer_cursor_embedded()) { 255 ComPtr<ABI::Windows::Graphics::Capture::IGraphicsCaptureSession2> session2; 256 if (SUCCEEDED(session_->QueryInterface(IID_PPV_ARGS(&session2)))) { 257 session2->put_IsCursorCaptureEnabled(false); 258 } 259 } 260 261 // Until Mozilla builds with Win 10 SDK v10.0.20348.0 or newer, this 262 // code will not build. Once we support the newer SDK, Bug 1868198 263 // exists to decide if we ever want to use this code since it is 264 // removing an indicator that capture is happening. 265 #if !defined(WEBRTC_MOZILLA_BUILD) 266 // By default, the WGC capture API adds a yellow border around the captured 267 // window or display to indicate that a capture is in progress. The section 268 // below is an attempt to remove this yellow border to make the capture 269 // experience more inline with the DXGI capture path. 270 // This requires 10.0.20348.0 or later, which practically means Windows 11. 271 ComPtr<ABI::Windows::Graphics::Capture::IGraphicsCaptureSession3> session3; 272 if (SUCCEEDED(session_->QueryInterface( 273 ABI::Windows::Graphics::Capture::IID_IGraphicsCaptureSession3, 274 &session3))) { 275 session3->put_IsBorderRequired(options.wgc_require_border()); 276 } 277 #endif 278 279 // Until Mozilla builds with SDK v10.0.26100.0 or newer, this 280 // code will not build. 281 #if !defined(WEBRTC_MOZILLA_BUILD) 282 // Windows 11 24H2 (10.0.26100.0) added 283 // `IGraphicsCaptureSession6::put_IncludeSecondaryWindows()`. See 284 // `wgc_include_secondary_windows()` in 285 // /modules/desktop_capture/desktop_capture_options.h for more details. 286 ComPtr<ABI::Windows::Graphics::Capture::IGraphicsCaptureSession6> session6; 287 if (SUCCEEDED(session_->QueryInterface( 288 ABI::Windows::Graphics::Capture::IID_IGraphicsCaptureSession6, 289 &session6))) { 290 session6->put_IncludeSecondaryWindows( 291 options.wgc_include_secondary_windows()); 292 } 293 #endif 294 295 allow_zero_hertz_ = options.allow_wgc_zero_hertz(); 296 297 hr = session_->StartCapture(); 298 if (FAILED(hr)) { 299 RTC_LOG(LS_ERROR) << "Failed to start CaptureSession: " << hr; 300 RecordStartCaptureResult(StartCaptureResult::kStartCaptureFailed); 301 return hr; 302 } 303 304 RecordStartCaptureResult(StartCaptureResult::kSuccess); 305 306 is_capture_started_ = true; 307 return hr; 308 } 309 310 bool WgcCaptureSession::WaitForFirstFrame() { 311 RTC_CHECK(!has_first_frame_arrived_); 312 313 ComPtr<WGC::IDirect3D11CaptureFrame> capture_frame = nullptr; 314 // Flush the `frame_pool_` buffers so that we can receive the most recent 315 // frames. 316 for (int i = 0; i < kNumBuffers; ++i) { 317 HRESULT hr = frame_pool_->TryGetNextFrame(&capture_frame); 318 if (FAILED(hr)) { 319 RTC_LOG(LS_ERROR) << "TryGetNextFrame failed: " << hr; 320 RecordWaitForFirstFrameResult( 321 WaitForFirstFrameResult::kTryGetNextFrameFailed); 322 return false; 323 } 324 } 325 326 if (FAILED(AddFrameArrivedEventHandler())) { 327 RecordWaitForFirstFrameResult( 328 WaitForFirstFrameResult::kAddFrameArrivedCallbackFailed); 329 return false; 330 } 331 332 RTC_CHECK(has_first_frame_arrived_event_); 333 int64_t first_frame_event_wait_start = TimeMillis(); 334 // Only start the frame polling once the first frame becomes available. 335 if (!has_first_frame_arrived_event_->Wait( 336 TimeDelta::Millis(kFirstFrameTimeoutMs))) { 337 RecordGetFirstFrameTime(kFirstFrameTimeoutMs); 338 RecordWaitForFirstFrameResult(WaitForFirstFrameResult::kWaitingTimedOut); 339 RTC_LOG(LS_ERROR) << "Timed out after waiting " << kFirstFrameTimeoutMs 340 << " ms for the first frame."; 341 return false; 342 } 343 344 RecordGetFirstFrameTime(TimeMillis() - first_frame_event_wait_start); 345 RecordWaitForFirstFrameResult(WaitForFirstFrameResult::kSuccess); 346 has_first_frame_arrived_ = true; 347 RemoveFrameArrivedEventHandler(); 348 return true; 349 } 350 351 void WgcCaptureSession::EnsureFrame() { 352 // We need to wait for the first frame because it might take some extra time 353 // for the `frame_pool_` to be populated and capture may fail because of too 354 // many `kFrameDropped` errors. 355 if (!has_first_frame_arrived_) { 356 if (!WaitForFirstFrame()) { 357 RecordGetFrameResult(GetFrameResult::kWaitForFirstFrameFailed); 358 return; 359 } 360 } 361 362 // Try to process the captured frame and copy it to the `queue_`. 363 HRESULT hr = ProcessFrame(); 364 if (SUCCEEDED(hr)) { 365 RTC_CHECK(queue_.current_frame()); 366 return; 367 } 368 369 // We failed to process the frame, but we do have a frame so just return that. 370 if (queue_.current_frame()) { 371 RTC_LOG(LS_VERBOSE) << "ProcessFrame failed, using existing frame: " << hr; 372 return; 373 } 374 375 // ProcessFrame failed and we don't have a current frame. This could indicate 376 // a startup path where we may need to try/wait a few times to ensure that we 377 // have a frame. We try to get a new frame from the frame pool for a maximum 378 // of 10 times after sleeping for 20ms. We choose 20ms as it's just a bit 379 // longer than 17ms (for 60fps*) and hopefully avoids unlucky timing causing 380 // us to wait two frames when we mostly seem to only need to wait for one. 381 // This approach should ensure that GetFrame() always delivers a valid frame 382 // with a max latency of 200ms and often after sleeping only once. 383 // The scheme is heuristic and based on manual testing. 384 // (*) On a modern system, the FPS / monitor refresh rate is usually larger 385 // than or equal to 60. 386 387 const int max_sleep_count = 10; 388 const int sleep_time_ms = 20; 389 390 int sleep_count = 0; 391 while (!queue_.current_frame() && sleep_count < max_sleep_count) { 392 sleep_count++; 393 Thread::SleepMs(sleep_time_ms); 394 hr = ProcessFrame(); 395 if (FAILED(hr)) { 396 RTC_DLOG(LS_WARNING) << "ProcessFrame failed during startup: " << hr; 397 } 398 } 399 RTC_LOG_IF(LS_ERROR, !queue_.current_frame()) 400 << "Unable to process a valid frame even after trying 10 times."; 401 } 402 403 bool WgcCaptureSession::GetFrame(std::unique_ptr<DesktopFrame>* output_frame, 404 bool source_should_be_capturable) { 405 RTC_DCHECK_RUN_ON(&sequence_checker_); 406 407 if (item_closed_) { 408 RTC_LOG(LS_ERROR) << "The target source has been closed."; 409 RecordGetFrameResult(GetFrameResult::kItemClosed); 410 return false; 411 } 412 413 // Try to process the captured frame and wait some if needed. Avoid trying 414 // if we know that the source will not be capturable. This can happen e.g. 415 // when captured window is minimized and if EnsureFrame() was called in this 416 // state a large amount of kFrameDropped errors would be logged. 417 if (source_should_be_capturable) { 418 EnsureFrame(); 419 } else { 420 // If the source is not capturable, we must reset `has_first_frame_arrived_` 421 // so that the next time the source becomes capturable we can wait for the 422 // first frame again. 423 if (has_first_frame_arrived_) { 424 has_first_frame_arrived_ = false; 425 } 426 } 427 428 // Return a NULL frame and false as `result` if we still don't have a valid 429 // frame. This will lead to a DesktopCapturer::Result::ERROR_PERMANENT being 430 // posted by the WGC capturer. 431 DesktopFrame* current_frame = queue_.current_frame(); 432 if (!current_frame) { 433 RTC_LOG(LS_ERROR) << "GetFrame failed."; 434 return false; 435 } 436 437 // Swap in the DesktopRegion in `damage_region_` which is updated in 438 // ProcessFrame(). The updated region is either empty or the full rect being 439 // captured where an empty damage region corresponds to "no change in content 440 // since last frame". 441 current_frame->mutable_updated_region()->Swap(&damage_region_); 442 damage_region_.Clear(); 443 444 // Emit the current frame. 445 std::unique_ptr<DesktopFrame> new_frame = queue_.current_frame()->Share(); 446 *output_frame = std::move(new_frame); 447 448 return true; 449 } 450 451 HRESULT WgcCaptureSession::CreateMappedTexture( 452 ComPtr<ID3D11Texture2D> src_texture, 453 UINT width, 454 UINT height) { 455 RTC_DCHECK_RUN_ON(&sequence_checker_); 456 457 D3D11_TEXTURE2D_DESC src_desc; 458 src_texture->GetDesc(&src_desc); 459 D3D11_TEXTURE2D_DESC map_desc; 460 map_desc.Width = width == 0 ? src_desc.Width : width; 461 map_desc.Height = height == 0 ? src_desc.Height : height; 462 map_desc.MipLevels = src_desc.MipLevels; 463 map_desc.ArraySize = src_desc.ArraySize; 464 map_desc.Format = src_desc.Format; 465 map_desc.SampleDesc = src_desc.SampleDesc; 466 map_desc.Usage = D3D11_USAGE_STAGING; 467 map_desc.BindFlags = 0; 468 map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; 469 map_desc.MiscFlags = 0; 470 return d3d11_device_->CreateTexture2D(&map_desc, nullptr, &mapped_texture_); 471 } 472 473 HRESULT WgcCaptureSession::ProcessFrame() { 474 RTC_DCHECK_RUN_ON(&sequence_checker_); 475 476 RTC_DCHECK(is_capture_started_); 477 478 ComPtr<WGC::IDirect3D11CaptureFrame> capture_frame; 479 HRESULT hr = frame_pool_->TryGetNextFrame(&capture_frame); 480 if (FAILED(hr)) { 481 RTC_LOG(LS_ERROR) << "TryGetNextFrame failed: " << hr; 482 RecordGetFrameResult(GetFrameResult::kTryGetNextFrameFailed); 483 return hr; 484 } 485 486 if (!capture_frame) { 487 if (!queue_.current_frame()) { 488 // The frame pool was empty and so is the external queue. 489 RTC_DLOG(LS_ERROR) << "Frame pool was empty => kFrameDropped."; 490 RecordGetFrameResult(GetFrameResult::kFrameDropped); 491 } else { 492 // The frame pool was empty but there is still one old frame available in 493 // external the queue. 494 RTC_DLOG(LS_WARNING) << "Frame pool was empty => kFramePoolEmpty."; 495 RecordGetFrameResult(GetFrameResult::kFramePoolEmpty); 496 } 497 return E_FAIL; 498 } 499 500 queue_.MoveToNextFrame(); 501 if (queue_.current_frame() && queue_.current_frame()->IsShared()) { 502 RTC_DLOG(LS_VERBOSE) << "Overwriting frame that is still shared."; 503 } 504 505 // We need to get `capture_frame` as an `ID3D11Texture2D` so that we can get 506 // the raw image data in the format required by the `DesktopFrame` interface. 507 ComPtr<ABI::Windows::Graphics::DirectX::Direct3D11::IDirect3DSurface> 508 d3d_surface; 509 hr = capture_frame->get_Surface(&d3d_surface); 510 if (FAILED(hr)) { 511 RecordGetFrameResult(GetFrameResult::kGetSurfaceFailed); 512 return hr; 513 } 514 515 ComPtr<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> 516 direct3DDxgiInterfaceAccess; 517 hr = d3d_surface->QueryInterface(IID_PPV_ARGS(&direct3DDxgiInterfaceAccess)); 518 if (FAILED(hr)) { 519 RecordGetFrameResult(GetFrameResult::kDxgiInterfaceAccessFailed); 520 return hr; 521 } 522 523 ComPtr<ID3D11Texture2D> texture_2D; 524 hr = direct3DDxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&texture_2D)); 525 if (FAILED(hr)) { 526 RecordGetFrameResult(GetFrameResult::kTexture2dCastFailed); 527 return hr; 528 } 529 530 if (!mapped_texture_) { 531 hr = CreateMappedTexture(texture_2D); 532 if (FAILED(hr)) { 533 RecordGetFrameResult(GetFrameResult::kCreateMappedTextureFailed); 534 return hr; 535 } 536 } 537 538 // We need to copy `texture_2D` into `mapped_texture_` as the latter has the 539 // D3D11_CPU_ACCESS_READ flag set, which lets us access the image data. 540 // Otherwise it would only be readable by the GPU. 541 ComPtr<ID3D11DeviceContext> d3d_context; 542 d3d11_device_->GetImmediateContext(&d3d_context); 543 544 ABI::Windows::Graphics::SizeInt32 new_size; 545 hr = capture_frame->get_ContentSize(&new_size); 546 if (FAILED(hr)) { 547 RecordGetFrameResult(GetFrameResult::kGetContentSizeFailed); 548 return hr; 549 } 550 551 // If the size changed, we must resize `mapped_texture_` and `frame_pool_` to 552 // fit the new size. This must be done before `CopySubresourceRegion` so that 553 // the textures are the same size. 554 if (SizeHasChanged(new_size, size_)) { 555 hr = CreateMappedTexture(texture_2D, new_size.Width, new_size.Height); 556 if (FAILED(hr)) { 557 RecordGetFrameResult(GetFrameResult::kResizeMappedTextureFailed); 558 return hr; 559 } 560 561 hr = frame_pool_->Recreate(direct3d_device_.Get(), kPixelFormat, 562 kNumBuffers, new_size); 563 if (FAILED(hr)) { 564 RecordGetFrameResult(GetFrameResult::kRecreateFramePoolFailed); 565 return hr; 566 } 567 } 568 569 // If the size has changed since the last capture, we must be sure to use 570 // the smaller dimensions. Otherwise we might overrun our buffer, or 571 // read stale data from the last frame. 572 int image_height = std::min(size_.Height, new_size.Height); 573 int image_width = std::min(size_.Width, new_size.Width); 574 575 D3D11_BOX copy_region; 576 copy_region.left = 0; 577 copy_region.top = 0; 578 copy_region.right = image_width; 579 copy_region.bottom = image_height; 580 // Our textures are 2D so we just want one "slice" of the box. 581 copy_region.front = 0; 582 copy_region.back = 1; 583 d3d_context->CopySubresourceRegion(mapped_texture_.Get(), 584 /*dst_subresource_index=*/0, /*dst_x=*/0, 585 /*dst_y=*/0, /*dst_z=*/0, texture_2D.Get(), 586 /*src_subresource_index=*/0, ©_region); 587 588 D3D11_MAPPED_SUBRESOURCE map_info; 589 hr = d3d_context->Map(mapped_texture_.Get(), /*subresource_index=*/0, 590 D3D11_MAP_READ, /*D3D11_MAP_FLAG_DO_NOT_WAIT=*/0, 591 &map_info); 592 if (FAILED(hr)) { 593 RecordGetFrameResult(GetFrameResult::kMapFrameFailed); 594 return hr; 595 } 596 597 // Allocate the current frame buffer only if it is not already allocated or 598 // if the size has changed. Note that we can't reallocate other buffers at 599 // this point, since the caller may still be reading from them. The queue can 600 // hold up to two frames. 601 DesktopSize image_size(image_width, image_height); 602 if (!queue_.current_frame() || 603 !queue_.current_frame()->size().equals(image_size)) { 604 std::unique_ptr<DesktopFrame> buffer = 605 std::make_unique<BasicDesktopFrame>(image_size); 606 queue_.ReplaceCurrentFrame(SharedDesktopFrame::Wrap(std::move(buffer))); 607 } 608 609 DesktopFrame* current_frame = queue_.current_frame(); 610 DesktopFrame* previous_frame = queue_.previous_frame(); 611 612 if (is_window_source_) { 613 // If the captured window moves to another screen, the HMONITOR associated 614 // with the captured window will change. Therefore, we need to get the value 615 // of HMONITOR per frame. 616 monitor_ = ::MonitorFromWindow(reinterpret_cast<HWND>(source_id_), 617 /*dwFlags=*/MONITOR_DEFAULTTONEAREST); 618 } else { 619 if (!monitor_.has_value()) { 620 HMONITOR monitor; 621 if (!GetHmonitorFromDeviceIndex(source_id_, &monitor)) { 622 RTC_LOG(LS_ERROR) << "Failed to get HMONITOR from device index."; 623 d3d_context->Unmap(mapped_texture_.Get(), 0); 624 return E_FAIL; 625 } 626 monitor_ = monitor; 627 } 628 } 629 630 // Captures the device scale factor of the monitor where the frame is captured 631 // from. This value is the same as the scale from windows settings. Valid 632 // values are some distinct numbers in the range of [1,5], for example, 633 // 1, 1.5, 2.5, etc. 634 DEVICE_SCALE_FACTOR device_scale_factor = DEVICE_SCALE_FACTOR_INVALID; 635 HRESULT scale_factor_hr = 636 GetScaleFactorForMonitor(monitor_.value(), &device_scale_factor); 637 RTC_LOG_IF(LS_ERROR, FAILED(scale_factor_hr)) 638 << "Failed to get scale factor for monitor: " << scale_factor_hr; 639 if (device_scale_factor != DEVICE_SCALE_FACTOR_INVALID) { 640 current_frame->set_device_scale_factor( 641 static_cast<float>(device_scale_factor) / 100.0f); 642 } 643 644 // Will be set to true while copying the frame data to the `current_frame` if 645 // we can already determine that the content of the new frame differs from the 646 // previous. The idea is to get a low-complexity indication of if the content 647 // is static or not without performing a full/deep memory comparison when 648 // updating the damaged region. 649 // `DoesWgcSkipStaticFrames()`: `TryGetNextFrame()` returns a frame 650 // successfully only if there is a region that has changed. This means that 651 // we can skip the full memory comparison if the running OS is Windows 11 652 // 24H2 or later. 653 bool frame_content_has_changed = DoesWgcSkipStaticFrames(); 654 655 // Check if the queue contains two frames whose content can be compared. 656 const bool frame_content_can_be_compared = FrameContentCanBeCompared(); 657 658 // Make a copy of the data pointed to by `map_info.pData` to the preallocated 659 // `current_frame` so we are free to unmap our texture. If possible, also 660 // perform a light-weight scan of the vertical line of pixels in the middle 661 // of the screen. A comparison is performed between two 32-bit pixels (RGBA); 662 // one from the current frame and one from the previous, and as soon as a 663 // difference is detected the scan stops and `frame_content_has_changed` is 664 // set to true. 665 uint8_t* src_data = static_cast<uint8_t*>(map_info.pData); 666 uint8_t* dst_data = current_frame->data(); 667 uint8_t* prev_data = 668 frame_content_can_be_compared ? previous_frame->data() : nullptr; 669 670 const int width_in_bytes = 671 current_frame->size().width() * DesktopFrame::kBytesPerPixel; 672 RTC_DCHECK_GE(current_frame->stride(), width_in_bytes); 673 RTC_DCHECK_GE(map_info.RowPitch, width_in_bytes); 674 const int middle_pixel_offset = 675 (image_width / 2) * DesktopFrame::kBytesPerPixel; 676 for (int i = 0; i < image_height; i++) { 677 memcpy(dst_data, src_data, width_in_bytes); 678 if (prev_data && !frame_content_has_changed) { 679 uint8_t* previous_pixel = prev_data + middle_pixel_offset; 680 uint8_t* current_pixel = dst_data + middle_pixel_offset; 681 frame_content_has_changed = 682 memcmp(previous_pixel, current_pixel, DesktopFrame::kBytesPerPixel); 683 prev_data += current_frame->stride(); 684 } 685 dst_data += current_frame->stride(); 686 src_data += map_info.RowPitch; 687 } 688 689 d3d_context->Unmap(mapped_texture_.Get(), 0); 690 691 if (allow_zero_hertz()) { 692 if (previous_frame) { 693 const int previous_frame_size = 694 previous_frame->stride() * previous_frame->size().height(); 695 const int current_frame_size = 696 current_frame->stride() * current_frame->size().height(); 697 698 // Compare the latest frame with the previous and check if the frames are 699 // equal (both contain the exact same pixel values). Avoid full memory 700 // comparison if indication of a changed frame already exists from the 701 // stage above. 702 if (current_frame_size == previous_frame_size) { 703 if (frame_content_has_changed) { 704 // Mark frame as damaged based on existing light-weight indicator. 705 // Avoids deep memcmp of complete frame and saves resources. 706 damage_region_.SetRect(DesktopRect::MakeSize(current_frame->size())); 707 } else { 708 // Perform full memory comparison for all bytes between the current 709 // and the previous frames. 710 const bool frames_are_equal = 711 !memcmp(current_frame->data(), previous_frame->data(), 712 current_frame_size); 713 if (!frames_are_equal) { 714 // TODO(https://crbug.com/1421242): If we had an API to report 715 // proper damage regions we should be doing AddRect() with a 716 // SetRect() call on a resize. 717 damage_region_.SetRect( 718 DesktopRect::MakeSize(current_frame->size())); 719 } 720 } 721 } else { 722 // Mark resized frames as damaged. 723 damage_region_.SetRect(DesktopRect::MakeSize(current_frame->size())); 724 } 725 } else { 726 // Mark a `damage_region_` even if there is no previous frame. This 727 // condition does not create any increased overhead but is useful while 728 // using FullScreenWindowDetector, where it would create a new 729 // WgcCaptureSession(with no previous frame) for the slide show window but 730 // the DesktopCaptureDevice instance might have already received frames 731 // from the editor window's WgcCaptureSession which would have activated 732 // the zero-hertz mode. 733 damage_region_.SetRect(DesktopRect::MakeSize(current_frame->size())); 734 } 735 } 736 737 size_ = new_size; 738 RecordGetFrameResult(GetFrameResult::kSuccess); 739 return hr; 740 } 741 742 HRESULT WgcCaptureSession::OnItemClosed(WGC::IGraphicsCaptureItem* sender, 743 IInspectable* event_args) { 744 RTC_DCHECK_RUN_ON(&sequence_checker_); 745 746 RTC_LOG(LS_INFO) << "Capture target has been closed."; 747 item_closed_ = true; 748 749 RemoveItemClosedEventHandler(); 750 751 // Do not attempt to free resources in the OnItemClosed handler, as this 752 // causes a race where we try to delete the item that is calling us. Removing 753 // the event handlers and setting `item_closed_` above is sufficient to ensure 754 // that the resources are no longer used, and the next time the capturer tries 755 // to get a frame, we will report a permanent failure and be destroyed. 756 return S_OK; 757 } 758 759 void WgcCaptureSession::RemoveEventHandlers() { 760 RemoveItemClosedEventHandler(); 761 RemoveFrameArrivedEventHandler(); 762 } 763 764 void WgcCaptureSession::RemoveItemClosedEventHandler() { 765 HRESULT hr; 766 if (item_ && item_closed_token_) { 767 hr = item_->remove_Closed(*item_closed_token_); 768 item_closed_token_.reset(); 769 if (FAILED(hr)) { 770 RTC_LOG(LS_WARNING) << "Failed to remove Closed event handler: " << hr; 771 } 772 } 773 } 774 775 void WgcCaptureSession::RemoveFrameArrivedEventHandler() { 776 RTC_DCHECK(frame_pool_); 777 if (frame_arrived_token_) { 778 HRESULT hr = frame_pool_->remove_FrameArrived(*frame_arrived_token_); 779 frame_arrived_token_.reset(); 780 has_first_frame_arrived_event_ = nullptr; 781 if (FAILED(hr)) { 782 RTC_LOG(LS_WARNING) << "Failed to remove FrameArrived event handler: " 783 << hr; 784 } 785 } 786 } 787 788 HRESULT WgcCaptureSession::AddFrameArrivedEventHandler() { 789 RTC_DCHECK(frame_pool_); 790 HRESULT hr = E_FAIL; 791 frame_arrived_token_ = std::make_unique<EventRegistrationToken>(); 792 has_first_frame_arrived_event_ = make_ref_counted<RefCountedEvent>( 793 /*manual_reset=*/true, /*initially_signaled=*/false); 794 auto frame_arrived_handler = Microsoft::WRL::Make<AgileFrameArrivedHandler>( 795 has_first_frame_arrived_event_); 796 hr = frame_pool_->add_FrameArrived(frame_arrived_handler.Get(), 797 frame_arrived_token_.get()); 798 if (FAILED(hr)) { 799 RTC_LOG(LS_WARNING) << "Failed to add FrameArrived event handler: " << hr; 800 frame_arrived_token_.reset(); 801 has_first_frame_arrived_event_ = nullptr; 802 } 803 return hr; 804 } 805 806 bool WgcCaptureSession::FrameContentCanBeCompared() { 807 DesktopFrame* current_frame = queue_.current_frame(); 808 DesktopFrame* previous_frame = queue_.previous_frame(); 809 if (!current_frame || !previous_frame) { 810 return false; 811 } 812 if (current_frame->stride() != previous_frame->stride()) { 813 return false; 814 } 815 return current_frame->size().equals(previous_frame->size()); 816 } 817 818 } // namespace webrtc