commit 2c38d9a8558669749e305d9911aa3390a3310872
parent 9e80bc22f2168fea0c1cc1b85fbac1dfe73a9d72
Author: Henrik Skupin <mail@hskupin.info>
Date: Tue, 11 Nov 2025 10:33:26 +0000
Bug 1848958 - [remote] Add support for mouse widget events to Marionette and WebDriver BiDi. r=jdescottes
Differential Revision: https://phabricator.services.mozilla.com/D265220
Diffstat:
8 files changed, 137 insertions(+), 59 deletions(-)
diff --git a/remote/marionette/actors/MarionetteCommandsChild.sys.mjs b/remote/marionette/actors/MarionetteCommandsChild.sys.mjs
@@ -87,7 +87,7 @@ export class MarionetteCommandsChild extends JSWindowActorChild {
lazy.event.sendKeyUp(details.eventData, win);
break;
case "synthesizeMouseAtPoint":
- lazy.event.synthesizeMouseAtPoint(
+ await lazy.event.synthesizeMouseAtPoint(
details.x,
details.y,
details.eventData,
diff --git a/remote/marionette/driver.sys.mjs b/remote/marionette/driver.sys.mjs
@@ -158,10 +158,12 @@ class ActionsHelper {
*/
dispatchEvent(eventName, browsingContext, details) {
if (
- eventName === "synthesizeWheelAtPoint" &&
- lazy.actions.useAsyncWheelEvents
+ (eventName === "synthesizeWheelAtPoint" &&
+ lazy.actions.useAsyncWheelEvents) ||
+ (eventName == "synthesizeMouseAtPoint" &&
+ lazy.actions.useAsyncMouseEvents)
) {
- browsingContext = browsingContext.topChromeWindow.browsingContext;
+ browsingContext = browsingContext.topChromeWindow?.browsingContext;
details.eventData.asyncEnabled = true;
}
diff --git a/remote/marionette/interaction.sys.mjs b/remote/marionette/interaction.sys.mjs
@@ -190,7 +190,7 @@ async function webdriverClickElement(el, a11y) {
interaction.selectOption(el);
} else {
// Synthesize a pointerMove action.
- lazy.event.synthesizeMouseAtPoint(
+ await lazy.event.synthesizeMouseAtPoint(
clickPoint.x,
clickPoint.y,
{
@@ -204,7 +204,7 @@ async function webdriverClickElement(el, a11y) {
// Special handling is required if the mousemove started a drag session.
// In this case, mousedown event shouldn't be fired, and the mouseup should
// end the session. Therefore, we should synthesize only mouseup.
- lazy.event.synthesizeMouseAtPoint(
+ await lazy.event.synthesizeMouseAtPoint(
clickPoint.x,
clickPoint.y,
{
@@ -218,7 +218,7 @@ async function webdriverClickElement(el, a11y) {
let clicked = interaction.flushEventLoop(containerEl);
// Synthesize a pointerDown + pointerUp action.
- lazy.event.synthesizeMouseAtPoint(
+ await lazy.event.synthesizeMouseAtPoint(
clickPoint.x,
clickPoint.y,
{ allowToHandleDragDrop: true },
@@ -278,7 +278,7 @@ async function seleniumClickElement(el, a11y) {
let rects = el.getClientRects();
let centre = lazy.dom.getInViewCentrePoint(rects[0], win);
let opts = {};
- lazy.event.synthesizeMouseAtPoint(centre.x, centre.y, opts, win);
+ await lazy.event.synthesizeMouseAtPoint(centre.x, centre.y, opts, win);
}
}
diff --git a/remote/shared/webdriver/Actions.sys.mjs b/remote/shared/webdriver/Actions.sys.mjs
@@ -49,6 +49,14 @@ const MODIFIER_NAME_LOOKUP = {
Meta: "meta",
};
+// Flag, that indicates if an async widget event should be used when dispatching a mouse event.
+XPCOMUtils.defineLazyPreferenceGetter(
+ actions,
+ "useAsyncMouseEvents",
+ "remote.events.async.mouse.enabled",
+ false
+);
+
// Flag, that indicates if an async widget event should be used when dispatching a wheel scroll event.
XPCOMUtils.defineLazyPreferenceGetter(
actions,
@@ -507,6 +515,10 @@ class KeyInputSource extends InputSource {
* Input state associated with a pointer-type device.
*/
class PointerInputSource extends InputSource {
+ #initialized;
+ #x;
+ #y;
+
static type = "pointer";
/**
@@ -521,9 +533,23 @@ class PointerInputSource extends InputSource {
super(id);
this.pointer = pointer;
- this.x = 0;
- this.y = 0;
this.pressed = new Set();
+
+ this.#initialized = false;
+ this.#x = 0;
+ this.#y = 0;
+ }
+
+ get initialized() {
+ return this.#initialized;
+ }
+
+ get x() {
+ return this.#x;
+ }
+
+ get y() {
+ return this.#y;
}
/**
@@ -544,6 +570,12 @@ class PointerInputSource extends InputSource {
return this.pressed.has(button);
}
+ moveTo(x, y) {
+ this.#initialized = true;
+ this.#x = x;
+ this.#y = y;
+ }
+
/**
* Add |button| to the set of pressed keys.
*
@@ -1368,14 +1400,15 @@ class PointerDownAction extends PointerAction {
* Promise that is resolved once the action is complete.
*/
async dispatch(state, inputSource, tickDuration, options) {
- lazy.logger.trace(
- `Dispatch ${this.constructor.name} ${inputSource.pointer.type} with id: ${this.id} button: ${this.button}`
- );
-
if (inputSource.isPressed(this.button)) {
return;
}
+ lazy.logger.trace(
+ `Dispatch ${this.constructor.name} ${inputSource.pointer.type} with id: ${this.id} ` +
+ `button: ${this.button} async: ${actions.useAsyncMouseEvents}`
+ );
+
inputSource.press(this.button);
await inputSource.pointer.pointerDown(state, inputSource, this, options);
@@ -1472,14 +1505,15 @@ class PointerUpAction extends PointerAction {
* Promise that is resolved once the action is complete.
*/
async dispatch(state, inputSource, tickDuration, options) {
- lazy.logger.trace(
- `Dispatch ${this.constructor.name} ${inputSource.pointer.type} with id: ${this.id} button: ${this.button}`
- );
-
if (!inputSource.isPressed(this.button)) {
return;
}
+ lazy.logger.trace(
+ `Dispatch ${this.constructor.name} ${inputSource.pointer.type} with id: ${this.id} ` +
+ `button: ${this.button} async: ${actions.useAsyncMouseEvents}`
+ );
+
inputSource.release(this.button);
await inputSource.pointer.pointerUp(state, inputSource, this, options);
@@ -1581,23 +1615,38 @@ class PointerMoveAction extends PointerAction {
* Promise that is resolved once the action is complete.
*/
async dispatch(state, inputSource, tickDuration, options) {
- const { assertInViewPort, context } = options;
-
- lazy.logger.trace(
- `Dispatch ${this.constructor.name} ${inputSource.pointer.type} with id: ${this.id} x: ${this.x} y: ${this.y}`
- );
+ const { assertInViewPort, context, toBrowserWindowCoordinates } = options;
- const target = await this.origin.getTargetCoordinates(
+ let moveCoordinates = await this.origin.getTargetCoordinates(
inputSource,
[this.x, this.y],
options
);
- await assertInViewPort(target, context);
+ await assertInViewPort(moveCoordinates, context);
+
+ lazy.logger.trace(
+ `Dispatch ${this.constructor.name} ${inputSource.pointer.type} with id: ${this.id} ` +
+ `x: ${moveCoordinates[0]} y: ${moveCoordinates[1]} ` +
+ `async: ${actions.useAsyncMouseEvents}`
+ );
+
+ // Only convert coordinates if these are for a content process, and are not
+ // relative to an already initialized pointer source.
+ if (
+ !(this.origin instanceof PointerOrigin && inputSource.initialized) &&
+ context.isContent &&
+ actions.useAsyncMouseEvents
+ ) {
+ moveCoordinates = await toBrowserWindowCoordinates(
+ moveCoordinates,
+ context
+ );
+ }
return moveOverTime(
[[inputSource.x, inputSource.y]],
- [target],
+ [moveCoordinates],
this.duration ?? tickDuration,
async _target =>
await this.performPointerMoveStep(state, inputSource, _target, options)
@@ -1626,13 +1675,14 @@ class PointerMoveAction extends PointerAction {
}
const target = targets[0];
- lazy.logger.trace(
- `PointerMoveAction.performPointerMoveStep ${JSON.stringify(target)}`
- );
if (target[0] == inputSource.x && target[1] == inputSource.y) {
return;
}
+ lazy.logger.trace(
+ `PointerMoveAction.performPointerMoveStep ${JSON.stringify(target)}`
+ );
+
await inputSource.pointer.pointerMove(
state,
inputSource,
@@ -1642,8 +1692,7 @@ class PointerMoveAction extends PointerAction {
options
);
- inputSource.x = target[0];
- inputSource.y = target[1];
+ inputSource.moveTo(target[0], target[1]);
}
/**
@@ -2238,8 +2287,7 @@ class PointerMoveTouchActionGroup extends TouchActionGroup {
const eventData = new MultiTouchEventData("touchmove");
for (const [inputSource, action, target] of perPointerData) {
- inputSource.x = target[0];
- inputSource.y = target[1];
+ inputSource.moveTo(target[0], target[1]);
eventData.addPointerEventData(inputSource, action);
eventData.update(state, inputSource);
}
diff --git a/remote/shared/webdriver/Event.sys.mjs b/remote/shared/webdriver/Event.sys.mjs
@@ -59,26 +59,40 @@ event.MouseButton = {
};
/**
- * Synthesise a mouse event at a point.
+ * Synthesize a mouse event in `win` at a point.
*
- * If the type is specified in opts, an mouse event of that type is
+ * If the type is specified in `event`, a mouse event of that type is
* fired. Otherwise, a mousedown followed by a mouseup is performed.
*
- * @param {number} left
- * Offset from viewport left, in CSS pixels
- * @param {number} top
- * Offset from viewport top, in CSS pixels
- * @param {object} opts
- * Object which may contain the properties "shiftKey", "ctrlKey",
- * "altKey", "metaKey", "accessKey", "clickCount", "button", and
- * "type".
- * @param {Window} win
- * Window object.
+ * @param {number} left - Value for the X offset in CSS pixels.
+ * @param {number} top - Value for the Y offset in CSS pixels.
+ * @param {module:EventUtils~MouseEventData} event - Details of the mouse event
+ * to dispatch.
+ * @param {DOMWindow} win - DOM window used to dispatch the event.
*
- * @returns {boolean} defaultPrevented
+ * @returns {Promise<boolean>} Promise that resolves to a boolean,
+ * indicating whether the event had preventDefault() called on it.
*/
-event.synthesizeMouseAtPoint = function (left, top, opts, win) {
- return _getEventUtils(win).synthesizeMouseAtPoint(left, top, opts, win);
+event.synthesizeMouseAtPoint = function (left, top, event, win) {
+ if (!event.asyncEnabled) {
+ return Promise.resolve(
+ _getEventUtils(win).synthesizeMouseAtPoint(left, top, event, win)
+ );
+ }
+
+ // A callback must be used when handling events with the `asyncEnabled`
+ // flag set to `true`, as these events are synthesized in the parent process.
+ // We need to wait for them to be fully dispatched to the content process
+ // before continuing.
+ return new Promise(resolve => {
+ const preventDefault = _getEventUtils(win).synthesizeMouseAtPoint(
+ left,
+ top,
+ event,
+ win,
+ () => resolve(preventDefault)
+ );
+ });
};
/**
diff --git a/remote/shared/webdriver/test/xpcshell/test_Actions.js b/remote/shared/webdriver/test/xpcshell/test_Actions.js
@@ -314,8 +314,7 @@ add_task(async function test_computePointerDestinationViewport() {
const actionItem = chain[0][0];
const inputSource = state.getInputSource(actionItem.id);
// these values should not affect the outcome
- inputSource.x = "99";
- inputSource.y = "10";
+ inputSource.moveTo(99, 10);
const target = await actionItem.origin.getTargetCoordinates(
inputSource,
[actionItem.x, actionItem.y],
@@ -343,8 +342,7 @@ add_task(async function test_computePointerDestinationPointer() {
);
const actionItem = chain[0][0];
const inputSource = state.getInputSource(actionItem.id);
- inputSource.x = 10;
- inputSource.y = 99;
+ inputSource.moveTo(10, 99);
const target = await actionItem.origin.getTargetCoordinates(
inputSource,
[actionItem.x, actionItem.y],
diff --git a/remote/webdriver-bidi/modules/root/input.sys.mjs b/remote/webdriver-bidi/modules/root/input.sys.mjs
@@ -9,7 +9,9 @@ const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
actions: "chrome://remote/content/shared/webdriver/Actions.sys.mjs",
assert: "chrome://remote/content/shared/webdriver/Assert.sys.mjs",
+ error: "chrome://remote/content/shared/webdriver/Errors.sys.mjs",
event: "chrome://remote/content/shared/webdriver/Event.sys.mjs",
+ NavigableManager: "chrome://remote/content/shared/NavigableManager.sys.mjs",
pprint: "chrome://remote/content/shared/Format.sys.mjs",
});
@@ -76,17 +78,31 @@ class InputModule extends RootBiDiModule {
* Promise that resolves once the event is dispatched.
*/
async #dispatchEvent(eventName, context, details) {
- if (
- eventName === "synthesizeWheelAtPoint" &&
- lazy.actions.useAsyncWheelEvents
- ) {
- details.eventData.asyncEnabled = true;
- }
+ details.eventData.asyncEnabled =
+ (eventName === "synthesizeWheelAtPoint" &&
+ lazy.actions.useAsyncWheelEvents) ||
+ (eventName == "synthesizeMouseAtPoint" &&
+ lazy.actions.useAsyncMouseEvents);
// TODO: Call the _dispatchEvent method of the windowglobal module once
// chrome support was added for the message handler.
if (details.eventData.asyncEnabled) {
+ if (!context || context.isDiscarded) {
+ const id = lazy.NavigableManager.getIdForBrowsingContext(context);
+ throw new lazy.error.NoSuchFrameError(
+ `Browsing Context with id ${id} not found`
+ );
+ }
+
switch (eventName) {
+ case "synthesizeMouseAtPoint":
+ await lazy.event.synthesizeMouseAtPoint(
+ details.x,
+ details.y,
+ details.eventData,
+ context.topChromeWindow
+ );
+ break;
case "synthesizeWheelAtPoint":
await lazy.event.synthesizeWheelAtPoint(
details.x,
diff --git a/remote/webdriver-bidi/modules/windowglobal/input.sys.mjs b/remote/webdriver-bidi/modules/windowglobal/input.sys.mjs
@@ -69,7 +69,7 @@ class InputModule extends WindowGlobalBiDiModule {
lazy.event.sendKeyUp(details.eventData, this.messageHandler.window);
break;
case "synthesizeMouseAtPoint":
- lazy.event.synthesizeMouseAtPoint(
+ await lazy.event.synthesizeMouseAtPoint(
details.x,
details.y,
details.eventData,
