From 010f41d15a4dedaa9e7fa303166df11c7f408fd0 Mon Sep 17 00:00:00 2001 From: pletch <17855860+pletch@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:31:14 -0400 Subject: [PATCH] feat(mouse): replay the moves the browser coalesced away during a drag A busy main thread does not delay pointer movement, it discards it: moves arriving while a handler runs are merged into one, and what is dispatched afterwards carries only the last position. A drag feels that immediately -- the path between two points is gone, so a dragged scrollbar thumb is dropped rather than merely lagged -- while a hover or a video degrades gracefully under the same load. getCoalescedEvents() returns what was merged. Replaying it in order restores the path, though not the timing: nothing running on a blocked thread can do that, and the positions still arrive together once the thread is free. What changes is that the remote end sees where the pointer went rather than only where it ended up. Deliberately additive. The mousemove listener is untouched and still delivers every position it did before, so touch, the compatibility mouse events it produces and the counter that swallows them all behave as they did; the dispatched position arrives twice and the second is dropped by the unchanged-coordinates check already in move(). Migrating this file to pointer events outright is the tidier shape and a far larger change -- every listener would have to move together, or synthetic events from touch would arrive with nothing left counting them down. Only while a button is held, because the path matters for a drag and the endpoint is all that matters for a hover, and replaying every merged move of an idle pointer would multiply a session's upstream mouse traffic for movement nobody can perceive. Capped and evenly sampled, because a pointer reporting at 1000Hz behind a thread blocked for 100ms offers a hundred positions for one drag, and answering a rendering problem with a traffic one is no answer. An ordinary pointer never reaches the cap. The pop-out monitor windows get the same replay. They bypass Guacamole.Mouse entirely -- it does not track the X axis correctly in a popup -- and map the pointer themselves, so a drag on a second head would otherwise lose its path exactly as the first one did. The sampler moves to a static so there is one definition of the rule rather than two that can drift, and that path needs one thing the main display does not: it sends unconditionally, where move() drops an unchanged position, so it keeps what it last sent and drops a repeat -- keyed on the buttons as well as the position, since a press and a release happen at one place and are not repeats of each other. Feature-detected on the event rather than on PointerEvent, because a pop-out window is a realm of its own and the constructor this page can see is not the one that made the event it is handed. That pointer handling lives in Guacamole.MonitorPointer rather than inline in client.html, because these windows cannot be opened without a live multi-monitor session: while it sat in the page the only thing a test could do was match the source text and hope, and the two things most worth checking there are a coordinate mapping and a suppression rule -- exactly the kind that look right in a diff and are off by a term. It takes the canvas, a function returning this monitor's rectangle in the combined framebuffer (a call rather than a captured value, since the layout changes under a running window) and a function to send a state. Writing those tests found a real defect in the wheel, which sends a momentary press and release: it built one state object, sent it, mutated it and sent it again, which works only while every consumer copies synchronously. It sends two states now. Tested rather than tried, because this path only runs once the browser has already thrown something away: the case it exists for is the one that cannot be reliably produced, and a broken version feels like the behaviour it replaced. Twenty tests cover the mapping, the button mask, the repeat suppression, the wheel's reset of it and the replay, plus a harness driving real pointer input through the debugger with the main thread deliberately blocked -- because the unit tests feed events shaped the way the code expects, and that assumption is the one worth checking against a browser. (cherry picked from commit d8ce743fe0801bd8c776281ee5541abf8040a339) --- static/client.html | 65 ++----- static/guac/MonitorPointer.js | 178 ++++++++++++++++++ static/guac/Mouse.js | 144 +++++++++++++++ tests/browser/drive-pointer.mjs | 113 ++++++++++++ tests/browser/monitor-pointer.html | 69 +++++++ tests/browser/serve.mjs | 69 +++++++ tests/monitor-pointer.mjs | 281 +++++++++++++++++++++++++++++ tests/mouse-coalesced.mjs | 251 ++++++++++++++++++++++++++ 8 files changed, 1118 insertions(+), 52 deletions(-) create mode 100644 static/guac/MonitorPointer.js create mode 100644 tests/browser/drive-pointer.mjs create mode 100644 tests/browser/monitor-pointer.html create mode 100644 tests/browser/serve.mjs create mode 100644 tests/monitor-pointer.mjs create mode 100644 tests/mouse-coalesced.mjs diff --git a/static/client.html b/static/client.html index fa8acb2..cf47719 100644 --- a/static/client.html +++ b/static/client.html @@ -186,6 +186,7 @@ + @@ -953,58 +954,18 @@ }, 500); }); - // Mouse: map the pointer from popup-window pixels to this - // monitor's native pixels using the canvas's LIVE on-screen - // geometry (getBoundingClientRect), then offset into the combined - // framebuffer. Reading the live rect each event avoids any stale - // cached scale/offset. Sent raw; guacd maps the combined - // coordinate back to this head. - // Native pointer handling (bypasses Guacamole.Mouse, which does - // not track the X axis correctly in a popup window). Map the raw - // client coordinate into the canvas's live on-screen rect as a - // 0..1 fraction, then into this monitor's native pixels and the - // combined framebuffer. Fraction-based, so it self-recalibrates - // on any resize. - function popupPointer(ev) { - var rect = monitorLayout[index]; - if (!rect) return; - var cr = canvas.getBoundingClientRect(); - var fx = cr.width > 0 ? (ev.clientX - cr.left) / cr.width : 0; - var fy = cr.height > 0 ? (ev.clientY - cr.top) / cr.height : 0; - if (fx < 0) fx = 0; else if (fx > 1) fx = 1; - if (fy < 0) fy = 0; else if (fy > 1) fy = 1; - client.sendMouseState(new Guacamole.Mouse.State({ - x: rect.left + fx * rect.width, - y: rect.top + fy * rect.height, - left: (ev.buttons & 1) !== 0, - middle: (ev.buttons & 4) !== 0, - right: (ev.buttons & 2) !== 0, - up: false, down: false - }), false); - ev.preventDefault(); - } - canvas.addEventListener('mousedown', popupPointer); - canvas.addEventListener('mousemove', popupPointer); - canvas.addEventListener('mouseup', popupPointer); - canvas.addEventListener('contextmenu', function(ev) { ev.preventDefault(); }); - // Scroll wheel -> momentary up/down button. - canvas.addEventListener('wheel', function(ev) { - var rect = monitorLayout[index]; - if (!rect) return; - var cr = canvas.getBoundingClientRect(); - var fx = cr.width > 0 ? (ev.clientX - cr.left) / cr.width : 0; - var fy = cr.height > 0 ? (ev.clientY - cr.top) / cr.height : 0; - if (fx < 0) fx = 0; else if (fx > 1) fx = 1; - if (fy < 0) fy = 0; else if (fy > 1) fy = 1; - var base = { x: rect.left + fx * rect.width, y: rect.top + fy * rect.height, - left: false, middle: false, right: false, up: false, down: false }; - var dir = ev.deltaY < 0 ? 'up' : 'down'; - base[dir] = true; - client.sendMouseState(new Guacamole.Mouse.State(base), false); - base[dir] = false; - client.sendMouseState(new Guacamole.Mouse.State(base), false); - ev.preventDefault(); - }, { passive: false }); + // Pointer, wheel and context menu. Guacamole.Mouse is not + // used here -- it does not track the X axis correctly in a + // popup -- so the mapping, the repeat suppression and the + // coalesced replay live in Guacamole.MonitorPointer, where a + // test can reach them. The rect is read per event rather than + // captured, because the layout changes under a running window. + Guacamole.MonitorPointer(canvas, + function () { return monitorLayout[index]; }, + function (state) { + client.sendMouseState(new Guacamole.Mouse.State(state), + false); + }); // Keyboard: forward to the same client (combined session). var keyboard = new Guacamole.Keyboard(win.document); diff --git a/static/guac/MonitorPointer.js b/static/guac/MonitorPointer.js new file mode 100644 index 0000000..ef2333f --- /dev/null +++ b/static/guac/MonitorPointer.js @@ -0,0 +1,178 @@ +/* + * Pointer handling for a pop-out monitor window. + * + * These windows do not use Guacamole.Mouse -- it does not track the X axis + * correctly in a popup -- so they map the pointer themselves: the raw client + * coordinate into the canvas's live on-screen rect as a 0..1 fraction, then + * into this monitor's native pixels and on into the combined framebuffer. + * Fraction-based and read from getBoundingClientRect() on every event, so it + * self-recalibrates on any resize rather than caching a scale that goes stale. + * + * It lives in its own file because the alternative is untestable. There is no + * way to open one of these windows without a live multi-monitor session, so + * inline in the page the only thing a test could do is match the source text + * and hope -- and the two things most worth checking here are a coordinate + * mapping and a suppression rule, which are exactly the kind that look right + * and are off by a term. + * + * Copyright (C) 2026 Sol1 Pty Ltd. Licensed under Apache 2.0. + */ + +var Guacamole = Guacamole || {}; + +/** + * Attaches pointer, wheel and context-menu handling to a pop-out monitor's + * canvas. + * + * @constructor + * @param {!Element} canvas + * The canvas this monitor is drawn on, in the pop-out window's document. + * + * @param {!function(): ?Object} getRect + * This monitor's rectangle within the combined framebuffer, as + * {left, top, width, height}, or null if the layout does not have one yet. + * Called per event rather than captured, because the layout changes under + * a running window. + * + * @param {!function(!Object)} sendState + * Called with a plain mouse state -- {x, y, left, middle, right, up, down} + * -- to be forwarded to the session. + */ +Guacamole.MonitorPointer = function MonitorPointer(canvas, getRect, sendState) { + + /** + * The last state sent, as a key, so the same one is not sent twice. + * + * Guacamole.Mouse drops an unchanged position inside move(); this path has + * no such check and needs one, because the coalesced replay below delivers + * the dispatched position and the mousemove that follows delivers it + * again. Keyed on the buttons as well as the position: a press and a + * release happen at one place and are not repeats of each other. + * + * @private + * @type {?string} + */ + var lastSent = null; + + /** + * Maps an event's client coordinate into the combined framebuffer and + * sends it, unless it is a repeat of what was last sent. + * + * @private + * @param {!Object} ev + */ + function pointer(ev) { + + var rect = getRect(); + if (!rect) + return; + + var cr = canvas.getBoundingClientRect(); + var fx = cr.width > 0 ? (ev.clientX - cr.left) / cr.width : 0; + var fy = cr.height > 0 ? (ev.clientY - cr.top) / cr.height : 0; + if (fx < 0) fx = 0; else if (fx > 1) fx = 1; + if (fy < 0) fy = 0; else if (fy > 1) fy = 1; + + var state = { + x : rect.left + fx * rect.width, + y : rect.top + fy * rect.height, + left : (ev.buttons & 1) !== 0, + middle : (ev.buttons & 4) !== 0, + right : (ev.buttons & 2) !== 0, + up : false, + down : false + }; + + var key = state.x + ',' + state.y + ',' + ev.buttons; + if (key !== lastSent) { + lastSent = key; + sendState(state); + } + + if (ev.preventDefault) + ev.preventDefault(); + + } + + /* Replay the moves the browser merged away during a drag, on the same + * terms as Guacamole.Mouse does for the main display: a busy main thread + * discards pointer movement rather than delaying it, and a dragged + * scrollbar thumb is dropped rather than lagged. Additive -- the mousemove + * listener below is unchanged and still delivers every position it did, + * and the one they both deliver is dropped by the check above. + * + * Feature-detected on the event rather than on PointerEvent, because a + * pop-out window is a realm of its own: the constructor the page that + * opened it can see is not the one that made this event. */ + canvas.addEventListener('pointermove', function (ev) { + + if (ev.pointerType !== 'mouse' || !ev.buttons) + return; + + if (!Guacamole.Mouse.coalescedMovement + || typeof ev.getCoalescedEvents !== 'function') + return; + + var merged = ev.getCoalescedEvents(); + if (!merged || merged.length <= 1) + return; + + var events = Guacamole.Mouse.sampleCoalesced(merged); + for (var i = 0; i < events.length; i++) + pointer(events[i]); + + }); + + canvas.addEventListener('mousedown', pointer); + canvas.addEventListener('mousemove', pointer); + canvas.addEventListener('mouseup', pointer); + + canvas.addEventListener('contextmenu', function (ev) { + ev.preventDefault(); + }); + + /* Scroll wheel -> momentary up/down button. */ + canvas.addEventListener('wheel', function (ev) { + + var rect = getRect(); + if (!rect) + return; + + var cr = canvas.getBoundingClientRect(); + var fx = cr.width > 0 ? (ev.clientX - cr.left) / cr.width : 0; + var fy = cr.height > 0 ? (ev.clientY - cr.top) / cr.height : 0; + if (fx < 0) fx = 0; else if (fx > 1) fx = 1; + if (fy < 0) fy = 0; else if (fy > 1) fy = 1; + + var dir = ev.deltaY < 0 ? 'up' : 'down'; + + /* Two states, not one mutated between the sends. Sending the same + * object twice and changing it in between only works while every + * consumer copies it synchronously -- client.html does, by wrapping it + * in a Guacamole.Mouse.State -- and silently sends the wrong thing to + * one that does not. A momentary button press is exactly the shape + * where that would be missed, since the second state is the one that + * survives. */ + function wheelState(pressed) { + var state = { + x : rect.left + fx * rect.width, + y : rect.top + fy * rect.height, + left : false, middle : false, right : false, + up : false, down : false + }; + state[dir] = pressed; + return state; + } + + sendState(wheelState(true)); + sendState(wheelState(false)); + + /* Sent behind pointer()'s back, so what it last saw is no longer what + * the session last got. */ + lastSent = null; + + ev.preventDefault(); + + }, { passive : false }); + +}; diff --git a/static/guac/Mouse.js b/static/guac/Mouse.js index 083616e..84064f5 100644 --- a/static/guac/Mouse.js +++ b/static/guac/Mouse.js @@ -136,6 +136,71 @@ Guacamole.Mouse = function Mouse(element) { }, false); + /* Recovers the moves the browser merged away. + * + * A busy main thread does not delay pointer movement, it *discards* it: + * moves that arrive while a handler is running are coalesced into one, and + * what is dispatched afterwards carries only the last position. A drag + * feels that immediately -- the path between two points is gone, so a + * dragged scrollbar thumb is dropped rather than merely lagged -- while a + * hover or a video degrades gracefully under the same load, which is why + * this is worth doing for one and not the other. + * + * getCoalescedEvents() returns the moves that were merged, the dispatched + * one last. Replaying them in order restores the path. It does not restore + * the timing: nothing running on a blocked thread can, and the positions + * still arrive together once the thread is free. The remote end sees where + * the pointer went rather than only where it ended up. + * + * This is deliberately additive. The mousemove listener above is untouched + * and still delivers every position it did before, so touch, the + * compatibility mouse events it produces and the ignore_mouse counter that + * swallows them all behave exactly as they did. The dispatched position + * arrives here first and again there, and the second is dropped by the + * unchanged-coordinates check in move(). Migrating this file to pointer + * events outright would be the tidier shape and a much larger change: + * every listener would have to move together, or synthetic mouse events + * from touch would arrive with nothing left counting them down. */ + if (typeof PointerEvent !== 'undefined' + && typeof PointerEvent.prototype.getCoalescedEvents === 'function' + && Guacamole.Mouse.coalescedMovement) { + + element.addEventListener("pointermove", function(e) { + + /* Touch has its own handling -- Guacamole.Touch and + * Guacamole.Mouse.Touchscreen -- and reaches the listener above + * as compatibility mouse events, which the counter below is there + * to swallow. Pen does not coalesce meaningfully. */ + if (e.pointerType !== 'mouse') + return; + + /* Recently touched: the listener above is counting synthetic + * events down and this must not interfere with either the count + * or what it is suppressing. */ + if (ignore_mouse) + return; + + /* Only while a button is held. The path matters for a drag and + * the endpoint is all that matters for a hover, and replaying + * every merged move of an idle pointer would multiply the + * upstream mouse traffic of a session that had nothing wrong with + * it. */ + if (!e.buttons) + return; + + var merged = e.getCoalescedEvents(); + if (!merged || merged.length <= 1) + return; + + var events = Guacamole.Mouse.sampleCoalesced(merged); + for (var i = 0; i < events.length; i++) + guac_mouse.move(Guacamole.Position.fromClientPosition(element, + events[i].clientX, events[i].clientY), events[i]); + + }, false); + + } + element.addEventListener("mousedown", function(e) { // Do not handle if ignoring events @@ -340,6 +405,85 @@ Guacamole.Mouse = function Mouse(element) { }; +/** + * Whether the moves the browser coalesced away during a drag are replayed, so + * that the remote end sees the path the pointer took rather than only where it + * ended up. See the pointermove listener above for what this does and does + * not recover. + * + * Settable as window.__mouseCoalesced, or as the mouseCoalesced key in + * localStorage to survive a reconnect -- which is the form that works on a + * live session, since client.html rebuilds its own URL on every launch and + * relaunch and drops anything added to it by hand. + * + * @type {!boolean} + */ +Guacamole.Mouse.coalescedMovement = (function coalescedMovement() { + + if (typeof window === 'undefined') + return true; + + if (window.__mouseCoalesced !== undefined) + return !!window.__mouseCoalesced; + + try { + var stored = window.localStorage + && window.localStorage.getItem('mouseCoalesced'); + if (stored === 'off' || stored === 'false') + return false; + } catch (e) { + /* Storage can be blocked outright; the global still works. */ + } + + return true; + +}()); + +/** + * The merged moves to replay, at most COALESCED_MAX of them, in order. + * + * A pointer reporting at 1000Hz behind a thread blocked for 100ms offers a + * hundred positions for one drag, and forwarding each as its own instruction + * would answer a rendering problem with a traffic one. Sampling evenly, and + * always keeping the last, holds the shape of the path at a bounded cost; an + * ordinary pointer never reaches the cap at all. + * + * @param {!Array} merged + * What getCoalescedEvents() returned. + * + * @returns {!Array} + * The events to replay, in order. + */ +Guacamole.Mouse.sampleCoalesced = function sampleCoalesced(merged) { + + if (merged.length <= Guacamole.Mouse.COALESCED_MAX) + return merged; + + var out = []; + var step = merged.length / Guacamole.Mouse.COALESCED_MAX; + + for (var i = 0; i < Guacamole.Mouse.COALESCED_MAX - 1; i++) + out.push(merged[Math.floor(i * step)]); + + /* The last is where the pointer actually is, so it is never the one + * dropped. */ + out.push(merged[merged.length - 1]); + + return out; + +}; + +/** + * The most merged moves replayed for one dispatched move. A pointer reporting + * at 1000Hz behind a thread blocked for 100ms offers a hundred positions for a + * single drag, and forwarding each as its own instruction would answer a + * rendering problem with a traffic one. An ordinary pointer never reaches + * this. + * + * @type {!number} + */ +Guacamole.Mouse.COALESCED_MAX = 16; + /** * The current state of a mouse, including position and buttons. * diff --git a/tests/browser/drive-pointer.mjs b/tests/browser/drive-pointer.mjs new file mode 100644 index 0000000..9c16c43 --- /dev/null +++ b/tests/browser/drive-pointer.mjs @@ -0,0 +1,113 @@ +#!/usr/bin/env node +/* + * Drives real pointer input at the pop-out monitor harness through the + * debugger, with the main thread deliberately blocked, so that the coalesced + * replay is exercised against the browser's own getCoalescedEvents(). + * + * The node tests feed it events shaped the way the code expects. This checks + * the assumption underneath them: that the browser merges moves at all when + * the thread is busy, and that what it hands back replays in order and inside + * the monitor it belongs to. + * + * Needs a Chrome listening on a debugging port and the harness server: + * node tests/browser/serve.mjs + * node tests/browser/drive-pointer.mjs [--port 9333] + */ + +const args = process.argv.slice(2); +const portAt = args.indexOf('--port'); +const port = portAt === -1 ? 9333 : +args[portAt + 1]; + +async function connect(wsUrl) { + const ws = new WebSocket(wsUrl); + await new Promise((ok, bad) => { ws.onopen = ok; ws.onerror = bad; }); + let next = 1; + const waiting = new Map(); + ws.onmessage = (m) => { + const msg = JSON.parse(m.data); + if (msg.id && waiting.has(msg.id)) { + const { ok, bad } = waiting.get(msg.id); + waiting.delete(msg.id); + msg.error ? bad(new Error(msg.error.message)) : ok(msg.result); + } + }; + return (method, params = {}) => new Promise((ok, bad) => { + const id = next++; + waiting.set(id, { ok, bad }); + ws.send(JSON.stringify({ id, method, params })); + }); +} + +const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json(); +const browser = await connect(version.webSocketDebuggerUrl); +const { targetId } = await browser('Target.createTarget', + { url: 'http://127.0.0.1:8099/monitor-pointer.html' }); +const list = await (await fetch(`http://127.0.0.1:${port}/json/list`)).json(); +const page = await connect(list.find((t) => t.id === targetId).webSocketDebuggerUrl); + +await page('Runtime.enable'); +await new Promise((r) => setTimeout(r, 1200)); + +const read = async (expression) => JSON.parse((await page('Runtime.evaluate', + { expression, returnByValue: true })).result.value); + +let failures = 0; +const check = (name, ok, detail = '') => { + if (ok) console.log(` ok ${name}`); + else { failures++; console.log(` FAIL ${name}${detail ? '\n ' + detail : ''}`); } +}; + +console.log('pop-out monitor pointer, real browser'); + +/* Press, then drag across the canvas while the main thread is busy, then + * release. The block is what makes the browser coalesce. */ +await page('Input.dispatchMouseEvent', { type: 'mousePressed', x: 150, y: 130, + button: 'left', buttons: 1, clickCount: 1, pointerType: 'mouse' }); + +page('Runtime.evaluate', { expression: 'window.__block(150)' }); +const moves = []; +for (let i = 0; i < 60; i++) + moves.push(page('Input.dispatchMouseEvent', { type: 'mouseMoved', + x: 150 + i * 10, y: 130 + i * 4, buttons: 1, + button: 'left', pointerType: 'mouse' })); +await Promise.all(moves); + +await new Promise((r) => setTimeout(r, 700)); +await page('Input.dispatchMouseEvent', { type: 'mouseReleased', x: 740, y: 366, + button: 'left', buttons: 0, clickCount: 1, pointerType: 'mouse' }); +await new Promise((r) => setTimeout(r, 300)); + +const sent = await read('JSON.stringify(window.__sent)'); +const maxCoalesced = await read('JSON.stringify(window.__maxCoalesced)'); + +console.log(` (the browser merged up to ${maxCoalesced} moves into one event;` + + ` ${sent.length} states were sent)`); + +check('the browser really did coalesce', maxCoalesced > 1, + `max coalesced was ${maxCoalesced} -- the thread was not busy enough, so` + + ' this run proves nothing either way'); + +check('real pointer input produced states', sent.length > 0); + +check('every position lands inside this monitor', + sent.every((s) => s.x >= 2560 && s.x <= 2560 + 1920 + && s.y >= 0 && s.y <= 1080), + JSON.stringify(sent.slice(0, 3))); + +check('the drag replays in order', + sent.filter((s) => s.left) + .every((s, i, a) => i === 0 || s.x >= a[i - 1].x), + 'positions went backwards, so the replay ran out of order'); + +check('no two consecutive states are identical', + sent.every((s, i) => i === 0 + || JSON.stringify(s) !== JSON.stringify(sent[i - 1])), + 'the repeat suppression let a duplicate through'); + +check('the release is reported with no button held', + sent.length > 0 && sent[sent.length - 1].left === false, + JSON.stringify(sent[sent.length - 1])); + +await browser('Target.closeTarget', { targetId }); +console.log(failures ? `\n${failures} failure(s)` : '\nall passed'); +process.exit(failures ? 1 : 0); diff --git a/tests/browser/monitor-pointer.html b/tests/browser/monitor-pointer.html new file mode 100644 index 0000000..f80d2dd --- /dev/null +++ b/tests/browser/monitor-pointer.html @@ -0,0 +1,69 @@ + + +