From f07620e71fd7b930ff654ca4d40da88ec3a2ae0a Mon Sep 17 00:00:00 2001 From: Youssef Date: Wed, 11 Nov 2026 17:18:42 +0200 Subject: [PATCH 1/6] ipc: pointer injection + agent integration - msg pointer [--button left|right|middle]: move the seat pointer in the output layout and optionally click there. Works in nested sessions (no uinput) and gives remote viewers/agents a coordinate input path. - docs/agents.md: how to drive driftwm with AT-SPI agents; docs/ipc.md: pointer command. - contrib/driftwm-agent: reference adapter (state/shot/focus/center/snap/click/type) and a WebSocket browser viewer with keyboard input. - cli docs regenerated. --- contrib/driftwm-agent/README.md | 70 +++++++++++++ contrib/driftwm-agent/package.json | 8 ++ contrib/driftwm-agent/public/index.html | 48 +++++++++ contrib/driftwm-agent/src/cli.ts | 130 +++++++++++++++++++++++ contrib/driftwm-agent/src/serve.ts | 131 ++++++++++++++++++++++++ docs/agents.md | 46 +++++++++ docs/cli.md | 16 +++ docs/ipc.md | 18 +++- src/ipc/client.rs | 14 +++ src/ipc/mod.rs | 51 +++++++++ src/ipc/protocol.rs | 14 +++ src/tests/cli_docs.rs | 7 ++ 12 files changed, 552 insertions(+), 1 deletion(-) create mode 100644 contrib/driftwm-agent/README.md create mode 100644 contrib/driftwm-agent/package.json create mode 100644 contrib/driftwm-agent/public/index.html create mode 100755 contrib/driftwm-agent/src/cli.ts create mode 100644 contrib/driftwm-agent/src/serve.ts create mode 100644 docs/agents.md diff --git a/contrib/driftwm-agent/README.md b/contrib/driftwm-agent/README.md new file mode 100644 index 00000000..e46f34ad --- /dev/null +++ b/contrib/driftwm-agent/README.md @@ -0,0 +1,70 @@ +# driftwm-agent + +Thin adapter between a **driftwm** Wayland session and agent tooling. Not a new agent framework: +it exposes driftwm's canvas (state / camera / zoom / canvas screenshot) next to +`agent-desktop`'s AT-SPI tree (snapshot / click), plus a browser viewer (noVNC-like). + +``` +driftwm IPC ──► driftwm-agent ──► CLI / WebSocket + state state, shot agents, browser + camera/zoom focus, center + screenshot snap, click, type +``` + +## Requirements + +- `driftwm` (built from source; set `DRIFTWM_BIN` if not in PATH) +- `agent-desktop` (npm, for AT-SPI snapshot/click) +- `wtype` (keyboard via virtual-keyboard protocol) +- `ydotool` + `ydotoold` (pointer; only works when driftwm owns libinput — see Limits) +- Bun 1.2+ + +## CLI + +```bash +bun run src/cli.ts state # camera, zoom, windows (id, app_id, pid, pos, size) +bun run src/cli.ts center thunar # pan camera to a window +bun run src/cli.ts shot --scale 1 # canvas PNG (1600x900 on a 1600x900 output) +bun run src/cli.ts shot --scale 2 # 3200x1800 - arbitrary capture resolution +bun run src/cli.ts snap thunar # agent-desktop AT-SPI snapshot (JSON) +bun run src/cli.ts click @s...:e7 # semantic AT-SPI click (no pointer needed) +bun run src/cli.ts type "hello" # wtype into the focused surface +bun run src/cli.ts msg zoom 1.5 # raw IPC passthrough +``` + +## Browser viewer (noVNC-like) + +```bash +bun run src/serve.ts --port 6081 --scale 1 [--token SECRET] [--view-only] [--host 0.0.0.0] +# open http://127.0.0.1:6081/ +``` + +- Frames: `driftwm msg screenshot -o -` (PNG, ~6 fps, canvas re-render — not a framebuffer grab). +- Input: pointer via ydotool, text via wtype, clicks via ydotool or AT-SPI. +- State overlay: camera / zoom / output size / window list. + +## Limits (measured 2026-11-11) + +- **Pointer input** goes through the `msg pointer` IPC command added on this branch + (`driftwm msg pointer [--button left]`), so it also works in nested sessions + where uinput events would go to the host compositor. `ydotool` remains a fallback for + native sessions. +- `agent-desktop`'s own `screenshot` refuses on Wayland (only GNOME/Mutter is measured); + the compositor's canvas screenshot replaces it. +- Canvas screenshots do not draw layer-shell panels (use `grim` for the literal output). +- Nested winit + llvmpipe costs ~172 MiB PSS idle; native udev on a GPU should be lower. + +## Remote options considered + +| Option | Verdict | +|---|---| +| this WebSocket streamer | works today, PNG frames, ~6 fps; best for agent/human on LAN | +| [Selkies](https://docs.selkies.io) (pixelflux + GStreamer/WebRTC) | production path: portal/PipeWire capture, H.264, browser; heavier stack | +| Sunshine + Moonlight | lowest latency (NVENC), native client, weak clipboard | +| wayvnc | **not compatible** (requires wlroots protocols; driftwm is Smithay) | +| KasmVNC / noVNC | X11 only | + +## Security + +Bind `127.0.0.1` (default) or a tailnet address; set `--token` before exposing it; `--view-only` +disables input entirely. diff --git a/contrib/driftwm-agent/package.json b/contrib/driftwm-agent/package.json new file mode 100644 index 00000000..1e462a12 --- /dev/null +++ b/contrib/driftwm-agent/package.json @@ -0,0 +1,8 @@ +{ + "name": "driftwm-agent", + "version": "0.1.0", + "private": true, + "type": "module", + "bin": { "driftwm-agent": "./src/cli.ts" }, + "scripts": { "start": "bun run src/cli.ts" } +} diff --git a/contrib/driftwm-agent/public/index.html b/contrib/driftwm-agent/public/index.html new file mode 100644 index 00000000..e835f942 --- /dev/null +++ b/contrib/driftwm-agent/public/index.html @@ -0,0 +1,48 @@ + +driftwm web + + +
driftwmconnecting…click to grab keyboard
+
+ diff --git a/contrib/driftwm-agent/src/cli.ts b/contrib/driftwm-agent/src/cli.ts new file mode 100755 index 00000000..44332641 --- /dev/null +++ b/contrib/driftwm-agent/src/cli.ts @@ -0,0 +1,130 @@ +#!/usr/bin/env bun +// driftwm-agent — thin adapter between a driftwm session and agent tooling. +// Not a new agent framework: it exposes driftwm's canvas (state/camera/zoom/screenshot) +// next to agent-desktop's AT-SPI tree (snapshot/click), so an external agent can use both. +import { spawnSync } from "node:child_process"; +import { existsSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const AD = process.env.AGENT_DESKTOP_BIN ?? + `${process.env.HOME}/.npm-global/lib/node_modules/agent-desktop/bin/agent-desktop`; + +function driftwmBin(): string { + if (process.env.DRIFTWM_BIN) return process.env.DRIFTWM_BIN; + const local = `${process.env.HOME}/Playground/driftwm/target/release/driftwm`; + if (existsSync(local)) return local; + const which = spawnSync("sh", ["-c", "command -v driftwm"], { encoding: "utf8" }); + if (which.status === 0 && which.stdout.trim()) return which.stdout.trim(); + fail("driftwm binary not found (set DRIFTWM_BIN)"); +} + +function fail(msg: string): never { + console.log(JSON.stringify({ ok: false, error: msg })); + process.exit(1); +} + +function run(cmd: string, args: string[]): { ok: boolean; out: string; err: string; code: number } { + const r = spawnSync(cmd, args, { encoding: "utf8" }); + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), err: (r.stderr ?? "").trim(), code: r.status ?? -1 }; +} + +function msg(args: string[]): any { + const bin = driftwmBin(); + const r = run(bin, ["msg", "--json", ...args]); + if (!r.ok) fail(`driftwm msg ${args.join(" ")} failed: ${r.err || r.out}`); + try { return JSON.parse(r.out); } catch { fail(`unparsable driftwm reply: ${r.out.slice(0, 200)}`); } +} + +function state(): any { + const reply = msg(["state"]); + return reply?.Ok?.State ?? fail("unexpected state reply"); +} + +function pngSize(path: string): { width: number; height: number } { + const buf = require("node:fs").readFileSync(path); + if (buf.length < 24 || buf.toString("ascii", 1, 4) !== "PNG") fail(`not a PNG: ${path}`); + return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; +} + +function findWindow(s: any, sel: string): any { + const byId = /^\d+$/.test(sel) ? s.windows.find((w: any) => w.id === Number(sel)) : undefined; + if (byId) return byId; + const needle = sel.toLowerCase(); + return s.windows.find((w: any) => String(w.app_id).toLowerCase().includes(needle) || + String(w.title).toLowerCase().includes(needle)); +} + +const [cmd, ...rest] = process.argv.slice(2); +switch (cmd) { + case "state": { + const s = state(); + console.log(JSON.stringify({ ok: true, camera: s.camera, zoom: s.zoom, layout: s.layout, windows: s.windows, outputs: s.outputs })); + break; + } + case "shot": { + const scale = (() => { const i = rest.indexOf("--scale"); return i >= 0 ? rest[i + 1] : "1"; })(); + const out = join(tmpdir(), `driftwm-agent-${Date.now()}.png`); + const bin = driftwmBin(); + const r = run(bin, ["msg", "screenshot", "-o", out, "--scale", scale]); + if (!r.ok) fail(`screenshot failed: ${r.err || r.out}`); + const size = pngSize(out); + console.log(JSON.stringify({ ok: true, path: out, scale: Number(scale), ...size })); + break; + } + case "focus": { + const sel = rest[0] ?? fail("focus needs "); + const bin = driftwmBin(); + const args = /^\d+$/.test(sel) ? ["--id", sel] : [sel]; + const r = run(bin, ["msg", "focus", ...args]); + if (!r.ok) fail(`focus failed: ${r.err || r.out}`); + console.log(JSON.stringify({ ok: true, focused: r.out })); + break; + } + case "center": { + const sel = rest[0] ?? fail("center needs "); + const s = state(); + const w = findWindow(s, sel) ?? fail(`no window matching ${sel}`); + const r = run(driftwmBin(), ["msg", "camera", String(w.position[0]), String(w.position[1])]); + if (!r.ok) fail(`camera failed: ${r.err || r.out}`); + console.log(JSON.stringify({ ok: true, window: { id: w.id, app_id: w.app_id, title: w.title }, camera: [w.position[0], w.position[1]] })); + break; + } + case "snap": { + const app = rest[0] ?? fail("snap needs "); + const r = run(AD, ["snapshot", "--app", app, "-i", "--compact"]); + if (!r.ok) fail(`agent-desktop snapshot failed: ${r.err || r.out}`); + console.log(r.out); + break; + } + case "click": { + const ref = rest[0] ?? fail("click needs "); + const r = run(AD, ["click", ref]); + console.log(r.out || JSON.stringify({ ok: false, error: r.err })); + if (!r.ok) process.exit(1); + break; + } + case "type": { + const text = rest.join(" "); + if (!text) fail("type needs "); + const which = spawnSync("sh", ["-c", "command -v wtype"], { encoding: "utf8" }); + if (which.status !== 0) fail("wtype not installed (apt install wtype)"); + const r = run("wtype", ["--", text]); + if (!r.ok) fail(`wtype failed: ${r.err}`); + console.log(JSON.stringify({ ok: true, typed: text.length })); + break; + } + case "msg": { + const bin = driftwmBin(); + const r = run(bin, ["msg", "--json", ...rest]); + console.log(r.out || JSON.stringify({ ok: false, error: r.err })); + if (!r.ok) process.exit(1); + break; + } + default: + console.log(JSON.stringify({ + ok: false, + usage: "driftwm-agent |center |snap |click |type |msg ...>", + })); + process.exit(2); +} diff --git a/contrib/driftwm-agent/src/serve.ts b/contrib/driftwm-agent/src/serve.ts new file mode 100644 index 00000000..4f4fc889 --- /dev/null +++ b/contrib/driftwm-agent/src/serve.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env bun +// driftwm-web — noVNC-like browser viewer for a driftwm session. +// Frames: `driftwm msg screenshot -o -` (canvas re-render, PNG) -> WebSocket binary. +// Input: pointer via `driftwm msg pointer` (IPC), text via wtype (virtual keyboard). +// Usage: bun run src/serve.ts [--port 6081] [--host 127.0.0.1] [--scale 1] [--token SECRET] [--view-only] +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; + +const args = process.argv.slice(2); +const arg = (name: string, dflt: string) => { const i = args.indexOf(`--${name}`); return i >= 0 ? args[i + 1] : dflt; }; +const has = (name: string) => args.includes(`--${name}`); + +const PORT = Number(arg("port", "6081")); +const HOST = arg("host", "127.0.0.1"); +const SCALE = Number(arg("scale", "1")); +const TOKEN = arg("token", ""); +const VIEW_ONLY = has("view-only"); +const DRIFTWM = process.env.DRIFTWM_BIN ?? `${process.env.HOME}/Playground/driftwm/target/release/driftwm`; + +// ydotoold is a user daemon; its socket lives in the user runtime dir, which is +// not necessarily the session XDG_RUNTIME_DIR a test harness uses. +if (!process.env.YDOTOOL_SOCKET) { + for (const p of [`/run/user/1000/.ydotool_socket`, `/tmp/.ydotool_socket`]) { + if (require("node:fs").existsSync(p)) { process.env.YDOTOOL_SOCKET = p; break; } + } +} + +function run(cmd: string, a: string[], maxBuffer = 32 * 1024 * 1024) { + return spawnSync(cmd, a, { maxBuffer, encoding: "buffer" }); +} + +let camera = { x: 0, y: 0 }, zoom = 1, output = { width: 1600, height: 900 }; +let windows: any[] = []; +let lastFrameAt = 0; + +type WS = { sendBinary(b: Uint8Array): void; send(s: string): void; close(): void }; +const clients = new Set(); + +function refreshState() { + const r = run(DRIFTWM, ["msg", "--json", "state"]); + if (r.status !== 0) return; + try { + const s = JSON.parse(r.stdout.toString()).Ok.State; + camera = { x: s.camera[0], y: s.camera[1] }; + zoom = s.zoom; + windows = s.windows; + const o = s.outputs.find((o: any) => o.active) ?? s.outputs[0]; + if (o) output = { width: o.size[0], height: o.size[1] }; + } catch {} +} + +function capture() { + if (clients.size === 0) return; + const r = run(DRIFTWM, ["msg", "screenshot", "-o", "-", "--scale", String(SCALE)]); + if (r.status !== 0 || !r.stdout.length) return; + lastFrameAt = Date.now(); + for (const c of clients) { try { c.sendBinary(r.stdout); } catch {} } +} + +function stateMessage() { + return JSON.stringify({ t: "state", camera, zoom, output, windows, scale: SCALE, viewOnly: VIEW_ONLY, frameAgeMs: Date.now() - lastFrameAt }); +} + +/** screen-space pixel from a canvas-view pixel in the received frame (frame is camera view at SCALE) */ +function toScreen(px: number, py: number) { + return { x: Math.round(px / SCALE), y: Math.round(py / SCALE) }; +} + +function cursor(): { x: number; y: number } { + const r = run(DRIFTWM, ["msg", "cursor"]); + if (r.status !== 0) return { x: 0, y: 0 }; + const m = r.stdout.toString().trim().match(/^(-?\d+)\s+(-?\d+)/); + return m ? { x: Number(m[1]), y: Number(m[2]) } : { x: 0, y: 0 }; +} +let lastPointer = { x: 0, y: 0 }; +function pointer(px: number, py: number) { + const p = toScreen(px, py); + lastPointer = p; + run(DRIFTWM, ["msg", "pointer", String(p.x), String(p.y)]); +} + +function click(button = "left") { + const p = lastPointer; + run(DRIFTWM, ["msg", "pointer", String(p.x), String(p.y), "--button", button]); +} + +function typeText(text: string) { + run("wtype", ["--", text]); +} + +function keyPress(key: string) { + run("wtype", ["-k", key]); +} + +Bun.serve({ + port: PORT, + hostname: HOST, + fetch(req, server) { + const url = new URL(req.url); + if (url.pathname === "/ws") { + if (TOKEN && url.searchParams.get("token") !== TOKEN) return new Response("forbidden", { status: 403 }); + if (server.upgrade(req)) return; + return new Response("upgrade failed", { status: 400 }); + } + if (url.pathname === "/" || url.pathname === "/index.html") { + return new Response(readFileSync(`${import.meta.dir}/../public/index.html`), { headers: { "content-type": "text/html" } }); + } + return new Response("not found", { status: 404 }); + }, + websocket: { + open(ws: any) { clients.add(ws); ws.send(stateMessage()); }, + close(ws: any) { clients.delete(ws); }, + message(ws: any, raw: any) { + if (VIEW_ONLY) return; + let m: any; try { m = JSON.parse(String(raw)); } catch { return; } + switch (m.t) { + case "pointer": pointer(Number(m.x) || 0, Number(m.y) || 0); break; + case "click": click(String(m.button ?? "left")); break; + case "text": typeText(String(m.text ?? "").slice(0, 512)); break; + case "key": keyPress(String(m.key ?? "")); break; + case "state": ws.send(stateMessage()); break; + } + }, + }, +}); + +refreshState(); +setInterval(refreshState, 1500); +setInterval(capture, 160); +setInterval(() => { const s = stateMessage(); for (const c of clients) { try { c.send(s); } catch {} } }, 1500); +console.log(JSON.stringify({ ok: true, listening: `http://${HOST}:${PORT}/`, scale: SCALE, token: !!TOKEN, viewOnly: VIEW_ONLY })); diff --git a/docs/agents.md b/docs/agents.md new file mode 100644 index 00000000..c46ec029 --- /dev/null +++ b/docs/agents.md @@ -0,0 +1,46 @@ +# Driving driftwm with AI agents + +driftwm is unusually well suited to computer-use agents: the canvas is queryable and +renderable over IPC, and windows carry a stable id plus their client pid. This page +describes the integration and the pieces added for it. + +## What the IPC gives an agent + +| Need | Command | +|---|---| +| Window inventory (id, app_id, title, canvas position/size, pid, focus) | `driftwm msg --json state` | +| Canvas screenshot at arbitrary scale, including off-screen content | `driftwm msg screenshot -o - --scale N` | +| Spatial navigation | `driftwm msg camera `, `driftwm msg zoom `, `driftwm msg focus ` | +| Pointer input without uinput | `driftwm msg pointer [--button left|right|middle]` | +| Stable window identity | `state.windows[].id` / `[].pid` | +| Bookmarks | `driftwm msg bookmark ...` | + +A screenshot is a **canvas re-render**, not a framebuffer copy: an agent does not need the +window on screen. `--scale 2` on a 1600×900 output yields 3200×1800, which is useful for +OCR/vision stages. + +`pointer` is deliberately part of the IPC rather than a Wayland protocol: it works in +nested sessions (where uinput events would go to the host compositor) and needs no extra +device permissions. The socket is user-only (0600); treat it as an input-injection +surface and keep it inside the session. + +## Glue for AT-SPI agents + +`agent-desktop` (AT-SPI) can see app windows inside driftwm and click them semantically +(no pointer). Pair it with driftwm's IPC: + +- map `agent-desktop` windows to driftwm ids by `pid`; +- use `driftwm msg focus ` to bring a window into view before snapshotting; +- use `driftwm msg screenshot` for observation (agent-desktop's own Wayland capture only + supports GNOME/Mutter); +- use `driftwm msg pointer ... --button left` when a coordinate click is required. + +A reference implementation of this glue (CLI + WebSocket viewer) lives in +[`contrib/driftwm-agent`](../contrib/driftwm-agent). + +## Remaining gaps + +1. **Video streaming.** Each `msg screenshot` re-renders; a remote viewer should throttle + (5–10 fps) or capture through `zwlr_screencopy`/portal for H.264/WebRTC. +2. **Security.** Input IPC is a deliberate capability; if it ever becomes configurable, + default it on only for the session owner. diff --git a/docs/cli.md b/docs/cli.md index e1a1a996..6bfe74b1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -65,6 +65,7 @@ driftwm msg --json focus --id 5 | [`bookmark`](#driftwm-msg-bookmark) | List bookmarks, get or set one by ``, or delete with `--delete` | | [`layout`](#driftwm-msg-layout) | Print the active keyboard layout (full XKB name, e.g. `English (US)`) | | [`cursor`](#driftwm-msg-cursor) | Print the pointer position in the global output layout (logical pixels, Y-down, the space slurp prints) and the output it is on | +| [`pointer`](#driftwm-msg-pointer) | Move the seat pointer to in the output layout (logical pixels, Y-down, the space `cursor` reports); with --button, click there | | [`action`](#driftwm-msg-action) | Run a config action, e.g. `action close-window`, `action switch-layout next` | | [`screenshot`](#driftwm-msg-screenshot) | Capture a canvas PNG. With no subcommand, captures the active output's current view of the canvas | | [`debug-counters`](#driftwm-msg-debug-counters) | Print internal collection sizes for leak diagnosis (unstable keys) | @@ -337,6 +338,21 @@ Print the pointer position in the global output layout (logical pixels, Y-down, driftwm msg --json cursor ``` +#### `driftwm msg pointer` + +``` +driftwm msg pointer [OPTIONS] +``` + +Move the seat pointer to in the output layout (logical pixels, Y-down, the space `cursor` reports); with --button, click there + +- `--button ` + +```bash +driftwm msg pointer 640 400 +driftwm msg pointer 640 400 --button left +``` + #### `driftwm msg action` ``` diff --git a/docs/ipc.md b/docs/ipc.md index 231b9e90..55d23a1e 100644 --- a/docs/ipc.md +++ b/docs/ipc.md @@ -9,7 +9,7 @@ JSON, so any language can speak it directly. Run `driftwm msg ` from inside a driftwm session. The commands — `camera`, `zoom`, `focus`, `move`, `resize`, `opacity`, `pin`, `close`, -`suspend`, `relaunch`, `layout`, `cursor`, `action`, `bookmark`, `screenshot`, +`suspend`, `relaunch`, `layout`, `cursor`, `pointer`, `action`, `bookmark`, `screenshot`, `state`, `subscribe`, and `debug-counters` — with their arguments, flags, and JSON reply shapes are documented in the generated [CLI reference](cli.md); `driftwm msg --help` prints the same for one command. The conventions @@ -130,6 +130,22 @@ reaches off-screen content; its four targets, flags, and caveats are in the [CLI reference](cli.md#driftwm-msg-screenshot) and their wire forms in [Requests](#requests). +### Pointer injection + +`pointer [--button left|right|middle]` moves the seat pointer to a point in the +global output layout (logical pixels, Y-down — the same space `cursor` reports) and, with +`--button`, clicks there. + +This is the input path for remote viewers and coordinate-based agents: it does not depend +on uinput, so it also works in nested sessions, where host input devices are not visible +to driftwm. The reply is the resulting pointer position: + +```bash +driftwm msg pointer 640 400 +driftwm msg --json pointer 640 400 --button left +# {"Ok":{"Pointer":{"x":640.0,"y":400.0,"output":"winit"}}} +``` + ### Subscribing to changes `subscribe` turns the connection into a live feed instead of polling: one event diff --git a/src/ipc/client.rs b/src/ipc/client.rs index afc51fdb..b1388722 100644 --- a/src/ipc/client.rs +++ b/src/ipc/client.rs @@ -203,6 +203,14 @@ pub enum Msg { /// /// `--json` reply: `{"Ok":{"Cursor":{"x":2600.5,"y":12.0,"output":"DP-1"}}}`. Cursor, + /// Move the seat pointer to in the output layout (logical pixels, + /// Y-down, the space `cursor` reports); with --button, click there. + Pointer { + x: f64, + y: f64, + #[arg(long, value_name = "left|right|middle")] + button: Option, + }, /// Run a config action, e.g. `action close-window`, `action switch-layout next`. /// /// Takes the same string you would write in a config keybinding, parsed with @@ -405,6 +413,11 @@ fn to_request(msg: &Msg) -> Result { Msg::Zoom { level } => Request::Zoom(*level), Msg::Layout { short } => Request::Layout { short: *short }, Msg::Cursor => Request::Cursor, + Msg::Pointer { x, y, button } => Request::Pointer { + x: *x, + y: *y, + button: button.clone(), + }, Msg::State => Request::State, Msg::DebugCounters => Request::DebugCounters, Msg::Subscribe => Request::Subscribe, @@ -623,6 +636,7 @@ fn print_response(response: Response) { Response::Zoom(zoom) => println!("{zoom}"), Response::Layout(layout) => println!("{layout}"), Response::Cursor { x, y, output } => println!("{x} {y} {output}"), + Response::Pointer { x, y, output } => println!("{x} {y} {output}"), Response::Focused(Some(w)) => { println!("#{} {}", w.id, w.app_id.as_deref().unwrap_or("(no app_id)")) } diff --git a/src/ipc/mod.rs b/src/ipc/mod.rs index 36ad97d4..82422a38 100644 --- a/src/ipc/mod.rs +++ b/src/ipc/mod.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use std::rc::Rc; use std::time::{Duration, Instant}; +use smithay::backend::input::ButtonState; +use smithay::input::pointer::ButtonEvent; use smithay::reexports::calloop::generic::Generic; use smithay::reexports::calloop::{Interest, LoopHandle, Mode, PostAction}; use smithay::reexports::wayland_server::Resource; @@ -268,6 +270,7 @@ pub(crate) fn dispatch(request: Request, state: &mut DriftWm) -> Reply { Request::Zoom(arg) => cmd_zoom(arg, state), Request::Layout { short } => cmd_layout(short, state), Request::Cursor => cmd_cursor(state), + Request::Pointer { x, y, button } => cmd_pointer(x, y, button, state), Request::State => Ok(cmd_state(state)), Request::DebugCounters => Ok(Response::DebugCounters(state.debug_counters())), // Handled in serve_connection, which has the raw stream Subscribe needs. @@ -398,6 +401,54 @@ fn cmd_cursor(state: &mut DriftWm) -> Reply { }) } +fn cmd_pointer(x: f64, y: f64, button: Option, state: &mut DriftWm) -> Reply { + let output = state.active_output().ok_or("no output")?; + let (camera, zoom, origin) = { + let os = crate::state::output_state(&output); + (os.camera, os.zoom, os.layout_position) + }; + let screen = Point::::from((x - origin.x as f64, y - origin.y as f64)); + let canvas = + driftwm::canvas::screen_to_canvas(driftwm::canvas::ScreenPos(screen), camera, zoom).0; + let serial = SERIAL_COUNTER.next_serial(); + let time = std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .map(|d| d.subsec_millis()) + .unwrap_or(0); + state.dispatch_absolute_motion(&output, screen, canvas, serial, time); + if let Some(name) = button { + let code = match name.as_str() { + "left" => 0x110, + "right" => 0x111, + "middle" => 0x112, + _ => return Err(format!("unknown button {name}")), + }; + let pointer = state.seat.get_pointer().ok_or("no pointer")?; + for button_state in [ButtonState::Pressed, ButtonState::Released] { + let serial = SERIAL_COUNTER.next_serial(); + pointer.button( + state, + &ButtonEvent { + button: code, + state: button_state, + serial, + time, + }, + ); + } + pointer.frame(state); + } + let pointer = state.seat.get_pointer().ok_or("no pointer")?; + let location = pointer.current_location(); + let screen = + driftwm::canvas::canvas_to_screen(driftwm::canvas::CanvasPos(location), camera, zoom).0; + Ok(Response::Pointer { + x: origin.x as f64 + screen.x, + y: origin.y as f64 + screen.y, + output: output.name(), + }) +} + fn cmd_layout(short: bool, state: &mut DriftWm) -> Reply { if !short { return Ok(Response::Layout(state.active_layout.clone())); diff --git a/src/ipc/protocol.rs b/src/ipc/protocol.rs index 9e402cd8..71ce2a80 100644 --- a/src/ipc/protocol.rs +++ b/src/ipc/protocol.rs @@ -35,6 +35,15 @@ pub enum Request { /// The pointer in the global output layout (screen space, not canvas), plus /// the output it is on. Cursor, + /// Move the seat pointer to a point in the global output layout (logical + /// pixels, Y-down — the space `cursor` reports), optionally clicking a + /// button there: `left`, `right` or `middle`. + Pointer { + x: f64, + y: f64, + #[serde(default)] + button: Option, + }, State, /// Sizes of the compositor's leak-prone internal collections, keyed by /// field name. A debug/introspection endpoint — the keys are unstable @@ -158,6 +167,11 @@ pub enum Response { y: f64, output: String, }, + Pointer { + x: f64, + y: f64, + output: String, + }, /// Boxed so the whole-state snapshot doesn't bloat every other (tiny) /// reply variant. Serde-transparent, so the wire shape is unchanged. State(Box), diff --git a/src/tests/cli_docs.rs b/src/tests/cli_docs.rs index b708d43f..b61d4c15 100644 --- a/src/tests/cli_docs.rs +++ b/src/tests/cli_docs.rs @@ -68,6 +68,13 @@ const EXAMPLES: &[(&str, &[&str])] = &[ "driftwm msg camera", &["driftwm msg camera", "driftwm msg camera 500 300"], ), + ( + "driftwm msg pointer", + &[ + "driftwm msg pointer 640 400", + "driftwm msg pointer 640 400 --button left", + ], + ), ("driftwm msg zoom", &["driftwm msg zoom 0.5"]), ( "driftwm msg bookmark", From 0b74184ecd68a855cfc85d6e4cdba485d8b74d7e Mon Sep 17 00:00:00 2001 From: Youssef Date: Wed, 11 Nov 2026 17:44:57 +0200 Subject: [PATCH 2/6] ipc: axis (scroll) + jpeg viewer path + wheel input - msg axis : scroll the focused surface by wheel notches (verified: wev sees axis_source wheel, value120, value) - contrib viewer: mouse-wheel forwarding, optional --jpeg (ffmpeg transcode, ~39 KB/frame at 1600x900) and --fps. - docs/ipc.md scrolling section, cli docs regenerated, hyphen values accepted for pointer/axis args. --- contrib/driftwm-agent/public/index.html | 5 +++++ contrib/driftwm-agent/src/serve.ts | 15 ++++++++++++--- docs/cli.md | 13 +++++++++++++ docs/ipc.md | 10 ++++++++++ src/ipc/client.rs | 10 ++++++++++ src/ipc/mod.rs | 24 ++++++++++++++++++++++-- src/ipc/protocol.rs | 10 ++++++++++ src/tests/cli_docs.rs | 1 + 8 files changed, 83 insertions(+), 5 deletions(-) diff --git a/contrib/driftwm-agent/public/index.html b/contrib/driftwm-agent/public/index.html index e835f942..8a4180dd 100644 --- a/contrib/driftwm-agent/public/index.html +++ b/contrib/driftwm-agent/public/index.html @@ -37,6 +37,11 @@ cv.addEventListener('mousemove', (ev) => { if (meta.viewOnly) return; const p = toFrame(ev); ws.send(JSON.stringify({ t: 'pointer', x: p.x, y: p.y })); }); cv.addEventListener('mousedown', (ev) => { if (meta.viewOnly) return; ev.preventDefault(); const p = toFrame(ev); ws.send(JSON.stringify({ t: 'pointer', x: p.x, y: p.y })); ws.send(JSON.stringify({ t: 'click', button: ev.button === 2 ? 'right' : ev.button === 1 ? 'middle' : 'left' })); }); cv.addEventListener('contextmenu', e => e.preventDefault()); +cv.addEventListener('wheel', (ev) => { + if (meta.viewOnly) return; + ev.preventDefault(); + ws.send(JSON.stringify({ t: 'axis', dx: ev.deltaX / 100, dy: ev.deltaY / 100 })); +}, { passive: false }); addEventListener('keydown', (ev) => { if (meta.viewOnly) return; if (ev.key.length === 1) { ws.send(JSON.stringify({ t: 'text', text: ev.key })); } diff --git a/contrib/driftwm-agent/src/serve.ts b/contrib/driftwm-agent/src/serve.ts index 4f4fc889..a4aba135 100644 --- a/contrib/driftwm-agent/src/serve.ts +++ b/contrib/driftwm-agent/src/serve.ts @@ -15,6 +15,8 @@ const HOST = arg("host", "127.0.0.1"); const SCALE = Number(arg("scale", "1")); const TOKEN = arg("token", ""); const VIEW_ONLY = has("view-only"); +const JPEG = has("jpeg"); +const FPS = Number(arg("fps", "6")); const DRIFTWM = process.env.DRIFTWM_BIN ?? `${process.env.HOME}/Playground/driftwm/target/release/driftwm`; // ydotoold is a user daemon; its socket lives in the user runtime dir, which is @@ -53,8 +55,14 @@ function capture() { if (clients.size === 0) return; const r = run(DRIFTWM, ["msg", "screenshot", "-o", "-", "--scale", String(SCALE)]); if (r.status !== 0 || !r.stdout.length) return; + let frame: Buffer = r.stdout; + if (JPEG) { + const j = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "image2pipe", "-i", "-", "-frames:v", "1", "-f", "mjpeg", "-q:v", "5", "pipe:1"], { input: r.stdout, maxBuffer: 32 * 1024 * 1024 }); + if (j.status !== 0 || !j.stdout?.length) return; + frame = j.stdout; + } lastFrameAt = Date.now(); - for (const c of clients) { try { c.sendBinary(r.stdout); } catch {} } + for (const c of clients) { try { c.sendBinary(frame); } catch {} } } function stateMessage() { @@ -118,6 +126,7 @@ Bun.serve({ case "click": click(String(m.button ?? "left")); break; case "text": typeText(String(m.text ?? "").slice(0, 512)); break; case "key": keyPress(String(m.key ?? "")); break; + case "axis": run(DRIFTWM, ["msg", "axis", String(Number(m.dx) || 0), String(Number(m.dy) || 0)]); break; case "state": ws.send(stateMessage()); break; } }, @@ -126,6 +135,6 @@ Bun.serve({ refreshState(); setInterval(refreshState, 1500); -setInterval(capture, 160); +setInterval(capture, Math.max(50, Math.round(1000 / Math.max(1, FPS)))); setInterval(() => { const s = stateMessage(); for (const c of clients) { try { c.send(s); } catch {} } }, 1500); -console.log(JSON.stringify({ ok: true, listening: `http://${HOST}:${PORT}/`, scale: SCALE, token: !!TOKEN, viewOnly: VIEW_ONLY })); +console.log(JSON.stringify({ ok: true, listening: `http://${HOST}:${PORT}/`, scale: SCALE, jpeg: JPEG, fps: FPS, token: !!TOKEN, viewOnly: VIEW_ONLY })); diff --git a/docs/cli.md b/docs/cli.md index 6bfe74b1..049cf8a1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -66,6 +66,7 @@ driftwm msg --json focus --id 5 | [`layout`](#driftwm-msg-layout) | Print the active keyboard layout (full XKB name, e.g. `English (US)`) | | [`cursor`](#driftwm-msg-cursor) | Print the pointer position in the global output layout (logical pixels, Y-down, the space slurp prints) and the output it is on | | [`pointer`](#driftwm-msg-pointer) | Move the seat pointer to in the output layout (logical pixels, Y-down, the space `cursor` reports); with --button, click there | +| [`axis`](#driftwm-msg-axis) | Scroll the focused surface by wheel notches (positive = right/down) | | [`action`](#driftwm-msg-action) | Run a config action, e.g. `action close-window`, `action switch-layout next` | | [`screenshot`](#driftwm-msg-screenshot) | Capture a canvas PNG. With no subcommand, captures the active output's current view of the canvas | | [`debug-counters`](#driftwm-msg-debug-counters) | Print internal collection sizes for leak diagnosis (unstable keys) | @@ -353,6 +354,18 @@ driftwm msg pointer 640 400 driftwm msg pointer 640 400 --button left ``` +#### `driftwm msg axis` + +``` +driftwm msg axis +``` + +Scroll the focused surface by wheel notches (positive = right/down) + +```bash +driftwm msg axis 0 -1 +``` + #### `driftwm msg action` ``` diff --git a/docs/ipc.md b/docs/ipc.md index 55d23a1e..71225e91 100644 --- a/docs/ipc.md +++ b/docs/ipc.md @@ -130,6 +130,16 @@ reaches off-screen content; its four targets, flags, and caveats are in the [CLI reference](cli.md#driftwm-msg-screenshot) and their wire forms in [Requests](#requests). +### Scrolling + +`axis ` scrolls the focused surface by wheel notches (positive = right/down). +It is what a remote viewer forwards from mouse-wheel events; the reply echoes the deltas. + +```bash +driftwm msg axis 0 -1 # one notch up +driftwm msg axis 0 3 # three notches down +``` + ### Pointer injection `pointer [--button left|right|middle]` moves the seat pointer to a point in the diff --git a/src/ipc/client.rs b/src/ipc/client.rs index b1388722..f75f83b8 100644 --- a/src/ipc/client.rs +++ b/src/ipc/client.rs @@ -211,6 +211,14 @@ pub enum Msg { #[arg(long, value_name = "left|right|middle")] button: Option, }, + /// Scroll the focused surface by wheel notches (positive = + /// right/down). + Axis { + #[arg(allow_hyphen_values = true)] + dx: f64, + #[arg(allow_hyphen_values = true)] + dy: f64, + }, /// Run a config action, e.g. `action close-window`, `action switch-layout next`. /// /// Takes the same string you would write in a config keybinding, parsed with @@ -418,6 +426,7 @@ fn to_request(msg: &Msg) -> Result { y: *y, button: button.clone(), }, + Msg::Axis { dx, dy } => Request::Axis { dx: *dx, dy: *dy }, Msg::State => Request::State, Msg::DebugCounters => Request::DebugCounters, Msg::Subscribe => Request::Subscribe, @@ -637,6 +646,7 @@ fn print_response(response: Response) { Response::Layout(layout) => println!("{layout}"), Response::Cursor { x, y, output } => println!("{x} {y} {output}"), Response::Pointer { x, y, output } => println!("{x} {y} {output}"), + Response::Axis { dx, dy } => println!("{dx} {dy}"), Response::Focused(Some(w)) => { println!("#{} {}", w.id, w.app_id.as_deref().unwrap_or("(no app_id)")) } diff --git a/src/ipc/mod.rs b/src/ipc/mod.rs index 82422a38..306a06f6 100644 --- a/src/ipc/mod.rs +++ b/src/ipc/mod.rs @@ -6,8 +6,8 @@ use std::path::PathBuf; use std::rc::Rc; use std::time::{Duration, Instant}; -use smithay::backend::input::ButtonState; -use smithay::input::pointer::ButtonEvent; +use smithay::backend::input::{Axis, AxisSource, ButtonState}; +use smithay::input::pointer::{AxisFrame, ButtonEvent}; use smithay::reexports::calloop::generic::Generic; use smithay::reexports::calloop::{Interest, LoopHandle, Mode, PostAction}; use smithay::reexports::wayland_server::Resource; @@ -271,6 +271,7 @@ pub(crate) fn dispatch(request: Request, state: &mut DriftWm) -> Reply { Request::Layout { short } => cmd_layout(short, state), Request::Cursor => cmd_cursor(state), Request::Pointer { x, y, button } => cmd_pointer(x, y, button, state), + Request::Axis { dx, dy } => cmd_axis(dx, dy, state), Request::State => Ok(cmd_state(state)), Request::DebugCounters => Ok(Response::DebugCounters(state.debug_counters())), // Handled in serve_connection, which has the raw stream Subscribe needs. @@ -449,6 +450,25 @@ fn cmd_pointer(x: f64, y: f64, button: Option, state: &mut DriftWm) -> R }) } +fn cmd_axis(dx: f64, dy: f64, state: &mut DriftWm) -> Reply { + let pointer = state.seat.get_pointer().ok_or("no pointer")?; + let time = std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .map(|d| d.subsec_millis()) + .unwrap_or(0); + let vertical = (dy * 120.0) as i32; + let horizontal = (dx * 120.0) as i32; + let frame = AxisFrame::new(time) + .source(AxisSource::Wheel) + .v120(Axis::Vertical, vertical) + .v120(Axis::Horizontal, horizontal) + .value(Axis::Vertical, dy * 15.0) + .value(Axis::Horizontal, dx * 15.0); + pointer.axis(state, frame); + pointer.frame(state); + Ok(Response::Axis { dx, dy }) +} + fn cmd_layout(short: bool, state: &mut DriftWm) -> Reply { if !short { return Ok(Response::Layout(state.active_layout.clone())); diff --git a/src/ipc/protocol.rs b/src/ipc/protocol.rs index 71ce2a80..29a50de3 100644 --- a/src/ipc/protocol.rs +++ b/src/ipc/protocol.rs @@ -44,6 +44,12 @@ pub enum Request { #[serde(default)] button: Option, }, + /// Scroll the focused surface: `dx` is horizontal, `dy` vertical, in wheel + /// notches (positive = right/down). + Axis { + dx: f64, + dy: f64, + }, State, /// Sizes of the compositor's leak-prone internal collections, keyed by /// field name. A debug/introspection endpoint — the keys are unstable @@ -172,6 +178,10 @@ pub enum Response { y: f64, output: String, }, + Axis { + dx: f64, + dy: f64, + }, /// Boxed so the whole-state snapshot doesn't bloat every other (tiny) /// reply variant. Serde-transparent, so the wire shape is unchanged. State(Box), diff --git a/src/tests/cli_docs.rs b/src/tests/cli_docs.rs index b61d4c15..3ba9a84e 100644 --- a/src/tests/cli_docs.rs +++ b/src/tests/cli_docs.rs @@ -75,6 +75,7 @@ const EXAMPLES: &[(&str, &[&str])] = &[ "driftwm msg pointer 640 400 --button left", ], ), + ("driftwm msg axis", &["driftwm msg axis 0 -1"]), ("driftwm msg zoom", &["driftwm msg zoom 0.5"]), ( "driftwm msg bookmark", From b4dc20ba9fac0da4c50002cb9dec96a019593eee Mon Sep 17 00:00:00 2001 From: Youssef Date: Wed, 11 Nov 2026 18:47:00 +0200 Subject: [PATCH 3/6] udev: --drm-device to select the KMS card (headless vkms) Without it the udev backend prefers the system primary GPU. On a headless box that card has no active scanout; an explicit device (e.g. /dev/dri/card0 from vkms) lets driftwm run a real 1920x1080 session with no monitor and no X server. Verified locally: driftwm --backend udev --drm-device /dev/dri/card0 on vkms, output 'Virtual-1' at 1920x1080@60, IPC state + 1920x1080 canvas screenshot. --- docs/cli.md | 1 + src/backend/udev.rs | 13 +++++++++---- src/main.rs | 7 ++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 049cf8a1..0aef14fb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -18,6 +18,7 @@ A trackpad-first infinite canvas Wayland compositor. With no subcommand, starts the compositor, auto-detecting the backend (udev on a TTY, winit when nested). The `msg` subcommand instead talks to an already-running instance over its IPC socket. - `--backend ` — Backend to use (default: udev on a TTY, winit if nested) +- `--drm-device ` — DRM device for the udev backend (e.g. /dev/dri/card0 for vkms). Without it the primary GPU is preferred - `--config ` — Use an alternate config file - `--check-config` — Validate the config and exit - `--session-file ` — Durable session file path. Overrides the default; lets a nested winit dev session opt into session restore (it skips it otherwise) diff --git a/src/backend/udev.rs b/src/backend/udev.rs index f0db29a2..23d9293e 100644 --- a/src/backend/udev.rs +++ b/src/backend/udev.rs @@ -409,6 +409,7 @@ pub(crate) fn render_if_needed(data: &mut DriftWm) { pub fn init_udev( event_loop: &mut EventLoop<'static, DriftWm>, data: &mut DriftWm, + drm_device: Option<&std::path::Path>, ) -> Result> { // 1. Create libseat session let (mut session, session_notifier) = LibSeatSession::new() @@ -429,10 +430,14 @@ pub fn init_udev( tracing::info!("System primary GPU: {}", p.display()); } - // Build ordered candidate list: primary GPU first, then all others. - // On hybrid graphics (iGPU + dGPU), the "primary" GPU may not have - // the display outputs, so we fall back to other devices. - let gpu_paths: Vec = { + // Build ordered candidate list: an explicit --drm-device wins and is used + // alone (headless vkms must not fall back to the monitor GPU); otherwise + // primary GPU first, then all others (on hybrid graphics the "primary" GPU + // may not have the display outputs). + let gpu_paths: Vec = if let Some(path) = drm_device { + tracing::info!("Using explicit DRM device: {}", path.display()); + vec![path.to_path_buf()] + } else { let mut paths = Vec::new(); if let Some(ref p) = primary_gpu_path { paths.push(p.clone()); diff --git a/src/main.rs b/src/main.rs index f75f748d..db86e92c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,6 +32,10 @@ struct Cli { /// Backend to use (default: udev on a TTY, winit if nested) #[arg(long, value_name = "udev|winit")] backend: Option, + /// DRM device for the udev backend (e.g. /dev/dri/card0 for vkms). + /// Without it the primary GPU is preferred. + #[arg(long, value_name = "PATH")] + drm_device: Option, /// Use an alternate config file #[arg(long, value_name = "PATH")] config: Option, @@ -191,7 +195,8 @@ fn main() -> Result<(), Box> { // Initialize backend BEFORE setting WAYLAND_DISPLAY. match backend_name.as_str() { "udev" => { - let dev = backend::udev::init_udev(&mut event_loop, &mut data)?; + let dev = + backend::udev::init_udev(&mut event_loop, &mut data, cli.drm_device.as_deref())?; data.udev_device = Some(dev); } _ => backend::winit::init_winit(&mut event_loop, &mut data)?, From 326dc4fb34faeb92948b9a50ce396b746bd95036 Mon Sep 17 00:00:00 2001 From: Youssef Date: Thu, 12 Nov 2026 12:46:26 +0200 Subject: [PATCH 4/6] agents: pid in Focused reply, layout origins in state, layout name fixes FocusedWindow gains pid (same socket-credential source as WindowInfo) so agents map focus to AT-SPI without a second state round-trip; stand-ins report None. OutputInfo gains layout_origin (the output top-left in global layout coords, same space msg cursor/pointer use). Reload and cold boot now refresh/seed active_layout display name (was stale English (US) / empty until first keypress). --- src/ipc/mod.rs | 20 +++++++++++-- src/ipc/protocol.rs | 19 +++++++++++- src/state/init.rs | 8 ++++- src/state/reload.rs | 7 +++++ src/tests/config_reload.rs | 43 +++++++++++++++++++++++++++ src/tests/ipc_focus.rs | 60 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 150 insertions(+), 7 deletions(-) diff --git a/src/ipc/mod.rs b/src/ipc/mod.rs index 306a06f6..93570e15 100644 --- a/src/ipc/mod.rs +++ b/src/ipc/mod.rs @@ -523,9 +523,14 @@ pub(crate) fn state_info(state: &mut DriftWm) -> StateInfo { .space .outputs() .map(|output| { - let (cam, z, active_bookmark) = { + let (cam, z, active_bookmark, layout_origin) = { let os = crate::state::output_state(output); - (os.camera, os.zoom, os.active_bookmark.clone()) + ( + os.camera, + os.zoom, + os.active_bookmark.clone(), + [os.layout_position.x, os.layout_position.y], + ) }; let logical = crate::state::output_logical_size(output); OutputInfo { @@ -535,6 +540,7 @@ pub(crate) fn state_info(state: &mut DriftWm) -> StateInfo { size: [logical.w, logical.h], active: active.as_ref() == Some(output), active_bookmark, + layout_origin, } }) .collect(); @@ -620,7 +626,7 @@ fn suspended_by_selector( } /// IPC handle + app_id for a suspended window (its stage `ElementId`), for the -/// `Focused` reply. +/// `Focused` reply. A stand-in has no client, so `pid` is always `None`. fn suspended_focused_info(state: &DriftWm, id: SuspendedId) -> Option { let element = state .stage @@ -629,6 +635,7 @@ fn suspended_focused_info(state: &DriftWm, id: SuspendedId) -> Option protocol::FocusedWindow { + use smithay::reexports::wayland_server::Resource; + let pid = window.wl_surface().and_then(|s| s.client()).and_then(|c| { + c.get_credentials(&state.display_handle) + .ok() + .and_then(|cred| u32::try_from(cred.pid).ok()) + }); protocol::FocusedWindow { id: state .stage @@ -691,6 +704,7 @@ fn focused_window_info( .expect("window from the stage has an id") .0, app_id: window.app_id_or_class(), + pid, } } diff --git a/src/ipc/protocol.rs b/src/ipc/protocol.rs index 29a50de3..f704ce30 100644 --- a/src/ipc/protocol.rs +++ b/src/ipc/protocol.rs @@ -228,6 +228,12 @@ pub type Reply = Result; pub struct FocusedWindow { pub id: u64, pub app_id: Option, + /// The PID of the process that opened the window's Wayland connection, from + /// the socket's credentials — the same value as [`WindowInfo::pid`]. Lets an + /// agent map a `msg focus` reply to the AT-SPI tree without a second + /// `msg state` round-trip. `None` for a suspended stand-in (no client). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pid: Option, } /// The full compositor state snapshot: the payload of a [`Response::State`] @@ -268,7 +274,10 @@ pub struct StateInfo { } /// One output's viewport in the `state` reply. `camera` is the viewport -/// center, Y-up; `size` is the output's logical size. +/// center, Y-up; `size` is the output's logical size. `layout_origin` is the +/// output's top-left in the global layout (logical pixels, Y-down) — the same +/// space `msg cursor`/`msg pointer` use, so an agent can map global screen +/// coords to outputs without guessing. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OutputInfo { pub name: String, @@ -276,6 +285,8 @@ pub struct OutputInfo { pub zoom: f64, pub size: [i32; 2], pub active: bool, + #[serde(default)] + pub layout_origin: [i32; 2], /// This output's active bookmark (nearest visible to its usable center), /// or `None` when no bookmark is in view. #[serde(default)] @@ -660,6 +671,7 @@ mod tests { zoom: 1.0, size: [1920, 1200], active: true, + layout_origin: [0, 0], active_bookmark: Some("home".into()), }], active_bookmark: Some("home".into()), @@ -686,6 +698,7 @@ mod tests { Ok(Response::Focused(Some(FocusedWindow { id: 5, app_id: Some("foot".into()), + pid: Some(4242), }))), Ok(Response::Size { width: 800, @@ -737,6 +750,10 @@ mod tests { assert!(obj.get(key).is_some(), "missing key {key}"); } assert_eq!(obj["windows"][0]["mode"], "Normal"); + assert!( + obj["outputs"][0].get("layout_origin").is_some(), + "missing outputs[].layout_origin" + ); } /// `pid` is a plain number when the window has a client, left out for a diff --git a/src/state/init.rs b/src/state/init.rs index ac424f30..d47e5a4f 100644 --- a/src/state/init.rs +++ b/src/state/init.rs @@ -215,6 +215,12 @@ impl DriftWm { let mut seat: Seat = seat_state.new_wl_seat(&dh, "seat-0"); let kb = &config.keyboard_layout; + // Seed for `msg layout`'s display name: the compositor only learns the + // real display name from a live keymap read (key events, + // switch-layout, reload refresh), so cold boot reports the configured + // code until the first of those swaps in the real name. Cloned before + // the fallback arm below, which may pin `config.keyboard_layout` to us. + let initial_layout = kb.layout.clone(); let xkb = XkbConfig { layout: &kb.layout, variant: &kb.variant, @@ -394,7 +400,7 @@ impl DriftWm { trackpad_send_events: None, state_file_cameras: HashMap::new(), state_file_last_write: Instant::now(), - active_layout: String::new(), + active_layout: initial_layout, state_file_layout: String::new(), state_file_windows: Vec::new(), state_file_layer_count: 0, diff --git a/src/state/reload.rs b/src/state/reload.rs index b4b8ba1c..4f13e23c 100644 --- a/src/state/reload.rs +++ b/src/state/reload.rs @@ -64,6 +64,13 @@ impl DriftWm { } else { tracing::info!("Config reload: keyboard layout updated"); driftwm::protocols::virtual_keyboard::seat_keymap_changed(self, &keyboard); + // The keymap just swapped under us, so refresh the display name + // `msg layout` reports — same read as the key-event path, or it + // stays stale until the next keypress. + self.active_layout = keyboard.with_xkb_state(self, |ctx| { + let xkb = ctx.xkb().lock().unwrap(); + xkb.layout_name(xkb.active_layout()).to_owned() + }); let mut mods = keyboard.modifier_state(); if mods.num_lock != num_lock { mods.num_lock = num_lock; diff --git a/src/tests/config_reload.rs b/src/tests/config_reload.rs index c66d38b2..d019cf0c 100644 --- a/src/tests/config_reload.rs +++ b/src/tests/config_reload.rs @@ -323,3 +323,46 @@ fn reload_keeps_the_displays_children_are_pointed_at() { ); assert_eq!(env.get("DISPLAY").map(String::as_str), Some(":3")); } + +/// `msg layout` reports the display name of the loaded keymap, so a layout +/// reload must refresh it the same way a key event does — otherwise the +/// reported name stays stale until the next keypress. +#[test] +fn reload_refreshes_the_layout_display_name_msg_layout_reports() { + use crate::ipc::dispatch; + use crate::ipc::protocol::{Request, Response}; + + fn layout(f: &mut Fixture, short: bool) -> String { + match dispatch(Request::Layout { short }, f.state()) { + Ok(Response::Layout(name)) => name, + other => panic!("expected a Layout reply, got {other:?}"), + } + } + + let mut f = Fixture::new(); + + f.state() + .reload_config_from_contents("[input.keyboard]\nlayout = \"de\"\n"); + assert_eq!( + layout(&mut f, false), + "German", + "`msg layout` must follow the reloaded keymap, not wait for a keypress" + ); + assert_eq!(layout(&mut f, true), "de"); + + f.state() + .reload_config_from_contents("[input.keyboard]\nlayout = \"us\"\n"); + assert_eq!(layout(&mut f, false), "English (US)"); + assert_eq!(layout(&mut f, true), "us"); + + // A layout the keymap compiler rejects keeps the last good config and the + // name that goes with it. + f.state() + .reload_config_from_contents("[input.keyboard]\nlayout = \"xx-bogus\"\n"); + assert!( + f.state().errors.contains_key(&ErrorSource::Keyboard), + "a rejected layout must raise the keyboard error" + ); + assert_eq!(f.state().config.keyboard_layout.layout, "us"); + assert_eq!(layout(&mut f, false), "English (US)"); +} diff --git a/src/tests/ipc_focus.rs b/src/tests/ipc_focus.rs index 6d6cbcee..d7b54f61 100644 --- a/src/tests/ipc_focus.rs +++ b/src/tests/ipc_focus.rs @@ -5,12 +5,11 @@ use smithay::utils::{Logical, Point, Size}; +use super::{Fixture, map_window, settle, window_by_app_id}; use crate::ipc::dispatch; use crate::ipc::protocol::{Request, Response, WindowSelector}; use crate::state::StageWindow; -use super::{Fixture, map_window, settle, window_by_app_id}; - /// Where an unrelated pan flight is heading: far enough right that a window /// near the canvas origin is off screen once it lands. const FAR_CAMERA: (f64, f64) = (5000.0, 0.0); @@ -49,6 +48,40 @@ fn focus(f: &mut Fixture, element: &StageWindow) { ); } +/// The `Focused` reply carries the client pid (same value as `state` +/// windows[]), so an agent can map focus to the AT-SPI tree without a second +/// `msg state` round-trip. Stand-ins have no client, so theirs is `None`. +#[test] +fn focus_reply_carries_the_client_pid() { + let mut f = Fixture::new(); + f.add_output(1, (1920, 1080)); + let id = f.add_client(); + map_window(&mut f, id, "app", (400, 300)); + let window = window_by_app_id(&mut f, "app").unwrap(); + let element = StageWindow::Client(window); + + let wid = f.state().stage.id_of(&element).expect("staged").0; + let reply = dispatch(Request::Focus(Some(WindowSelector::Id(wid))), f.state()); + let info = match reply { + Ok(Response::Focused(Some(info))) => info, + other => panic!("expected a Focused reply, got {other:?}"), + }; + let state_pid = f + .state() + .window_inventory() + .into_iter() + .find(|w| w.id == wid) + .and_then(|w| w.pid); + assert_eq!( + info.pid, state_pid, + "Focused pid must match the state inventory pid" + ); + assert!( + info.pid.is_some(), + "a live client focus reply must carry a pid" + ); +} + #[test] fn focus_on_a_window_a_pan_flight_is_leaving_behind_pans_to_it() { let mut f = Fixture::new(); @@ -93,3 +126,26 @@ fn focus_on_a_stand_in_a_pan_flight_is_leaving_behind_pans_to_it() { f.state().dismiss_suspended(sid); } + +/// `state` outputs[] publish each output's global-layout origin (the same +/// space `msg cursor`/`msg pointer` use), so an agent can map global screen +/// coords to outputs. Outputs tile left-to-right by add order. +#[test] +fn state_outputs_publish_global_layout_origins() { + use crate::ipc::state_info; + + let mut f = Fixture::new(); + f.add_output(1, (1920, 1080)); + f.add_output(2, (1280, 720)); + + let state = state_info(f.state()); + assert_eq!(state.outputs.len(), 2); + + // First output anchors the layout; the second sits to its right. + assert_eq!(state.outputs[0].layout_origin, [0, 0]); + assert_eq!( + state.outputs[1].layout_origin, + [1920, 0], + "second output must start where the 1920-wide first ends" + ); +} From c9837478d9a4dad15ee7a584275b7343417d7a17 Mon Sep 17 00:00:00 2001 From: Youssef Date: Thu, 12 Nov 2026 16:08:40 +0200 Subject: [PATCH 5/6] docs: the IPC is the agent surface, not a separate adapter The agent integration shipped in the compositor itself: inventory, pid, focus+reveal, pointer/axis injection and canvas capture are all 'driftwm msg' subcommands, in-process. The contrib/driftwm-agent CLI was pure indirection over those commands plus a wtype/ydotool shim and a viewer that duplicates RustDesk, so it is removed rather than maintained as a second interface. agents.md now documents the built-in surface and how a standard AT-SPI client pairs with it: map windows by pid, focus to reveal, capture via the compositor. Notes the two upstream limits that are not ours to fix: agent-desktop lists displays only through Mutter DisplayConfig, and xdg-desktop-portal-wlr writes its Screenshot result to a hardcoded /tmp/out.png (ScreenCast through the same portal works against our screencopy). --- contrib/driftwm-agent/README.md | 70 ------------ contrib/driftwm-agent/package.json | 8 -- contrib/driftwm-agent/public/index.html | 53 --------- contrib/driftwm-agent/src/cli.ts | 130 ---------------------- contrib/driftwm-agent/src/serve.ts | 140 ------------------------ docs/agents.md | 39 +++---- 6 files changed, 20 insertions(+), 420 deletions(-) delete mode 100644 contrib/driftwm-agent/README.md delete mode 100644 contrib/driftwm-agent/package.json delete mode 100644 contrib/driftwm-agent/public/index.html delete mode 100755 contrib/driftwm-agent/src/cli.ts delete mode 100644 contrib/driftwm-agent/src/serve.ts diff --git a/contrib/driftwm-agent/README.md b/contrib/driftwm-agent/README.md deleted file mode 100644 index e46f34ad..00000000 --- a/contrib/driftwm-agent/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# driftwm-agent - -Thin adapter between a **driftwm** Wayland session and agent tooling. Not a new agent framework: -it exposes driftwm's canvas (state / camera / zoom / canvas screenshot) next to -`agent-desktop`'s AT-SPI tree (snapshot / click), plus a browser viewer (noVNC-like). - -``` -driftwm IPC ──► driftwm-agent ──► CLI / WebSocket - state state, shot agents, browser - camera/zoom focus, center - screenshot snap, click, type -``` - -## Requirements - -- `driftwm` (built from source; set `DRIFTWM_BIN` if not in PATH) -- `agent-desktop` (npm, for AT-SPI snapshot/click) -- `wtype` (keyboard via virtual-keyboard protocol) -- `ydotool` + `ydotoold` (pointer; only works when driftwm owns libinput — see Limits) -- Bun 1.2+ - -## CLI - -```bash -bun run src/cli.ts state # camera, zoom, windows (id, app_id, pid, pos, size) -bun run src/cli.ts center thunar # pan camera to a window -bun run src/cli.ts shot --scale 1 # canvas PNG (1600x900 on a 1600x900 output) -bun run src/cli.ts shot --scale 2 # 3200x1800 - arbitrary capture resolution -bun run src/cli.ts snap thunar # agent-desktop AT-SPI snapshot (JSON) -bun run src/cli.ts click @s...:e7 # semantic AT-SPI click (no pointer needed) -bun run src/cli.ts type "hello" # wtype into the focused surface -bun run src/cli.ts msg zoom 1.5 # raw IPC passthrough -``` - -## Browser viewer (noVNC-like) - -```bash -bun run src/serve.ts --port 6081 --scale 1 [--token SECRET] [--view-only] [--host 0.0.0.0] -# open http://127.0.0.1:6081/ -``` - -- Frames: `driftwm msg screenshot -o -` (PNG, ~6 fps, canvas re-render — not a framebuffer grab). -- Input: pointer via ydotool, text via wtype, clicks via ydotool or AT-SPI. -- State overlay: camera / zoom / output size / window list. - -## Limits (measured 2026-11-11) - -- **Pointer input** goes through the `msg pointer` IPC command added on this branch - (`driftwm msg pointer [--button left]`), so it also works in nested sessions - where uinput events would go to the host compositor. `ydotool` remains a fallback for - native sessions. -- `agent-desktop`'s own `screenshot` refuses on Wayland (only GNOME/Mutter is measured); - the compositor's canvas screenshot replaces it. -- Canvas screenshots do not draw layer-shell panels (use `grim` for the literal output). -- Nested winit + llvmpipe costs ~172 MiB PSS idle; native udev on a GPU should be lower. - -## Remote options considered - -| Option | Verdict | -|---|---| -| this WebSocket streamer | works today, PNG frames, ~6 fps; best for agent/human on LAN | -| [Selkies](https://docs.selkies.io) (pixelflux + GStreamer/WebRTC) | production path: portal/PipeWire capture, H.264, browser; heavier stack | -| Sunshine + Moonlight | lowest latency (NVENC), native client, weak clipboard | -| wayvnc | **not compatible** (requires wlroots protocols; driftwm is Smithay) | -| KasmVNC / noVNC | X11 only | - -## Security - -Bind `127.0.0.1` (default) or a tailnet address; set `--token` before exposing it; `--view-only` -disables input entirely. diff --git a/contrib/driftwm-agent/package.json b/contrib/driftwm-agent/package.json deleted file mode 100644 index 1e462a12..00000000 --- a/contrib/driftwm-agent/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "driftwm-agent", - "version": "0.1.0", - "private": true, - "type": "module", - "bin": { "driftwm-agent": "./src/cli.ts" }, - "scripts": { "start": "bun run src/cli.ts" } -} diff --git a/contrib/driftwm-agent/public/index.html b/contrib/driftwm-agent/public/index.html deleted file mode 100644 index 8a4180dd..00000000 --- a/contrib/driftwm-agent/public/index.html +++ /dev/null @@ -1,53 +0,0 @@ - -driftwm web - - -
driftwmconnecting…click to grab keyboard
-
- diff --git a/contrib/driftwm-agent/src/cli.ts b/contrib/driftwm-agent/src/cli.ts deleted file mode 100755 index 44332641..00000000 --- a/contrib/driftwm-agent/src/cli.ts +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bun -// driftwm-agent — thin adapter between a driftwm session and agent tooling. -// Not a new agent framework: it exposes driftwm's canvas (state/camera/zoom/screenshot) -// next to agent-desktop's AT-SPI tree (snapshot/click), so an external agent can use both. -import { spawnSync } from "node:child_process"; -import { existsSync, statSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -const AD = process.env.AGENT_DESKTOP_BIN ?? - `${process.env.HOME}/.npm-global/lib/node_modules/agent-desktop/bin/agent-desktop`; - -function driftwmBin(): string { - if (process.env.DRIFTWM_BIN) return process.env.DRIFTWM_BIN; - const local = `${process.env.HOME}/Playground/driftwm/target/release/driftwm`; - if (existsSync(local)) return local; - const which = spawnSync("sh", ["-c", "command -v driftwm"], { encoding: "utf8" }); - if (which.status === 0 && which.stdout.trim()) return which.stdout.trim(); - fail("driftwm binary not found (set DRIFTWM_BIN)"); -} - -function fail(msg: string): never { - console.log(JSON.stringify({ ok: false, error: msg })); - process.exit(1); -} - -function run(cmd: string, args: string[]): { ok: boolean; out: string; err: string; code: number } { - const r = spawnSync(cmd, args, { encoding: "utf8" }); - return { ok: r.status === 0, out: (r.stdout ?? "").trim(), err: (r.stderr ?? "").trim(), code: r.status ?? -1 }; -} - -function msg(args: string[]): any { - const bin = driftwmBin(); - const r = run(bin, ["msg", "--json", ...args]); - if (!r.ok) fail(`driftwm msg ${args.join(" ")} failed: ${r.err || r.out}`); - try { return JSON.parse(r.out); } catch { fail(`unparsable driftwm reply: ${r.out.slice(0, 200)}`); } -} - -function state(): any { - const reply = msg(["state"]); - return reply?.Ok?.State ?? fail("unexpected state reply"); -} - -function pngSize(path: string): { width: number; height: number } { - const buf = require("node:fs").readFileSync(path); - if (buf.length < 24 || buf.toString("ascii", 1, 4) !== "PNG") fail(`not a PNG: ${path}`); - return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; -} - -function findWindow(s: any, sel: string): any { - const byId = /^\d+$/.test(sel) ? s.windows.find((w: any) => w.id === Number(sel)) : undefined; - if (byId) return byId; - const needle = sel.toLowerCase(); - return s.windows.find((w: any) => String(w.app_id).toLowerCase().includes(needle) || - String(w.title).toLowerCase().includes(needle)); -} - -const [cmd, ...rest] = process.argv.slice(2); -switch (cmd) { - case "state": { - const s = state(); - console.log(JSON.stringify({ ok: true, camera: s.camera, zoom: s.zoom, layout: s.layout, windows: s.windows, outputs: s.outputs })); - break; - } - case "shot": { - const scale = (() => { const i = rest.indexOf("--scale"); return i >= 0 ? rest[i + 1] : "1"; })(); - const out = join(tmpdir(), `driftwm-agent-${Date.now()}.png`); - const bin = driftwmBin(); - const r = run(bin, ["msg", "screenshot", "-o", out, "--scale", scale]); - if (!r.ok) fail(`screenshot failed: ${r.err || r.out}`); - const size = pngSize(out); - console.log(JSON.stringify({ ok: true, path: out, scale: Number(scale), ...size })); - break; - } - case "focus": { - const sel = rest[0] ?? fail("focus needs "); - const bin = driftwmBin(); - const args = /^\d+$/.test(sel) ? ["--id", sel] : [sel]; - const r = run(bin, ["msg", "focus", ...args]); - if (!r.ok) fail(`focus failed: ${r.err || r.out}`); - console.log(JSON.stringify({ ok: true, focused: r.out })); - break; - } - case "center": { - const sel = rest[0] ?? fail("center needs "); - const s = state(); - const w = findWindow(s, sel) ?? fail(`no window matching ${sel}`); - const r = run(driftwmBin(), ["msg", "camera", String(w.position[0]), String(w.position[1])]); - if (!r.ok) fail(`camera failed: ${r.err || r.out}`); - console.log(JSON.stringify({ ok: true, window: { id: w.id, app_id: w.app_id, title: w.title }, camera: [w.position[0], w.position[1]] })); - break; - } - case "snap": { - const app = rest[0] ?? fail("snap needs "); - const r = run(AD, ["snapshot", "--app", app, "-i", "--compact"]); - if (!r.ok) fail(`agent-desktop snapshot failed: ${r.err || r.out}`); - console.log(r.out); - break; - } - case "click": { - const ref = rest[0] ?? fail("click needs "); - const r = run(AD, ["click", ref]); - console.log(r.out || JSON.stringify({ ok: false, error: r.err })); - if (!r.ok) process.exit(1); - break; - } - case "type": { - const text = rest.join(" "); - if (!text) fail("type needs "); - const which = spawnSync("sh", ["-c", "command -v wtype"], { encoding: "utf8" }); - if (which.status !== 0) fail("wtype not installed (apt install wtype)"); - const r = run("wtype", ["--", text]); - if (!r.ok) fail(`wtype failed: ${r.err}`); - console.log(JSON.stringify({ ok: true, typed: text.length })); - break; - } - case "msg": { - const bin = driftwmBin(); - const r = run(bin, ["msg", "--json", ...rest]); - console.log(r.out || JSON.stringify({ ok: false, error: r.err })); - if (!r.ok) process.exit(1); - break; - } - default: - console.log(JSON.stringify({ - ok: false, - usage: "driftwm-agent |center |snap |click |type |msg ...>", - })); - process.exit(2); -} diff --git a/contrib/driftwm-agent/src/serve.ts b/contrib/driftwm-agent/src/serve.ts deleted file mode 100644 index a4aba135..00000000 --- a/contrib/driftwm-agent/src/serve.ts +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bun -// driftwm-web — noVNC-like browser viewer for a driftwm session. -// Frames: `driftwm msg screenshot -o -` (canvas re-render, PNG) -> WebSocket binary. -// Input: pointer via `driftwm msg pointer` (IPC), text via wtype (virtual keyboard). -// Usage: bun run src/serve.ts [--port 6081] [--host 127.0.0.1] [--scale 1] [--token SECRET] [--view-only] -import { spawnSync } from "node:child_process"; -import { readFileSync } from "node:fs"; - -const args = process.argv.slice(2); -const arg = (name: string, dflt: string) => { const i = args.indexOf(`--${name}`); return i >= 0 ? args[i + 1] : dflt; }; -const has = (name: string) => args.includes(`--${name}`); - -const PORT = Number(arg("port", "6081")); -const HOST = arg("host", "127.0.0.1"); -const SCALE = Number(arg("scale", "1")); -const TOKEN = arg("token", ""); -const VIEW_ONLY = has("view-only"); -const JPEG = has("jpeg"); -const FPS = Number(arg("fps", "6")); -const DRIFTWM = process.env.DRIFTWM_BIN ?? `${process.env.HOME}/Playground/driftwm/target/release/driftwm`; - -// ydotoold is a user daemon; its socket lives in the user runtime dir, which is -// not necessarily the session XDG_RUNTIME_DIR a test harness uses. -if (!process.env.YDOTOOL_SOCKET) { - for (const p of [`/run/user/1000/.ydotool_socket`, `/tmp/.ydotool_socket`]) { - if (require("node:fs").existsSync(p)) { process.env.YDOTOOL_SOCKET = p; break; } - } -} - -function run(cmd: string, a: string[], maxBuffer = 32 * 1024 * 1024) { - return spawnSync(cmd, a, { maxBuffer, encoding: "buffer" }); -} - -let camera = { x: 0, y: 0 }, zoom = 1, output = { width: 1600, height: 900 }; -let windows: any[] = []; -let lastFrameAt = 0; - -type WS = { sendBinary(b: Uint8Array): void; send(s: string): void; close(): void }; -const clients = new Set(); - -function refreshState() { - const r = run(DRIFTWM, ["msg", "--json", "state"]); - if (r.status !== 0) return; - try { - const s = JSON.parse(r.stdout.toString()).Ok.State; - camera = { x: s.camera[0], y: s.camera[1] }; - zoom = s.zoom; - windows = s.windows; - const o = s.outputs.find((o: any) => o.active) ?? s.outputs[0]; - if (o) output = { width: o.size[0], height: o.size[1] }; - } catch {} -} - -function capture() { - if (clients.size === 0) return; - const r = run(DRIFTWM, ["msg", "screenshot", "-o", "-", "--scale", String(SCALE)]); - if (r.status !== 0 || !r.stdout.length) return; - let frame: Buffer = r.stdout; - if (JPEG) { - const j = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "image2pipe", "-i", "-", "-frames:v", "1", "-f", "mjpeg", "-q:v", "5", "pipe:1"], { input: r.stdout, maxBuffer: 32 * 1024 * 1024 }); - if (j.status !== 0 || !j.stdout?.length) return; - frame = j.stdout; - } - lastFrameAt = Date.now(); - for (const c of clients) { try { c.sendBinary(frame); } catch {} } -} - -function stateMessage() { - return JSON.stringify({ t: "state", camera, zoom, output, windows, scale: SCALE, viewOnly: VIEW_ONLY, frameAgeMs: Date.now() - lastFrameAt }); -} - -/** screen-space pixel from a canvas-view pixel in the received frame (frame is camera view at SCALE) */ -function toScreen(px: number, py: number) { - return { x: Math.round(px / SCALE), y: Math.round(py / SCALE) }; -} - -function cursor(): { x: number; y: number } { - const r = run(DRIFTWM, ["msg", "cursor"]); - if (r.status !== 0) return { x: 0, y: 0 }; - const m = r.stdout.toString().trim().match(/^(-?\d+)\s+(-?\d+)/); - return m ? { x: Number(m[1]), y: Number(m[2]) } : { x: 0, y: 0 }; -} -let lastPointer = { x: 0, y: 0 }; -function pointer(px: number, py: number) { - const p = toScreen(px, py); - lastPointer = p; - run(DRIFTWM, ["msg", "pointer", String(p.x), String(p.y)]); -} - -function click(button = "left") { - const p = lastPointer; - run(DRIFTWM, ["msg", "pointer", String(p.x), String(p.y), "--button", button]); -} - -function typeText(text: string) { - run("wtype", ["--", text]); -} - -function keyPress(key: string) { - run("wtype", ["-k", key]); -} - -Bun.serve({ - port: PORT, - hostname: HOST, - fetch(req, server) { - const url = new URL(req.url); - if (url.pathname === "/ws") { - if (TOKEN && url.searchParams.get("token") !== TOKEN) return new Response("forbidden", { status: 403 }); - if (server.upgrade(req)) return; - return new Response("upgrade failed", { status: 400 }); - } - if (url.pathname === "/" || url.pathname === "/index.html") { - return new Response(readFileSync(`${import.meta.dir}/../public/index.html`), { headers: { "content-type": "text/html" } }); - } - return new Response("not found", { status: 404 }); - }, - websocket: { - open(ws: any) { clients.add(ws); ws.send(stateMessage()); }, - close(ws: any) { clients.delete(ws); }, - message(ws: any, raw: any) { - if (VIEW_ONLY) return; - let m: any; try { m = JSON.parse(String(raw)); } catch { return; } - switch (m.t) { - case "pointer": pointer(Number(m.x) || 0, Number(m.y) || 0); break; - case "click": click(String(m.button ?? "left")); break; - case "text": typeText(String(m.text ?? "").slice(0, 512)); break; - case "key": keyPress(String(m.key ?? "")); break; - case "axis": run(DRIFTWM, ["msg", "axis", String(Number(m.dx) || 0), String(Number(m.dy) || 0)]); break; - case "state": ws.send(stateMessage()); break; - } - }, - }, -}); - -refreshState(); -setInterval(refreshState, 1500); -setInterval(capture, Math.max(50, Math.round(1000 / Math.max(1, FPS)))); -setInterval(() => { const s = stateMessage(); for (const c of clients) { try { c.send(s); } catch {} } }, 1500); -console.log(JSON.stringify({ ok: true, listening: `http://${HOST}:${PORT}/`, scale: SCALE, jpeg: JPEG, fps: FPS, token: !!TOKEN, viewOnly: VIEW_ONLY })); diff --git a/docs/agents.md b/docs/agents.md index c46ec029..254d56a6 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -1,8 +1,8 @@ # Driving driftwm with AI agents -driftwm is unusually well suited to computer-use agents: the canvas is queryable and -renderable over IPC, and windows carry a stable id plus their client pid. This page -describes the integration and the pieces added for it. +driftwm is well suited to computer-use agents: the canvas is queryable and renderable over +its own IPC, and windows carry a stable id plus their client pid. Everything an agent needs +ships in the compositor — there is no adapter process, no shim and no second daemon. ## What the IPC gives an agent @@ -12,6 +12,7 @@ describes the integration and the pieces added for it. | Canvas screenshot at arbitrary scale, including off-screen content | `driftwm msg screenshot -o - --scale N` | | Spatial navigation | `driftwm msg camera `, `driftwm msg zoom `, `driftwm msg focus ` | | Pointer input without uinput | `driftwm msg pointer [--button left|right|middle]` | +| Scroll | `driftwm msg axis ` | | Stable window identity | `state.windows[].id` / `[].pid` | | Bookmarks | `driftwm msg bookmark ...` | @@ -21,22 +22,22 @@ OCR/vision stages. `pointer` is deliberately part of the IPC rather than a Wayland protocol: it works in nested sessions (where uinput events would go to the host compositor) and needs no extra -device permissions. The socket is user-only (0600); treat it as an input-injection -surface and keep it inside the session. - -## Glue for AT-SPI agents - -`agent-desktop` (AT-SPI) can see app windows inside driftwm and click them semantically -(no pointer). Pair it with driftwm's IPC: - -- map `agent-desktop` windows to driftwm ids by `pid`; -- use `driftwm msg focus ` to bring a window into view before snapshotting; -- use `driftwm msg screenshot` for observation (agent-desktop's own Wayland capture only - supports GNOME/Mutter); -- use `driftwm msg pointer ... --button left` when a coordinate click is required. - -A reference implementation of this glue (CLI + WebSocket viewer) lives in -[`contrib/driftwm-agent`](../contrib/driftwm-agent). +device permissions. The socket is user-only (0600); treat it as an input-injection surface +and keep it inside the session. + +## Pairing with agent-desktop + +`agent-desktop` is a standard AT-SPI client: it reads the accessibility tree and drives +elements semantically. It needs no driftwm-specific glue. + +- map its windows to driftwm ids by `pid`, which `state` and the `focus` reply both carry; +- `driftwm msg focus ` activates and reveals a window (foreign-toplevel `activate` + and `xdg-activation` both land on the same path), then its snapshot sees the window; +- `driftwm msg screenshot` is the observation channel. agent-desktop's own Wayland capture + has two known limits: display *listing* is Mutter-only, and the wlr portal's `Screenshot` + implementation writes to a hardcoded `/tmp/out.png`, which an agent that requires + home-relative files will reject. `ScreenCast` through `xdg-desktop-portal-wlr` works + against driftwm's `zwlr_screencopy` surface, which is what remote viewers use. ## Remaining gaps From 4494e505ed5814ed13bbb44a2ca3aed1b8545111 Mon Sep 17 00:00:00 2001 From: Youssef Date: Thu, 12 Nov 2026 18:14:13 +0200 Subject: [PATCH 6/6] ipc: take the scroll distance in pixels to match pointer `msg pointer ` takes output-layout logical pixels and `cursor` reports the same space, but `msg axis ` took wheel notches, so a caller had to know two units to drive one pointer. Reported by Granitosaurus on #341. `axis` now takes the same logical pixels. The frame still carries both `axis_value120` and `axis`, so clients that scroll discretely keep getting whole notches: a notch is 15 px, which is the ratio the old notch-based call already used internally, so one notch in is still one notch out. --- docs/cli.md | 4 ++-- docs/ipc.md | 12 ++++++++---- src/ipc/client.rs | 4 ++-- src/ipc/mod.rs | 26 ++++++++++++++++++++++---- src/ipc/protocol.rs | 4 ++-- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 0aef14fb..81b34d81 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -67,7 +67,7 @@ driftwm msg --json focus --id 5 | [`layout`](#driftwm-msg-layout) | Print the active keyboard layout (full XKB name, e.g. `English (US)`) | | [`cursor`](#driftwm-msg-cursor) | Print the pointer position in the global output layout (logical pixels, Y-down, the space slurp prints) and the output it is on | | [`pointer`](#driftwm-msg-pointer) | Move the seat pointer to in the output layout (logical pixels, Y-down, the space `cursor` reports); with --button, click there | -| [`axis`](#driftwm-msg-axis) | Scroll the focused surface by wheel notches (positive = right/down) | +| [`axis`](#driftwm-msg-axis) | Scroll the focused surface by in output-layout logical pixels, the space `pointer` and `cursor` use (positive = right/down) | | [`action`](#driftwm-msg-action) | Run a config action, e.g. `action close-window`, `action switch-layout next` | | [`screenshot`](#driftwm-msg-screenshot) | Capture a canvas PNG. With no subcommand, captures the active output's current view of the canvas | | [`debug-counters`](#driftwm-msg-debug-counters) | Print internal collection sizes for leak diagnosis (unstable keys) | @@ -361,7 +361,7 @@ driftwm msg pointer 640 400 --button left driftwm msg axis ``` -Scroll the focused surface by wheel notches (positive = right/down) +Scroll the focused surface by in output-layout logical pixels, the space `pointer` and `cursor` use (positive = right/down) ```bash driftwm msg axis 0 -1 diff --git a/docs/ipc.md b/docs/ipc.md index 71225e91..972afbec 100644 --- a/docs/ipc.md +++ b/docs/ipc.md @@ -132,12 +132,16 @@ reaches off-screen content; its four targets, flags, and caveats are in the ### Scrolling -`axis ` scrolls the focused surface by wheel notches (positive = right/down). -It is what a remote viewer forwards from mouse-wheel events; the reply echoes the deltas. +`axis ` scrolls the focused surface by a distance in output-layout logical +pixels — the same space `pointer ` takes and `cursor` reports (positive = +right/down). It is what a remote viewer forwards from mouse-wheel events; the reply +echoes the deltas. A notch is 15 px, and the frame carries both `axis_value120` (for +clients that scroll discretely) and `axis` (for clients that scroll smoothly), so one +call serves both kinds of client. ```bash -driftwm msg axis 0 -1 # one notch up -driftwm msg axis 0 3 # three notches down +driftwm msg axis 0 -15 # one notch up +driftwm msg axis 0 45 # three notches down ``` ### Pointer injection diff --git a/src/ipc/client.rs b/src/ipc/client.rs index f75f83b8..b7a2a8d6 100644 --- a/src/ipc/client.rs +++ b/src/ipc/client.rs @@ -211,8 +211,8 @@ pub enum Msg { #[arg(long, value_name = "left|right|middle")] button: Option, }, - /// Scroll the focused surface by wheel notches (positive = - /// right/down). + /// Scroll the focused surface by in output-layout logical + /// pixels, the space `pointer` and `cursor` use (positive = right/down). Axis { #[arg(allow_hyphen_values = true)] dx: f64, diff --git a/src/ipc/mod.rs b/src/ipc/mod.rs index 93570e15..b96ce8bb 100644 --- a/src/ipc/mod.rs +++ b/src/ipc/mod.rs @@ -456,19 +456,29 @@ fn cmd_axis(dx: f64, dy: f64, state: &mut DriftWm) -> Reply { .duration_since(std::time::SystemTime::UNIX_EPOCH) .map(|d| d.subsec_millis()) .unwrap_or(0); - let vertical = (dy * 120.0) as i32; - let horizontal = (dx * 120.0) as i32; + let vertical = notches(dy); + let horizontal = notches(dx); let frame = AxisFrame::new(time) .source(AxisSource::Wheel) .v120(Axis::Vertical, vertical) .v120(Axis::Horizontal, horizontal) - .value(Axis::Vertical, dy * 15.0) - .value(Axis::Horizontal, dx * 15.0); + .value(Axis::Vertical, dy) + .value(Axis::Horizontal, dx); pointer.axis(state, frame); pointer.frame(state); Ok(Response::Axis { dx, dy }) } +/// Pixels a wheel notch is worth, so a caller that thinks in pixels still gets +/// discrete wheel scrolling in clients that read `v120` instead of `value`. +const AXIS_PIXELS_PER_NOTCH: f64 = 15.0; + +/// `v120` for a pixel distance: 120 units per notch, so one notch in lands as +/// one notch out and a fractional pixel distance stays proportional. +fn notches(pixels: f64) -> i32 { + (pixels / AXIS_PIXELS_PER_NOTCH * 120.0) as i32 +} + fn cmd_layout(short: bool, state: &mut DriftWm) -> Reply { if !short { return Ok(Response::Layout(state.active_layout.clone())); @@ -1550,4 +1560,12 @@ mod tests { assert_eq!(layout_code("us,ru", 2), None); assert_eq!(layout_code("us,", 1), None); } + + #[test] + fn a_wheel_notch_is_fifteen_pixels_of_scroll() { + assert_eq!(super::notches(15.0), 120); + assert_eq!(super::notches(-15.0), -120); + assert_eq!(super::notches(45.0), 360); + assert_eq!(super::notches(0.0), 0); + } } diff --git a/src/ipc/protocol.rs b/src/ipc/protocol.rs index f704ce30..a20e250f 100644 --- a/src/ipc/protocol.rs +++ b/src/ipc/protocol.rs @@ -44,8 +44,8 @@ pub enum Request { #[serde(default)] button: Option, }, - /// Scroll the focused surface: `dx` is horizontal, `dy` vertical, in wheel - /// notches (positive = right/down). + /// Scroll the focused surface: `dx` is horizontal, `dy` vertical, in + /// output-layout logical pixels (positive = right/down). Axis { dx: f64, dy: f64,