From 3c5c5207995726d11bdc6a304479cfdb68a29a44 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Fri, 26 Jun 2026 17:02:35 -0500 Subject: [PATCH] work on linux release Linux release not fully working as well as windows, will investigate later --- GETTING_STARTED_WITH_LINUX.md | 151 +++++++++++++++++++ README.md | 2 + app/capture.js | 259 ++++++++++++++++++++++++++++----- app/main.js | 35 ++++- app/renderer/capture-worker.js | 46 +++--- app/stream-backend.js | 28 +++- tests/unit/capture.test.js | 98 ++++++++++++- 7 files changed, 550 insertions(+), 69 deletions(-) create mode 100644 GETTING_STARTED_WITH_LINUX.md diff --git a/GETTING_STARTED_WITH_LINUX.md b/GETTING_STARTED_WITH_LINUX.md new file mode 100644 index 0000000..51e787b --- /dev/null +++ b/GETTING_STARTED_WITH_LINUX.md @@ -0,0 +1,151 @@ +# Getting Started with StepForge on Linux + +> ⚠️ **Work in progress.** Linux support is still under active development. +> Expect rough edges — especially on Wayland (see the limitations below). X11 / +> Xorg is the most complete path today. Please report issues. + +StepForge was built on Windows, where the OS lets an app watch every click and +grab the screen freely. Linux is more restrictive, and **how much works depends +on whether you are running X11 (Xorg) or Wayland.** This guide explains the +difference, how to get the best experience, and how to enable per-click capture. + +## TL;DR + +| | **X11 / "Ubuntu on Xorg"** | **Wayland (default on Ubuntu)** | +|---|---|---| +| Screenshot per click | ✅ Yes | ✅ Yes (needs `input` group) | +| Red circle on the click | ✅ Yes | ❌ No (Wayland hides the cursor position) | +| "Share your screen" prompt | Never | Once per recording session | +| Setup needed | None | Add yourself to the `input` group | + +**If you want the full Windows-like experience (click capture *with* the red +marker), use an Xorg session — see [Option A](#option-a-best-experience--use-xorg).** + +--- + +## 1. Check which session you are running + +```bash +echo $XDG_SESSION_TYPE +``` + +- `x11` → you're on Xorg. Everything works, including the red click marker. No setup needed. +- `wayland` → you're on Wayland. Read on. + +--- + +## Option A (best experience) — use Xorg + +On Xorg, StepForge captures a screenshot **on every click** and draws the **red +marker** at the exact click position, exactly like Windows. No dependencies, no +permissions, no portal dialogs. + +To switch: + +1. Log out. +2. On the login (password) screen, click the **⚙ gear icon** in the bottom-right corner. +3. Choose **"Ubuntu on Xorg"**. +4. Log back in. + +That's it — open StepForge and record. (To go back to Wayland later, pick +"Ubuntu" at the gear menu again.) + +--- + +## Option B — stay on Wayland + +Wayland deliberately blocks apps from monitoring global input and from grabbing +the screen silently. StepForge works around this as far as the platform allows: + +### Screen capture + +The first time you press **Start recording**, the system shows a **"Share your +screen"** dialog (the XDG desktop portal). Pick your screen and click **Share**. +This happens **once per recording session** — not per screenshot. The shared +stream stays open until you stop recording. + +> If you never see steps appear, make sure you actually picked a screen and +> clicked **Share** in that dialog. + +### Per-click capture (requires the `input` group) + +By default on Wayland, StepForge cannot see your clicks, so it falls back to +**capturing a screenshot every few seconds** (timed capture). + +To get a screenshot **on every click** instead, give your user read access to +the mouse devices by joining the `input` group: + +```bash +sudo usermod -aG input "$USER" +``` + +Then **log out and log back in** (group membership only applies to new sessions). +Verify it took effect: + +```bash +groups | tr ' ' '\n' | grep input # should print: input +``` + +Now StepForge reads mouse buttons directly from the kernel (`/dev/input`) and +captures a screenshot on each click. + +> **No red marker on Wayland.** Even with per-click capture working, Wayland +> does not tell apps *where* the pointer is, so StepForge cannot draw the circle +> at the click. The screenshot is still captured per click — just without the +> marker. If you need the marker, use [Option A (Xorg)](#option-a-best-experience--use-xorg). + +### Adjusting the timed-capture interval + +If you don't enable the `input` group, StepForge captures on a timer. Change the +interval in **Settings → Capture** (`capture.autoIntervalSec`, default 5 +seconds). + +--- + +## How StepForge picks a capture method (for reference) + +On launch StepForge chooses the best available click source: + +1. **Windows** — low-level mouse hook (position + timing). +2. **X11** — `xinput` (position + timing → full red marker). +3. **Linux evdev** (`/dev/input`) — button presses on X11 *and* Wayland, no + position on Wayland. Used when `xinput` can't see clicks (i.e. Wayland), if + you're in the `input` group. +4. **Timed capture** — the always-works fallback (a screenshot every N seconds) + when no click source is available. + +Screen frames come from a single long-lived capture stream per recording, so +clicks/timer ticks never re-open the screen-share dialog. + +--- + +## Troubleshooting + +**"It asks to share my screen every time."** +You're likely on an older build. Update to the current version — the screen +stream is now opened once per recording session. If it persists, confirm +`echo $XDG_SESSION_TYPE` and that you clicked **Share** (not Cancel) in the dialog. + +**"Recording captures a couple of steps then stops."** +Fixed in the current version (a slow, GPU-less PNG encode used to trip a +failure guard and tear down the stream). Update and retry. + +**"The window disappeared and I can't stop the recording."** +On Linux the window **minimizes** while recording (GNOME's system tray is +unreliable). Bring it back from the **taskbar / dock**, then click **Stop +recording**. + +**"No steps at all on Wayland, even after picking a screen."** +Run from a terminal with logging and look for the diagnostic lines: + +```bash +STEPFORGE_CAPTURE_LOG=1 npm start +``` + +- `[stepforge] screen-capture stream active …` — the stream is up. +- `[stepforge] per-click capture via evdev on N device(s) …` — clicks are wired up. +- `[stepforge] no readable mouse input devices …` — you need the `input` group (see above). + +**Harmless console noise.** Lines like `vaInitialize failed`, `Frame latency is +negative`, and `StatusNotifierItem … already exported` come from Chromium/GNOME, +not StepForge, and don't affect recording. diff --git a/README.md b/README.md index 2f42cd6..06026d7 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ using only Node built-ins. For a windows installation, see [docs/windows_installation](docs/windows_installation.md) or for a developer/more in depth walkthrough, see [docs/GETTING_STARTED.md](docs/GETTING_STARTED.md). +On **Linux** (⚠️ work in progress — X11 vs Wayland, enabling per-click capture, the screen-share prompt), see [GETTING_STARTED_WITH_LINUX.md](GETTING_STARTED_WITH_LINUX.md). + Requirements: Node.js 20+ and npm (Electron is the only dependency). ```bash diff --git a/app/capture.js b/app/capture.js index 100884c..771308c 100644 --- a/app/capture.js +++ b/app/capture.js @@ -1,6 +1,7 @@ 'use strict'; const path = require('node:path'); +const fs = require('node:fs'); const { spawn, execFileSync } = require('node:child_process'); const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron'); const raster = require('../core/raster'); @@ -116,6 +117,74 @@ function isWayland() { && (process.env.XDG_SESSION_TYPE === 'wayland' || Boolean(process.env.WAYLAND_DISPLAY)); } +// ---- evdev (Linux kernel input) click reader -------------------------------- +// Reading /dev/input/event* directly sees mouse-button presses on BOTH X11 and +// Wayland, because it taps the kernel input layer below the display server — +// the one global-click source that survives Wayland's security model. It needs +// read access to the device nodes (the user must be in the `input` group). +// +// Each event is a fixed-size `struct input_event`: a timeval, then u16 type, +// u16 code, s32 value. The timeval is two `long`s, so the record is 24 bytes on +// 64-bit and 16 on 32-bit; type/code/value always sit in the last 8 bytes. +const EV_KEY = 0x01; +const EVDEV_PRESS = 1; +// BTN_LEFT/RIGHT/MIDDLE -> the same button-N naming the xinput path emits +// (1=left, 2=middle, 3=right) so downstream debounce/marker logic is identical. +const EVDEV_BUTTONS = { 272: 'button-1', 273: 'button-3', 274: 'button-2' }; +const EVDEV_RECORD_SIZE = (process.arch === 'x64' || process.arch === 'arm64' + || process.arch === 'ppc64' || process.arch === 's390x' || process.arch === 'loong64' + || process.arch === 'riscv64') ? 24 : 16; + +/** + * Decode a buffer of packed input_event records, returning the button presses + * found and any trailing partial record (a device read can split mid-record). + * Pure and size-parameterised so it is unit-testable without real devices. + */ +function decodeEvdevButtonPresses(buffer, recordSize = EVDEV_RECORD_SIZE) { + const presses = []; + let offset = 0; + while (buffer.length - offset >= recordSize) { + const type = buffer.readUInt16LE(offset + recordSize - 8); + const code = buffer.readUInt16LE(offset + recordSize - 6); + const value = buffer.readInt32LE(offset + recordSize - 4); + offset += recordSize; + if (type === EV_KEY && value === EVDEV_PRESS && EVDEV_BUTTONS[code]) { + presses.push(EVDEV_BUTTONS[code]); + } + } + return { presses, rest: buffer.subarray(offset) }; +} + +/** + * The /dev/input/event* nodes for pointing devices that are readable by this + * process. /proc/bus/input/devices lists every device with its Handlers line; + * a pointing device exposes a `mouseN` handler, and the matching `eventN` is + * the node to read. Unreadable nodes (no `input` group membership) are skipped. + */ +function readableEvdevMouseNodes() { + const nodes = []; + let table; + try { + table = fs.readFileSync('/proc/bus/input/devices', 'utf8'); + } catch { + return nodes; + } + for (const block of table.split('\n\n')) { + const handlers = /H:\s*Handlers=([^\n]*)/.exec(block); + if (!handlers || !/\bmouse\d+\b/.test(handlers[1])) continue; + const event = /\bevent(\d+)\b/.exec(handlers[1]); + if (!event) continue; + const node = `/dev/input/event${event[1]}`; + try { + fs.accessSync(node, fs.constants.R_OK); + nodes.push(node); + } catch { + // Not readable — user is not in the `input` group for this node. + } + } + return nodes; +} + class CaptureService { constructor({ store, @@ -133,6 +202,9 @@ class CaptureService { // the global `screen` directly so coordinate handling stays testable. this.screen = screenApi; this.textIntel = textIntel; + // Cached display-server detection. A method (onWayland) reads this so tests + // can flip platform behavior without touching process.env. + this._wayland = isWayland(); this.session = null; // { guideId, paused, count, intervalSec } this.intervalTimer = null; this.clickWatcher = null; @@ -193,15 +265,35 @@ class CaptureService { clickCaptureAvailable() { if (this._clickAvail === undefined) { - // Wayland: xinput connects via XWayland and only sees X11-bridge clicks. - // Native Wayland app events are invisible to it, so we can't reliably - // detect clicks — disable this path and fall back to interval capture. + // Three click sources, in order of fidelity: + // - Windows: the low-level mouse hook (position + timing); + // - X11: xinput test-xi2 (position + timing) — but it can't see native + // Wayland clicks, only XWayland ones, so it's gated to non-Wayland; + // - Linux evdev (/dev/input): button presses on X11 AND Wayland, but no + // cursor position on Wayland — used for per-click capture there (no + // marker). Requires the user to be in the `input` group. this._clickAvail = process.platform === 'win32' - || (process.platform === 'linux' && !isWayland() && hasBinary('xinput')); + || (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput')) + || (process.platform === 'linux' && readableEvdevMouseNodes().length > 0); } return this._clickAvail; } + /** Whether this is a Wayland session (cached; overridable in tests). */ + onWayland() { + return this._wayland; + } + + /** + * Whether the in-process frame loop is a usable fallback recorder. It grabs + * via desktopCapturer.getSources(), which on Wayland is broken (throws) and + * pops the portal — so the loop is viable only off Wayland. On Wayland the + * portal-backed stream backend is the sole capture path. + */ + canUseFrameLoop() { + return !this.onWayland(); + } + startSession(guideId, { intervalSec = null } = {}) { this.finishSession(); // Default trigger: clicks when the platform supports it, otherwise an @@ -335,7 +427,14 @@ class CaptureService { const sec = this.session && this.session.intervalSec; if (sec > 0) { this.intervalTimer = setInterval(() => { - this.sessionCapture('interval').catch(() => {}); + // Don't let a slow capture (e.g. a multi-second software PNG encode on + // a GPU-less host) overlap with the next tick — overlapping requests + // would pile up and could trip the backend's failure counter. + if (this.intervalCapturing) return; + this.intervalCapturing = true; + this.sessionCapture('interval') + .catch(() => {}) + .finally(() => { this.intervalCapturing = false; }); }, sec * 1000); } } @@ -402,12 +501,13 @@ class CaptureService { if (!this.session || this.session.paused) { this.warmingUp = false; return; } } if (wantHide && win && !win.isDestroyed() && win.isVisible()) { - // On Linux without a working system tray (common on Wayland GNOME - // without AppIndicator) the user has no way to bring back a hidden - // window mid-session. Minimize instead: the window disappears from - // screen (so it won't appear in fullscreen screenshots) but remains - // accessible via the taskbar. - if (process.platform === 'linux' && (!this.tray || this.tray.isDestroyed())) { + // On Linux, always minimize rather than hide. GNOME's system tray + // (StatusNotifier) is unreliable — it can fail or half-export over + // dbus — so a hidden window can be impossible to bring back, leaving + // the user unable to stop the recording. A minimized window is always + // restorable from the taskbar, and minimized windows aren't rendered + // so they still stay out of the fullscreen capture. + if (process.platform === 'linux') { win.minimize(); } else { win.hide(); @@ -509,22 +609,44 @@ class CaptureService { if (!sessionLive) return { ok: false, reason: 'session ended before the fallback shot' }; } - // For non-click triggers (interval, hotkey, manual) pull a frame from the - // stream backend's ring buffer when available. This avoids a + // For non-click triggers (interval, hotkey, manual) pull the latest frame + // from the stream backend's ring buffer when available. This avoids a // desktopCapturer.getSources() call per capture — on Linux/Wayland that - // call goes through the XDG portal and can show a dialog every time. + // call goes through the XDG portal and shows a dialog every time. + // + // No clickPos: a timed capture has no click position, and passing a cursor + // point here is actively harmful on Wayland — getCursorScreenPoint() can + // return a stale/out-of-bounds point, which makes the backend reject the + // frame (wrong display / out of bounds) and fall through to a getSources() + // shot, i.e. a portal dialog on every interval tick. if (trigger !== 'click' && this.streamBackend && this.streamBackend.isActive()) { const frame = await this.streamBackend.frameForClick({ - clickPos: this.screen.getCursorScreenPoint(), + clickPos: null, clickAt: Date.now(), strict: false, // no pre/post-click constraint for timed captures leadMs: 0, + failable: false, // a slow timed-capture encode must not kill the stream }).catch(() => null); if (frame) { const result = await this.storeFrameAsStep(this.session.guideId, 'fullscreen', frame); if (result.ok) this.noteStepAdded(result.step, trigger); + clog(trigger, 'capture stored from stream; total', this.session && this.session.count); return result; } + clog(trigger, 'capture: no frame from stream this tick — will retry next tick'); + } else if (trigger !== 'click' && this.onWayland()) { + clog(trigger, 'capture: stream backend not active (active=', + Boolean(this.streamBackend && this.streamBackend.isActive()), ')'); + } + + // On Wayland the only screen-grab fallback below is desktopCapturer + // .getSources(), which pops the XDG portal dialog every call. For the + // automatic timed triggers that would mean a dialog on every tick, so skip + // the fallback and wait for the open stream to deliver a frame on a later + // tick. Explicit captures (manual, and click on X11) still fall through — + // one dialog for one deliberate action. + if (this.onWayland() && (trigger === 'interval' || trigger === 'hotkey')) { + return { ok: false, reason: 'waiting for the screen-share stream' }; } if (this.shooting) return { ok: false, reason: 'capture already in progress' }; @@ -688,7 +810,11 @@ class CaptureService { }; if (this.streamBackend && this.streamBackend.isActive() && grabMode === 'fullscreen') { - const frame = await this.streamBackend.frameForClick({ clickPos, clickAt: clickTime, strict, leadMs }); + // On Wayland the stream is the only capture path (no frame-loop fallback), + // so a slow PNG encode must not let the 2-strikes rule tear it down. + const frame = await this.streamBackend.frameForClick({ + clickPos, clickAt: clickTime, strict, leadMs, failable: !this.onWayland(), + }); if (frame) return frame; // No qualifying frame (or the backend just went unhealthy): fall // through to the loop buffer / fresh-shot fallbacks below. @@ -741,8 +867,11 @@ class CaptureService { async startClickFrameBackend() { const mode = this.settings.get('capture.mode') || 'fullscreen'; // The worker streams screens; window-mode grabs need the loop's - // source-filtering logic. - if (this.settings.get('capture.streamCapture') === false || mode === 'window') { + // source-filtering logic. But the loop isn't viable on Wayland (getSources + // is broken/portal), so there we always take the stream backend regardless + // of the streamCapture/window settings. + if (this.canUseFrameLoop() + && (this.settings.get('capture.streamCapture') === false || mode === 'window')) { this.startFrameLoop(); return; } @@ -762,7 +891,11 @@ class CaptureService { onUnhealthy: () => this.degradeToFrameLoop(), }); const displays = this.screen.getAllDisplays(); - const sources = await desktopCapturer.getSources({ + // On Wayland, desktopCapturer.getSources() both fails to yield usable + // source ids AND pops the portal dialog. Skip it entirely and drive the + // worker through getDisplayMedia (the portal picker chooses the screen). + const useDisplayMedia = this.onWayland(); + const sources = useDisplayMedia ? [] : await desktopCapturer.getSources({ types: ['screen'], thumbnailSize: { width: 1, height: 1 }, // ids only — skip thumbnail work }); @@ -770,33 +903,37 @@ class CaptureService { displays, sources: sources.map((s) => ({ id: s.id, display_id: s.display_id })), sampleMs: this.settings.get('capture.frameSampleMs') || 100, + useDisplayMedia, }); const stale = gen !== this.captureGen; if (!ok || stale || !this.session || this.session.paused) { backend.stop(); if (!stale && this.session && !this.session.paused) { - if (this.clickCaptureAvailable()) { + if (this.canUseFrameLoop()) { console.error('[stepforge] stream capture backend failed to start — using in-process frame loop'); this.startFrameLoop(); } else { - // Without click capture the frame loop would spam getSources() - // every 200ms (triggering the portal dialog on Linux/Wayland). - // Interval/hotkey captures will take individual fresh shots instead. - console.error('[stepforge] stream capture backend failed to start — interval/hotkey captures will use fresh shots'); + // On Wayland the frame loop would spam getSources() (portal) with + // nothing usable, so there's no fallback — the recording needs the + // portal stream. Tell the user how to recover. + console.error('[stepforge] screen-share stream did not start — pick a screen in the share dialog, or stop and start recording again'); } } return; } this.streamBackend = backend; - clog('stream capture backend active'); + // Visible in normal output (one line per recording): confirms the screen + // stream came up, so a "nothing records" report can be told apart from a + // stream that never started (which logs the failure paths above). + console.log(`[stepforge] screen-capture stream active (${useDisplayMedia ? 'getDisplayMedia/portal' : 'desktopCapturer'})`); this.notify('capture:state', this.state()); } catch (err) { if (gen === this.captureGen && this.session && !this.session.paused) { - if (this.clickCaptureAvailable()) { + if (this.canUseFrameLoop()) { console.error(`[stepforge] stream capture backend error (${err && err.message}) — using in-process frame loop`); this.startFrameLoop(); } else { - console.error(`[stepforge] stream capture backend error (${err && err.message}) — interval/hotkey captures will use fresh shots`); + console.error(`[stepforge] screen-share stream error (${err && err.message}) — stop and start recording again`); } } } finally { @@ -822,14 +959,13 @@ class CaptureService { */ degradeToFrameLoop() { this.streamBackend = null; - if (this.clickCaptureAvailable()) { + if (this.canUseFrameLoop()) { console.error('[stepforge] stream capture backend unhealthy — falling back to in-process frame loop'); if (this.session && !this.session.paused) this.startFrameLoop(); } else { - // No click capture means no click watcher either, so the frame loop - // is pointless and would spam getSources() every 200ms. Individual - // per-capture shots are the correct fallback. - console.error('[stepforge] stream capture backend unhealthy — continuing with per-capture fresh shots'); + // On Wayland the frame loop isn't viable (getSources is broken/portal), + // so there's nothing to fall back to — the stream is the only path. + console.error('[stepforge] screen-share stream stopped — stop and start recording again to re-share'); } this.notify('capture:state', this.state()); } @@ -839,7 +975,7 @@ class CaptureService { try { this.clickWatcherBuf = ''; this.linuxEvent = null; - if (process.platform === 'linux' && !isWayland() && hasBinary('xinput')) { + if (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput')) { // Stream raw button events from the X server; one capture per press. // xinput block-buffers stdout when piped, so a press event can sit // in its buffer until later motion events flush it — by then the @@ -854,6 +990,13 @@ class CaptureService { this.clickWatcher.stdout.on('data', (chunk) => { this.ingestClickWatcherChunk(chunk.toString(), 'linux'); }); + } else if (process.platform === 'linux' && readableEvdevMouseNodes().length > 0) { + // Wayland (or X11 without xinput): read mouse buttons from the kernel + // input layer. This is the only global click source on Wayland, but it + // carries no cursor position — onOsClick gets a null point, so steps are + // captured per click without a marker. (X11 prefers the xinput branch + // above, which does carry root coordinates for the marker.) + this.startEvdevWatcher(); } else if (process.platform === 'win32') { // Use a low-level Windows mouse hook instead of polling // GetAsyncKeyState. The low bit from GetAsyncKeyState can be consumed @@ -1249,12 +1392,54 @@ public static class SFHook { try { this.clickWatcher.kill(); } catch { /* already gone */ } this.clickWatcher = null; } + this.stopEvdevWatcher(); this.clickWatcherBuf = ''; this.linuxEvent = null; this.discardPendingRawClick(); this.lastAcceptedClickByButton.clear(); } + /** + * Open every readable mouse device node and turn button-down events into + * onOsClick calls. One physical mouse is normally one node; the leading-edge + * debounce in onOsClick collapses any cross-node duplicates. Frames are still + * served by the stream backend; this only supplies the click *trigger*. + */ + startEvdevWatcher() { + const nodes = readableEvdevMouseNodes(); + this.evdevStreams = []; + for (const node of nodes) { + try { + const stream = fs.createReadStream(node); + let buf = Buffer.alloc(0); + stream.on('data', (chunk) => { + buf = buf.length ? Buffer.concat([buf, chunk]) : chunk; + const { presses, rest } = decodeEvdevButtonPresses(buf); + buf = rest; + for (const button of presses) this.onOsClick(Date.now(), null, button); + }); + // A device can disappear (unplugged); just drop that stream. + stream.on('error', () => {}); + this.evdevStreams.push(stream); + } catch { + // Node became unreadable between enumeration and open — skip it. + } + } + if (!this.evdevStreams.length) { + console.error('[stepforge] no readable mouse input devices — add your user to the "input" group for per-click capture: sudo usermod -aG input "$USER" (then log out and back in)'); + } else { + console.log(`[stepforge] per-click capture via evdev on ${this.evdevStreams.length} device(s)${this.onWayland() ? ' (Wayland: no click marker)' : ''}`); + } + } + + stopEvdevWatcher() { + if (!this.evdevStreams) return; + for (const stream of this.evdevStreams) { + try { stream.destroy(); } catch { /* already closed */ } + } + this.evdevStreams = null; + } + /** * Buffer stdout chunks and only parse complete lines: a chunk boundary * can split an event line in half, which used to corrupt press/release @@ -1461,7 +1646,11 @@ public static class SFHook { // filtered by the cursor-position check in sessionCapture, not by // window focus — WSLg reports focus unreliably.) let clickPos = osPoint ? this.osPointToDip(osPoint) : null; - if (!clickPos) clickPos = this.screen.getCursorScreenPoint(); + // Read the live cursor as a fallback only off Wayland: Wayland refuses to + // report the global pointer position (getCursorScreenPoint returns 0,0), so + // a fallback there would stamp every click marker in the top-left corner. + // Leaving clickPos null means the step is captured with no (wrong) marker. + if (!clickPos && !this.onWayland()) clickPos = this.screen.getCursorScreenPoint(); clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos); this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y) ? { x: osPoint.x, y: osPoint.y } @@ -1826,3 +2015,5 @@ public static class SFHook { } module.exports = CaptureService; +// Exposed for unit tests (pure, no device access). +module.exports.decodeEvdevButtonPresses = decodeEvdevButtonPresses; diff --git a/app/main.js b/app/main.js index 6b02d6c..c4342db 100644 --- a/app/main.js +++ b/app/main.js @@ -5,7 +5,7 @@ const fs = require('node:fs'); const os = require('node:os'); const { app, BrowserWindow, ipcMain, dialog, shell, nativeTheme, globalShortcut, - clipboard, nativeImage, screen, powerSaveBlocker, session, + clipboard, nativeImage, screen, powerSaveBlocker, session, desktopCapturer, } = require('electron'); const { GuideStore } = require('../core/store'); @@ -95,6 +95,11 @@ function createWindow() { contextIsolation: true, nodeIntegration: false, spellcheck: Boolean(settings.get('spellcheck')), + // During a recording the window is minimized (Linux) or hidden (Windows). + // A throttled renderer stops processing capture:added events, so the step + // list and capture bar appear "stuck" even though steps are saved. Keep + // the renderer live so the UI updates in real time while recording. + backgroundThrottling: false, }, }); mainWindow.loadFile(path.join(__dirname, 'renderer', 'index.html')); @@ -835,14 +840,34 @@ if (!gotLock) { textIntel, }); - // Allow the hidden capture-worker renderer to open a desktop media stream - // via getUserMedia. Electron 29+ requires an explicit permission grant for - // display-capture in renderer windows; without it the getUserMedia call - // fails, the stream backend never starts, and every capture falls back to + // Allow the hidden capture-worker renderer to open a desktop media stream. + // Electron 29+ requires an explicit permission grant for display-capture in + // renderer windows; without it getUserMedia/getDisplayMedia fails, the + // stream backend never starts, and every capture falls back to // desktopCapturer.getSources() — which triggers the portal dialog on Linux // on every single capture. StepForge is fully local/offline so allowing // all permissions for our own content is safe. session.defaultSession.setPermissionCheckHandler(() => true); + session.defaultSession.setPermissionRequestHandler((_wc, _perm, cb) => cb(true)); + + // On GNOME Wayland the only working screen-capture path is the portal-backed + // getDisplayMedia (desktopCapturer source ids fail with "device not found"). + // The worker calls getDisplayMedia; this handler answers it. Calling + // getSources() *inside* the handler is the documented Wayland path: it + // drives the XDG portal picker (shown once when a recording starts), and + // the chosen source then streams for the whole session. (useSystemPicker is + // macOS-only today, harmless elsewhere.) + session.defaultSession.setDisplayMediaRequestHandler((request, callback) => { + desktopCapturer.getSources({ types: ['screen'] }) + .then((sources) => { + console.log(`[stepforge] display-media request resolved: ${sources.length} screen source(s)`); + callback(sources.length ? { video: sources[0] } : {}); + }) + .catch((err) => { + console.error(`[stepforge] display-media getSources failed: ${err && err.message}`); + callback({}); + }); + }, { useSystemPicker: true }); applyTheme(); setupIpc(); diff --git a/app/renderer/capture-worker.js b/app/renderer/capture-worker.js index 8726474..6fc34c4 100644 --- a/app/renderer/capture-worker.js +++ b/app/renderer/capture-worker.js @@ -47,11 +47,6 @@ async function startStream(cmd) { const key = String(cmd.displayId); stopStream(key); - const display = cmd.display || {}; - const scale = display.scaleFactor || 1; - const bounds = display.bounds || { width: 1280, height: 720 }; - const physWidth = Math.round(bounds.width * scale); - const physHeight = Math.round(bounds.height * scale); const state = { displayId: cmd.displayId, media: null, @@ -68,20 +63,33 @@ }; streams.set(key, state); try { - // The chromeMediaSource constraint is Electron's bridge from a - // desktopCapturer source id to a live media stream. The legacy - // `mandatory` wrapper was removed in Electron 29 (Chromium 116+); - // constraints must now be flat (no mandatory/optional nesting). - state.media = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: { - chromeMediaSource: 'desktop', - chromeMediaSourceId: cmd.sourceId, - // Sampling cadence is controlled by the setInterval timer, so the - // actual capture rate is sampleMs-driven regardless of display - // refresh rate. Resolution is driven by the source itself. - }, - }); + if (cmd.useDisplayMedia) { + // GNOME Wayland path: desktopCapturer source ids fail with + // getUserMedia, so go through the portal-backed getDisplayMedia. The + // main process installs a setDisplayMediaRequestHandler that answers + // this request; the OS portal picker chooses the screen. The stream + // then stays open for the whole session — one prompt, not one per shot. + state.media = await navigator.mediaDevices.getDisplayMedia({ + audio: false, + video: true, + }); + } else { + // The chromeMediaSource constraint is Electron's bridge from a + // desktopCapturer source id to a live media stream. The legacy + // `mandatory` wrapper was removed in Electron 29 (Chromium 116+); + // constraints must now be flat (no mandatory/optional nesting). This + // path works on Windows and X11. + state.media = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + chromeMediaSource: 'desktop', + chromeMediaSourceId: cmd.sourceId, + // Sampling cadence is controlled by the setInterval timer, so the + // actual capture rate is sampleMs-driven regardless of display + // refresh rate. Resolution is driven by the source itself. + }, + }); + } const video = document.createElement('video'); video.muted = true; video.srcObject = state.media; diff --git a/app/stream-backend.js b/app/stream-backend.js index 32b91d2..302da4f 100644 --- a/app/stream-backend.js +++ b/app/stream-backend.js @@ -83,9 +83,19 @@ class StreamCaptureBackend { * Spin up the worker and one stream per display that has a matching screen * source. Resolves true when at least one stream is delivering frames. */ - async start({ displays = [], sources = [], sampleMs = DEFAULT_SAMPLE_MS, retentionMs = null, frameLimit = null } = {}) { + async start({ + displays = [], sources = [], sampleMs = DEFAULT_SAMPLE_MS, + retentionMs = null, frameLimit = null, useDisplayMedia = false, + } = {}) { if (this.host) return this.active; - const pairs = pairDisplaysToSources(displays, sources); + // On GNOME Wayland, desktopCapturer source ids can't be reopened with + // getUserMedia ("Requested device not found") — the portal-backed + // getDisplayMedia path is the only one that works. There's no per-display + // source id in that mode, so capture a single stream against the primary + // display (the portal picker decides which screen is actually shared). + const pairs = useDisplayMedia + ? (displays.length ? [{ display: displays[0], sourceId: null }] : []) + : pairDisplaysToSources(displays, sources); if (!pairs.length) return false; try { this.host = await this.createHost((msg) => this.handleWorkerEvent(msg)); @@ -99,6 +109,7 @@ class StreamCaptureBackend { type: 'start-stream', displayId: display.id, sourceId, + useDisplayMedia, // The worker needs the physical pixel size to request a full-res // stream; bounds stay in DIP for marker math back in the main process. display: { @@ -170,7 +181,7 @@ class StreamCaptureBackend { clearTimeout(pending.timer); pending.timer = setTimeout(() => { this.settleRequest(msg.requestId, null); - this.noteFailure(); + if (pending.failable !== false) this.noteFailure(); }, this.encodeTimeoutMs); return; } @@ -213,7 +224,7 @@ class StreamCaptureBackend { * Resolves null when no frame qualifies (caller falls back) — and also on * timeout, which additionally counts toward unhealthiness. */ - frameForClick({ clickPos = null, clickAt = Date.now(), strict = true, leadMs = 0 } = {}) { + frameForClick({ clickPos = null, clickAt = Date.now(), strict = true, leadMs = 0, failable = true } = {}) { if (!this.active || !this.host) return Promise.resolve(null); const displays = [...this.streams.values()].filter((s) => s.ready).map((s) => s.display); const display = clickPos ? displayForDipPoint(clickPos, displays) : (displays[0] || null); @@ -225,10 +236,15 @@ class StreamCaptureBackend { if (clickPos && !pointInBounds(clickPos, display.bounds)) return Promise.resolve(null); const requestId = this.nextRequestId++; return new Promise((resolve) => { - const pending = { resolve, display, timer: null }; + // failable=false: a timeout resolves null but does NOT count toward + // unhealthiness. Interval/timed captures use this so a single slow PNG + // encode (common on software-rendered hosts with no GPU) can't trip the + // 2-strikes rule and tear down an otherwise-healthy stream — the bug + // that left Wayland recordings "stuck after two captures". + const pending = { resolve, display, timer: null, failable }; pending.timer = setTimeout(() => { this.settleRequest(requestId, null); - this.noteFailure(); + if (failable) this.noteFailure(); }, this.ackTimeoutMs); this.requests.set(requestId, pending); this.hostSend({ diff --git a/tests/unit/capture.test.js b/tests/unit/capture.test.js index 5e0414f..cbb7067 100644 --- a/tests/unit/capture.test.js +++ b/tests/unit/capture.test.js @@ -596,6 +596,7 @@ test('armRecording warms while visible, then hides and arms the session', async isVisible() { return this.visible; }, isMinimized() { return false; }, hide() { this.visible = false; }, + minimize() { this.visible = false; }, show() { this.visible = true; }, focus() {}, getTitle() { return 'StepForge'; }, getBounds() { return { x: 0, y: 0, width: 800, height: 600 }; }, @@ -637,6 +638,7 @@ test('a slow recorder start still arms within the warmup cap', async () => { isVisible() { return this.visible; }, isMinimized() { return false; }, hide() { this.visible = false; }, + minimize() { this.visible = false; }, show() { this.visible = true; }, focus() {}, getTitle() { return 'StepForge'; }, getBounds() { return { x: 0, y: 0, width: 800, height: 600 }; }, @@ -746,6 +748,7 @@ test('clicks without event coordinates fall back to a live cursor read', () => { getAllDisplays: () => [], }, }); + service._wayland = false; // X11/Windows: the live-cursor fallback applies service.session = { guideId: 'guide-cursor', paused: false, count: 0, intervalSec: 0 }; const seen = []; service.enqueueClickCapture = (clickPos) => { @@ -757,6 +760,64 @@ test('clicks without event coordinates fall back to a live cursor read', () => { assert.deepEqual(seen, [{ x: 11, y: 22 }]); }); +test('on Wayland an evdev click does not stamp a bogus top-left cursor position', () => { + // Wayland's getCursorScreenPoint returns {0,0}; a fallback read would mark + // every click in the corner. The click must capture with a null position + // (no marker) instead. + const service = makeService({ + screenApi: { + getCursorScreenPoint: () => ({ x: 0, y: 0 }), + getAllDisplays: () => [], + }, + }); + service._wayland = true; + service.session = { guideId: 'guide-wl-click', paused: false, count: 0, intervalSec: 0 }; + const seen = []; + service.enqueueClickCapture = (clickPos) => { seen.push(clickPos); }; + + service.onOsClick(1770000000000, null, 'button-1'); + + assert.deepEqual(seen, [null], 'no position is passed, so no marker is drawn'); +}); + +// ---- evdev decoding --------------------------------------------------------------- + +const { decodeEvdevButtonPresses } = CaptureService; + +// Pack a 64-bit struct input_event (24 bytes): 16-byte timeval, u16 type, +// u16 code, s32 value. +function evdevRecord(type, code, value) { + const b = Buffer.alloc(24); + b.writeUInt16LE(type, 16); + b.writeUInt16LE(code, 18); + b.writeInt32LE(value, 20); + return b; +} + +test('evdev decoder extracts left/right/middle button presses, ignores the rest', () => { + const EV_KEY = 0x01; + const EV_REL = 0x02; + const buf = Buffer.concat([ + evdevRecord(EV_KEY, 272, 1), // BTN_LEFT down -> button-1 + evdevRecord(EV_KEY, 272, 0), // BTN_LEFT up -> ignored (release) + evdevRecord(EV_REL, 0, 5), // motion -> ignored (not a key) + evdevRecord(EV_KEY, 273, 1), // BTN_RIGHT down -> button-3 + evdevRecord(EV_KEY, 274, 1), // BTN_MIDDLE down-> button-2 + evdevRecord(EV_KEY, 0x110 + 8, 1), // a non-mouse key -> ignored + ]); + const { presses, rest } = decodeEvdevButtonPresses(buf, 24); + assert.deepEqual(presses, ['button-1', 'button-3', 'button-2']); + assert.equal(rest.length, 0); +}); + +test('evdev decoder keeps a trailing partial record for the next chunk', () => { + const full = evdevRecord(0x01, 272, 1); + const buf = Buffer.concat([full, full.subarray(0, 10)]); // 1.4 records + const { presses, rest } = decodeEvdevButtonPresses(buf, 24); + assert.deepEqual(presses, ['button-1']); + assert.equal(rest.length, 10, 'the half record is returned to be completed later'); +}); + // ---- watcher loss ----------------------------------------------------------------- test('losing the click watcher mid-session falls back to interval capture', () => { @@ -1004,6 +1065,7 @@ test('clicks during an in-flight pre-click grab wait for the frame instead of be test('click frames come from the stream backend when it is active', async () => { const service = makeService(); + service._wayland = false; // X11/Windows: click frame-requests are failable const clickAt = Date.now(); service.session = { guideId: 'guide-stream', paused: false, count: 0, intervalSec: 0 }; const requests = []; @@ -1036,8 +1098,8 @@ test('click frames come from the stream backend when it is active', async () => assert.equal(result.ok, true); assert.deepEqual(added, ['stream-frame']); - assert.deepEqual(requests, [{ clickPos: { x: 10, y: 10 }, clickAt, strict: true, leadMs: 0 }], - 'the worker receives the hook-time click timestamp, strictness, and lead'); + assert.deepEqual(requests, [{ clickPos: { x: 10, y: 10 }, clickAt, strict: true, leadMs: 0, failable: true }], + 'the worker receives the hook-time click timestamp, strictness, lead, and failability'); }); test('a stream backend with no qualifying frame falls through to the fresh-shot path', async () => { @@ -1080,6 +1142,9 @@ test('an unhealthy stream backend degrades to the in-process frame loop', () => const service = makeService(); service.session = { guideId: 'guide-degrade', paused: false, count: 0, intervalSec: 0 }; service.streamBackend = { isActive: () => true, stop: () => {} }; + // Off Wayland the in-process loop is viable (getSources works), so an + // unhealthy worker must degrade to the loop rather than silently stopping. + service._wayland = false; let loopStarted = false; service.startFrameLoop = () => { loopStarted = true; }; const states = []; @@ -1092,6 +1157,24 @@ test('an unhealthy stream backend degrades to the in-process frame loop', () => assert.ok(states.includes('capture:state')); }); +test('an unhealthy stream backend does NOT start the frame loop on Wayland', () => { + // On Wayland the 200ms getSources() frame loop is broken (throws) and pops + // the portal, so it's not a usable fallback — the portal stream is the only + // capture path and the user must re-share if it stops. + const service = makeService(); + service.session = { guideId: 'guide-degrade-wl', paused: false, count: 0, intervalSec: 0 }; + service.streamBackend = { isActive: () => true, stop: () => {} }; + service._wayland = true; + let loopStarted = false; + service.startFrameLoop = () => { loopStarted = true; }; + service.notify = () => {}; + + service.degradeToFrameLoop(); + + assert.equal(service.streamBackend, null); + assert.equal(loopStarted, false, 'no frame loop on Wayland — getSources is unusable there'); +}); + test('session state reports which frame recorder is serving clicks', () => { const service = makeService(); service.session = { guideId: 'guide-state', paused: false, count: 0, intervalSec: 0 }; @@ -1156,7 +1239,12 @@ test('a new session starts paused and does not hide the window until "Start reco isDestroyed() { return this.destroyed; }, isVisible() { return this.visible; }, isMinimized() { return this.minimized; }, + // The window is "tucked away" for recording either by hide() (with a tray) + // or minimize() (Linux without a tray). Both make it not visible; assert on + // visibility so the test is independent of which mechanism the platform picks. hide() { this.visible = false; this.hidden += 1; }, + minimize() { this.visible = false; this.minimized = true; }, + restore() { this.minimized = false; this.visible = true; }, show() { this.visible = true; this.shown += 1; }, showInactive() { this.visible = true; this.shown += 1; }, focus() {}, @@ -1171,15 +1259,15 @@ test('a new session starts paused and does not hide the window until "Start reco assert.equal(service.session.paused, true, 'sessions start paused'); assert.equal(service.state().paused, true); - assert.equal(win.hidden, 0, 'window must stay visible until recording starts'); + assert.equal(win.visible, true, 'window must stay visible until recording starts'); // User clicks "Start recording" (the resume action). service.togglePause(false); assert.equal(service.session.paused, false); - assert.equal(win.hidden, 0, 'hide is deferred until the resume path runs'); + assert.equal(win.visible, true, 'tuck-away is deferred until the resume path runs'); await new Promise((r) => setTimeout(r, 25)); - assert.equal(win.hidden, 1, 'window hides once recording actually starts'); + assert.equal(win.visible, false, 'window is tucked away once recording actually starts'); } finally { service.finishSession(); }