diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ca7eea1..ef2e527 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: version: - description: 'Release tag, e.g. v0.1.1 (the tag is created at the current commit on this branch)' + description: 'Release tag, e.g. v0.3.2.1 (the tag is created at the current commit on this branch)' required: true type: string prerelease: @@ -42,7 +42,7 @@ jobs: shell: bash env: VERSION: ${{ inputs.version }} - run: npm version "${VERSION#v}" --no-git-tag-version --allow-same-version + run: node scripts/stamp-version.js "${VERSION#v}" - name: Configure Windows code signing shell: bash diff --git a/README.md b/README.md index 386dae1..8200c4a 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ using only Node built-ins. For a Windows installation, see [docs/windows_installation](docs/windows_installation.md) or for a developer/more in depth walkthrough, see [docs/GETTING_STARTED.md](docs/GETTING_STARTED.md). +On **Linux** (⚠️ work in progress — X11 vs Wayland, enabling per-click capture, the screen-share prompt), see [docs/GETTING_STARTED_WITH_LINUX.md](docs/GETTING_STARTED_WITH_LINUX.md). + Requirements: Node.js 20+ and npm (Electron is the only dependency). ```bash diff --git a/TODO.md b/TODO.md index 97ce594..b6feb53 100644 --- a/TODO.md +++ b/TODO.md @@ -1 +1 @@ -In the future, planned updates include a re-vamp of the settings panel, text box's (important, tip, warning, and note) are not properly moving to the intended areas and I will look into locking the size (or being able to control the size) of images throughout varias export formats. +In the future, planned updates include a re-vamp of the settings panel, text box's (important, tip, warning, and note) are not properly moving to the intended areas and I will look into locking the size (or being able to control the size) of images throughout varias export formats. OCR or local ai would be pretty cool.... diff --git a/app/capture.js b/app/capture.js index eebe825..5fd2701 100644 --- a/app/capture.js +++ b/app/capture.js @@ -1,6 +1,7 @@ 'use strict'; const path = require('node:path'); +const fs = require('node:fs'); const { spawn, execFileSync } = require('node:child_process'); const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron'); const raster = require('../core/raster'); @@ -107,6 +108,88 @@ function hasBinary(name) { } } +// On Wayland, xinput only sees XWayland (X11-bridge) events — native Wayland +// app clicks are delivered via the Wayland protocol and never reach xinput. +// Treating it as "available" would leave the session with no click capture AND +// no interval fallback, so zero steps get captured. +function isWayland() { + if (process.platform !== 'linux') return false; + // XDG_SESSION_TYPE is the authoritative session hint when present. Some + // desktops still export WAYLAND_DISPLAY even when the active session is X11, + // so only fall back to it when XDG_SESSION_TYPE is unavailable. + const sessionType = String(process.env.XDG_SESSION_TYPE || '').toLowerCase(); + if (sessionType) return sessionType === 'wayland'; + return Boolean(process.env.WAYLAND_DISPLAY); +} + +// ---- evdev (Linux kernel input) click reader -------------------------------- +// Reading /dev/input/event* directly sees mouse-button presses on BOTH X11 and +// Wayland, because it taps the kernel input layer below the display server — +// the one global-click source that survives Wayland's security model. It needs +// read access to the device nodes (the user must be in the `input` group). +// +// Each event is a fixed-size `struct input_event`: a timeval, then u16 type, +// u16 code, s32 value. The timeval is two `long`s, so the record is 24 bytes on +// 64-bit and 16 on 32-bit; type/code/value always sit in the last 8 bytes. +const EV_KEY = 0x01; +const EVDEV_PRESS = 1; +// BTN_LEFT/RIGHT/MIDDLE -> the same button-N naming the xinput path emits +// (1=left, 2=middle, 3=right) so downstream debounce/marker logic is identical. +const EVDEV_BUTTONS = { 272: 'button-1', 273: 'button-3', 274: 'button-2' }; +const EVDEV_RECORD_SIZE = (process.arch === 'x64' || process.arch === 'arm64' + || process.arch === 'ppc64' || process.arch === 's390x' || process.arch === 'loong64' + || process.arch === 'riscv64') ? 24 : 16; + +/** + * Decode a buffer of packed input_event records, returning the button presses + * found and any trailing partial record (a device read can split mid-record). + * Pure and size-parameterised so it is unit-testable without real devices. + */ +function decodeEvdevButtonPresses(buffer, recordSize = EVDEV_RECORD_SIZE) { + const presses = []; + let offset = 0; + while (buffer.length - offset >= recordSize) { + const type = buffer.readUInt16LE(offset + recordSize - 8); + const code = buffer.readUInt16LE(offset + recordSize - 6); + const value = buffer.readInt32LE(offset + recordSize - 4); + offset += recordSize; + if (type === EV_KEY && value === EVDEV_PRESS && EVDEV_BUTTONS[code]) { + presses.push(EVDEV_BUTTONS[code]); + } + } + return { presses, rest: buffer.subarray(offset) }; +} + +/** + * The /dev/input/event* nodes for pointing devices that are readable by this + * process. /proc/bus/input/devices lists every device with its Handlers line; + * a pointing device exposes a `mouseN` handler, and the matching `eventN` is + * the node to read. Unreadable nodes (no `input` group membership) are skipped. + */ +function readableEvdevMouseNodes() { + const nodes = []; + let table; + try { + table = fs.readFileSync('/proc/bus/input/devices', 'utf8'); + } catch { + return nodes; + } + for (const block of table.split('\n\n')) { + const handlers = /H:\s*Handlers=([^\n]*)/.exec(block); + if (!handlers || !/\bmouse\d+\b/.test(handlers[1])) continue; + const event = /\bevent(\d+)\b/.exec(handlers[1]); + if (!event) continue; + const node = `/dev/input/event${event[1]}`; + try { + fs.accessSync(node, fs.constants.R_OK); + nodes.push(node); + } catch { + // Not readable — user is not in the `input` group for this node. + } + } + return nodes; +} + class CaptureService { constructor({ store, @@ -124,6 +207,9 @@ class CaptureService { // the global `screen` directly so coordinate handling stays testable. this.screen = screenApi; this.textIntel = textIntel; + // Cached display-server detection. A method (onWayland) reads this so tests + // can flip platform behavior without touching process.env. + this._wayland = isWayland(); this.session = null; // { guideId, paused, count, intervalSec } this.intervalTimer = null; this.clickWatcher = null; @@ -182,21 +268,62 @@ class CaptureService { return this.settings.get('capture.strictClickFrames') !== false; } + fallbackCaptureTrigger() { + const raw = String(this.settings.get('capture.fallbackTrigger') || 'interval').toLowerCase(); + return raw === 'hotkey' ? 'hotkey' : 'interval'; + } + + fallbackIntervalSec() { + const raw = Number(this.settings.get('capture.autoIntervalSec')); + return Number.isFinite(raw) && raw > 0 ? raw : 5; + } + clickCaptureAvailable() { if (this._clickAvail === undefined) { - this._clickAvail = process.platform === 'win32' || (process.platform === 'linux' && hasBinary('xinput')); + // Three click sources, in order of fidelity: + // - Windows: the low-level mouse hook (position + timing); + // - X11: xinput test-xi2 (position + timing) — but it can't see native + // Wayland clicks, only XWayland ones, so it's gated to non-Wayland; + // - Linux evdev (/dev/input): button presses on X11 AND Wayland, but no + // cursor position on Wayland — used for per-click capture there (no + // marker). Requires the user to be in the `input` group. + this._clickAvail = process.platform === 'win32' + || (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput')) + || (process.platform === 'linux' && readableEvdevMouseNodes().length > 0); } return this._clickAvail; } + /** Whether this is a Wayland session (cached; overridable in tests). */ + onWayland() { + return this._wayland; + } + + /** + * Whether the in-process frame loop is a usable fallback recorder. It grabs + * via desktopCapturer.getSources(), which on Wayland is broken (throws) and + * pops the portal — so the loop is viable only off Wayland. On Wayland the + * portal-backed stream backend is the sole capture path. + */ + canUseFrameLoop() { + return !this.onWayland(); + } + startSession(guideId, { intervalSec = null } = {}) { this.finishSession(); - // Default trigger: clicks when the platform supports it, otherwise an - // interval so a session always produces steps even if the global hotkey - // never fires (common under Wayland/WSLg). + // Default trigger: clicks when the platform supports it, otherwise the + // user-selected fallback (timer or hotkey-only). That keeps Linux from + // silently dropping into an unwanted 5-second timer when click capture + // is unavailable. let interval = intervalSec; if (interval == null) { - interval = this.clickCaptureAvailable() ? 0 : (this.settings.get('capture.autoIntervalSec') || 5); + if (this.clickCaptureAvailable()) { + interval = 0; + } else if (this.fallbackCaptureTrigger() === 'hotkey') { + interval = 0; + } else { + interval = this.fallbackIntervalSec(); + } } // Sessions start paused: nothing hides and no capturing happens until // the user explicitly presses "Start recording" in the capture bar, so @@ -300,6 +427,7 @@ class CaptureService { showWindow() { const win = this.getWindow(); if (win && !win.isDestroyed()) { + if (win.isMinimized()) win.restore(); win.show(); win.focus(); } @@ -321,7 +449,14 @@ class CaptureService { const sec = this.session && this.session.intervalSec; if (sec > 0) { this.intervalTimer = setInterval(() => { - this.sessionCapture('interval').catch(() => {}); + // Don't let a slow capture (e.g. a multi-second software PNG encode on + // a GPU-less host) overlap with the next tick — overlapping requests + // would pile up and could trip the backend's failure counter. + if (this.intervalCapturing) return; + this.intervalCapturing = true; + this.sessionCapture('interval') + .catch(() => {}) + .finally(() => { this.intervalCapturing = false; }); }, sec * 1000); } } @@ -358,8 +493,13 @@ class CaptureService { armRecording() { const win = this.getWindow(); const wantHide = Boolean(this.hiddenForSession && win && !win.isDestroyed()); - const recorderWanted = this.settings.get('capture.captureOutsideClicks') !== false - && this.clickCaptureAvailable(); + // Always start the frame recorder when stream capture is enabled — it + // buffers frames for click captures AND is used for interval/hotkey + // captures to avoid calling desktopCapturer.getSources() on every capture. + // On Linux/Wayland, each getSources() call goes through the XDG portal and + // shows a permission dialog; the stream backend eliminates that by keeping + // a live video stream open for the duration of the recording session. + const recorderWanted = this.settings.get('capture.streamCapture') !== false; // Recording is not "live" until the window is hidden and the buffer is // primed. While warming up, the window is still visible and over the // user's work, so clicks in this period are ignored (onOsClick checks @@ -383,7 +523,17 @@ class CaptureService { if (!this.session || this.session.paused) { this.warmingUp = false; return; } } if (wantHide && win && !win.isDestroyed() && win.isVisible()) { - win.hide(); + // On Linux, always minimize rather than hide. GNOME's system tray + // (StatusNotifier) is unreliable — it can fail or half-export over + // dbus — so a hidden window can be impossible to bring back, leaving + // the user unable to stop the recording. A minimized window is always + // restorable from the taskbar, and minimized windows aren't rendered + // so they still stay out of the fullscreen capture. + if (process.platform === 'linux') { + win.minimize(); + } else { + win.hide(); + } // Let a couple of frames of the now-unobscured screen land before // the user's first click, so that frame shows their work, not the // app window that was just dismissed. @@ -481,6 +631,46 @@ class CaptureService { if (!sessionLive) return { ok: false, reason: 'session ended before the fallback shot' }; } + // For non-click triggers (interval, hotkey, manual) pull the latest frame + // from the stream backend's ring buffer when available. This avoids a + // desktopCapturer.getSources() call per capture — on Linux/Wayland that + // call goes through the XDG portal and shows a dialog every time. + // + // No clickPos: a timed capture has no click position, and passing a cursor + // point here is actively harmful on Wayland — getCursorScreenPoint() can + // return a stale/out-of-bounds point, which makes the backend reject the + // frame (wrong display / out of bounds) and fall through to a getSources() + // shot, i.e. a portal dialog on every interval tick. + if (trigger !== 'click' && this.streamBackend && this.streamBackend.isActive()) { + const frame = await this.streamBackend.frameForClick({ + clickPos: null, + clickAt: Date.now(), + strict: false, // no pre/post-click constraint for timed captures + leadMs: 0, + failable: false, // a slow timed-capture encode must not kill the stream + }).catch(() => null); + if (frame) { + const result = await this.storeFrameAsStep(this.session.guideId, 'fullscreen', frame); + if (result.ok) this.noteStepAdded(result.step, trigger); + clog(trigger, 'capture stored from stream; total', this.session && this.session.count); + return result; + } + clog(trigger, 'capture: no frame from stream this tick — will retry next tick'); + } else if (trigger !== 'click' && this.onWayland()) { + clog(trigger, 'capture: stream backend not active (active=', + Boolean(this.streamBackend && this.streamBackend.isActive()), ')'); + } + + // On Wayland the only screen-grab fallback below is desktopCapturer + // .getSources(), which pops the XDG portal dialog every call. For the + // automatic timed triggers that would mean a dialog on every tick, so skip + // the fallback and wait for the open stream to deliver a frame on a later + // tick. Explicit captures (manual, and click on X11) still fall through — + // one dialog for one deliberate action. + if (this.onWayland() && (trigger === 'interval' || trigger === 'hotkey')) { + return { ok: false, reason: 'waiting for the screen-share stream' }; + } + if (this.shooting) return { ok: false, reason: 'capture already in progress' }; this.shooting = true; try { @@ -642,7 +832,11 @@ class CaptureService { }; if (this.streamBackend && this.streamBackend.isActive() && grabMode === 'fullscreen') { - const frame = await this.streamBackend.frameForClick({ clickPos, clickAt: clickTime, strict, leadMs }); + // On Wayland the stream is the only capture path (no frame-loop fallback), + // so a slow PNG encode must not let the 2-strikes rule tear it down. + const frame = await this.streamBackend.frameForClick({ + clickPos, clickAt: clickTime, strict, leadMs, failable: !this.onWayland(), + }); if (frame) return frame; // No qualifying frame (or the backend just went unhealthy): fall // through to the loop buffer / fresh-shot fallbacks below. @@ -695,8 +889,11 @@ class CaptureService { async startClickFrameBackend() { const mode = this.settings.get('capture.mode') || 'fullscreen'; // The worker streams screens; window-mode grabs need the loop's - // source-filtering logic. - if (this.settings.get('capture.streamCapture') === false || mode === 'window') { + // source-filtering logic. But the loop isn't viable on Wayland (getSources + // is broken/portal), so there we always take the stream backend regardless + // of the streamCapture/window settings. + if (this.canUseFrameLoop() + && (this.settings.get('capture.streamCapture') === false || mode === 'window')) { this.startFrameLoop(); return; } @@ -716,7 +913,11 @@ class CaptureService { onUnhealthy: () => this.degradeToFrameLoop(), }); const displays = this.screen.getAllDisplays(); - const sources = await desktopCapturer.getSources({ + // On Wayland, desktopCapturer.getSources() both fails to yield usable + // source ids AND pops the portal dialog. Skip it entirely and drive the + // worker through getDisplayMedia (the portal picker chooses the screen). + const useDisplayMedia = this.onWayland(); + const sources = useDisplayMedia ? [] : await desktopCapturer.getSources({ types: ['screen'], thumbnailSize: { width: 1, height: 1 }, // ids only — skip thumbnail work }); @@ -724,23 +925,38 @@ class CaptureService { displays, sources: sources.map((s) => ({ id: s.id, display_id: s.display_id })), sampleMs: this.settings.get('capture.frameSampleMs') || 100, + useDisplayMedia, }); const stale = gen !== this.captureGen; if (!ok || stale || !this.session || this.session.paused) { backend.stop(); if (!stale && this.session && !this.session.paused) { - console.error('[stepforge] stream capture backend failed to start — using in-process frame loop'); - this.startFrameLoop(); + if (this.canUseFrameLoop()) { + console.error('[stepforge] stream capture backend failed to start — using in-process frame loop'); + this.startFrameLoop(); + } else { + // On Wayland the frame loop would spam getSources() (portal) with + // nothing usable, so there's no fallback — the recording needs the + // portal stream. Tell the user how to recover. + console.error('[stepforge] screen-share stream did not start — pick a screen in the share dialog, or stop and start recording again'); + } } return; } this.streamBackend = backend; - clog('stream capture backend active'); + // Visible in normal output (one line per recording): confirms the screen + // stream came up, so a "nothing records" report can be told apart from a + // stream that never started (which logs the failure paths above). + console.log(`[stepforge] screen-capture stream active (${useDisplayMedia ? 'getDisplayMedia/portal' : 'desktopCapturer'})`); this.notify('capture:state', this.state()); } catch (err) { if (gen === this.captureGen && this.session && !this.session.paused) { - console.error(`[stepforge] stream capture backend error (${err && err.message}) — using in-process frame loop`); - this.startFrameLoop(); + if (this.canUseFrameLoop()) { + console.error(`[stepforge] stream capture backend error (${err && err.message}) — using in-process frame loop`); + this.startFrameLoop(); + } else { + console.error(`[stepforge] screen-share stream error (${err && err.message}) — stop and start recording again`); + } } } finally { if (gen === this.captureGen) this.streamBackendStarting = false; @@ -765,8 +981,14 @@ class CaptureService { */ degradeToFrameLoop() { this.streamBackend = null; - console.error('[stepforge] stream capture backend unhealthy — falling back to in-process frame loop'); - if (this.session && !this.session.paused) this.startFrameLoop(); + if (this.canUseFrameLoop()) { + console.error('[stepforge] stream capture backend unhealthy — falling back to in-process frame loop'); + if (this.session && !this.session.paused) this.startFrameLoop(); + } else { + // On Wayland the frame loop isn't viable (getSources is broken/portal), + // so there's nothing to fall back to — the stream is the only path. + console.error('[stepforge] screen-share stream stopped — stop and start recording again to re-share'); + } this.notify('capture:state', this.state()); } @@ -775,13 +997,14 @@ class CaptureService { try { this.clickWatcherBuf = ''; this.linuxEvent = null; - if (process.platform === 'linux' && hasBinary('xinput')) { + if (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput')) { // Stream raw button events from the X server; one capture per press. // xinput block-buffers stdout when piped, so a press event can sit // in its buffer until later motion events flush it — by then the // cursor read in onOsClick lands where the mouse moved *after* the // click. stdbuf -oL forces line-buffering so events (and the cursor // read) line up with the actual click instant. + // (Skipped on Wayland: xinput only sees XWayland events — see isWayland.) const argv = hasBinary('stdbuf') ? ['stdbuf', '-oL', 'xinput', 'test-xi2', '--root'] : ['xinput', 'test-xi2', '--root']; @@ -789,6 +1012,13 @@ class CaptureService { this.clickWatcher.stdout.on('data', (chunk) => { this.ingestClickWatcherChunk(chunk.toString(), 'linux'); }); + } else if (process.platform === 'linux' && readableEvdevMouseNodes().length > 0) { + // Wayland (or X11 without xinput): read mouse buttons from the kernel + // input layer. This is the only global click source on Wayland, but it + // carries no cursor position — onOsClick gets a null point, so steps are + // captured per click without a marker. (X11 prefers the xinput branch + // above, which does carry root coordinates for the marker.) + this.startEvdevWatcher(); } else if (process.platform === 'win32') { // Use a low-level Windows mouse hook instead of polling // GetAsyncKeyState. The low bit from GetAsyncKeyState can be consumed @@ -1173,7 +1403,9 @@ public static class SFHook { console.error(`[stepforge] click watcher stopped${detail ? `: ${detail}` : ''}`); if (!this.session) return; if (!this.session.intervalSec) { - this.session.intervalSec = this.settings.get('capture.autoIntervalSec') || 5; + this.session.intervalSec = this.fallbackCaptureTrigger() === 'hotkey' + ? 0 + : this.fallbackIntervalSec(); this.applyInterval(); } this.notify('capture:state', this.state()); @@ -1184,12 +1416,54 @@ public static class SFHook { try { this.clickWatcher.kill(); } catch { /* already gone */ } this.clickWatcher = null; } + this.stopEvdevWatcher(); this.clickWatcherBuf = ''; this.linuxEvent = null; this.discardPendingRawClick(); this.lastAcceptedClickByButton.clear(); } + /** + * Open every readable mouse device node and turn button-down events into + * onOsClick calls. One physical mouse is normally one node; the leading-edge + * debounce in onOsClick collapses any cross-node duplicates. Frames are still + * served by the stream backend; this only supplies the click *trigger*. + */ + startEvdevWatcher() { + const nodes = readableEvdevMouseNodes(); + this.evdevStreams = []; + for (const node of nodes) { + try { + const stream = fs.createReadStream(node); + let buf = Buffer.alloc(0); + stream.on('data', (chunk) => { + buf = buf.length ? Buffer.concat([buf, chunk]) : chunk; + const { presses, rest } = decodeEvdevButtonPresses(buf); + buf = rest; + for (const button of presses) this.onOsClick(Date.now(), null, button); + }); + // A device can disappear (unplugged); just drop that stream. + stream.on('error', () => {}); + this.evdevStreams.push(stream); + } catch { + // Node became unreadable between enumeration and open — skip it. + } + } + if (!this.evdevStreams.length) { + console.error('[stepforge] no readable mouse input devices — add your user to the "input" group for per-click capture: sudo usermod -aG input "$USER" (then log out and back in)'); + } else { + console.log(`[stepforge] per-click capture via evdev on ${this.evdevStreams.length} device(s)${this.onWayland() ? ' (Wayland: no click marker)' : ''}`); + } + } + + stopEvdevWatcher() { + if (!this.evdevStreams) return; + for (const stream of this.evdevStreams) { + try { stream.destroy(); } catch { /* already closed */ } + } + this.evdevStreams = null; + } + /** * Buffer stdout chunks and only parse complete lines: a chunk boundary * can split an event line in half, which used to corrupt press/release @@ -1396,7 +1670,11 @@ public static class SFHook { // filtered by the cursor-position check in sessionCapture, not by // window focus — WSLg reports focus unreliably.) let clickPos = osPoint ? this.osPointToDip(osPoint) : null; - if (!clickPos) clickPos = this.screen.getCursorScreenPoint(); + // Read the live cursor as a fallback only off Wayland: Wayland refuses to + // report the global pointer position (getCursorScreenPoint returns 0,0), so + // a fallback there would stamp every click marker in the top-left corner. + // Leaving clickPos null means the step is captured with no (wrong) marker. + if (!clickPos && !this.onWayland()) clickPos = this.screen.getCursorScreenPoint(); clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos); this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y) ? { x: osPoint.x, y: osPoint.y } @@ -1428,20 +1706,32 @@ public static class SFHook { * scaled away from 100% and on secondary monitors. */ osPointToDip(osPoint) { - if (this.screen && typeof this.screen.screenToDipPoint === 'function') { - try { - const dip = this.screen.screenToDipPoint(osPoint); - if (dip && Number.isFinite(dip.x) && Number.isFinite(dip.y)) return dip; - } catch { /* fall through to manual conversion */ } - } + let geometryDip = null; try { const displays = this.screen && typeof this.screen.getAllDisplays === 'function' ? this.screen.getAllDisplays() : []; - const dip = physicalToDip(osPoint, displays); - if (dip) return dip; - } catch { /* no display geometry available */ } - return osPoint; + geometryDip = physicalToDip(osPoint, displays); + } catch { + geometryDip = null; + } + if (this.screen && typeof this.screen.screenToDipPoint === 'function') { + try { + const dip = this.screen.screenToDipPoint(osPoint); + if (dip && Number.isFinite(dip.x) && Number.isFinite(dip.y)) { + if (!geometryDip) return dip; + const offByX = Math.abs(dip.x - geometryDip.x); + const offByY = Math.abs(dip.y - geometryDip.y); + // Some Windows/Electron combinations have been observed to return a + // raw physical point here. That keeps the click marker off-screen on + // scaled displays, so trust the geometry path when the two disagree + // by more than a tiny rounding margin. + if (offByX <= 1 && offByY <= 1) return dip; + return geometryDip; + } + } catch { /* fall through to manual conversion */ } + } + return geometryDip || osPoint; } /** @@ -1761,3 +2051,5 @@ public static class SFHook { } module.exports = CaptureService; +// Exposed for unit tests (pure, no device access). +module.exports.decodeEvdevButtonPresses = decodeEvdevButtonPresses; diff --git a/app/main.js b/app/main.js index 2e52984..e73a9f2 100644 --- a/app/main.js +++ b/app/main.js @@ -5,7 +5,7 @@ const fs = require('node:fs'); const os = require('node:os'); const { app, BrowserWindow, ipcMain, dialog, shell, nativeTheme, globalShortcut, - clipboard, nativeImage, screen, powerSaveBlocker, + clipboard, nativeImage, screen, powerSaveBlocker, session, desktopCapturer, } = require('electron'); const { GuideStore } = require('../core/store'); @@ -21,6 +21,7 @@ const { readLock } = require('../core/locks'); const CaptureService = require('./capture'); const { TextIntelService } = require('./text-intel'); const { keepProcessesResponsive } = require('./win-power'); +const PACKAGE_JSON = require(path.join(__dirname, '..', 'package.json')); const APP_ID = 'com.stepforge.app'; @@ -95,6 +96,11 @@ function createWindow() { contextIsolation: true, nodeIntegration: false, spellcheck: Boolean(settings.get('spellcheck')), + // During a recording the window is minimized (Linux) or hidden (Windows). + // A throttled renderer stops processing capture:added events, so the step + // list and capture bar appear "stuck" even though steps are saved. Keep + // the renderer live so the UI updates in real time while recording. + backgroundThrottling: false, }, }); mainWindow.loadFile(path.join(__dirname, 'renderer', 'index.html')); @@ -757,6 +763,7 @@ function setupIpc() { h('shell:showItemInFolder', ({ target }) => shell.showItemInFolder(target)); h('app:info', () => ({ version: app.getVersion(), + buildVersion: PACKAGE_JSON.buildVersion || app.getVersion(), dataDir: store.root, platform: process.platform, })); @@ -835,6 +842,35 @@ if (!gotLock) { textIntel, }); + // Allow the hidden capture-worker renderer to open a desktop media stream. + // Electron 29+ requires an explicit permission grant for display-capture in + // renderer windows; without it getUserMedia/getDisplayMedia fails, the + // stream backend never starts, and every capture falls back to + // desktopCapturer.getSources() — which triggers the portal dialog on Linux + // on every single capture. StepForge is fully local/offline so allowing + // all permissions for our own content is safe. + session.defaultSession.setPermissionCheckHandler(() => true); + session.defaultSession.setPermissionRequestHandler((_wc, _perm, cb) => cb(true)); + + // On GNOME Wayland the only working screen-capture path is the portal-backed + // getDisplayMedia (desktopCapturer source ids fail with "device not found"). + // The worker calls getDisplayMedia; this handler answers it. Calling + // getSources() *inside* the handler is the documented Wayland path: it + // drives the XDG portal picker (shown once when a recording starts), and + // the chosen source then streams for the whole session. (useSystemPicker is + // macOS-only today, harmless elsewhere.) + session.defaultSession.setDisplayMediaRequestHandler((request, callback) => { + desktopCapturer.getSources({ types: ['screen'] }) + .then((sources) => { + console.log(`[stepforge] display-media request resolved: ${sources.length} screen source(s)`); + callback(sources.length ? { video: sources[0] } : {}); + }) + .catch((err) => { + console.error(`[stepforge] display-media getSources failed: ${err && err.message}`); + callback({}); + }); + }, { useSystemPicker: true }); + applyTheme(); setupIpc(); createWindow(); diff --git a/app/renderer/app.js b/app/renderer/app.js index cba9994..760ca6d 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -122,6 +122,7 @@ class StepForgeApp { api.library.trashList(), ]); this.state.info = info; + document.body.classList.toggle('platform-linux', info.platform === 'linux'); this.state.settings = settings; this.state.library = { guides: library.guides || [], @@ -263,15 +264,26 @@ class StepForgeApp { updateCaptureState(state) { this.captureState = state || { active: false }; clearNode(this.captureStatus); - // The capture bar only makes sense alongside the editor it's recording - // into — hide it everywhere else (e.g. the library) even if a session - // is still active in the background. - if (!this.captureState.active || this.state.view !== 'editor') { + // The capture bar is editor-only — hide it everywhere else (library, welcome). + if (this.state.view !== 'editor') { this.captureStatus.classList.add('hidden'); return; } this.captureStatus.classList.remove('hidden'); const s = this.captureState; + + // No active session: show a button to start a new one for the open guide. + if (!s.active) { + this.captureStatus.append( + el('span', {}, 'Recording - stopped'), + el('button', { + type: 'button', + onClick: () => this.armCaptureSession(this.editor.guideId), + }, 'New recording'), + ); + return; + } + const send = (payload) => api.capture.session(payload).then((next) => this.updateCaptureState(next)); // What is currently triggering captures, so the user knows what to do. @@ -389,7 +401,7 @@ class StepForgeApp { el('div', { style: { fontWeight: 650 } }, folderLabel), q ? el('div.muted', {}, `Search: ${q}`) : el('div.muted', {}, `${this.state.library.guides.length} guides`), ), - el('div.muted', {}, this.state.info ? `StepForge ${this.state.info.version}` : ''), + el('div.muted', {}, this.state.info ? `StepForge ${this.state.info.buildVersion || this.state.info.version}` : ''), ), this.domBulkBar = el('div', {}), this.domLibraryResults = el('div', {}), diff --git a/app/renderer/capture-worker.js b/app/renderer/capture-worker.js index 0f87084..f65ecbf 100644 --- a/app/renderer/capture-worker.js +++ b/app/renderer/capture-worker.js @@ -68,24 +68,39 @@ }; streams.set(key, state); try { - // The chromeMediaSource constraint set is Electron's documented bridge - // from a desktopCapturer source id to a live media stream. - state.media = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: { - mandatory: { - chromeMediaSource: 'desktop', - chromeMediaSourceId: cmd.sourceId, - minWidth: physWidth, - maxWidth: physWidth, - minHeight: physHeight, - maxHeight: physHeight, - // No maxFrameRate: sampling cadence is controlled by the setInterval - // timer below, so the actual capture rate is always sampleMs-driven - // regardless of display refresh rate or power mode. + if (cmd.useDisplayMedia) { + // GNOME Wayland path: desktopCapturer source ids fail with + // getUserMedia, so go through the portal-backed getDisplayMedia. The + // main process installs a setDisplayMediaRequestHandler that answers + // this request; the OS portal picker chooses the screen. The stream + // then stays open for the whole session — one prompt, not one per shot. + state.media = await navigator.mediaDevices.getDisplayMedia({ + audio: false, + video: true, + }); + } else { + // Keep the legacy desktop-capture constraint wrapper here: it binds + // the stream to the exact desktop source chosen in the main process. + // Without it Chromium can treat the request like a normal media + // request and pick the default camera device instead. + state.media = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + mandatory: { + chromeMediaSource: 'desktop', + chromeMediaSourceId: cmd.sourceId, + minWidth: physWidth, + maxWidth: physWidth, + minHeight: physHeight, + maxHeight: physHeight, + // No maxFrameRate: sampling cadence is controlled by the + // setInterval timer below, so the actual capture rate is always + // sampleMs-driven regardless of display refresh rate or power + // mode. + }, }, - }, - }); + }); + } const video = document.createElement('video'); video.muted = true; video.srcObject = state.media; diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index e92181a..7b72745 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -312,14 +312,30 @@ function showSettingsDialog({ const previewCount = makeInput(settings.exports?.previewStepCount ?? 3, 'number', { min: 1, step: 1 }); const openFolder = el('input', { type: 'checkbox', checked: Boolean(settings.exports?.openFolderAfterExport) }); const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) }); + const fallbackTrigger = makeSelect(settings.capture?.fallbackTrigger || 'interval', [ + { value: 'interval', label: 'Timed interval' }, + { value: 'hotkey', label: 'Hotkey only' }, + ]); + const autoIntervalSec = makeInput(settings.capture?.autoIntervalSec ?? 5, 'number', { min: 1, step: 1 }); const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) }); const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 }); const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) }); const aiAutoDoc = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.autoDoc) }); const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434'); const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b'); - const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.'); + const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Vision-capable models can also inspect the screenshot attached to each step.'); const testAiBtn = el('button', { type: 'button' }, 'Test connection'); + const persistOllamaModel = debounce(() => { + const model = ollamaModel.value.trim(); + // Keep the last model choice even if the dialog is dismissed without a full save. + void api.settings.set({ keyPath: 'ai.ollama.model', value: model }).catch(() => {}); + }, 250); + + const syncFallbackUi = () => { + autoIntervalSec.disabled = fallbackTrigger.value === 'hotkey'; + }; + fallbackTrigger.addEventListener('change', syncFallbackUi); + syncFallbackUi(); const updateAiStatus = (message, { error = false } = {}) => { aiStatus.textContent = message; @@ -341,7 +357,9 @@ function showSettingsDialog({ return; } if (result.installed) { - updateAiStatus(`Connected to ${result.host} with ${result.model}.`); + updateAiStatus(result.vision + ? `Connected to ${result.host} with ${result.model}. It can inspect screenshots.` + : `Connected to ${result.host} with ${result.model}. This model is text-only, so StepForge will use OCR and metadata only.`); } else { updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true }); } @@ -351,6 +369,8 @@ function showSettingsDialog({ setButtonLoading(testAiBtn, false); } }; + ollamaModel.addEventListener('input', () => persistOllamaModel()); + ollamaModel.addEventListener('blur', () => persistOllamaModel.flush()); const placeholderRows = el('div', { className: 'placeholder-rows' }); const rows = []; @@ -394,9 +414,14 @@ function showSettingsDialog({ labeledRow('Delay (ms)', delayMs), labeledRow('Click marker', clickMarker), labeledRow('Capture outside clicks', captureOutside), + labeledRow('When clicks are unavailable', fallbackTrigger), + labeledRow('Timer interval (seconds)', autoIntervalSec), labeledRow('Confirm simple capture', confirmSimple), labeledRow('Capture hotkey', captureHotkey), labeledRow('Pause / resume hotkey', pauseHotkey), + el('div.muted', {}, + 'Hotkey fallback uses the Capture hotkey. Timer fallback uses the interval above.', + ), ), el('fieldset', {}, el('legend', {}, 'Editor'), @@ -446,6 +471,8 @@ function showSettingsDialog({ delayMs: Number(delayMs.value || 0), mode: captureMode.value, clickMarker: clickMarker.checked, + fallbackTrigger: fallbackTrigger.value === 'hotkey' ? 'hotkey' : 'interval', + autoIntervalSec: Math.max(1, Number(autoIntervalSec.value || 5)), hotkeyCapture: captureHotkey.value.trim(), hotkeyPauseResume: pauseHotkey.value.trim(), captureOutsideClicks: captureOutside.checked, diff --git a/app/renderer/style.css b/app/renderer/style.css index 5ea2b5a..833c9e8 100644 --- a/app/renderer/style.css +++ b/app/renderer/style.css @@ -49,6 +49,16 @@ body { overflow: hidden; } +body.platform-linux button { + padding: 5px 11px; +} + +body.platform-linux input, +body.platform-linux select, +body.platform-linux textarea { + padding: 6px 9px; +} + *::selection { background: rgba(0, 104, 255, 0.2); } #app { display: flex; flex-direction: column; height: 100vh; } @@ -183,6 +193,21 @@ kbd { color: #fff; } +body.platform-linux #topbar { + height: 52px; + gap: 10px; + padding: 0 14px; +} + +body.platform-linux #capture-status { + padding: 5px 9px; + gap: 7px; +} + +body.platform-linux #capture-status button { + padding: 2px 7px; +} + .library, .editor { flex: 1; min-height: 0; display: flex; } .lib-side { diff --git a/app/stream-backend.js b/app/stream-backend.js index dcf7369..302da4f 100644 --- a/app/stream-backend.js +++ b/app/stream-backend.js @@ -83,9 +83,19 @@ class StreamCaptureBackend { * Spin up the worker and one stream per display that has a matching screen * source. Resolves true when at least one stream is delivering frames. */ - async start({ displays = [], sources = [], sampleMs = DEFAULT_SAMPLE_MS, retentionMs = null, frameLimit = null } = {}) { + async start({ + displays = [], sources = [], sampleMs = DEFAULT_SAMPLE_MS, + retentionMs = null, frameLimit = null, useDisplayMedia = false, + } = {}) { if (this.host) return this.active; - const pairs = pairDisplaysToSources(displays, sources); + // On GNOME Wayland, desktopCapturer source ids can't be reopened with + // getUserMedia ("Requested device not found") — the portal-backed + // getDisplayMedia path is the only one that works. There's no per-display + // source id in that mode, so capture a single stream against the primary + // display (the portal picker decides which screen is actually shared). + const pairs = useDisplayMedia + ? (displays.length ? [{ display: displays[0], sourceId: null }] : []) + : pairDisplaysToSources(displays, sources); if (!pairs.length) return false; try { this.host = await this.createHost((msg) => this.handleWorkerEvent(msg)); @@ -99,6 +109,7 @@ class StreamCaptureBackend { type: 'start-stream', displayId: display.id, sourceId, + useDisplayMedia, // The worker needs the physical pixel size to request a full-res // stream; bounds stay in DIP for marker math back in the main process. display: { @@ -154,6 +165,9 @@ class StreamCaptureBackend { stream.ready = msg.type === 'stream-ready'; stream.failed = msg.type === 'stream-error'; } + if (msg.type === 'stream-error') { + console.error(`[stepforge] capture worker stream-error display=${msg.displayId}: ${msg.reason}`); + } for (const check of [...this.startWaiters]) check(); return; } @@ -167,7 +181,7 @@ class StreamCaptureBackend { clearTimeout(pending.timer); pending.timer = setTimeout(() => { this.settleRequest(msg.requestId, null); - this.noteFailure(); + if (pending.failable !== false) this.noteFailure(); }, this.encodeTimeoutMs); return; } @@ -210,7 +224,7 @@ class StreamCaptureBackend { * Resolves null when no frame qualifies (caller falls back) — and also on * timeout, which additionally counts toward unhealthiness. */ - frameForClick({ clickPos = null, clickAt = Date.now(), strict = true, leadMs = 0 } = {}) { + frameForClick({ clickPos = null, clickAt = Date.now(), strict = true, leadMs = 0, failable = true } = {}) { if (!this.active || !this.host) return Promise.resolve(null); const displays = [...this.streams.values()].filter((s) => s.ready).map((s) => s.display); const display = clickPos ? displayForDipPoint(clickPos, displays) : (displays[0] || null); @@ -222,10 +236,15 @@ class StreamCaptureBackend { if (clickPos && !pointInBounds(clickPos, display.bounds)) return Promise.resolve(null); const requestId = this.nextRequestId++; return new Promise((resolve) => { - const pending = { resolve, display, timer: null }; + // failable=false: a timeout resolves null but does NOT count toward + // unhealthiness. Interval/timed captures use this so a single slow PNG + // encode (common on software-rendered hosts with no GPU) can't trip the + // 2-strikes rule and tear down an otherwise-healthy stream — the bug + // that left Wayland recordings "stuck after two captures". + const pending = { resolve, display, timer: null, failable }; pending.timer = setTimeout(() => { this.settleRequest(requestId, null); - this.noteFailure(); + if (failable) this.noteFailure(); }, this.ackTimeoutMs); this.requests.set(requestId, pending); this.hostSend({ diff --git a/app/text-intel.js b/app/text-intel.js index 961db39..0388642 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -35,6 +35,15 @@ function clamp(v, min, max) { return Math.min(max, Math.max(min, v)); } +function modelLooksVisionCapable(model) { + const clean = normalizeWhitespace(model).toLowerCase(); + if (!clean) return false; + return clean.includes('vision') + || clean.includes('llava') + || clean.includes('gemma4') + || /(^|[^a-z0-9])qwen[23](?:\.[0-9]+)?vl([^a-z0-9]|$)/.test(clean); +} + let createWorkerImpl = null; function loadCreateWorker() { if (createWorkerImpl) return createWorkerImpl; @@ -64,6 +73,7 @@ class TextIntelService { this.workerPromise = null; this.workerQueue = Promise.resolve(); this.ocrDataDir = path.join(dataDir, 'ocr', 'eng'); + this.modelCapabilityCache = new Map(); } async shutdown() { @@ -372,15 +382,63 @@ public static class Win32 { const data = await res.json(); const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : []; const installed = config.ollama.model ? models.includes(config.ollama.model) : false; + const vision = installed ? await this.modelSupportsVision({ + host: config.ollama.host, + model: config.ollama.model, + }) : false; return { ok: true, installed, + vision, models, host: config.ollama.host, model: config.ollama.model, }; } + async modelCapabilities({ host, model }) { + const normalizedHost = normalizeOllamaHost(host); + const normalizedModel = normalizeWhitespace(model); + if (!normalizedHost || !normalizedModel) return []; + const cacheKey = `${normalizedHost}::${normalizedModel}`; + if (this.modelCapabilityCache.has(cacheKey)) { + return this.modelCapabilityCache.get(cacheKey); + } + const url = new URL('/api/show', `${normalizedHost.replace(/\/+$/, '')}/`); + let capabilities = []; + try { + const response = await this.fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: normalizedModel }), + }); + if (response.ok) { + const payload = await response.json(); + capabilities = Array.isArray(payload?.capabilities) + ? payload.capabilities.map((cap) => normalizeWhitespace(cap).toLowerCase()).filter(Boolean) + : []; + } + } catch { + capabilities = []; + } + if (!capabilities.includes('vision') && modelLooksVisionCapable(normalizedModel)) { + capabilities = [...capabilities, 'vision']; + } + this.modelCapabilityCache.set(cacheKey, capabilities); + return capabilities; + } + + async modelSupportsVision({ host, model }) { + const capabilities = await this.modelCapabilities({ host, model }); + return capabilities.includes('vision'); + } + + readStepImageBase64(guideId, stepId) { + const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original'); + if (!imagePath || !fs.existsSync(imagePath)) return ''; + return fs.readFileSync(imagePath).toString('base64'); + } + async callOllamaText({ host, model, prompt, systemPrompt }) { const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); const response = await this.fetch(url, { @@ -403,8 +461,12 @@ public static class Win32 { return content.trim(); } - async callOllama({ host, model, prompt, systemPrompt }) { + async callOllama({ host, model, prompt, systemPrompt, images = [] }) { const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); + const userMessage = { role: 'user', content: prompt }; + if (Array.isArray(images) && images.length) { + userMessage.images = images; + } const response = await this.fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -414,7 +476,7 @@ public static class Win32 { format: 'json', messages: [ { role: 'system', content: systemPrompt }, - { role: 'user', content: prompt }, + userMessage, ], options: { temperature: 0.2, @@ -460,6 +522,14 @@ public static class Win32 { return { ok: false, reason: 'Block not found.' }; } + const screenshotBase64 = step.image ? this.readStepImageBase64(guideId, stepId) : ''; + const screenshotAttached = Boolean(screenshotBase64) + ? await this.modelSupportsVision({ + host: config.ollama.host, + model: config.ollama.model, + }) + : false; + let captureContext = null; // Use stored capture metadata when available (best context, from capture time). // Fall back to re-running OCR on the stored image only when metadata is absent. @@ -514,6 +584,7 @@ public static class Win32 { step, captureContext, block: currentBlock, + screenshotAttached, }); const raw = await this.callOllama({ @@ -521,6 +592,7 @@ public static class Win32 { model: config.ollama.model, prompt, systemPrompt, + images: screenshotAttached ? [screenshotBase64] : [], }); const patch = normalizeAiPatch(raw); const updated = applyAiPatchToStep(step, patch, { target, blockId }); diff --git a/build/installer.nsh b/build/installer.nsh new file mode 100644 index 0000000..5391d8e --- /dev/null +++ b/build/installer.nsh @@ -0,0 +1,110 @@ +!include LogicLib.nsh +!include nsDialogs.nsh + +!ifndef BUILD_UNINSTALLER + Var StepForgeDesktopShortcutCheckbox + Var StepForgeDesktopShortcutState + + ; Assisted installer page for the desktop shortcut choice. + !macro customInit + StrCpy $StepForgeDesktopShortcutState "true" + ${If} ${isNoDesktopShortcut} + StrCpy $StepForgeDesktopShortcutState "false" + ${EndIf} + !macroend + + !macro customHeader + Function StepForgeDesktopShortcutPagePre + !insertmacro MUI_PAGE_FUNCTION_CUSTOM PRE + !insertmacro MUI_HEADER_TEXT "Desktop Icon" "Choose whether StepForge creates a desktop icon." + + nsDialogs::Create 1018 + Pop $0 + ${If} $0 == error + Abort + ${EndIf} + + ${NSD_CreateLabel} 0u 0u 280u 24u "StepForge can create a desktop icon for quick access from the desktop." + Pop $0 + + ${NSD_CreateCheckbox} 0u 34u 280u 12u "Create a desktop icon" + Pop $StepForgeDesktopShortcutCheckbox + + StrCpy $StepForgeDesktopShortcutState "true" + ${If} ${isNoDesktopShortcut} + StrCpy $StepForgeDesktopShortcutState "false" + EnableWindow $StepForgeDesktopShortcutCheckbox 0 + ${Else} + ReadRegStr $0 SHELL_CONTEXT "${INSTALL_REGISTRY_KEY}" CreateDesktopShortcut + ${If} $0 == "false" + StrCpy $StepForgeDesktopShortcutState "false" + ${Else} + ReadRegStr $1 SHELL_CONTEXT "${INSTALL_REGISTRY_KEY}" ShortcutName + ${If} $1 == "" + StrCpy $1 "${SHORTCUT_NAME}" + ${EndIf} + + ${If} ${FileExists} "$DESKTOP\$1.lnk" + StrCpy $StepForgeDesktopShortcutState "true" + ${ElseIf} ${FileExists} "$INSTDIR\${APP_EXECUTABLE_FILENAME}" + StrCpy $StepForgeDesktopShortcutState "false" + ${EndIf} + ${EndIf} + ${EndIf} + + ${If} $StepForgeDesktopShortcutState == "false" + SendMessage $StepForgeDesktopShortcutCheckbox ${BM_SETCHECK} ${BST_UNCHECKED} 0 + ${Else} + SendMessage $StepForgeDesktopShortcutCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + ${EndIf} + + !insertmacro MUI_PAGE_FUNCTION_CUSTOM SHOW + nsDialogs::Show + FunctionEnd + + Function StepForgeDesktopShortcutPageLeave + !insertmacro MUI_PAGE_FUNCTION_CUSTOM LEAVE + + SendMessage $StepForgeDesktopShortcutCheckbox ${BM_GETCHECK} 0 0 $StepForgeDesktopShortcutState + ${If} $StepForgeDesktopShortcutState == ${BST_UNCHECKED} + StrCpy $StepForgeDesktopShortcutState "false" + ${Else} + StrCpy $StepForgeDesktopShortcutState "true" + ${EndIf} + FunctionEnd + !macroend + + !macro customPageAfterChangeDir + !insertmacro MUI_PAGE_INIT + PageEx custom + PageCallbacks StepForgeDesktopShortcutPagePre StepForgeDesktopShortcutPageLeave + Caption " " + PageExEnd + !macroend + + !macro customInstall + ; Reconcile the desktop shortcut after the default installer logic runs. + ${If} ${isNoDesktopShortcut} + StrCpy $StepForgeDesktopShortcutState "false" + ${EndIf} + + WriteRegStr SHELL_CONTEXT "${INSTALL_REGISTRY_KEY}" CreateDesktopShortcut "$StepForgeDesktopShortcutState" + + ${If} $StepForgeDesktopShortcutState == "false" + Delete "$newDesktopLink" + Delete "$oldDesktopLink" + ${Else} + ${IfNot} ${FileExists} "$newDesktopLink" + ${If} ${FileExists} "$oldDesktopLink" + Rename "$oldDesktopLink" "$newDesktopLink" + ${Else} + CreateShortCut "$newDesktopLink" "$appExe" "" "$appExe" 0 "" "" "${APP_DESCRIPTION}" + ${EndIf} + ClearErrors + WinShell::SetLnkAUMI "$newDesktopLink" "${APP_ID}" + ${EndIf} + ${EndIf} + + System::Call 'shell32::SHChangeNotify(i 0x08000000, i 0, i 0, i 0)' + !macroend +!endif diff --git a/core/settings.js b/core/settings.js index 02b1552..e21b483 100644 --- a/core/settings.js +++ b/core/settings.js @@ -18,6 +18,9 @@ const DEFAULT_SETTINGS = { hotkeyPauseResume: 'CommandOrControl+Shift+2', captureOutsideClicks: true, confirmSimpleCapture: false, + // Fallback trigger when click capture is unavailable: keep the old timer + // fallback by default, but let users switch to hotkey-only recordings. + fallbackTrigger: 'interval', // interval | hotkey // Leading-edge click debounce (ms): clicks of the same button closer // together than this collapse into one step, so accidental fast/double // clicks don't each become a step. Clicks spaced further apart always diff --git a/core/text-intel.js b/core/text-intel.js index 90ac831..e7cf21d 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -673,6 +673,7 @@ function buildAiPrompt({ step = null, captureContext = null, block = null, + screenshotAttached = false, } = {}) { const hasDraftTitle = step && !isPlaceholderTitle(step.title); const hasDraftDesc = step && Boolean(htmlToText(step.descriptionHtml || '')); @@ -717,6 +718,7 @@ function buildAiPrompt({ (!hasDraftTitle || target === 'description') && captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null, ] : []), + screenshotAttached ? 'Screenshot: attached to this request.' : null, draftTitleLine, draftDescLine, ].filter(Boolean); @@ -770,6 +772,9 @@ function buildAiPrompt({ richContext ? '- Use the OCR text, window title, app name, and element info to make the documentation specific.' : '- Context is limited. Use the app name or window title if available; generate a reasonable action title.', + screenshotAttached + ? '- A screenshot is attached. Use it together with the OCR and metadata to resolve visual details, but do not mention the screenshot in the output.' + : '- No screenshot is attached. Rely on OCR, the window title, app name, and element info.', '- Do NOT generate blocks that describe the technical capture process or mention OCR.', '- Do NOT invent details not supported by the capture context.', '- If the target is one block, only rewrite that block.', diff --git a/docs/GETTING_STARTED_WITH_LINUX.md b/docs/GETTING_STARTED_WITH_LINUX.md new file mode 100644 index 0000000..1128177 --- /dev/null +++ b/docs/GETTING_STARTED_WITH_LINUX.md @@ -0,0 +1,156 @@ +# Getting Started with StepForge on Linux + +> ⚠️ **Work in progress.** Linux support is still under active development. +> Expect rough edges — especially on Wayland (see the limitations below). X11 / +> Xorg is the most complete path today. Please report issues. + +StepForge was built on Windows, where the OS lets an app watch every click and +grab the screen freely. Linux is more restrictive, and **how much works depends +on whether you are running X11 (Xorg) or Wayland.** This guide explains the +difference, how to get the best experience, and how to enable per-click capture. + +## TL;DR + +| | **X11 / "Ubuntu on Xorg"** | **Wayland (default on Ubuntu)** | +|---|---|---| +| Screenshot per click | ✅ Yes | ✅ Yes (needs `input` group) | +| Red circle on the click | ✅ Yes | ❌ No (Wayland hides the cursor position) | +| "Share your screen" prompt | Never | Once per recording session | +| Setup needed | None | Add yourself to the `input` group | + +**If you want the full Windows-like experience (click capture *with* the red +marker), use an Xorg session — see [Option A](#option-a-best-experience--use-xorg).** + +--- + +## 1. Check which session you are running + +```bash +echo $XDG_SESSION_TYPE +``` + +- `x11` → you're on Xorg. Everything works, including the red click marker. No setup needed. +- `wayland` → you're on Wayland. Read on. + +--- + +## Option A (best experience) — use Xorg + +On Xorg, StepForge captures a screenshot **on every click** and draws the **red +marker** at the exact click position, exactly like Windows. No dependencies, no +permissions, no portal dialogs. + +To switch: + +1. Log out. +2. On the login (password) screen, click the **⚙ gear icon** in the bottom-right corner. +3. Choose **"Ubuntu on Xorg"**. +4. Log back in. + +That's it — open StepForge and record. (To go back to Wayland later, pick +"Ubuntu" at the gear menu again.) + +--- + +## Option B — stay on Wayland + +Wayland deliberately blocks apps from monitoring global input and from grabbing +the screen silently. StepForge works around this as far as the platform allows: + +### Screen capture + +The first time you press **Start recording**, the system shows a **"Share your +screen"** dialog (the XDG desktop portal). Pick your screen and click **Share**. +This happens **once per recording session** — not per screenshot. The shared +stream stays open until you stop recording. + +> If you never see steps appear, make sure you actually picked a screen and +> clicked **Share** in that dialog. + +### Per-click capture (requires the `input` group) + +By default on Wayland, StepForge cannot see your clicks, so it falls back to +**capturing a screenshot every few seconds** (timed capture). + +To get a screenshot **on every click** instead, give your user read access to +the mouse devices by joining the `input` group: + +```bash +sudo usermod -aG input "$USER" +``` + +Then **log out and log back in** (group membership only applies to new sessions). +Verify it took effect: + +```bash +groups | tr ' ' '\n' | grep input # should print: input +``` + +Now StepForge reads mouse buttons directly from the kernel (`/dev/input`) and +captures a screenshot on each click. + +> **No red marker on Wayland.** Even with per-click capture working, Wayland +> does not tell apps *where* the pointer is, so StepForge cannot draw the circle +> at the click. The screenshot is still captured per click — just without the +> marker. If you need the marker, use [Option A (Xorg)](#option-a-best-experience--use-xorg). + +### Adjusting the timed-capture interval + +If you don't enable the `input` group, StepForge captures on a timer. Change the +fallback in **Settings → Capture**: + +- `When clicks are unavailable` -> `Hotkey only` to use the Capture hotkey + instead of a timer. +- `When clicks are unavailable` -> `Timed interval`, then set + `Timer interval (seconds)` (`capture.autoIntervalSec`, default 5 seconds) + if you want timed captures. + +--- + +## How StepForge picks a capture method (for reference) + +On launch StepForge chooses the best available click source: + +1. **Windows** — low-level mouse hook (position + timing). +2. **X11** — `xinput` (position + timing → full red marker). +3. **Linux evdev** (`/dev/input`) — button presses on X11 *and* Wayland, no + position on Wayland. Used when `xinput` can't see clicks (i.e. Wayland), if + you're in the `input` group. +4. **Timed capture** — the always-works fallback (a screenshot every N seconds) + when no click source is available. + +Screen frames come from a single long-lived capture stream per recording, so +clicks/timer ticks never re-open the screen-share dialog. + +--- + +## Troubleshooting + +**"It asks to share my screen every time."** +You're likely on an older build. Update to the current version — the screen +stream is now opened once per recording session. If it persists, confirm +`echo $XDG_SESSION_TYPE` and that you clicked **Share** (not Cancel) in the dialog. + +**"Recording captures a couple of steps then stops."** +Fixed in the current version (a slow, GPU-less PNG encode used to trip a +failure guard and tear down the stream). Update and retry. + +**"The window disappeared and I can't stop the recording."** +On Linux the window **minimizes** while recording (GNOME's system tray is +unreliable). Bring it back from the **taskbar / dock**, then click **Stop +recording**. + +**"No steps at all on Wayland, even after picking a screen."** +Run from a terminal with logging and look for the diagnostic lines: + +```bash +STEPFORGE_CAPTURE_LOG=1 npm start +``` + +- `[stepforge] screen-capture stream active …` — the stream is up. +- `[stepforge] per-click capture via evdev on N device(s) …` — clicks are wired up. +- `[stepforge] no readable mouse input devices …` — you need the `input` group (see above). + +**Harmless console noise.** Lines like `vaInitialize failed`, `Frame latency is +negative`, and `StatusNotifierItem … already exported` come from Chromium/GNOME, +not StepForge, and don't affect recording. diff --git a/docs/getting_started_with_ai.md b/docs/getting_started_with_ai.md index 193218e..817d38d 100644 --- a/docs/getting_started_with_ai.md +++ b/docs/getting_started_with_ai.md @@ -22,6 +22,14 @@ ollama pull llama3.2:1b That model is small enough to feel responsive on modest hardware, but still good enough for human-sounding titles and short text blocks. +If you want StepForge to send the screenshot itself to the model, pull a vision-capable model instead: + +```bash +ollama pull gemma3 +``` + +That model can inspect pictures as well as text, so it is better when you want the AI to read the UI directly from the screenshot. + If you need something even smaller, try: ```bash @@ -44,7 +52,7 @@ Set: * `Enable AI text filling` to on * `Ollama host` to your local Ollama server -* `Ollama model` to `llama3.2:1b` or the smaller model you pulled +* `Ollama model` to `llama3.2:1b` for text-only mode, or `llama3.2-vision` if you want screenshot-aware AI The default host is: @@ -73,3 +81,4 @@ You can also use `More -> Generate all text fields with AI` to fill the whole st * Capture titles are still generated automatically without AI. * AI generation only works when `Enable AI text filling` is turned on. * The app always uses local OCR around the click area first, then local AI only when you ask for it. +* When the selected Ollama model supports vision, StepForge also sends the screenshot to the model so it can cross-check OCR and visual context. diff --git a/package-lock.json b/package-lock.json index 9cc3468..2c6eee8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "stepforge", - "version": "0.1.0", + "version": "0.3.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "stepforge", - "version": "0.1.0", + "version": "0.3.2", "license": "MPL-2.0", "dependencies": { "@tesseract.js-data/eng": "^1.0.0", diff --git a/package.json b/package.json index 41fa808..61d17b7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "stepforge", - "version": "0.1.0", + "version": "0.3.2", + "buildVersion": "0.3.2.1", "description": "Fully offline desktop tool for capturing, annotating, and exporting step-by-step guides.", "main": "app/main.js", "author": "StepForge tyler@twestbrook.com", @@ -8,7 +9,7 @@ "private": true, "scripts": { "start": "node scripts/start-electron.js", - "test": "node --test tests/unit/", + "test": "node scripts/run-unit-tests.js", "sample": "node scripts/make-sample-guide.js", "package:windows": "node scripts/package-windows.js", "build": "bash scripts/build-release.sh", diff --git a/scripts/bootstrap-offline.sh b/scripts/bootstrap-offline.sh index 8dd2dfd..ffb8f68 100644 --- a/scripts/bootstrap-offline.sh +++ b/scripts/bootstrap-offline.sh @@ -20,5 +20,5 @@ fi node - <<'NODE' const pkg = require('./package.json'); -console.log(`StepForge ${pkg.version} bootstrap OK`); +console.log(`StepForge ${pkg.buildVersion || pkg.version} bootstrap OK`); NODE diff --git a/scripts/build-release.sh b/scripts/build-release.sh index 60546d0..e5c51d3 100644 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -70,6 +70,7 @@ for (const rel of walk(examplesRoot, examplesRoot)) { } const pkg = require(path.join(rootDir, 'package.json')); +const buildVersion = pkg.buildVersion || pkg.version; const { execSync } = require('node:child_process'); function toolAvailable(cmd) { @@ -88,7 +89,8 @@ const toolRows = Object.entries(tools) const report = `# StepForge Build Report -Version: ${pkg.version} +Build version: ${buildVersion} +Package version: ${pkg.version} Generated: ${new Date().toISOString()} Host: ${process.platform} ${process.arch} (node ${process.version}) @@ -133,6 +135,7 @@ fs.writeFileSync(manifestFile, JSON.stringify({ version: 1, generatedAt: new Date().toISOString(), packageVersion: pkg.version, + buildVersion, files, }, null, 2) + '\n'); NODE diff --git a/scripts/electron-launcher.js b/scripts/electron-launcher.js index a23a993..518c166 100644 --- a/scripts/electron-launcher.js +++ b/scripts/electron-launcher.js @@ -60,6 +60,26 @@ function sanitizeElectronEnv(baseEnv = process.env) { return env; } +function linuxSandboxLaunchArgs({ + electronPath, + platform = process.platform, + statSync = fs.statSync, +} = {}) { + if (platform !== 'linux') return []; + if (!electronPath) return ['--no-sandbox']; + + const helperPath = path.join(path.dirname(electronPath), 'chrome-sandbox'); + try { + const stat = statSync(helperPath); + const ownedByRoot = stat.uid === 0; + const hasSetuid = Boolean(stat.mode & 0o4000); + if (ownedByRoot && hasSetuid) return []; + } catch { + // Missing or unreadable helper: fall back to the unsandboxed launcher. + } + return ['--no-sandbox']; +} + function electronBinaryCandidates({ packageRoot, distDir, platform }) { const candidatePaths = []; const pathHint = packageRoot ? readElectronPathHint(packageRoot) : null; @@ -292,6 +312,7 @@ module.exports = { runNpmRebuild, runNpmInstall, sanitizeElectronEnv, + linuxSandboxLaunchArgs, resolveElectronBinary, resolveElectronPackageRoot, platformBinaryCandidates, diff --git a/scripts/package-linux.sh b/scripts/package-linux.sh index 1be9c66..cf0fc0d 100644 --- a/scripts/package-linux.sh +++ b/scripts/package-linux.sh @@ -3,7 +3,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -VERSION="$(node -p "require('${ROOT_DIR}/package.json').version" 2>/dev/null || echo 0.0.0)" +VERSION="$(node -p "const pkg=require('${ROOT_DIR}/package.json'); pkg.buildVersion || pkg.version" 2>/dev/null || echo 0.0.0)" OUT_DIR="${STEPFORGE_PACKAGE_DIR:-$ROOT_DIR/build/artifacts}" mkdir -p "$OUT_DIR" WORK_DIR="$(mktemp -d "${OUT_DIR%/}/.pkg.XXXXXX")" @@ -46,8 +46,21 @@ fi cat > "$WORK_DIR/usr/bin/stepforge" <<'EOF' #!/usr/bin/env sh APP_DIR=/opt/stepforge +ELECTRON="$APP_DIR/node_modules/.bin/electron" +SANDBOX_HELPER="$APP_DIR/node_modules/electron/dist/chrome-sandbox" cd "$APP_DIR" || exit 1 -exec "$APP_DIR/node_modules/.bin/electron" "$APP_DIR" "$@" +if command -v stat >/dev/null 2>&1 && [ -e "$SANDBOX_HELPER" ]; then + helper_uid="$(stat -c '%u' "$SANDBOX_HELPER" 2>/dev/null || echo '')" + helper_mode="$(stat -c '%a' "$SANDBOX_HELPER" 2>/dev/null || echo '')" + if [ "$helper_uid" = "0" ] && [ -n "$helper_mode" ]; then + helper_mode_num=$((8#$helper_mode)) + if [ $((helper_mode_num & 04000)) -ne 0 ]; then + exec "$ELECTRON" "$APP_DIR" "$@" + fi + fi +fi +printf '%s\n' '[stepforge] Electron sandbox helper is not configured for this install; starting with --no-sandbox' >&2 +exec "$ELECTRON" --no-sandbox "$APP_DIR" "$@" EOF chmod 0755 "$WORK_DIR/usr/bin/stepforge" diff --git a/scripts/package-windows.js b/scripts/package-windows.js index 4d8883f..18dba03 100644 --- a/scripts/package-windows.js +++ b/scripts/package-windows.js @@ -10,6 +10,7 @@ const { build, Platform } = require('electron-builder'); const ROOT_DIR = path.resolve(__dirname, '..'); const PACKAGE_JSON = require(path.join(ROOT_DIR, 'package.json')); const APP_ID = 'com.stepforge.app'; +const BUILD_VERSION = PACKAGE_JSON.buildVersion || PACKAGE_JSON.version; function findInstallerExe(dir) { if (!fs.existsSync(dir)) return null; @@ -32,6 +33,7 @@ function createWindowsInstallerConfig(outputDir) { return { appId: APP_ID, productName: 'StepForge', + buildVersion: BUILD_VERSION, directories: { output: outputDir, }, @@ -52,6 +54,8 @@ function createWindowsInstallerConfig(outputDir) { createDesktopShortcut: true, createStartMenuShortcut: true, shortcutName: 'StepForge', + artifactName: '${productName} Setup ${buildVersion}.${ext}', + include: 'build/installer.nsh', }, }; } @@ -86,7 +90,7 @@ async function buildWindowsInstaller() { const releaseInstaller = path.join(releaseDir, path.basename(builtInstaller)); fs.copyFileSync(builtInstaller, releaseInstaller); - console.log(`StepForge ${PACKAGE_JSON.version} Windows installer written to ${releaseInstaller}`); + console.log(`StepForge ${BUILD_VERSION} Windows installer written to ${releaseInstaller}`); } if (require.main === module) { diff --git a/scripts/run-unit-tests.js b/scripts/run-unit-tests.js new file mode 100644 index 0000000..a7dbcd0 --- /dev/null +++ b/scripts/run-unit-tests.js @@ -0,0 +1,42 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +function collectTestFiles(rootDir) { + const files = []; + if (!fs.existsSync(rootDir)) return files; + + const walk = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(full); + continue; + } + if (/\.(test|spec)\.(js|mjs|cjs)$/.test(entry.name)) files.push(full); + } + }; + + walk(rootDir); + files.sort((a, b) => a.localeCompare(b)); + return files; +} + +const root = path.join(process.cwd(), 'tests', 'unit'); +const tests = collectTestFiles(root); + +if (!tests.length) { + console.log('No unit test files found under tests/unit, skipping unit tests.'); + process.exit(0); +} + +const result = spawnSync(process.execPath, ['--test', ...tests], { stdio: 'inherit' }); + +if (result.error) { + console.error(result.error.message); + process.exit(result.status ?? 1); +} + +process.exit(result.status ?? 0); diff --git a/scripts/stamp-version.js b/scripts/stamp-version.js new file mode 100644 index 0000000..60af74f --- /dev/null +++ b/scripts/stamp-version.js @@ -0,0 +1,52 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +function writeJson(file, value) { + fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n'); +} + +function stampVersion(rootDir, version) { + if (!version || typeof version !== 'string') { + throw new Error('version is required'); + } + + const normalized = version.replace(/^v/i, ''); + const parts = normalized.split('.'); + const isFourPartBuild = parts.length === 4 && parts.every((part) => /^\d+$/.test(part)); + const packageVersion = isFourPartBuild ? parts.slice(0, 3).join('.') : normalized; + const buildVersion = normalized; + + const pkgPath = path.join(rootDir, 'package.json'); + const pkg = readJson(pkgPath); + pkg.version = packageVersion; + pkg.buildVersion = buildVersion; + writeJson(pkgPath, pkg); + + const lockPath = path.join(rootDir, 'package-lock.json'); + if (!fs.existsSync(lockPath)) return; + + const lock = readJson(lockPath); + lock.version = packageVersion; + if (lock.packages && lock.packages['']) { + lock.packages[''].version = packageVersion; + } + writeJson(lockPath, lock); +} + +if (require.main === module) { + try { + const version = process.argv[2] || process.env.VERSION; + stampVersion(process.cwd(), version); + } catch (err) { + console.error(err && err.message ? err.message : err); + process.exitCode = 1; + } +} + +module.exports = { stampVersion }; diff --git a/scripts/start-electron.js b/scripts/start-electron.js index cec232f..c4c004a 100644 --- a/scripts/start-electron.js +++ b/scripts/start-electron.js @@ -3,7 +3,11 @@ const { spawn } = require('node:child_process'); -const { resolveElectronBinary, sanitizeElectronEnv } = require('./electron-launcher'); +const { + linuxSandboxLaunchArgs, + resolveElectronBinary, + sanitizeElectronEnv, +} = require('./electron-launcher'); let electronPath; try { @@ -13,8 +17,19 @@ try { process.exit(1); } const env = sanitizeElectronEnv(); +const sandboxArgs = linuxSandboxLaunchArgs({ electronPath }); +if (sandboxArgs.includes('--no-sandbox')) { + console.warn('[stepforge] Electron sandbox helper is not configured for this install; starting with --no-sandbox'); +} -const child = spawn(electronPath, ['.'], { +// On Linux, prefer the native Ozone path when available and enable PipeWire- +// based screen capture so desktopCapturer can go through the XDG Desktop +// Portal on Wayland without affecting X11. +const extraArgs = process.platform === 'linux' + ? ['--enable-features=WebRTCPipeWireCapturer', '--ozone-platform-hint=auto', ...sandboxArgs] + : []; + +const child = spawn(electronPath, [...extraArgs, '.'], { stdio: 'inherit', env, windowsHide: false, diff --git a/tests/unit/capture-worker.test.js b/tests/unit/capture-worker.test.js new file mode 100644 index 0000000..ab1bf04 --- /dev/null +++ b/tests/unit/capture-worker.test.js @@ -0,0 +1,146 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); + +test('capture worker requests the selected desktop source, not a plain camera stream', async () => { + const scriptPath = path.join(__dirname, '../../app/renderer/capture-worker.js'); + const script = fs.readFileSync(scriptPath, 'utf8'); + + const mediaCalls = []; + const messages = []; + let onCommand = null; + let resolveStreamReady; + const streamReady = new Promise((resolve) => { + resolveStreamReady = resolve; + }); + + const context = { + console: { + error() {}, + log() {}, + warn() {}, + }, + captureWorkerBridge: { + onCommand(fn) { + onCommand = fn; + }, + send(msg) { + messages.push(msg); + if (msg.type === 'stream-ready') resolveStreamReady(); + }, + }, + StepForgeClickFrames: { + FrameRing: class { + constructor() { + this._frames = []; + } + + push(frame) { + this._frames.push(frame); + } + + frames() { + return [...this._frames]; + } + + latest() { + return this._frames[this._frames.length - 1] || null; + } + + clear() { + this._frames = []; + } + }, + selectFrameForClick() { + return null; + }, + }, + navigator: { + mediaDevices: { + async getUserMedia(constraints) { + mediaCalls.push(constraints); + return { + getTracks() { + return [{ stop() {} }]; + }, + }; + }, + async getDisplayMedia() { + throw new Error('unexpected getDisplayMedia call'); + }, + }, + }, + document: { + createElement(tag) { + assert.equal(tag, 'video'); + return { + muted: false, + srcObject: null, + readyState: 2, + play: async () => {}, + videoWidth: 1920, + videoHeight: 1080, + }; + }, + }, + createImageBitmap: async () => ({ width: 1920, height: 1080, close() {} }), + setInterval: () => 1, + clearInterval: () => {}, + OffscreenCanvas: class { + constructor(width, height) { + this.width = width; + this.height = height; + } + + getContext() { + return { drawImage() {} }; + } + + async convertToBlob() { + return { + async arrayBuffer() { + return new ArrayBuffer(0); + }, + }; + } + }, + }; + + vm.createContext(context); + vm.runInContext(script, context, { filename: scriptPath }); + + assert.equal(typeof onCommand, 'function', 'worker should register a command handler'); + + onCommand({ + type: 'start-stream', + displayId: 7, + sourceId: 'screen:1:0', + display: { + bounds: { width: 1920, height: 1080 }, + scaleFactor: 1, + }, + sampleMs: 50, + }); + await streamReady; + + assert.equal(mediaCalls.length, 1); + const constraints = JSON.parse(JSON.stringify(mediaCalls[0])); + assert.deepEqual(constraints, { + audio: false, + video: { + mandatory: { + chromeMediaSource: 'desktop', + chromeMediaSourceId: 'screen:1:0', + minWidth: 1920, + maxWidth: 1920, + minHeight: 1080, + maxHeight: 1080, + }, + }, + }); + assert.ok(messages.some((msg) => msg.type === 'stream-ready')); +}); diff --git a/tests/unit/capture.test.js b/tests/unit/capture.test.js index 5e0414f..0c5e55c 100644 --- a/tests/unit/capture.test.js +++ b/tests/unit/capture.test.js @@ -33,6 +33,22 @@ function makeService({ settings: settingsOverrides, screenApi } = {}) { }); } +test('XDG_SESSION_TYPE=x11 wins over a stray WAYLAND_DISPLAY value', () => { + const prevSessionType = process.env.XDG_SESSION_TYPE; + const prevWaylandDisplay = process.env.WAYLAND_DISPLAY; + try { + process.env.XDG_SESSION_TYPE = 'x11'; + process.env.WAYLAND_DISPLAY = 'wayland-0'; + const service = makeService(); + assert.equal(service.onWayland(), false); + } finally { + if (prevSessionType === undefined) delete process.env.XDG_SESSION_TYPE; + else process.env.XDG_SESSION_TYPE = prevSessionType; + if (prevWaylandDisplay === undefined) delete process.env.WAYLAND_DISPLAY; + else process.env.WAYLAND_DISPLAY = prevWaylandDisplay; + } +}); + // The raw/regular twin window plus margin: how long a test must wait for a // held Linux raw press to fire when no coordinate twin arrives. const TWIN_FLUSH_MS = 60; @@ -596,6 +612,7 @@ test('armRecording warms while visible, then hides and arms the session', async isVisible() { return this.visible; }, isMinimized() { return false; }, hide() { this.visible = false; }, + minimize() { this.visible = false; }, show() { this.visible = true; }, focus() {}, getTitle() { return 'StepForge'; }, getBounds() { return { x: 0, y: 0, width: 800, height: 600 }; }, @@ -637,6 +654,7 @@ test('a slow recorder start still arms within the warmup cap', async () => { isVisible() { return this.visible; }, isMinimized() { return false; }, hide() { this.visible = false; }, + minimize() { this.visible = false; }, show() { this.visible = true; }, focus() {}, getTitle() { return 'StepForge'; }, getBounds() { return { x: 0, y: 0, width: 800, height: 600 }; }, @@ -718,6 +736,27 @@ test('hook coordinates are converted physical → DIP via screenToDipPoint when assert.deepEqual(seen, [{ x: 640, y: 320 }]); }); +test('bogus Windows screenToDipPoint results fall back to display geometry', () => { + const service = makeService({ + screenApi: { + screenToDipPoint: (p) => ({ x: p.x, y: p.y }), // raw physical point: wrong on scaled displays + getAllDisplays: () => [ + { id: 1, scaleFactor: 2, bounds: { x: 0, y: 0, width: 1440, height: 900 } }, + ], + getCursorScreenPoint: () => { throw new Error('must not fall back to a cursor read'); }, + }, + }); + service.session = { guideId: 'guide-dip-fallback', paused: false, count: 0, intervalSec: 0 }; + const seen = []; + service.enqueueClickCapture = (clickPos) => { + seen.push(clickPos); + }; + + service.onOsClick(1770000000000, { x: 1500, y: 900 }, 'left'); + + assert.deepEqual(seen, [{ x: 750, y: 450 }]); +}); + test('without screenToDipPoint, coordinates convert via display geometry (Linux/X11)', () => { const service = makeService({ screenApi: { @@ -746,6 +785,7 @@ test('clicks without event coordinates fall back to a live cursor read', () => { getAllDisplays: () => [], }, }); + service._wayland = false; // X11/Windows: the live-cursor fallback applies service.session = { guideId: 'guide-cursor', paused: false, count: 0, intervalSec: 0 }; const seen = []; service.enqueueClickCapture = (clickPos) => { @@ -757,6 +797,64 @@ test('clicks without event coordinates fall back to a live cursor read', () => { assert.deepEqual(seen, [{ x: 11, y: 22 }]); }); +test('on Wayland an evdev click does not stamp a bogus top-left cursor position', () => { + // Wayland's getCursorScreenPoint returns {0,0}; a fallback read would mark + // every click in the corner. The click must capture with a null position + // (no marker) instead. + const service = makeService({ + screenApi: { + getCursorScreenPoint: () => ({ x: 0, y: 0 }), + getAllDisplays: () => [], + }, + }); + service._wayland = true; + service.session = { guideId: 'guide-wl-click', paused: false, count: 0, intervalSec: 0 }; + const seen = []; + service.enqueueClickCapture = (clickPos) => { seen.push(clickPos); }; + + service.onOsClick(1770000000000, null, 'button-1'); + + assert.deepEqual(seen, [null], 'no position is passed, so no marker is drawn'); +}); + +// ---- evdev decoding --------------------------------------------------------------- + +const { decodeEvdevButtonPresses } = CaptureService; + +// Pack a 64-bit struct input_event (24 bytes): 16-byte timeval, u16 type, +// u16 code, s32 value. +function evdevRecord(type, code, value) { + const b = Buffer.alloc(24); + b.writeUInt16LE(type, 16); + b.writeUInt16LE(code, 18); + b.writeInt32LE(value, 20); + return b; +} + +test('evdev decoder extracts left/right/middle button presses, ignores the rest', () => { + const EV_KEY = 0x01; + const EV_REL = 0x02; + const buf = Buffer.concat([ + evdevRecord(EV_KEY, 272, 1), // BTN_LEFT down -> button-1 + evdevRecord(EV_KEY, 272, 0), // BTN_LEFT up -> ignored (release) + evdevRecord(EV_REL, 0, 5), // motion -> ignored (not a key) + evdevRecord(EV_KEY, 273, 1), // BTN_RIGHT down -> button-3 + evdevRecord(EV_KEY, 274, 1), // BTN_MIDDLE down-> button-2 + evdevRecord(EV_KEY, 0x110 + 8, 1), // a non-mouse key -> ignored + ]); + const { presses, rest } = decodeEvdevButtonPresses(buf, 24); + assert.deepEqual(presses, ['button-1', 'button-3', 'button-2']); + assert.equal(rest.length, 0); +}); + +test('evdev decoder keeps a trailing partial record for the next chunk', () => { + const full = evdevRecord(0x01, 272, 1); + const buf = Buffer.concat([full, full.subarray(0, 10)]); // 1.4 records + const { presses, rest } = decodeEvdevButtonPresses(buf, 24); + assert.deepEqual(presses, ['button-1']); + assert.equal(rest.length, 10, 'the half record is returned to be completed later'); +}); + // ---- watcher loss ----------------------------------------------------------------- test('losing the click watcher mid-session falls back to interval capture', () => { @@ -779,6 +877,49 @@ test('losing the click watcher mid-session falls back to interval capture', () = } }); +test('losing the click watcher mid-session can fall back to hotkey-only', () => { + const service = makeService(); + service.settings.get = (key) => { + if (key === 'capture.fallbackTrigger') return 'hotkey'; + if (key === 'capture.autoIntervalSec') return 3; + return null; + }; + service.session = { guideId: 'guide-loss-hotkey', paused: false, count: 0, intervalSec: 0 }; + const states = []; + service.notify = (channel, payload) => { + states.push({ channel, payload }); + }; + + try { + service.handleClickWatcherLoss('exited with code 1'); + + assert.equal(service.session.intervalSec, 0, + 'hotkey fallback must not silently start the 5-second timer'); + assert.equal(service.intervalTimer, null); + assert.ok(states.some((s) => s.channel === 'capture:state')); + } finally { + service.finishSession(); + } +}); + +test('starting a session with hotkey fallback keeps the timer off when clicks are unavailable', () => { + const service = makeService(); + service.clickCaptureAvailable = () => false; + service.settings.get = (key) => { + if (key === 'capture.fallbackTrigger') return 'hotkey'; + if (key === 'capture.autoIntervalSec') return 7; + if (key === 'capture.captureOutsideClicks') return false; + return null; + }; + + service.startSession('guide-hotkey-fallback'); + + assert.equal(service.session.intervalSec, 0); + assert.equal(service.state().clickCaptureAvailable, false); + assert.equal(service.state().clickCapture, false); + service.finishSession(); +}); + // ---- strict frame selection ----------------------------------------------------- test('a click is served instantly from the freshly buffered frame', async () => { @@ -1004,6 +1145,7 @@ test('clicks during an in-flight pre-click grab wait for the frame instead of be test('click frames come from the stream backend when it is active', async () => { const service = makeService(); + service._wayland = false; // X11/Windows: click frame-requests are failable const clickAt = Date.now(); service.session = { guideId: 'guide-stream', paused: false, count: 0, intervalSec: 0 }; const requests = []; @@ -1036,8 +1178,8 @@ test('click frames come from the stream backend when it is active', async () => assert.equal(result.ok, true); assert.deepEqual(added, ['stream-frame']); - assert.deepEqual(requests, [{ clickPos: { x: 10, y: 10 }, clickAt, strict: true, leadMs: 0 }], - 'the worker receives the hook-time click timestamp, strictness, and lead'); + assert.deepEqual(requests, [{ clickPos: { x: 10, y: 10 }, clickAt, strict: true, leadMs: 0, failable: true }], + 'the worker receives the hook-time click timestamp, strictness, lead, and failability'); }); test('a stream backend with no qualifying frame falls through to the fresh-shot path', async () => { @@ -1080,6 +1222,9 @@ test('an unhealthy stream backend degrades to the in-process frame loop', () => const service = makeService(); service.session = { guideId: 'guide-degrade', paused: false, count: 0, intervalSec: 0 }; service.streamBackend = { isActive: () => true, stop: () => {} }; + // Off Wayland the in-process loop is viable (getSources works), so an + // unhealthy worker must degrade to the loop rather than silently stopping. + service._wayland = false; let loopStarted = false; service.startFrameLoop = () => { loopStarted = true; }; const states = []; @@ -1092,6 +1237,24 @@ test('an unhealthy stream backend degrades to the in-process frame loop', () => assert.ok(states.includes('capture:state')); }); +test('an unhealthy stream backend does NOT start the frame loop on Wayland', () => { + // On Wayland the 200ms getSources() frame loop is broken (throws) and pops + // the portal, so it's not a usable fallback — the portal stream is the only + // capture path and the user must re-share if it stops. + const service = makeService(); + service.session = { guideId: 'guide-degrade-wl', paused: false, count: 0, intervalSec: 0 }; + service.streamBackend = { isActive: () => true, stop: () => {} }; + service._wayland = true; + let loopStarted = false; + service.startFrameLoop = () => { loopStarted = true; }; + service.notify = () => {}; + + service.degradeToFrameLoop(); + + assert.equal(service.streamBackend, null); + assert.equal(loopStarted, false, 'no frame loop on Wayland — getSources is unusable there'); +}); + test('session state reports which frame recorder is serving clicks', () => { const service = makeService(); service.session = { guideId: 'guide-state', paused: false, count: 0, intervalSec: 0 }; @@ -1156,7 +1319,12 @@ test('a new session starts paused and does not hide the window until "Start reco isDestroyed() { return this.destroyed; }, isVisible() { return this.visible; }, isMinimized() { return this.minimized; }, + // The window is "tucked away" for recording either by hide() (with a tray) + // or minimize() (Linux without a tray). Both make it not visible; assert on + // visibility so the test is independent of which mechanism the platform picks. hide() { this.visible = false; this.hidden += 1; }, + minimize() { this.visible = false; this.minimized = true; }, + restore() { this.minimized = false; this.visible = true; }, show() { this.visible = true; this.shown += 1; }, showInactive() { this.visible = true; this.shown += 1; }, focus() {}, @@ -1171,15 +1339,15 @@ test('a new session starts paused and does not hide the window until "Start reco assert.equal(service.session.paused, true, 'sessions start paused'); assert.equal(service.state().paused, true); - assert.equal(win.hidden, 0, 'window must stay visible until recording starts'); + assert.equal(win.visible, true, 'window must stay visible until recording starts'); // User clicks "Start recording" (the resume action). service.togglePause(false); assert.equal(service.session.paused, false); - assert.equal(win.hidden, 0, 'hide is deferred until the resume path runs'); + assert.equal(win.visible, true, 'tuck-away is deferred until the resume path runs'); await new Promise((r) => setTimeout(r, 25)); - assert.equal(win.hidden, 1, 'window hides once recording actually starts'); + assert.equal(win.visible, false, 'window is tucked away once recording actually starts'); } finally { service.finishSession(); } diff --git a/tests/unit/electron-launcher.test.js b/tests/unit/electron-launcher.test.js index 3cefaa1..3044d87 100644 --- a/tests/unit/electron-launcher.test.js +++ b/tests/unit/electron-launcher.test.js @@ -7,6 +7,7 @@ const path = require('node:path'); const { buildMissingElectronError, + linuxSandboxLaunchArgs, repairElectronInstall, resolveElectronBinary, } = require('../../scripts/electron-launcher'); @@ -171,3 +172,21 @@ test('reports a helpful error when the runtime is missing', (t) => { assert.match(message, /Electron could not be started/); assert.match(message, /Expected the binary in:/); }); + +test('uses --no-sandbox when the Linux sandbox helper is not root-owned and setuid', () => { + const args = linuxSandboxLaunchArgs({ + electronPath: '/tmp/stepforge/node_modules/electron/dist/electron', + platform: 'linux', + statSync: () => ({ uid: 1000, mode: 0o100755 }), + }); + assert.deepEqual(args, ['--no-sandbox']); +}); + +test('keeps the sandbox enabled when the Linux helper is root-owned and setuid', () => { + const args = linuxSandboxLaunchArgs({ + electronPath: '/tmp/stepforge/node_modules/electron/dist/electron', + platform: 'linux', + statSync: () => ({ uid: 0, mode: 0o104755 }), + }); + assert.deepEqual(args, []); +}); diff --git a/tests/unit/package-windows.test.js b/tests/unit/package-windows.test.js index 4699b40..ecad59b 100644 --- a/tests/unit/package-windows.test.js +++ b/tests/unit/package-windows.test.js @@ -21,6 +21,9 @@ test('Windows packaging uses an assisted NSIS installer', (t) => { assert.equal(config.nsis.createDesktopShortcut, true); assert.equal(config.nsis.createStartMenuShortcut, true); assert.equal(config.nsis.shortcutName, 'StepForge'); + assert.equal(config.buildVersion, '0.3.2.1'); + assert.equal(config.nsis.artifactName, '${productName} Setup ${buildVersion}.${ext}'); + assert.equal(config.nsis.include, 'build/installer.nsh'); assert.equal(config.asar, true); assert.ok(config.files.includes('app/**/*')); assert.ok(config.files.includes('core/**/*')); @@ -34,7 +37,7 @@ test('Windows packaging uses an assisted NSIS installer', (t) => { const tmp = makeTmpDir('windows-installer'); t.after(() => rmrf(tmp)); fs.mkdirSync(path.join(tmp, 'nested', 'deeper'), { recursive: true }); - const installer = path.join(tmp, 'nested', 'deeper', 'StepForge Setup 0.2.0.exe'); + const installer = path.join(tmp, 'nested', 'deeper', 'StepForge Setup 0.3.2.1.exe'); fs.writeFileSync(installer, Buffer.from('fake installer')); assert.equal(findInstallerExe(tmp), installer); }); diff --git a/tests/unit/placeholders.test.js b/tests/unit/placeholders.test.js index 2b15ac3..6b700d8 100644 --- a/tests/unit/placeholders.test.js +++ b/tests/unit/placeholders.test.js @@ -56,13 +56,16 @@ test('settings persist, deep-merge with defaults, and store global placeholders' assert.equal(s1.get('appearance'), DEFAULT_SETTINGS.appearance); s1.set('appearance', 'dark'); s1.set('capture.delayMs', 1500); + s1.set('ai.ollama.model', 'qwen3:0.6b'); s1.setGlobalPlaceholders({ Company: 'Acme', Author: 'Tyler' }); // A fresh instance reads back the changed values merged over defaults. const s2 = new Settings(dir); assert.equal(s2.get('appearance'), 'dark'); assert.equal(s2.get('capture.delayMs'), 1500); + assert.equal(s2.get('ai.ollama.model'), 'qwen3:0.6b'); assert.equal(s2.get('capture.clickMarker'), DEFAULT_SETTINGS.capture.clickMarker); + assert.equal(s2.get('capture.fallbackTrigger'), DEFAULT_SETTINGS.capture.fallbackTrigger); assert.deepEqual(s2.getGlobalPlaceholders(), { Company: 'Acme', Author: 'Tyler' }); }); diff --git a/tests/unit/stamp-version.test.js b/tests/unit/stamp-version.test.js new file mode 100644 index 0000000..9b49509 --- /dev/null +++ b/tests/unit/stamp-version.test.js @@ -0,0 +1,46 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { makeTmpDir, rmrf } = require('./helpers'); +const { stampVersion } = require('../../scripts/stamp-version'); + +test('stampVersion splits build labels into package and build versions', () => { + const root = makeTmpDir('stamp-version'); + try { + fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ + name: 'stepforge', + version: '0.1.0', + private: true, + buildVersion: '0.1.0', + }, null, 2)); + + fs.writeFileSync(path.join(root, 'package-lock.json'), JSON.stringify({ + name: 'stepforge', + version: '0.1.0', + lockfileVersion: 3, + requires: true, + packages: { + '': { + name: 'stepforge', + version: '0.1.0', + }, + }, + }, null, 2)); + + stampVersion(root, '0.3.2.1'); + + const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')); + const lock = JSON.parse(fs.readFileSync(path.join(root, 'package-lock.json'), 'utf8')); + + assert.equal(pkg.version, '0.3.2'); + assert.equal(pkg.buildVersion, '0.3.2.1'); + assert.equal(lock.version, '0.3.2'); + assert.equal(lock.packages[''].version, '0.3.2'); + } finally { + rmrf(root); + } +}); diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 86cd3ff..379799b 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -1,5 +1,7 @@ 'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); const test = require('node:test'); const assert = require('node:assert/strict'); @@ -309,21 +311,112 @@ test('ollama connection test reports installed models', async (t) => { settings: makeSettings(), getWindow: () => null, dataDir: root, - fetchImpl: async () => ({ - ok: true, - json: async () => ({ - models: [ - { name: 'llama3.2:1b' }, - { name: 'qwen3:0.6b' }, - ], - }), - }), + fetchImpl: async (url) => { + const pathname = new URL(url).pathname; + if (pathname === '/api/tags') { + return { + ok: true, + json: async () => ({ + models: [ + { name: 'llama3.2:1b' }, + { name: 'qwen3:0.6b' }, + ], + }), + }; + } + if (pathname === '/api/show') { + return { + ok: true, + json: async () => ({ + capabilities: ['completion', 'vision'], + }), + }; + } + throw new Error(`unexpected fetch: ${pathname}`); + }, }); const result = await service.testAiConnection(); assert.equal(result.ok, true); assert.equal(result.installed, true); assert.equal(result.model, 'llama3.2:1b'); + assert.equal(result.vision, true); +}); + +test('vision-capable models receive the screenshot in the chat request', async (t) => { + const root = makeTmpDir('text-intel-ai-vision'); + t.after(() => rmrf(root)); + const imagePath = path.join(root, 'step.png'); + fs.writeFileSync(imagePath, Buffer.from('fake screenshot bytes')); + + const step = createStep({ + title: 'Old title', + descriptionHtml: '

Old text

', + image: { + originalPath: 'original.png', + workingPath: 'working.png', + size: { width: 10, height: 10 }, + }, + captureMetadata: { + windowTitle: 'Settings', + appName: 'chrome', + ocrText: 'Open settings', + titleCandidate: 'Open settings', + mode: 'fullscreen', + }, + }); + const fetchCalls = []; + const service = new TextIntelService({ + store: { + settingsDir: root, + getGuide: () => ({ guideId: 'g1', title: 'Guide', descriptionHtml: '', stepsOrder: ['s1'] }), + getStep: () => step, + stepImagePath: () => imagePath, + saveStep: (_, next) => next, + }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: async (url, init = {}) => { + const pathname = new URL(url).pathname; + fetchCalls.push({ pathname, init }); + if (pathname === '/api/show') { + return { + ok: true, + json: async () => ({ + capabilities: ['completion', 'vision'], + }), + }; + } + if (pathname === '/api/chat') { + return { + ok: true, + json: async () => ({ + message: { + content: JSON.stringify({ + title: 'Open settings', + description: 'Use the AI tab.', + }), + }, + }), + }; + } + throw new Error(`unexpected fetch: ${pathname}`); + }, + }); + + const result = await service.generateStepPatch({ + guideId: 'g1', + stepId: 's1', + target: 'all', + }); + + assert.equal(result.ok, true); + const chatCall = fetchCalls.find((call) => call.pathname === '/api/chat'); + assert.ok(chatCall, 'expected an Ollama chat request'); + const body = JSON.parse(chatCall.init.body); + assert.deepEqual(body.messages[1].images, [fs.readFileSync(imagePath).toString('base64')]); + assert.match(body.messages[1].content, /Screenshot: attached/i); }); test('invalid ollama output fails safely without saving the step', async (t) => {