This commit is contained in:
@@ -8,7 +8,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Release tag, e.g. v0.1.1 (the tag is created at the current commit on this branch)'
|
||||
description: 'Release tag, e.g. v0.3.2.1 (the tag is created at the current commit on this branch)'
|
||||
required: true
|
||||
type: string
|
||||
prerelease:
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
shell: bash
|
||||
env:
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: npm version "${VERSION#v}" --no-git-tag-version --allow-same-version
|
||||
run: node scripts/stamp-version.js "${VERSION#v}"
|
||||
|
||||
- name: Configure Windows code signing
|
||||
shell: bash
|
||||
|
||||
@@ -64,6 +64,8 @@ using only Node built-ins.
|
||||
|
||||
For a Windows installation, see [docs/windows_installation](docs/windows_installation.md) or for a developer/more in depth walkthrough, see [docs/GETTING_STARTED.md](docs/GETTING_STARTED.md).
|
||||
|
||||
On **Linux** (⚠️ work in progress — X11 vs Wayland, enabling per-click capture, the screen-share prompt), see [docs/GETTING_STARTED_WITH_LINUX.md](docs/GETTING_STARTED_WITH_LINUX.md).
|
||||
|
||||
Requirements: Node.js 20+ and npm (Electron is the only dependency).
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1 +1 @@
|
||||
In the future, planned updates include a re-vamp of the settings panel, text box's (important, tip, warning, and note) are not properly moving to the intended areas and I will look into locking the size (or being able to control the size) of images throughout varias export formats.
|
||||
In the future, planned updates include a re-vamp of the settings panel, text box's (important, tip, warning, and note) are not properly moving to the intended areas and I will look into locking the size (or being able to control the size) of images throughout varias export formats. OCR or local ai would be pretty cool....
|
||||
|
||||
+325
-33
@@ -1,6 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const path = require('node:path');
|
||||
const fs = require('node:fs');
|
||||
const { spawn, execFileSync } = require('node:child_process');
|
||||
const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron');
|
||||
const raster = require('../core/raster');
|
||||
@@ -107,6 +108,88 @@ function hasBinary(name) {
|
||||
}
|
||||
}
|
||||
|
||||
// On Wayland, xinput only sees XWayland (X11-bridge) events — native Wayland
|
||||
// app clicks are delivered via the Wayland protocol and never reach xinput.
|
||||
// Treating it as "available" would leave the session with no click capture AND
|
||||
// no interval fallback, so zero steps get captured.
|
||||
function isWayland() {
|
||||
if (process.platform !== 'linux') return false;
|
||||
// XDG_SESSION_TYPE is the authoritative session hint when present. Some
|
||||
// desktops still export WAYLAND_DISPLAY even when the active session is X11,
|
||||
// so only fall back to it when XDG_SESSION_TYPE is unavailable.
|
||||
const sessionType = String(process.env.XDG_SESSION_TYPE || '').toLowerCase();
|
||||
if (sessionType) return sessionType === 'wayland';
|
||||
return Boolean(process.env.WAYLAND_DISPLAY);
|
||||
}
|
||||
|
||||
// ---- evdev (Linux kernel input) click reader --------------------------------
|
||||
// Reading /dev/input/event* directly sees mouse-button presses on BOTH X11 and
|
||||
// Wayland, because it taps the kernel input layer below the display server —
|
||||
// the one global-click source that survives Wayland's security model. It needs
|
||||
// read access to the device nodes (the user must be in the `input` group).
|
||||
//
|
||||
// Each event is a fixed-size `struct input_event`: a timeval, then u16 type,
|
||||
// u16 code, s32 value. The timeval is two `long`s, so the record is 24 bytes on
|
||||
// 64-bit and 16 on 32-bit; type/code/value always sit in the last 8 bytes.
|
||||
const EV_KEY = 0x01;
|
||||
const EVDEV_PRESS = 1;
|
||||
// BTN_LEFT/RIGHT/MIDDLE -> the same button-N naming the xinput path emits
|
||||
// (1=left, 2=middle, 3=right) so downstream debounce/marker logic is identical.
|
||||
const EVDEV_BUTTONS = { 272: 'button-1', 273: 'button-3', 274: 'button-2' };
|
||||
const EVDEV_RECORD_SIZE = (process.arch === 'x64' || process.arch === 'arm64'
|
||||
|| process.arch === 'ppc64' || process.arch === 's390x' || process.arch === 'loong64'
|
||||
|| process.arch === 'riscv64') ? 24 : 16;
|
||||
|
||||
/**
|
||||
* Decode a buffer of packed input_event records, returning the button presses
|
||||
* found and any trailing partial record (a device read can split mid-record).
|
||||
* Pure and size-parameterised so it is unit-testable without real devices.
|
||||
*/
|
||||
function decodeEvdevButtonPresses(buffer, recordSize = EVDEV_RECORD_SIZE) {
|
||||
const presses = [];
|
||||
let offset = 0;
|
||||
while (buffer.length - offset >= recordSize) {
|
||||
const type = buffer.readUInt16LE(offset + recordSize - 8);
|
||||
const code = buffer.readUInt16LE(offset + recordSize - 6);
|
||||
const value = buffer.readInt32LE(offset + recordSize - 4);
|
||||
offset += recordSize;
|
||||
if (type === EV_KEY && value === EVDEV_PRESS && EVDEV_BUTTONS[code]) {
|
||||
presses.push(EVDEV_BUTTONS[code]);
|
||||
}
|
||||
}
|
||||
return { presses, rest: buffer.subarray(offset) };
|
||||
}
|
||||
|
||||
/**
|
||||
* The /dev/input/event* nodes for pointing devices that are readable by this
|
||||
* process. /proc/bus/input/devices lists every device with its Handlers line;
|
||||
* a pointing device exposes a `mouseN` handler, and the matching `eventN` is
|
||||
* the node to read. Unreadable nodes (no `input` group membership) are skipped.
|
||||
*/
|
||||
function readableEvdevMouseNodes() {
|
||||
const nodes = [];
|
||||
let table;
|
||||
try {
|
||||
table = fs.readFileSync('/proc/bus/input/devices', 'utf8');
|
||||
} catch {
|
||||
return nodes;
|
||||
}
|
||||
for (const block of table.split('\n\n')) {
|
||||
const handlers = /H:\s*Handlers=([^\n]*)/.exec(block);
|
||||
if (!handlers || !/\bmouse\d+\b/.test(handlers[1])) continue;
|
||||
const event = /\bevent(\d+)\b/.exec(handlers[1]);
|
||||
if (!event) continue;
|
||||
const node = `/dev/input/event${event[1]}`;
|
||||
try {
|
||||
fs.accessSync(node, fs.constants.R_OK);
|
||||
nodes.push(node);
|
||||
} catch {
|
||||
// Not readable — user is not in the `input` group for this node.
|
||||
}
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
class CaptureService {
|
||||
constructor({
|
||||
store,
|
||||
@@ -124,6 +207,9 @@ class CaptureService {
|
||||
// the global `screen` directly so coordinate handling stays testable.
|
||||
this.screen = screenApi;
|
||||
this.textIntel = textIntel;
|
||||
// Cached display-server detection. A method (onWayland) reads this so tests
|
||||
// can flip platform behavior without touching process.env.
|
||||
this._wayland = isWayland();
|
||||
this.session = null; // { guideId, paused, count, intervalSec }
|
||||
this.intervalTimer = null;
|
||||
this.clickWatcher = null;
|
||||
@@ -182,21 +268,62 @@ class CaptureService {
|
||||
return this.settings.get('capture.strictClickFrames') !== false;
|
||||
}
|
||||
|
||||
fallbackCaptureTrigger() {
|
||||
const raw = String(this.settings.get('capture.fallbackTrigger') || 'interval').toLowerCase();
|
||||
return raw === 'hotkey' ? 'hotkey' : 'interval';
|
||||
}
|
||||
|
||||
fallbackIntervalSec() {
|
||||
const raw = Number(this.settings.get('capture.autoIntervalSec'));
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 5;
|
||||
}
|
||||
|
||||
clickCaptureAvailable() {
|
||||
if (this._clickAvail === undefined) {
|
||||
this._clickAvail = process.platform === 'win32' || (process.platform === 'linux' && hasBinary('xinput'));
|
||||
// Three click sources, in order of fidelity:
|
||||
// - Windows: the low-level mouse hook (position + timing);
|
||||
// - X11: xinput test-xi2 (position + timing) — but it can't see native
|
||||
// Wayland clicks, only XWayland ones, so it's gated to non-Wayland;
|
||||
// - Linux evdev (/dev/input): button presses on X11 AND Wayland, but no
|
||||
// cursor position on Wayland — used for per-click capture there (no
|
||||
// marker). Requires the user to be in the `input` group.
|
||||
this._clickAvail = process.platform === 'win32'
|
||||
|| (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput'))
|
||||
|| (process.platform === 'linux' && readableEvdevMouseNodes().length > 0);
|
||||
}
|
||||
return this._clickAvail;
|
||||
}
|
||||
|
||||
/** Whether this is a Wayland session (cached; overridable in tests). */
|
||||
onWayland() {
|
||||
return this._wayland;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the in-process frame loop is a usable fallback recorder. It grabs
|
||||
* via desktopCapturer.getSources(), which on Wayland is broken (throws) and
|
||||
* pops the portal — so the loop is viable only off Wayland. On Wayland the
|
||||
* portal-backed stream backend is the sole capture path.
|
||||
*/
|
||||
canUseFrameLoop() {
|
||||
return !this.onWayland();
|
||||
}
|
||||
|
||||
startSession(guideId, { intervalSec = null } = {}) {
|
||||
this.finishSession();
|
||||
// Default trigger: clicks when the platform supports it, otherwise an
|
||||
// interval so a session always produces steps even if the global hotkey
|
||||
// never fires (common under Wayland/WSLg).
|
||||
// Default trigger: clicks when the platform supports it, otherwise the
|
||||
// user-selected fallback (timer or hotkey-only). That keeps Linux from
|
||||
// silently dropping into an unwanted 5-second timer when click capture
|
||||
// is unavailable.
|
||||
let interval = intervalSec;
|
||||
if (interval == null) {
|
||||
interval = this.clickCaptureAvailable() ? 0 : (this.settings.get('capture.autoIntervalSec') || 5);
|
||||
if (this.clickCaptureAvailable()) {
|
||||
interval = 0;
|
||||
} else if (this.fallbackCaptureTrigger() === 'hotkey') {
|
||||
interval = 0;
|
||||
} else {
|
||||
interval = this.fallbackIntervalSec();
|
||||
}
|
||||
}
|
||||
// Sessions start paused: nothing hides and no capturing happens until
|
||||
// the user explicitly presses "Start recording" in the capture bar, so
|
||||
@@ -300,6 +427,7 @@ class CaptureService {
|
||||
showWindow() {
|
||||
const win = this.getWindow();
|
||||
if (win && !win.isDestroyed()) {
|
||||
if (win.isMinimized()) win.restore();
|
||||
win.show();
|
||||
win.focus();
|
||||
}
|
||||
@@ -321,7 +449,14 @@ class CaptureService {
|
||||
const sec = this.session && this.session.intervalSec;
|
||||
if (sec > 0) {
|
||||
this.intervalTimer = setInterval(() => {
|
||||
this.sessionCapture('interval').catch(() => {});
|
||||
// Don't let a slow capture (e.g. a multi-second software PNG encode on
|
||||
// a GPU-less host) overlap with the next tick — overlapping requests
|
||||
// would pile up and could trip the backend's failure counter.
|
||||
if (this.intervalCapturing) return;
|
||||
this.intervalCapturing = true;
|
||||
this.sessionCapture('interval')
|
||||
.catch(() => {})
|
||||
.finally(() => { this.intervalCapturing = false; });
|
||||
}, sec * 1000);
|
||||
}
|
||||
}
|
||||
@@ -358,8 +493,13 @@ class CaptureService {
|
||||
armRecording() {
|
||||
const win = this.getWindow();
|
||||
const wantHide = Boolean(this.hiddenForSession && win && !win.isDestroyed());
|
||||
const recorderWanted = this.settings.get('capture.captureOutsideClicks') !== false
|
||||
&& this.clickCaptureAvailable();
|
||||
// Always start the frame recorder when stream capture is enabled — it
|
||||
// buffers frames for click captures AND is used for interval/hotkey
|
||||
// captures to avoid calling desktopCapturer.getSources() on every capture.
|
||||
// On Linux/Wayland, each getSources() call goes through the XDG portal and
|
||||
// shows a permission dialog; the stream backend eliminates that by keeping
|
||||
// a live video stream open for the duration of the recording session.
|
||||
const recorderWanted = this.settings.get('capture.streamCapture') !== false;
|
||||
// Recording is not "live" until the window is hidden and the buffer is
|
||||
// primed. While warming up, the window is still visible and over the
|
||||
// user's work, so clicks in this period are ignored (onOsClick checks
|
||||
@@ -383,7 +523,17 @@ class CaptureService {
|
||||
if (!this.session || this.session.paused) { this.warmingUp = false; return; }
|
||||
}
|
||||
if (wantHide && win && !win.isDestroyed() && win.isVisible()) {
|
||||
win.hide();
|
||||
// On Linux, always minimize rather than hide. GNOME's system tray
|
||||
// (StatusNotifier) is unreliable — it can fail or half-export over
|
||||
// dbus — so a hidden window can be impossible to bring back, leaving
|
||||
// the user unable to stop the recording. A minimized window is always
|
||||
// restorable from the taskbar, and minimized windows aren't rendered
|
||||
// so they still stay out of the fullscreen capture.
|
||||
if (process.platform === 'linux') {
|
||||
win.minimize();
|
||||
} else {
|
||||
win.hide();
|
||||
}
|
||||
// Let a couple of frames of the now-unobscured screen land before
|
||||
// the user's first click, so that frame shows their work, not the
|
||||
// app window that was just dismissed.
|
||||
@@ -481,6 +631,46 @@ class CaptureService {
|
||||
if (!sessionLive) return { ok: false, reason: 'session ended before the fallback shot' };
|
||||
}
|
||||
|
||||
// For non-click triggers (interval, hotkey, manual) pull the latest frame
|
||||
// from the stream backend's ring buffer when available. This avoids a
|
||||
// desktopCapturer.getSources() call per capture — on Linux/Wayland that
|
||||
// call goes through the XDG portal and shows a dialog every time.
|
||||
//
|
||||
// No clickPos: a timed capture has no click position, and passing a cursor
|
||||
// point here is actively harmful on Wayland — getCursorScreenPoint() can
|
||||
// return a stale/out-of-bounds point, which makes the backend reject the
|
||||
// frame (wrong display / out of bounds) and fall through to a getSources()
|
||||
// shot, i.e. a portal dialog on every interval tick.
|
||||
if (trigger !== 'click' && this.streamBackend && this.streamBackend.isActive()) {
|
||||
const frame = await this.streamBackend.frameForClick({
|
||||
clickPos: null,
|
||||
clickAt: Date.now(),
|
||||
strict: false, // no pre/post-click constraint for timed captures
|
||||
leadMs: 0,
|
||||
failable: false, // a slow timed-capture encode must not kill the stream
|
||||
}).catch(() => null);
|
||||
if (frame) {
|
||||
const result = await this.storeFrameAsStep(this.session.guideId, 'fullscreen', frame);
|
||||
if (result.ok) this.noteStepAdded(result.step, trigger);
|
||||
clog(trigger, 'capture stored from stream; total', this.session && this.session.count);
|
||||
return result;
|
||||
}
|
||||
clog(trigger, 'capture: no frame from stream this tick — will retry next tick');
|
||||
} else if (trigger !== 'click' && this.onWayland()) {
|
||||
clog(trigger, 'capture: stream backend not active (active=',
|
||||
Boolean(this.streamBackend && this.streamBackend.isActive()), ')');
|
||||
}
|
||||
|
||||
// On Wayland the only screen-grab fallback below is desktopCapturer
|
||||
// .getSources(), which pops the XDG portal dialog every call. For the
|
||||
// automatic timed triggers that would mean a dialog on every tick, so skip
|
||||
// the fallback and wait for the open stream to deliver a frame on a later
|
||||
// tick. Explicit captures (manual, and click on X11) still fall through —
|
||||
// one dialog for one deliberate action.
|
||||
if (this.onWayland() && (trigger === 'interval' || trigger === 'hotkey')) {
|
||||
return { ok: false, reason: 'waiting for the screen-share stream' };
|
||||
}
|
||||
|
||||
if (this.shooting) return { ok: false, reason: 'capture already in progress' };
|
||||
this.shooting = true;
|
||||
try {
|
||||
@@ -642,7 +832,11 @@ class CaptureService {
|
||||
};
|
||||
|
||||
if (this.streamBackend && this.streamBackend.isActive() && grabMode === 'fullscreen') {
|
||||
const frame = await this.streamBackend.frameForClick({ clickPos, clickAt: clickTime, strict, leadMs });
|
||||
// On Wayland the stream is the only capture path (no frame-loop fallback),
|
||||
// so a slow PNG encode must not let the 2-strikes rule tear it down.
|
||||
const frame = await this.streamBackend.frameForClick({
|
||||
clickPos, clickAt: clickTime, strict, leadMs, failable: !this.onWayland(),
|
||||
});
|
||||
if (frame) return frame;
|
||||
// No qualifying frame (or the backend just went unhealthy): fall
|
||||
// through to the loop buffer / fresh-shot fallbacks below.
|
||||
@@ -695,8 +889,11 @@ class CaptureService {
|
||||
async startClickFrameBackend() {
|
||||
const mode = this.settings.get('capture.mode') || 'fullscreen';
|
||||
// The worker streams screens; window-mode grabs need the loop's
|
||||
// source-filtering logic.
|
||||
if (this.settings.get('capture.streamCapture') === false || mode === 'window') {
|
||||
// source-filtering logic. But the loop isn't viable on Wayland (getSources
|
||||
// is broken/portal), so there we always take the stream backend regardless
|
||||
// of the streamCapture/window settings.
|
||||
if (this.canUseFrameLoop()
|
||||
&& (this.settings.get('capture.streamCapture') === false || mode === 'window')) {
|
||||
this.startFrameLoop();
|
||||
return;
|
||||
}
|
||||
@@ -716,7 +913,11 @@ class CaptureService {
|
||||
onUnhealthy: () => this.degradeToFrameLoop(),
|
||||
});
|
||||
const displays = this.screen.getAllDisplays();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
// On Wayland, desktopCapturer.getSources() both fails to yield usable
|
||||
// source ids AND pops the portal dialog. Skip it entirely and drive the
|
||||
// worker through getDisplayMedia (the portal picker chooses the screen).
|
||||
const useDisplayMedia = this.onWayland();
|
||||
const sources = useDisplayMedia ? [] : await desktopCapturer.getSources({
|
||||
types: ['screen'],
|
||||
thumbnailSize: { width: 1, height: 1 }, // ids only — skip thumbnail work
|
||||
});
|
||||
@@ -724,23 +925,38 @@ class CaptureService {
|
||||
displays,
|
||||
sources: sources.map((s) => ({ id: s.id, display_id: s.display_id })),
|
||||
sampleMs: this.settings.get('capture.frameSampleMs') || 100,
|
||||
useDisplayMedia,
|
||||
});
|
||||
const stale = gen !== this.captureGen;
|
||||
if (!ok || stale || !this.session || this.session.paused) {
|
||||
backend.stop();
|
||||
if (!stale && this.session && !this.session.paused) {
|
||||
console.error('[stepforge] stream capture backend failed to start — using in-process frame loop');
|
||||
this.startFrameLoop();
|
||||
if (this.canUseFrameLoop()) {
|
||||
console.error('[stepforge] stream capture backend failed to start — using in-process frame loop');
|
||||
this.startFrameLoop();
|
||||
} else {
|
||||
// On Wayland the frame loop would spam getSources() (portal) with
|
||||
// nothing usable, so there's no fallback — the recording needs the
|
||||
// portal stream. Tell the user how to recover.
|
||||
console.error('[stepforge] screen-share stream did not start — pick a screen in the share dialog, or stop and start recording again');
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.streamBackend = backend;
|
||||
clog('stream capture backend active');
|
||||
// Visible in normal output (one line per recording): confirms the screen
|
||||
// stream came up, so a "nothing records" report can be told apart from a
|
||||
// stream that never started (which logs the failure paths above).
|
||||
console.log(`[stepforge] screen-capture stream active (${useDisplayMedia ? 'getDisplayMedia/portal' : 'desktopCapturer'})`);
|
||||
this.notify('capture:state', this.state());
|
||||
} catch (err) {
|
||||
if (gen === this.captureGen && this.session && !this.session.paused) {
|
||||
console.error(`[stepforge] stream capture backend error (${err && err.message}) — using in-process frame loop`);
|
||||
this.startFrameLoop();
|
||||
if (this.canUseFrameLoop()) {
|
||||
console.error(`[stepforge] stream capture backend error (${err && err.message}) — using in-process frame loop`);
|
||||
this.startFrameLoop();
|
||||
} else {
|
||||
console.error(`[stepforge] screen-share stream error (${err && err.message}) — stop and start recording again`);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (gen === this.captureGen) this.streamBackendStarting = false;
|
||||
@@ -765,8 +981,14 @@ class CaptureService {
|
||||
*/
|
||||
degradeToFrameLoop() {
|
||||
this.streamBackend = null;
|
||||
console.error('[stepforge] stream capture backend unhealthy — falling back to in-process frame loop');
|
||||
if (this.session && !this.session.paused) this.startFrameLoop();
|
||||
if (this.canUseFrameLoop()) {
|
||||
console.error('[stepforge] stream capture backend unhealthy — falling back to in-process frame loop');
|
||||
if (this.session && !this.session.paused) this.startFrameLoop();
|
||||
} else {
|
||||
// On Wayland the frame loop isn't viable (getSources is broken/portal),
|
||||
// so there's nothing to fall back to — the stream is the only path.
|
||||
console.error('[stepforge] screen-share stream stopped — stop and start recording again to re-share');
|
||||
}
|
||||
this.notify('capture:state', this.state());
|
||||
}
|
||||
|
||||
@@ -775,13 +997,14 @@ class CaptureService {
|
||||
try {
|
||||
this.clickWatcherBuf = '';
|
||||
this.linuxEvent = null;
|
||||
if (process.platform === 'linux' && hasBinary('xinput')) {
|
||||
if (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput')) {
|
||||
// Stream raw button events from the X server; one capture per press.
|
||||
// xinput block-buffers stdout when piped, so a press event can sit
|
||||
// in its buffer until later motion events flush it — by then the
|
||||
// cursor read in onOsClick lands where the mouse moved *after* the
|
||||
// click. stdbuf -oL forces line-buffering so events (and the cursor
|
||||
// read) line up with the actual click instant.
|
||||
// (Skipped on Wayland: xinput only sees XWayland events — see isWayland.)
|
||||
const argv = hasBinary('stdbuf')
|
||||
? ['stdbuf', '-oL', 'xinput', 'test-xi2', '--root']
|
||||
: ['xinput', 'test-xi2', '--root'];
|
||||
@@ -789,6 +1012,13 @@ class CaptureService {
|
||||
this.clickWatcher.stdout.on('data', (chunk) => {
|
||||
this.ingestClickWatcherChunk(chunk.toString(), 'linux');
|
||||
});
|
||||
} else if (process.platform === 'linux' && readableEvdevMouseNodes().length > 0) {
|
||||
// Wayland (or X11 without xinput): read mouse buttons from the kernel
|
||||
// input layer. This is the only global click source on Wayland, but it
|
||||
// carries no cursor position — onOsClick gets a null point, so steps are
|
||||
// captured per click without a marker. (X11 prefers the xinput branch
|
||||
// above, which does carry root coordinates for the marker.)
|
||||
this.startEvdevWatcher();
|
||||
} else if (process.platform === 'win32') {
|
||||
// Use a low-level Windows mouse hook instead of polling
|
||||
// GetAsyncKeyState. The low bit from GetAsyncKeyState can be consumed
|
||||
@@ -1173,7 +1403,9 @@ public static class SFHook {
|
||||
console.error(`[stepforge] click watcher stopped${detail ? `: ${detail}` : ''}`);
|
||||
if (!this.session) return;
|
||||
if (!this.session.intervalSec) {
|
||||
this.session.intervalSec = this.settings.get('capture.autoIntervalSec') || 5;
|
||||
this.session.intervalSec = this.fallbackCaptureTrigger() === 'hotkey'
|
||||
? 0
|
||||
: this.fallbackIntervalSec();
|
||||
this.applyInterval();
|
||||
}
|
||||
this.notify('capture:state', this.state());
|
||||
@@ -1184,12 +1416,54 @@ public static class SFHook {
|
||||
try { this.clickWatcher.kill(); } catch { /* already gone */ }
|
||||
this.clickWatcher = null;
|
||||
}
|
||||
this.stopEvdevWatcher();
|
||||
this.clickWatcherBuf = '';
|
||||
this.linuxEvent = null;
|
||||
this.discardPendingRawClick();
|
||||
this.lastAcceptedClickByButton.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Open every readable mouse device node and turn button-down events into
|
||||
* onOsClick calls. One physical mouse is normally one node; the leading-edge
|
||||
* debounce in onOsClick collapses any cross-node duplicates. Frames are still
|
||||
* served by the stream backend; this only supplies the click *trigger*.
|
||||
*/
|
||||
startEvdevWatcher() {
|
||||
const nodes = readableEvdevMouseNodes();
|
||||
this.evdevStreams = [];
|
||||
for (const node of nodes) {
|
||||
try {
|
||||
const stream = fs.createReadStream(node);
|
||||
let buf = Buffer.alloc(0);
|
||||
stream.on('data', (chunk) => {
|
||||
buf = buf.length ? Buffer.concat([buf, chunk]) : chunk;
|
||||
const { presses, rest } = decodeEvdevButtonPresses(buf);
|
||||
buf = rest;
|
||||
for (const button of presses) this.onOsClick(Date.now(), null, button);
|
||||
});
|
||||
// A device can disappear (unplugged); just drop that stream.
|
||||
stream.on('error', () => {});
|
||||
this.evdevStreams.push(stream);
|
||||
} catch {
|
||||
// Node became unreadable between enumeration and open — skip it.
|
||||
}
|
||||
}
|
||||
if (!this.evdevStreams.length) {
|
||||
console.error('[stepforge] no readable mouse input devices — add your user to the "input" group for per-click capture: sudo usermod -aG input "$USER" (then log out and back in)');
|
||||
} else {
|
||||
console.log(`[stepforge] per-click capture via evdev on ${this.evdevStreams.length} device(s)${this.onWayland() ? ' (Wayland: no click marker)' : ''}`);
|
||||
}
|
||||
}
|
||||
|
||||
stopEvdevWatcher() {
|
||||
if (!this.evdevStreams) return;
|
||||
for (const stream of this.evdevStreams) {
|
||||
try { stream.destroy(); } catch { /* already closed */ }
|
||||
}
|
||||
this.evdevStreams = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Buffer stdout chunks and only parse complete lines: a chunk boundary
|
||||
* can split an event line in half, which used to corrupt press/release
|
||||
@@ -1396,7 +1670,11 @@ public static class SFHook {
|
||||
// filtered by the cursor-position check in sessionCapture, not by
|
||||
// window focus — WSLg reports focus unreliably.)
|
||||
let clickPos = osPoint ? this.osPointToDip(osPoint) : null;
|
||||
if (!clickPos) clickPos = this.screen.getCursorScreenPoint();
|
||||
// Read the live cursor as a fallback only off Wayland: Wayland refuses to
|
||||
// report the global pointer position (getCursorScreenPoint returns 0,0), so
|
||||
// a fallback there would stamp every click marker in the top-left corner.
|
||||
// Leaving clickPos null means the step is captured with no (wrong) marker.
|
||||
if (!clickPos && !this.onWayland()) clickPos = this.screen.getCursorScreenPoint();
|
||||
clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos);
|
||||
this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y)
|
||||
? { x: osPoint.x, y: osPoint.y }
|
||||
@@ -1428,20 +1706,32 @@ public static class SFHook {
|
||||
* scaled away from 100% and on secondary monitors.
|
||||
*/
|
||||
osPointToDip(osPoint) {
|
||||
if (this.screen && typeof this.screen.screenToDipPoint === 'function') {
|
||||
try {
|
||||
const dip = this.screen.screenToDipPoint(osPoint);
|
||||
if (dip && Number.isFinite(dip.x) && Number.isFinite(dip.y)) return dip;
|
||||
} catch { /* fall through to manual conversion */ }
|
||||
}
|
||||
let geometryDip = null;
|
||||
try {
|
||||
const displays = this.screen && typeof this.screen.getAllDisplays === 'function'
|
||||
? this.screen.getAllDisplays()
|
||||
: [];
|
||||
const dip = physicalToDip(osPoint, displays);
|
||||
if (dip) return dip;
|
||||
} catch { /* no display geometry available */ }
|
||||
return osPoint;
|
||||
geometryDip = physicalToDip(osPoint, displays);
|
||||
} catch {
|
||||
geometryDip = null;
|
||||
}
|
||||
if (this.screen && typeof this.screen.screenToDipPoint === 'function') {
|
||||
try {
|
||||
const dip = this.screen.screenToDipPoint(osPoint);
|
||||
if (dip && Number.isFinite(dip.x) && Number.isFinite(dip.y)) {
|
||||
if (!geometryDip) return dip;
|
||||
const offByX = Math.abs(dip.x - geometryDip.x);
|
||||
const offByY = Math.abs(dip.y - geometryDip.y);
|
||||
// Some Windows/Electron combinations have been observed to return a
|
||||
// raw physical point here. That keeps the click marker off-screen on
|
||||
// scaled displays, so trust the geometry path when the two disagree
|
||||
// by more than a tiny rounding margin.
|
||||
if (offByX <= 1 && offByY <= 1) return dip;
|
||||
return geometryDip;
|
||||
}
|
||||
} catch { /* fall through to manual conversion */ }
|
||||
}
|
||||
return geometryDip || osPoint;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1761,3 +2051,5 @@ public static class SFHook {
|
||||
}
|
||||
|
||||
module.exports = CaptureService;
|
||||
// Exposed for unit tests (pure, no device access).
|
||||
module.exports.decodeEvdevButtonPresses = decodeEvdevButtonPresses;
|
||||
|
||||
+37
-1
@@ -5,7 +5,7 @@ const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const {
|
||||
app, BrowserWindow, ipcMain, dialog, shell, nativeTheme, globalShortcut,
|
||||
clipboard, nativeImage, screen, powerSaveBlocker,
|
||||
clipboard, nativeImage, screen, powerSaveBlocker, session, desktopCapturer,
|
||||
} = require('electron');
|
||||
|
||||
const { GuideStore } = require('../core/store');
|
||||
@@ -21,6 +21,7 @@ const { readLock } = require('../core/locks');
|
||||
const CaptureService = require('./capture');
|
||||
const { TextIntelService } = require('./text-intel');
|
||||
const { keepProcessesResponsive } = require('./win-power');
|
||||
const PACKAGE_JSON = require(path.join(__dirname, '..', 'package.json'));
|
||||
|
||||
const APP_ID = 'com.stepforge.app';
|
||||
|
||||
@@ -95,6 +96,11 @@ function createWindow() {
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
spellcheck: Boolean(settings.get('spellcheck')),
|
||||
// During a recording the window is minimized (Linux) or hidden (Windows).
|
||||
// A throttled renderer stops processing capture:added events, so the step
|
||||
// list and capture bar appear "stuck" even though steps are saved. Keep
|
||||
// the renderer live so the UI updates in real time while recording.
|
||||
backgroundThrottling: false,
|
||||
},
|
||||
});
|
||||
mainWindow.loadFile(path.join(__dirname, 'renderer', 'index.html'));
|
||||
@@ -757,6 +763,7 @@ function setupIpc() {
|
||||
h('shell:showItemInFolder', ({ target }) => shell.showItemInFolder(target));
|
||||
h('app:info', () => ({
|
||||
version: app.getVersion(),
|
||||
buildVersion: PACKAGE_JSON.buildVersion || app.getVersion(),
|
||||
dataDir: store.root,
|
||||
platform: process.platform,
|
||||
}));
|
||||
@@ -835,6 +842,35 @@ if (!gotLock) {
|
||||
textIntel,
|
||||
});
|
||||
|
||||
// Allow the hidden capture-worker renderer to open a desktop media stream.
|
||||
// Electron 29+ requires an explicit permission grant for display-capture in
|
||||
// renderer windows; without it getUserMedia/getDisplayMedia fails, the
|
||||
// stream backend never starts, and every capture falls back to
|
||||
// desktopCapturer.getSources() — which triggers the portal dialog on Linux
|
||||
// on every single capture. StepForge is fully local/offline so allowing
|
||||
// all permissions for our own content is safe.
|
||||
session.defaultSession.setPermissionCheckHandler(() => true);
|
||||
session.defaultSession.setPermissionRequestHandler((_wc, _perm, cb) => cb(true));
|
||||
|
||||
// On GNOME Wayland the only working screen-capture path is the portal-backed
|
||||
// getDisplayMedia (desktopCapturer source ids fail with "device not found").
|
||||
// The worker calls getDisplayMedia; this handler answers it. Calling
|
||||
// getSources() *inside* the handler is the documented Wayland path: it
|
||||
// drives the XDG portal picker (shown once when a recording starts), and
|
||||
// the chosen source then streams for the whole session. (useSystemPicker is
|
||||
// macOS-only today, harmless elsewhere.)
|
||||
session.defaultSession.setDisplayMediaRequestHandler((request, callback) => {
|
||||
desktopCapturer.getSources({ types: ['screen'] })
|
||||
.then((sources) => {
|
||||
console.log(`[stepforge] display-media request resolved: ${sources.length} screen source(s)`);
|
||||
callback(sources.length ? { video: sources[0] } : {});
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error(`[stepforge] display-media getSources failed: ${err && err.message}`);
|
||||
callback({});
|
||||
});
|
||||
}, { useSystemPicker: true });
|
||||
|
||||
applyTheme();
|
||||
setupIpc();
|
||||
createWindow();
|
||||
|
||||
+17
-5
@@ -122,6 +122,7 @@ class StepForgeApp {
|
||||
api.library.trashList(),
|
||||
]);
|
||||
this.state.info = info;
|
||||
document.body.classList.toggle('platform-linux', info.platform === 'linux');
|
||||
this.state.settings = settings;
|
||||
this.state.library = {
|
||||
guides: library.guides || [],
|
||||
@@ -263,15 +264,26 @@ class StepForgeApp {
|
||||
updateCaptureState(state) {
|
||||
this.captureState = state || { active: false };
|
||||
clearNode(this.captureStatus);
|
||||
// The capture bar only makes sense alongside the editor it's recording
|
||||
// into — hide it everywhere else (e.g. the library) even if a session
|
||||
// is still active in the background.
|
||||
if (!this.captureState.active || this.state.view !== 'editor') {
|
||||
// The capture bar is editor-only — hide it everywhere else (library, welcome).
|
||||
if (this.state.view !== 'editor') {
|
||||
this.captureStatus.classList.add('hidden');
|
||||
return;
|
||||
}
|
||||
this.captureStatus.classList.remove('hidden');
|
||||
const s = this.captureState;
|
||||
|
||||
// No active session: show a button to start a new one for the open guide.
|
||||
if (!s.active) {
|
||||
this.captureStatus.append(
|
||||
el('span', {}, 'Recording - stopped'),
|
||||
el('button', {
|
||||
type: 'button',
|
||||
onClick: () => this.armCaptureSession(this.editor.guideId),
|
||||
}, 'New recording'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const send = (payload) => api.capture.session(payload).then((next) => this.updateCaptureState(next));
|
||||
|
||||
// What is currently triggering captures, so the user knows what to do.
|
||||
@@ -389,7 +401,7 @@ class StepForgeApp {
|
||||
el('div', { style: { fontWeight: 650 } }, folderLabel),
|
||||
q ? el('div.muted', {}, `Search: ${q}`) : el('div.muted', {}, `${this.state.library.guides.length} guides`),
|
||||
),
|
||||
el('div.muted', {}, this.state.info ? `StepForge ${this.state.info.version}` : ''),
|
||||
el('div.muted', {}, this.state.info ? `StepForge ${this.state.info.buildVersion || this.state.info.version}` : ''),
|
||||
),
|
||||
this.domBulkBar = el('div', {}),
|
||||
this.domLibraryResults = el('div', {}),
|
||||
|
||||
@@ -68,24 +68,39 @@
|
||||
};
|
||||
streams.set(key, state);
|
||||
try {
|
||||
// The chromeMediaSource constraint set is Electron's documented bridge
|
||||
// from a desktopCapturer source id to a live media stream.
|
||||
state.media = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: {
|
||||
mandatory: {
|
||||
chromeMediaSource: 'desktop',
|
||||
chromeMediaSourceId: cmd.sourceId,
|
||||
minWidth: physWidth,
|
||||
maxWidth: physWidth,
|
||||
minHeight: physHeight,
|
||||
maxHeight: physHeight,
|
||||
// No maxFrameRate: sampling cadence is controlled by the setInterval
|
||||
// timer below, so the actual capture rate is always sampleMs-driven
|
||||
// regardless of display refresh rate or power mode.
|
||||
if (cmd.useDisplayMedia) {
|
||||
// GNOME Wayland path: desktopCapturer source ids fail with
|
||||
// getUserMedia, so go through the portal-backed getDisplayMedia. The
|
||||
// main process installs a setDisplayMediaRequestHandler that answers
|
||||
// this request; the OS portal picker chooses the screen. The stream
|
||||
// then stays open for the whole session — one prompt, not one per shot.
|
||||
state.media = await navigator.mediaDevices.getDisplayMedia({
|
||||
audio: false,
|
||||
video: true,
|
||||
});
|
||||
} else {
|
||||
// Keep the legacy desktop-capture constraint wrapper here: it binds
|
||||
// the stream to the exact desktop source chosen in the main process.
|
||||
// Without it Chromium can treat the request like a normal media
|
||||
// request and pick the default camera device instead.
|
||||
state.media = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: {
|
||||
mandatory: {
|
||||
chromeMediaSource: 'desktop',
|
||||
chromeMediaSourceId: cmd.sourceId,
|
||||
minWidth: physWidth,
|
||||
maxWidth: physWidth,
|
||||
minHeight: physHeight,
|
||||
maxHeight: physHeight,
|
||||
// No maxFrameRate: sampling cadence is controlled by the
|
||||
// setInterval timer below, so the actual capture rate is always
|
||||
// sampleMs-driven regardless of display refresh rate or power
|
||||
// mode.
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
const video = document.createElement('video');
|
||||
video.muted = true;
|
||||
video.srcObject = state.media;
|
||||
|
||||
+29
-2
@@ -312,14 +312,30 @@ function showSettingsDialog({
|
||||
const previewCount = makeInput(settings.exports?.previewStepCount ?? 3, 'number', { min: 1, step: 1 });
|
||||
const openFolder = el('input', { type: 'checkbox', checked: Boolean(settings.exports?.openFolderAfterExport) });
|
||||
const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) });
|
||||
const fallbackTrigger = makeSelect(settings.capture?.fallbackTrigger || 'interval', [
|
||||
{ value: 'interval', label: 'Timed interval' },
|
||||
{ value: 'hotkey', label: 'Hotkey only' },
|
||||
]);
|
||||
const autoIntervalSec = makeInput(settings.capture?.autoIntervalSec ?? 5, 'number', { min: 1, step: 1 });
|
||||
const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) });
|
||||
const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 });
|
||||
const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) });
|
||||
const aiAutoDoc = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.autoDoc) });
|
||||
const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434');
|
||||
const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b');
|
||||
const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.');
|
||||
const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Vision-capable models can also inspect the screenshot attached to each step.');
|
||||
const testAiBtn = el('button', { type: 'button' }, 'Test connection');
|
||||
const persistOllamaModel = debounce(() => {
|
||||
const model = ollamaModel.value.trim();
|
||||
// Keep the last model choice even if the dialog is dismissed without a full save.
|
||||
void api.settings.set({ keyPath: 'ai.ollama.model', value: model }).catch(() => {});
|
||||
}, 250);
|
||||
|
||||
const syncFallbackUi = () => {
|
||||
autoIntervalSec.disabled = fallbackTrigger.value === 'hotkey';
|
||||
};
|
||||
fallbackTrigger.addEventListener('change', syncFallbackUi);
|
||||
syncFallbackUi();
|
||||
|
||||
const updateAiStatus = (message, { error = false } = {}) => {
|
||||
aiStatus.textContent = message;
|
||||
@@ -341,7 +357,9 @@ function showSettingsDialog({
|
||||
return;
|
||||
}
|
||||
if (result.installed) {
|
||||
updateAiStatus(`Connected to ${result.host} with ${result.model}.`);
|
||||
updateAiStatus(result.vision
|
||||
? `Connected to ${result.host} with ${result.model}. It can inspect screenshots.`
|
||||
: `Connected to ${result.host} with ${result.model}. This model is text-only, so StepForge will use OCR and metadata only.`);
|
||||
} else {
|
||||
updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true });
|
||||
}
|
||||
@@ -351,6 +369,8 @@ function showSettingsDialog({
|
||||
setButtonLoading(testAiBtn, false);
|
||||
}
|
||||
};
|
||||
ollamaModel.addEventListener('input', () => persistOllamaModel());
|
||||
ollamaModel.addEventListener('blur', () => persistOllamaModel.flush());
|
||||
|
||||
const placeholderRows = el('div', { className: 'placeholder-rows' });
|
||||
const rows = [];
|
||||
@@ -394,9 +414,14 @@ function showSettingsDialog({
|
||||
labeledRow('Delay (ms)', delayMs),
|
||||
labeledRow('Click marker', clickMarker),
|
||||
labeledRow('Capture outside clicks', captureOutside),
|
||||
labeledRow('When clicks are unavailable', fallbackTrigger),
|
||||
labeledRow('Timer interval (seconds)', autoIntervalSec),
|
||||
labeledRow('Confirm simple capture', confirmSimple),
|
||||
labeledRow('Capture hotkey', captureHotkey),
|
||||
labeledRow('Pause / resume hotkey', pauseHotkey),
|
||||
el('div.muted', {},
|
||||
'Hotkey fallback uses the Capture hotkey. Timer fallback uses the interval above.',
|
||||
),
|
||||
),
|
||||
el('fieldset', {},
|
||||
el('legend', {}, 'Editor'),
|
||||
@@ -446,6 +471,8 @@ function showSettingsDialog({
|
||||
delayMs: Number(delayMs.value || 0),
|
||||
mode: captureMode.value,
|
||||
clickMarker: clickMarker.checked,
|
||||
fallbackTrigger: fallbackTrigger.value === 'hotkey' ? 'hotkey' : 'interval',
|
||||
autoIntervalSec: Math.max(1, Number(autoIntervalSec.value || 5)),
|
||||
hotkeyCapture: captureHotkey.value.trim(),
|
||||
hotkeyPauseResume: pauseHotkey.value.trim(),
|
||||
captureOutsideClicks: captureOutside.checked,
|
||||
|
||||
@@ -49,6 +49,16 @@ body {
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
body.platform-linux button {
|
||||
padding: 5px 11px;
|
||||
}
|
||||
|
||||
body.platform-linux input,
|
||||
body.platform-linux select,
|
||||
body.platform-linux textarea {
|
||||
padding: 6px 9px;
|
||||
}
|
||||
|
||||
*::selection { background: rgba(0, 104, 255, 0.2); }
|
||||
|
||||
#app { display: flex; flex-direction: column; height: 100vh; }
|
||||
@@ -183,6 +193,21 @@ kbd {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
body.platform-linux #topbar {
|
||||
height: 52px;
|
||||
gap: 10px;
|
||||
padding: 0 14px;
|
||||
}
|
||||
|
||||
body.platform-linux #capture-status {
|
||||
padding: 5px 9px;
|
||||
gap: 7px;
|
||||
}
|
||||
|
||||
body.platform-linux #capture-status button {
|
||||
padding: 2px 7px;
|
||||
}
|
||||
|
||||
.library, .editor { flex: 1; min-height: 0; display: flex; }
|
||||
|
||||
.lib-side {
|
||||
|
||||
+25
-6
@@ -83,9 +83,19 @@ class StreamCaptureBackend {
|
||||
* Spin up the worker and one stream per display that has a matching screen
|
||||
* source. Resolves true when at least one stream is delivering frames.
|
||||
*/
|
||||
async start({ displays = [], sources = [], sampleMs = DEFAULT_SAMPLE_MS, retentionMs = null, frameLimit = null } = {}) {
|
||||
async start({
|
||||
displays = [], sources = [], sampleMs = DEFAULT_SAMPLE_MS,
|
||||
retentionMs = null, frameLimit = null, useDisplayMedia = false,
|
||||
} = {}) {
|
||||
if (this.host) return this.active;
|
||||
const pairs = pairDisplaysToSources(displays, sources);
|
||||
// On GNOME Wayland, desktopCapturer source ids can't be reopened with
|
||||
// getUserMedia ("Requested device not found") — the portal-backed
|
||||
// getDisplayMedia path is the only one that works. There's no per-display
|
||||
// source id in that mode, so capture a single stream against the primary
|
||||
// display (the portal picker decides which screen is actually shared).
|
||||
const pairs = useDisplayMedia
|
||||
? (displays.length ? [{ display: displays[0], sourceId: null }] : [])
|
||||
: pairDisplaysToSources(displays, sources);
|
||||
if (!pairs.length) return false;
|
||||
try {
|
||||
this.host = await this.createHost((msg) => this.handleWorkerEvent(msg));
|
||||
@@ -99,6 +109,7 @@ class StreamCaptureBackend {
|
||||
type: 'start-stream',
|
||||
displayId: display.id,
|
||||
sourceId,
|
||||
useDisplayMedia,
|
||||
// The worker needs the physical pixel size to request a full-res
|
||||
// stream; bounds stay in DIP for marker math back in the main process.
|
||||
display: {
|
||||
@@ -154,6 +165,9 @@ class StreamCaptureBackend {
|
||||
stream.ready = msg.type === 'stream-ready';
|
||||
stream.failed = msg.type === 'stream-error';
|
||||
}
|
||||
if (msg.type === 'stream-error') {
|
||||
console.error(`[stepforge] capture worker stream-error display=${msg.displayId}: ${msg.reason}`);
|
||||
}
|
||||
for (const check of [...this.startWaiters]) check();
|
||||
return;
|
||||
}
|
||||
@@ -167,7 +181,7 @@ class StreamCaptureBackend {
|
||||
clearTimeout(pending.timer);
|
||||
pending.timer = setTimeout(() => {
|
||||
this.settleRequest(msg.requestId, null);
|
||||
this.noteFailure();
|
||||
if (pending.failable !== false) this.noteFailure();
|
||||
}, this.encodeTimeoutMs);
|
||||
return;
|
||||
}
|
||||
@@ -210,7 +224,7 @@ class StreamCaptureBackend {
|
||||
* Resolves null when no frame qualifies (caller falls back) — and also on
|
||||
* timeout, which additionally counts toward unhealthiness.
|
||||
*/
|
||||
frameForClick({ clickPos = null, clickAt = Date.now(), strict = true, leadMs = 0 } = {}) {
|
||||
frameForClick({ clickPos = null, clickAt = Date.now(), strict = true, leadMs = 0, failable = true } = {}) {
|
||||
if (!this.active || !this.host) return Promise.resolve(null);
|
||||
const displays = [...this.streams.values()].filter((s) => s.ready).map((s) => s.display);
|
||||
const display = clickPos ? displayForDipPoint(clickPos, displays) : (displays[0] || null);
|
||||
@@ -222,10 +236,15 @@ class StreamCaptureBackend {
|
||||
if (clickPos && !pointInBounds(clickPos, display.bounds)) return Promise.resolve(null);
|
||||
const requestId = this.nextRequestId++;
|
||||
return new Promise((resolve) => {
|
||||
const pending = { resolve, display, timer: null };
|
||||
// failable=false: a timeout resolves null but does NOT count toward
|
||||
// unhealthiness. Interval/timed captures use this so a single slow PNG
|
||||
// encode (common on software-rendered hosts with no GPU) can't trip the
|
||||
// 2-strikes rule and tear down an otherwise-healthy stream — the bug
|
||||
// that left Wayland recordings "stuck after two captures".
|
||||
const pending = { resolve, display, timer: null, failable };
|
||||
pending.timer = setTimeout(() => {
|
||||
this.settleRequest(requestId, null);
|
||||
this.noteFailure();
|
||||
if (failable) this.noteFailure();
|
||||
}, this.ackTimeoutMs);
|
||||
this.requests.set(requestId, pending);
|
||||
this.hostSend({
|
||||
|
||||
+74
-2
@@ -35,6 +35,15 @@ function clamp(v, min, max) {
|
||||
return Math.min(max, Math.max(min, v));
|
||||
}
|
||||
|
||||
function modelLooksVisionCapable(model) {
|
||||
const clean = normalizeWhitespace(model).toLowerCase();
|
||||
if (!clean) return false;
|
||||
return clean.includes('vision')
|
||||
|| clean.includes('llava')
|
||||
|| clean.includes('gemma4')
|
||||
|| /(^|[^a-z0-9])qwen[23](?:\.[0-9]+)?vl([^a-z0-9]|$)/.test(clean);
|
||||
}
|
||||
|
||||
let createWorkerImpl = null;
|
||||
function loadCreateWorker() {
|
||||
if (createWorkerImpl) return createWorkerImpl;
|
||||
@@ -64,6 +73,7 @@ class TextIntelService {
|
||||
this.workerPromise = null;
|
||||
this.workerQueue = Promise.resolve();
|
||||
this.ocrDataDir = path.join(dataDir, 'ocr', 'eng');
|
||||
this.modelCapabilityCache = new Map();
|
||||
}
|
||||
|
||||
async shutdown() {
|
||||
@@ -372,15 +382,63 @@ public static class Win32 {
|
||||
const data = await res.json();
|
||||
const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : [];
|
||||
const installed = config.ollama.model ? models.includes(config.ollama.model) : false;
|
||||
const vision = installed ? await this.modelSupportsVision({
|
||||
host: config.ollama.host,
|
||||
model: config.ollama.model,
|
||||
}) : false;
|
||||
return {
|
||||
ok: true,
|
||||
installed,
|
||||
vision,
|
||||
models,
|
||||
host: config.ollama.host,
|
||||
model: config.ollama.model,
|
||||
};
|
||||
}
|
||||
|
||||
async modelCapabilities({ host, model }) {
|
||||
const normalizedHost = normalizeOllamaHost(host);
|
||||
const normalizedModel = normalizeWhitespace(model);
|
||||
if (!normalizedHost || !normalizedModel) return [];
|
||||
const cacheKey = `${normalizedHost}::${normalizedModel}`;
|
||||
if (this.modelCapabilityCache.has(cacheKey)) {
|
||||
return this.modelCapabilityCache.get(cacheKey);
|
||||
}
|
||||
const url = new URL('/api/show', `${normalizedHost.replace(/\/+$/, '')}/`);
|
||||
let capabilities = [];
|
||||
try {
|
||||
const response = await this.fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model: normalizedModel }),
|
||||
});
|
||||
if (response.ok) {
|
||||
const payload = await response.json();
|
||||
capabilities = Array.isArray(payload?.capabilities)
|
||||
? payload.capabilities.map((cap) => normalizeWhitespace(cap).toLowerCase()).filter(Boolean)
|
||||
: [];
|
||||
}
|
||||
} catch {
|
||||
capabilities = [];
|
||||
}
|
||||
if (!capabilities.includes('vision') && modelLooksVisionCapable(normalizedModel)) {
|
||||
capabilities = [...capabilities, 'vision'];
|
||||
}
|
||||
this.modelCapabilityCache.set(cacheKey, capabilities);
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
async modelSupportsVision({ host, model }) {
|
||||
const capabilities = await this.modelCapabilities({ host, model });
|
||||
return capabilities.includes('vision');
|
||||
}
|
||||
|
||||
readStepImageBase64(guideId, stepId) {
|
||||
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
|
||||
if (!imagePath || !fs.existsSync(imagePath)) return '';
|
||||
return fs.readFileSync(imagePath).toString('base64');
|
||||
}
|
||||
|
||||
async callOllamaText({ host, model, prompt, systemPrompt }) {
|
||||
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
|
||||
const response = await this.fetch(url, {
|
||||
@@ -403,8 +461,12 @@ public static class Win32 {
|
||||
return content.trim();
|
||||
}
|
||||
|
||||
async callOllama({ host, model, prompt, systemPrompt }) {
|
||||
async callOllama({ host, model, prompt, systemPrompt, images = [] }) {
|
||||
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
|
||||
const userMessage = { role: 'user', content: prompt };
|
||||
if (Array.isArray(images) && images.length) {
|
||||
userMessage.images = images;
|
||||
}
|
||||
const response = await this.fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -414,7 +476,7 @@ public static class Win32 {
|
||||
format: 'json',
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: prompt },
|
||||
userMessage,
|
||||
],
|
||||
options: {
|
||||
temperature: 0.2,
|
||||
@@ -460,6 +522,14 @@ public static class Win32 {
|
||||
return { ok: false, reason: 'Block not found.' };
|
||||
}
|
||||
|
||||
const screenshotBase64 = step.image ? this.readStepImageBase64(guideId, stepId) : '';
|
||||
const screenshotAttached = Boolean(screenshotBase64)
|
||||
? await this.modelSupportsVision({
|
||||
host: config.ollama.host,
|
||||
model: config.ollama.model,
|
||||
})
|
||||
: false;
|
||||
|
||||
let captureContext = null;
|
||||
// Use stored capture metadata when available (best context, from capture time).
|
||||
// Fall back to re-running OCR on the stored image only when metadata is absent.
|
||||
@@ -514,6 +584,7 @@ public static class Win32 {
|
||||
step,
|
||||
captureContext,
|
||||
block: currentBlock,
|
||||
screenshotAttached,
|
||||
});
|
||||
|
||||
const raw = await this.callOllama({
|
||||
@@ -521,6 +592,7 @@ public static class Win32 {
|
||||
model: config.ollama.model,
|
||||
prompt,
|
||||
systemPrompt,
|
||||
images: screenshotAttached ? [screenshotBase64] : [],
|
||||
});
|
||||
const patch = normalizeAiPatch(raw);
|
||||
const updated = applyAiPatchToStep(step, patch, { target, blockId });
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
!include LogicLib.nsh
|
||||
!include nsDialogs.nsh
|
||||
|
||||
!ifndef BUILD_UNINSTALLER
|
||||
Var StepForgeDesktopShortcutCheckbox
|
||||
Var StepForgeDesktopShortcutState
|
||||
|
||||
; Assisted installer page for the desktop shortcut choice.
|
||||
!macro customInit
|
||||
StrCpy $StepForgeDesktopShortcutState "true"
|
||||
${If} ${isNoDesktopShortcut}
|
||||
StrCpy $StepForgeDesktopShortcutState "false"
|
||||
${EndIf}
|
||||
!macroend
|
||||
|
||||
!macro customHeader
|
||||
Function StepForgeDesktopShortcutPagePre
|
||||
!insertmacro MUI_PAGE_FUNCTION_CUSTOM PRE
|
||||
!insertmacro MUI_HEADER_TEXT "Desktop Icon" "Choose whether StepForge creates a desktop icon."
|
||||
|
||||
nsDialogs::Create 1018
|
||||
Pop $0
|
||||
${If} $0 == error
|
||||
Abort
|
||||
${EndIf}
|
||||
|
||||
${NSD_CreateLabel} 0u 0u 280u 24u "StepForge can create a desktop icon for quick access from the desktop."
|
||||
Pop $0
|
||||
|
||||
${NSD_CreateCheckbox} 0u 34u 280u 12u "Create a desktop icon"
|
||||
Pop $StepForgeDesktopShortcutCheckbox
|
||||
|
||||
StrCpy $StepForgeDesktopShortcutState "true"
|
||||
${If} ${isNoDesktopShortcut}
|
||||
StrCpy $StepForgeDesktopShortcutState "false"
|
||||
EnableWindow $StepForgeDesktopShortcutCheckbox 0
|
||||
${Else}
|
||||
ReadRegStr $0 SHELL_CONTEXT "${INSTALL_REGISTRY_KEY}" CreateDesktopShortcut
|
||||
${If} $0 == "false"
|
||||
StrCpy $StepForgeDesktopShortcutState "false"
|
||||
${Else}
|
||||
ReadRegStr $1 SHELL_CONTEXT "${INSTALL_REGISTRY_KEY}" ShortcutName
|
||||
${If} $1 == ""
|
||||
StrCpy $1 "${SHORTCUT_NAME}"
|
||||
${EndIf}
|
||||
|
||||
${If} ${FileExists} "$DESKTOP\$1.lnk"
|
||||
StrCpy $StepForgeDesktopShortcutState "true"
|
||||
${ElseIf} ${FileExists} "$INSTDIR\${APP_EXECUTABLE_FILENAME}"
|
||||
StrCpy $StepForgeDesktopShortcutState "false"
|
||||
${EndIf}
|
||||
${EndIf}
|
||||
${EndIf}
|
||||
|
||||
${If} $StepForgeDesktopShortcutState == "false"
|
||||
SendMessage $StepForgeDesktopShortcutCheckbox ${BM_SETCHECK} ${BST_UNCHECKED} 0
|
||||
${Else}
|
||||
SendMessage $StepForgeDesktopShortcutCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0
|
||||
${EndIf}
|
||||
|
||||
!insertmacro MUI_PAGE_FUNCTION_CUSTOM SHOW
|
||||
nsDialogs::Show
|
||||
FunctionEnd
|
||||
|
||||
Function StepForgeDesktopShortcutPageLeave
|
||||
!insertmacro MUI_PAGE_FUNCTION_CUSTOM LEAVE
|
||||
|
||||
SendMessage $StepForgeDesktopShortcutCheckbox ${BM_GETCHECK} 0 0 $StepForgeDesktopShortcutState
|
||||
${If} $StepForgeDesktopShortcutState == ${BST_UNCHECKED}
|
||||
StrCpy $StepForgeDesktopShortcutState "false"
|
||||
${Else}
|
||||
StrCpy $StepForgeDesktopShortcutState "true"
|
||||
${EndIf}
|
||||
FunctionEnd
|
||||
!macroend
|
||||
|
||||
!macro customPageAfterChangeDir
|
||||
!insertmacro MUI_PAGE_INIT
|
||||
PageEx custom
|
||||
PageCallbacks StepForgeDesktopShortcutPagePre StepForgeDesktopShortcutPageLeave
|
||||
Caption " "
|
||||
PageExEnd
|
||||
!macroend
|
||||
|
||||
!macro customInstall
|
||||
; Reconcile the desktop shortcut after the default installer logic runs.
|
||||
${If} ${isNoDesktopShortcut}
|
||||
StrCpy $StepForgeDesktopShortcutState "false"
|
||||
${EndIf}
|
||||
|
||||
WriteRegStr SHELL_CONTEXT "${INSTALL_REGISTRY_KEY}" CreateDesktopShortcut "$StepForgeDesktopShortcutState"
|
||||
|
||||
${If} $StepForgeDesktopShortcutState == "false"
|
||||
Delete "$newDesktopLink"
|
||||
Delete "$oldDesktopLink"
|
||||
${Else}
|
||||
${IfNot} ${FileExists} "$newDesktopLink"
|
||||
${If} ${FileExists} "$oldDesktopLink"
|
||||
Rename "$oldDesktopLink" "$newDesktopLink"
|
||||
${Else}
|
||||
CreateShortCut "$newDesktopLink" "$appExe" "" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
|
||||
${EndIf}
|
||||
ClearErrors
|
||||
WinShell::SetLnkAUMI "$newDesktopLink" "${APP_ID}"
|
||||
${EndIf}
|
||||
${EndIf}
|
||||
|
||||
System::Call 'shell32::SHChangeNotify(i 0x08000000, i 0, i 0, i 0)'
|
||||
!macroend
|
||||
!endif
|
||||
@@ -18,6 +18,9 @@ const DEFAULT_SETTINGS = {
|
||||
hotkeyPauseResume: 'CommandOrControl+Shift+2',
|
||||
captureOutsideClicks: true,
|
||||
confirmSimpleCapture: false,
|
||||
// Fallback trigger when click capture is unavailable: keep the old timer
|
||||
// fallback by default, but let users switch to hotkey-only recordings.
|
||||
fallbackTrigger: 'interval', // interval | hotkey
|
||||
// Leading-edge click debounce (ms): clicks of the same button closer
|
||||
// together than this collapse into one step, so accidental fast/double
|
||||
// clicks don't each become a step. Clicks spaced further apart always
|
||||
|
||||
@@ -673,6 +673,7 @@ function buildAiPrompt({
|
||||
step = null,
|
||||
captureContext = null,
|
||||
block = null,
|
||||
screenshotAttached = false,
|
||||
} = {}) {
|
||||
const hasDraftTitle = step && !isPlaceholderTitle(step.title);
|
||||
const hasDraftDesc = step && Boolean(htmlToText(step.descriptionHtml || ''));
|
||||
@@ -717,6 +718,7 @@ function buildAiPrompt({
|
||||
(!hasDraftTitle || target === 'description') && captureContext.titleCandidate
|
||||
? `Suggested title: ${captureContext.titleCandidate}` : null,
|
||||
] : []),
|
||||
screenshotAttached ? 'Screenshot: attached to this request.' : null,
|
||||
draftTitleLine,
|
||||
draftDescLine,
|
||||
].filter(Boolean);
|
||||
@@ -770,6 +772,9 @@ function buildAiPrompt({
|
||||
richContext
|
||||
? '- Use the OCR text, window title, app name, and element info to make the documentation specific.'
|
||||
: '- Context is limited. Use the app name or window title if available; generate a reasonable action title.',
|
||||
screenshotAttached
|
||||
? '- A screenshot is attached. Use it together with the OCR and metadata to resolve visual details, but do not mention the screenshot in the output.'
|
||||
: '- No screenshot is attached. Rely on OCR, the window title, app name, and element info.',
|
||||
'- Do NOT generate blocks that describe the technical capture process or mention OCR.',
|
||||
'- Do NOT invent details not supported by the capture context.',
|
||||
'- If the target is one block, only rewrite that block.',
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# Getting Started with StepForge on Linux
|
||||
|
||||
> ⚠️ **Work in progress.** Linux support is still under active development.
|
||||
> Expect rough edges — especially on Wayland (see the limitations below). X11 /
|
||||
> Xorg is the most complete path today. Please report issues.
|
||||
|
||||
StepForge was built on Windows, where the OS lets an app watch every click and
|
||||
grab the screen freely. Linux is more restrictive, and **how much works depends
|
||||
on whether you are running X11 (Xorg) or Wayland.** This guide explains the
|
||||
difference, how to get the best experience, and how to enable per-click capture.
|
||||
|
||||
## TL;DR
|
||||
|
||||
| | **X11 / "Ubuntu on Xorg"** | **Wayland (default on Ubuntu)** |
|
||||
|---|---|---|
|
||||
| Screenshot per click | ✅ Yes | ✅ Yes (needs `input` group) |
|
||||
| Red circle on the click | ✅ Yes | ❌ No (Wayland hides the cursor position) |
|
||||
| "Share your screen" prompt | Never | Once per recording session |
|
||||
| Setup needed | None | Add yourself to the `input` group |
|
||||
|
||||
**If you want the full Windows-like experience (click capture *with* the red
|
||||
marker), use an Xorg session — see [Option A](#option-a-best-experience--use-xorg).**
|
||||
|
||||
---
|
||||
|
||||
## 1. Check which session you are running
|
||||
|
||||
```bash
|
||||
echo $XDG_SESSION_TYPE
|
||||
```
|
||||
|
||||
- `x11` → you're on Xorg. Everything works, including the red click marker. No setup needed.
|
||||
- `wayland` → you're on Wayland. Read on.
|
||||
|
||||
---
|
||||
|
||||
## Option A (best experience) — use Xorg
|
||||
|
||||
On Xorg, StepForge captures a screenshot **on every click** and draws the **red
|
||||
marker** at the exact click position, exactly like Windows. No dependencies, no
|
||||
permissions, no portal dialogs.
|
||||
|
||||
To switch:
|
||||
|
||||
1. Log out.
|
||||
2. On the login (password) screen, click the **⚙ gear icon** in the bottom-right corner.
|
||||
3. Choose **"Ubuntu on Xorg"**.
|
||||
4. Log back in.
|
||||
|
||||
That's it — open StepForge and record. (To go back to Wayland later, pick
|
||||
"Ubuntu" at the gear menu again.)
|
||||
|
||||
---
|
||||
|
||||
## Option B — stay on Wayland
|
||||
|
||||
Wayland deliberately blocks apps from monitoring global input and from grabbing
|
||||
the screen silently. StepForge works around this as far as the platform allows:
|
||||
|
||||
### Screen capture
|
||||
|
||||
The first time you press **Start recording**, the system shows a **"Share your
|
||||
screen"** dialog (the XDG desktop portal). Pick your screen and click **Share**.
|
||||
This happens **once per recording session** — not per screenshot. The shared
|
||||
stream stays open until you stop recording.
|
||||
|
||||
> If you never see steps appear, make sure you actually picked a screen and
|
||||
> clicked **Share** in that dialog.
|
||||
|
||||
### Per-click capture (requires the `input` group)
|
||||
|
||||
By default on Wayland, StepForge cannot see your clicks, so it falls back to
|
||||
**capturing a screenshot every few seconds** (timed capture).
|
||||
|
||||
To get a screenshot **on every click** instead, give your user read access to
|
||||
the mouse devices by joining the `input` group:
|
||||
|
||||
```bash
|
||||
sudo usermod -aG input "$USER"
|
||||
```
|
||||
|
||||
Then **log out and log back in** (group membership only applies to new sessions).
|
||||
Verify it took effect:
|
||||
|
||||
```bash
|
||||
groups | tr ' ' '\n' | grep input # should print: input
|
||||
```
|
||||
|
||||
Now StepForge reads mouse buttons directly from the kernel (`/dev/input`) and
|
||||
captures a screenshot on each click.
|
||||
|
||||
> **No red marker on Wayland.** Even with per-click capture working, Wayland
|
||||
> does not tell apps *where* the pointer is, so StepForge cannot draw the circle
|
||||
> at the click. The screenshot is still captured per click — just without the
|
||||
> marker. If you need the marker, use [Option A (Xorg)](#option-a-best-experience--use-xorg).
|
||||
|
||||
### Adjusting the timed-capture interval
|
||||
|
||||
If you don't enable the `input` group, StepForge captures on a timer. Change the
|
||||
fallback in **Settings → Capture**:
|
||||
|
||||
- `When clicks are unavailable` -> `Hotkey only` to use the Capture hotkey
|
||||
instead of a timer.
|
||||
- `When clicks are unavailable` -> `Timed interval`, then set
|
||||
`Timer interval (seconds)` (`capture.autoIntervalSec`, default 5 seconds)
|
||||
if you want timed captures.
|
||||
|
||||
---
|
||||
|
||||
## How StepForge picks a capture method (for reference)
|
||||
|
||||
On launch StepForge chooses the best available click source:
|
||||
|
||||
1. **Windows** — low-level mouse hook (position + timing).
|
||||
2. **X11** — `xinput` (position + timing → full red marker).
|
||||
3. **Linux evdev** (`/dev/input`) — button presses on X11 *and* Wayland, no
|
||||
position on Wayland. Used when `xinput` can't see clicks (i.e. Wayland), if
|
||||
you're in the `input` group.
|
||||
4. **Timed capture** — the always-works fallback (a screenshot every N seconds)
|
||||
when no click source is available.
|
||||
|
||||
Screen frames come from a single long-lived capture stream per recording, so
|
||||
clicks/timer ticks never re-open the screen-share dialog.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"It asks to share my screen every time."**
|
||||
You're likely on an older build. Update to the current version — the screen
|
||||
stream is now opened once per recording session. If it persists, confirm
|
||||
`echo $XDG_SESSION_TYPE` and that you clicked **Share** (not Cancel) in the dialog.
|
||||
|
||||
**"Recording captures a couple of steps then stops."**
|
||||
Fixed in the current version (a slow, GPU-less PNG encode used to trip a
|
||||
failure guard and tear down the stream). Update and retry.
|
||||
|
||||
**"The window disappeared and I can't stop the recording."**
|
||||
On Linux the window **minimizes** while recording (GNOME's system tray is
|
||||
unreliable). Bring it back from the **taskbar / dock**, then click **Stop
|
||||
recording**.
|
||||
|
||||
**"No steps at all on Wayland, even after picking a screen."**
|
||||
Run from a terminal with logging and look for the diagnostic lines:
|
||||
|
||||
```bash
|
||||
STEPFORGE_CAPTURE_LOG=1 npm start
|
||||
```
|
||||
|
||||
- `[stepforge] screen-capture stream active …` — the stream is up.
|
||||
- `[stepforge] per-click capture via evdev on N device(s) …` — clicks are wired up.
|
||||
- `[stepforge] no readable mouse input devices …` — you need the `input` group (see above).
|
||||
|
||||
**Harmless console noise.** Lines like `vaInitialize failed`, `Frame latency is
|
||||
negative`, and `StatusNotifierItem … already exported` come from Chromium/GNOME,
|
||||
not StepForge, and don't affect recording.
|
||||
@@ -22,6 +22,14 @@ ollama pull llama3.2:1b
|
||||
|
||||
That model is small enough to feel responsive on modest hardware, but still good enough for human-sounding titles and short text blocks.
|
||||
|
||||
If you want StepForge to send the screenshot itself to the model, pull a vision-capable model instead:
|
||||
|
||||
```bash
|
||||
ollama pull gemma3
|
||||
```
|
||||
|
||||
That model can inspect pictures as well as text, so it is better when you want the AI to read the UI directly from the screenshot.
|
||||
|
||||
If you need something even smaller, try:
|
||||
|
||||
```bash
|
||||
@@ -44,7 +52,7 @@ Set:
|
||||
|
||||
* `Enable AI text filling` to on
|
||||
* `Ollama host` to your local Ollama server
|
||||
* `Ollama model` to `llama3.2:1b` or the smaller model you pulled
|
||||
* `Ollama model` to `llama3.2:1b` for text-only mode, or `llama3.2-vision` if you want screenshot-aware AI
|
||||
|
||||
The default host is:
|
||||
|
||||
@@ -73,3 +81,4 @@ You can also use `More -> Generate all text fields with AI` to fill the whole st
|
||||
* Capture titles are still generated automatically without AI.
|
||||
* AI generation only works when `Enable AI text filling` is turned on.
|
||||
* The app always uses local OCR around the click area first, then local AI only when you ask for it.
|
||||
* When the selected Ollama model supports vision, StepForge also sends the screenshot to the model so it can cross-check OCR and visual context.
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "stepforge",
|
||||
"version": "0.1.0",
|
||||
"version": "0.3.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "stepforge",
|
||||
"version": "0.1.0",
|
||||
"version": "0.3.2",
|
||||
"license": "MPL-2.0",
|
||||
"dependencies": {
|
||||
"@tesseract.js-data/eng": "^1.0.0",
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"name": "stepforge",
|
||||
"version": "0.1.0",
|
||||
"version": "0.3.2",
|
||||
"buildVersion": "0.3.2.1",
|
||||
"description": "Fully offline desktop tool for capturing, annotating, and exporting step-by-step guides.",
|
||||
"main": "app/main.js",
|
||||
"author": "StepForge [email protected]",
|
||||
@@ -8,7 +9,7 @@
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "node scripts/start-electron.js",
|
||||
"test": "node --test tests/unit/",
|
||||
"test": "node scripts/run-unit-tests.js",
|
||||
"sample": "node scripts/make-sample-guide.js",
|
||||
"package:windows": "node scripts/package-windows.js",
|
||||
"build": "bash scripts/build-release.sh",
|
||||
|
||||
@@ -20,5 +20,5 @@ fi
|
||||
|
||||
node - <<'NODE'
|
||||
const pkg = require('./package.json');
|
||||
console.log(`StepForge ${pkg.version} bootstrap OK`);
|
||||
console.log(`StepForge ${pkg.buildVersion || pkg.version} bootstrap OK`);
|
||||
NODE
|
||||
|
||||
@@ -70,6 +70,7 @@ for (const rel of walk(examplesRoot, examplesRoot)) {
|
||||
}
|
||||
|
||||
const pkg = require(path.join(rootDir, 'package.json'));
|
||||
const buildVersion = pkg.buildVersion || pkg.version;
|
||||
|
||||
const { execSync } = require('node:child_process');
|
||||
function toolAvailable(cmd) {
|
||||
@@ -88,7 +89,8 @@ const toolRows = Object.entries(tools)
|
||||
|
||||
const report = `# StepForge Build Report
|
||||
|
||||
Version: ${pkg.version}
|
||||
Build version: ${buildVersion}
|
||||
Package version: ${pkg.version}
|
||||
Generated: ${new Date().toISOString()}
|
||||
Host: ${process.platform} ${process.arch} (node ${process.version})
|
||||
|
||||
@@ -133,6 +135,7 @@ fs.writeFileSync(manifestFile, JSON.stringify({
|
||||
version: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
packageVersion: pkg.version,
|
||||
buildVersion,
|
||||
files,
|
||||
}, null, 2) + '\n');
|
||||
NODE
|
||||
|
||||
@@ -60,6 +60,26 @@ function sanitizeElectronEnv(baseEnv = process.env) {
|
||||
return env;
|
||||
}
|
||||
|
||||
function linuxSandboxLaunchArgs({
|
||||
electronPath,
|
||||
platform = process.platform,
|
||||
statSync = fs.statSync,
|
||||
} = {}) {
|
||||
if (platform !== 'linux') return [];
|
||||
if (!electronPath) return ['--no-sandbox'];
|
||||
|
||||
const helperPath = path.join(path.dirname(electronPath), 'chrome-sandbox');
|
||||
try {
|
||||
const stat = statSync(helperPath);
|
||||
const ownedByRoot = stat.uid === 0;
|
||||
const hasSetuid = Boolean(stat.mode & 0o4000);
|
||||
if (ownedByRoot && hasSetuid) return [];
|
||||
} catch {
|
||||
// Missing or unreadable helper: fall back to the unsandboxed launcher.
|
||||
}
|
||||
return ['--no-sandbox'];
|
||||
}
|
||||
|
||||
function electronBinaryCandidates({ packageRoot, distDir, platform }) {
|
||||
const candidatePaths = [];
|
||||
const pathHint = packageRoot ? readElectronPathHint(packageRoot) : null;
|
||||
@@ -292,6 +312,7 @@ module.exports = {
|
||||
runNpmRebuild,
|
||||
runNpmInstall,
|
||||
sanitizeElectronEnv,
|
||||
linuxSandboxLaunchArgs,
|
||||
resolveElectronBinary,
|
||||
resolveElectronPackageRoot,
|
||||
platformBinaryCandidates,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
VERSION="$(node -p "require('${ROOT_DIR}/package.json').version" 2>/dev/null || echo 0.0.0)"
|
||||
VERSION="$(node -p "const pkg=require('${ROOT_DIR}/package.json'); pkg.buildVersion || pkg.version" 2>/dev/null || echo 0.0.0)"
|
||||
OUT_DIR="${STEPFORGE_PACKAGE_DIR:-$ROOT_DIR/build/artifacts}"
|
||||
mkdir -p "$OUT_DIR"
|
||||
WORK_DIR="$(mktemp -d "${OUT_DIR%/}/.pkg.XXXXXX")"
|
||||
@@ -46,8 +46,21 @@ fi
|
||||
cat > "$WORK_DIR/usr/bin/stepforge" <<'EOF'
|
||||
#!/usr/bin/env sh
|
||||
APP_DIR=/opt/stepforge
|
||||
ELECTRON="$APP_DIR/node_modules/.bin/electron"
|
||||
SANDBOX_HELPER="$APP_DIR/node_modules/electron/dist/chrome-sandbox"
|
||||
cd "$APP_DIR" || exit 1
|
||||
exec "$APP_DIR/node_modules/.bin/electron" "$APP_DIR" "$@"
|
||||
if command -v stat >/dev/null 2>&1 && [ -e "$SANDBOX_HELPER" ]; then
|
||||
helper_uid="$(stat -c '%u' "$SANDBOX_HELPER" 2>/dev/null || echo '')"
|
||||
helper_mode="$(stat -c '%a' "$SANDBOX_HELPER" 2>/dev/null || echo '')"
|
||||
if [ "$helper_uid" = "0" ] && [ -n "$helper_mode" ]; then
|
||||
helper_mode_num=$((8#$helper_mode))
|
||||
if [ $((helper_mode_num & 04000)) -ne 0 ]; then
|
||||
exec "$ELECTRON" "$APP_DIR" "$@"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' '[stepforge] Electron sandbox helper is not configured for this install; starting with --no-sandbox' >&2
|
||||
exec "$ELECTRON" --no-sandbox "$APP_DIR" "$@"
|
||||
EOF
|
||||
chmod 0755 "$WORK_DIR/usr/bin/stepforge"
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ const { build, Platform } = require('electron-builder');
|
||||
const ROOT_DIR = path.resolve(__dirname, '..');
|
||||
const PACKAGE_JSON = require(path.join(ROOT_DIR, 'package.json'));
|
||||
const APP_ID = 'com.stepforge.app';
|
||||
const BUILD_VERSION = PACKAGE_JSON.buildVersion || PACKAGE_JSON.version;
|
||||
|
||||
function findInstallerExe(dir) {
|
||||
if (!fs.existsSync(dir)) return null;
|
||||
@@ -32,6 +33,7 @@ function createWindowsInstallerConfig(outputDir) {
|
||||
return {
|
||||
appId: APP_ID,
|
||||
productName: 'StepForge',
|
||||
buildVersion: BUILD_VERSION,
|
||||
directories: {
|
||||
output: outputDir,
|
||||
},
|
||||
@@ -52,6 +54,8 @@ function createWindowsInstallerConfig(outputDir) {
|
||||
createDesktopShortcut: true,
|
||||
createStartMenuShortcut: true,
|
||||
shortcutName: 'StepForge',
|
||||
artifactName: '${productName} Setup ${buildVersion}.${ext}',
|
||||
include: 'build/installer.nsh',
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -86,7 +90,7 @@ async function buildWindowsInstaller() {
|
||||
const releaseInstaller = path.join(releaseDir, path.basename(builtInstaller));
|
||||
fs.copyFileSync(builtInstaller, releaseInstaller);
|
||||
|
||||
console.log(`StepForge ${PACKAGE_JSON.version} Windows installer written to ${releaseInstaller}`);
|
||||
console.log(`StepForge ${BUILD_VERSION} Windows installer written to ${releaseInstaller}`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
|
||||
function collectTestFiles(rootDir) {
|
||||
const files = [];
|
||||
if (!fs.existsSync(rootDir)) return files;
|
||||
|
||||
const walk = (dir) => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(full);
|
||||
continue;
|
||||
}
|
||||
if (/\.(test|spec)\.(js|mjs|cjs)$/.test(entry.name)) files.push(full);
|
||||
}
|
||||
};
|
||||
|
||||
walk(rootDir);
|
||||
files.sort((a, b) => a.localeCompare(b));
|
||||
return files;
|
||||
}
|
||||
|
||||
const root = path.join(process.cwd(), 'tests', 'unit');
|
||||
const tests = collectTestFiles(root);
|
||||
|
||||
if (!tests.length) {
|
||||
console.log('No unit test files found under tests/unit, skipping unit tests.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const result = spawnSync(process.execPath, ['--test', ...tests], { stdio: 'inherit' });
|
||||
|
||||
if (result.error) {
|
||||
console.error(result.error.message);
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
|
||||
process.exit(result.status ?? 0);
|
||||
@@ -0,0 +1,52 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
function readJson(file) {
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
}
|
||||
|
||||
function writeJson(file, value) {
|
||||
fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n');
|
||||
}
|
||||
|
||||
function stampVersion(rootDir, version) {
|
||||
if (!version || typeof version !== 'string') {
|
||||
throw new Error('version is required');
|
||||
}
|
||||
|
||||
const normalized = version.replace(/^v/i, '');
|
||||
const parts = normalized.split('.');
|
||||
const isFourPartBuild = parts.length === 4 && parts.every((part) => /^\d+$/.test(part));
|
||||
const packageVersion = isFourPartBuild ? parts.slice(0, 3).join('.') : normalized;
|
||||
const buildVersion = normalized;
|
||||
|
||||
const pkgPath = path.join(rootDir, 'package.json');
|
||||
const pkg = readJson(pkgPath);
|
||||
pkg.version = packageVersion;
|
||||
pkg.buildVersion = buildVersion;
|
||||
writeJson(pkgPath, pkg);
|
||||
|
||||
const lockPath = path.join(rootDir, 'package-lock.json');
|
||||
if (!fs.existsSync(lockPath)) return;
|
||||
|
||||
const lock = readJson(lockPath);
|
||||
lock.version = packageVersion;
|
||||
if (lock.packages && lock.packages['']) {
|
||||
lock.packages[''].version = packageVersion;
|
||||
}
|
||||
writeJson(lockPath, lock);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
const version = process.argv[2] || process.env.VERSION;
|
||||
stampVersion(process.cwd(), version);
|
||||
} catch (err) {
|
||||
console.error(err && err.message ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { stampVersion };
|
||||
@@ -3,7 +3,11 @@
|
||||
|
||||
const { spawn } = require('node:child_process');
|
||||
|
||||
const { resolveElectronBinary, sanitizeElectronEnv } = require('./electron-launcher');
|
||||
const {
|
||||
linuxSandboxLaunchArgs,
|
||||
resolveElectronBinary,
|
||||
sanitizeElectronEnv,
|
||||
} = require('./electron-launcher');
|
||||
|
||||
let electronPath;
|
||||
try {
|
||||
@@ -13,8 +17,19 @@ try {
|
||||
process.exit(1);
|
||||
}
|
||||
const env = sanitizeElectronEnv();
|
||||
const sandboxArgs = linuxSandboxLaunchArgs({ electronPath });
|
||||
if (sandboxArgs.includes('--no-sandbox')) {
|
||||
console.warn('[stepforge] Electron sandbox helper is not configured for this install; starting with --no-sandbox');
|
||||
}
|
||||
|
||||
const child = spawn(electronPath, ['.'], {
|
||||
// On Linux, prefer the native Ozone path when available and enable PipeWire-
|
||||
// based screen capture so desktopCapturer can go through the XDG Desktop
|
||||
// Portal on Wayland without affecting X11.
|
||||
const extraArgs = process.platform === 'linux'
|
||||
? ['--enable-features=WebRTCPipeWireCapturer', '--ozone-platform-hint=auto', ...sandboxArgs]
|
||||
: [];
|
||||
|
||||
const child = spawn(electronPath, [...extraArgs, '.'], {
|
||||
stdio: 'inherit',
|
||||
env,
|
||||
windowsHide: false,
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
|
||||
test('capture worker requests the selected desktop source, not a plain camera stream', async () => {
|
||||
const scriptPath = path.join(__dirname, '../../app/renderer/capture-worker.js');
|
||||
const script = fs.readFileSync(scriptPath, 'utf8');
|
||||
|
||||
const mediaCalls = [];
|
||||
const messages = [];
|
||||
let onCommand = null;
|
||||
let resolveStreamReady;
|
||||
const streamReady = new Promise((resolve) => {
|
||||
resolveStreamReady = resolve;
|
||||
});
|
||||
|
||||
const context = {
|
||||
console: {
|
||||
error() {},
|
||||
log() {},
|
||||
warn() {},
|
||||
},
|
||||
captureWorkerBridge: {
|
||||
onCommand(fn) {
|
||||
onCommand = fn;
|
||||
},
|
||||
send(msg) {
|
||||
messages.push(msg);
|
||||
if (msg.type === 'stream-ready') resolveStreamReady();
|
||||
},
|
||||
},
|
||||
StepForgeClickFrames: {
|
||||
FrameRing: class {
|
||||
constructor() {
|
||||
this._frames = [];
|
||||
}
|
||||
|
||||
push(frame) {
|
||||
this._frames.push(frame);
|
||||
}
|
||||
|
||||
frames() {
|
||||
return [...this._frames];
|
||||
}
|
||||
|
||||
latest() {
|
||||
return this._frames[this._frames.length - 1] || null;
|
||||
}
|
||||
|
||||
clear() {
|
||||
this._frames = [];
|
||||
}
|
||||
},
|
||||
selectFrameForClick() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
navigator: {
|
||||
mediaDevices: {
|
||||
async getUserMedia(constraints) {
|
||||
mediaCalls.push(constraints);
|
||||
return {
|
||||
getTracks() {
|
||||
return [{ stop() {} }];
|
||||
},
|
||||
};
|
||||
},
|
||||
async getDisplayMedia() {
|
||||
throw new Error('unexpected getDisplayMedia call');
|
||||
},
|
||||
},
|
||||
},
|
||||
document: {
|
||||
createElement(tag) {
|
||||
assert.equal(tag, 'video');
|
||||
return {
|
||||
muted: false,
|
||||
srcObject: null,
|
||||
readyState: 2,
|
||||
play: async () => {},
|
||||
videoWidth: 1920,
|
||||
videoHeight: 1080,
|
||||
};
|
||||
},
|
||||
},
|
||||
createImageBitmap: async () => ({ width: 1920, height: 1080, close() {} }),
|
||||
setInterval: () => 1,
|
||||
clearInterval: () => {},
|
||||
OffscreenCanvas: class {
|
||||
constructor(width, height) {
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
}
|
||||
|
||||
getContext() {
|
||||
return { drawImage() {} };
|
||||
}
|
||||
|
||||
async convertToBlob() {
|
||||
return {
|
||||
async arrayBuffer() {
|
||||
return new ArrayBuffer(0);
|
||||
},
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
vm.createContext(context);
|
||||
vm.runInContext(script, context, { filename: scriptPath });
|
||||
|
||||
assert.equal(typeof onCommand, 'function', 'worker should register a command handler');
|
||||
|
||||
onCommand({
|
||||
type: 'start-stream',
|
||||
displayId: 7,
|
||||
sourceId: 'screen:1:0',
|
||||
display: {
|
||||
bounds: { width: 1920, height: 1080 },
|
||||
scaleFactor: 1,
|
||||
},
|
||||
sampleMs: 50,
|
||||
});
|
||||
await streamReady;
|
||||
|
||||
assert.equal(mediaCalls.length, 1);
|
||||
const constraints = JSON.parse(JSON.stringify(mediaCalls[0]));
|
||||
assert.deepEqual(constraints, {
|
||||
audio: false,
|
||||
video: {
|
||||
mandatory: {
|
||||
chromeMediaSource: 'desktop',
|
||||
chromeMediaSourceId: 'screen:1:0',
|
||||
minWidth: 1920,
|
||||
maxWidth: 1920,
|
||||
minHeight: 1080,
|
||||
maxHeight: 1080,
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.ok(messages.some((msg) => msg.type === 'stream-ready'));
|
||||
});
|
||||
+173
-5
@@ -33,6 +33,22 @@ function makeService({ settings: settingsOverrides, screenApi } = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
test('XDG_SESSION_TYPE=x11 wins over a stray WAYLAND_DISPLAY value', () => {
|
||||
const prevSessionType = process.env.XDG_SESSION_TYPE;
|
||||
const prevWaylandDisplay = process.env.WAYLAND_DISPLAY;
|
||||
try {
|
||||
process.env.XDG_SESSION_TYPE = 'x11';
|
||||
process.env.WAYLAND_DISPLAY = 'wayland-0';
|
||||
const service = makeService();
|
||||
assert.equal(service.onWayland(), false);
|
||||
} finally {
|
||||
if (prevSessionType === undefined) delete process.env.XDG_SESSION_TYPE;
|
||||
else process.env.XDG_SESSION_TYPE = prevSessionType;
|
||||
if (prevWaylandDisplay === undefined) delete process.env.WAYLAND_DISPLAY;
|
||||
else process.env.WAYLAND_DISPLAY = prevWaylandDisplay;
|
||||
}
|
||||
});
|
||||
|
||||
// The raw/regular twin window plus margin: how long a test must wait for a
|
||||
// held Linux raw press to fire when no coordinate twin arrives.
|
||||
const TWIN_FLUSH_MS = 60;
|
||||
@@ -596,6 +612,7 @@ test('armRecording warms while visible, then hides and arms the session', async
|
||||
isVisible() { return this.visible; },
|
||||
isMinimized() { return false; },
|
||||
hide() { this.visible = false; },
|
||||
minimize() { this.visible = false; },
|
||||
show() { this.visible = true; },
|
||||
focus() {}, getTitle() { return 'StepForge'; },
|
||||
getBounds() { return { x: 0, y: 0, width: 800, height: 600 }; },
|
||||
@@ -637,6 +654,7 @@ test('a slow recorder start still arms within the warmup cap', async () => {
|
||||
isVisible() { return this.visible; },
|
||||
isMinimized() { return false; },
|
||||
hide() { this.visible = false; },
|
||||
minimize() { this.visible = false; },
|
||||
show() { this.visible = true; },
|
||||
focus() {}, getTitle() { return 'StepForge'; },
|
||||
getBounds() { return { x: 0, y: 0, width: 800, height: 600 }; },
|
||||
@@ -718,6 +736,27 @@ test('hook coordinates are converted physical → DIP via screenToDipPoint when
|
||||
assert.deepEqual(seen, [{ x: 640, y: 320 }]);
|
||||
});
|
||||
|
||||
test('bogus Windows screenToDipPoint results fall back to display geometry', () => {
|
||||
const service = makeService({
|
||||
screenApi: {
|
||||
screenToDipPoint: (p) => ({ x: p.x, y: p.y }), // raw physical point: wrong on scaled displays
|
||||
getAllDisplays: () => [
|
||||
{ id: 1, scaleFactor: 2, bounds: { x: 0, y: 0, width: 1440, height: 900 } },
|
||||
],
|
||||
getCursorScreenPoint: () => { throw new Error('must not fall back to a cursor read'); },
|
||||
},
|
||||
});
|
||||
service.session = { guideId: 'guide-dip-fallback', paused: false, count: 0, intervalSec: 0 };
|
||||
const seen = [];
|
||||
service.enqueueClickCapture = (clickPos) => {
|
||||
seen.push(clickPos);
|
||||
};
|
||||
|
||||
service.onOsClick(1770000000000, { x: 1500, y: 900 }, 'left');
|
||||
|
||||
assert.deepEqual(seen, [{ x: 750, y: 450 }]);
|
||||
});
|
||||
|
||||
test('without screenToDipPoint, coordinates convert via display geometry (Linux/X11)', () => {
|
||||
const service = makeService({
|
||||
screenApi: {
|
||||
@@ -746,6 +785,7 @@ test('clicks without event coordinates fall back to a live cursor read', () => {
|
||||
getAllDisplays: () => [],
|
||||
},
|
||||
});
|
||||
service._wayland = false; // X11/Windows: the live-cursor fallback applies
|
||||
service.session = { guideId: 'guide-cursor', paused: false, count: 0, intervalSec: 0 };
|
||||
const seen = [];
|
||||
service.enqueueClickCapture = (clickPos) => {
|
||||
@@ -757,6 +797,64 @@ test('clicks without event coordinates fall back to a live cursor read', () => {
|
||||
assert.deepEqual(seen, [{ x: 11, y: 22 }]);
|
||||
});
|
||||
|
||||
test('on Wayland an evdev click does not stamp a bogus top-left cursor position', () => {
|
||||
// Wayland's getCursorScreenPoint returns {0,0}; a fallback read would mark
|
||||
// every click in the corner. The click must capture with a null position
|
||||
// (no marker) instead.
|
||||
const service = makeService({
|
||||
screenApi: {
|
||||
getCursorScreenPoint: () => ({ x: 0, y: 0 }),
|
||||
getAllDisplays: () => [],
|
||||
},
|
||||
});
|
||||
service._wayland = true;
|
||||
service.session = { guideId: 'guide-wl-click', paused: false, count: 0, intervalSec: 0 };
|
||||
const seen = [];
|
||||
service.enqueueClickCapture = (clickPos) => { seen.push(clickPos); };
|
||||
|
||||
service.onOsClick(1770000000000, null, 'button-1');
|
||||
|
||||
assert.deepEqual(seen, [null], 'no position is passed, so no marker is drawn');
|
||||
});
|
||||
|
||||
// ---- evdev decoding ---------------------------------------------------------------
|
||||
|
||||
const { decodeEvdevButtonPresses } = CaptureService;
|
||||
|
||||
// Pack a 64-bit struct input_event (24 bytes): 16-byte timeval, u16 type,
|
||||
// u16 code, s32 value.
|
||||
function evdevRecord(type, code, value) {
|
||||
const b = Buffer.alloc(24);
|
||||
b.writeUInt16LE(type, 16);
|
||||
b.writeUInt16LE(code, 18);
|
||||
b.writeInt32LE(value, 20);
|
||||
return b;
|
||||
}
|
||||
|
||||
test('evdev decoder extracts left/right/middle button presses, ignores the rest', () => {
|
||||
const EV_KEY = 0x01;
|
||||
const EV_REL = 0x02;
|
||||
const buf = Buffer.concat([
|
||||
evdevRecord(EV_KEY, 272, 1), // BTN_LEFT down -> button-1
|
||||
evdevRecord(EV_KEY, 272, 0), // BTN_LEFT up -> ignored (release)
|
||||
evdevRecord(EV_REL, 0, 5), // motion -> ignored (not a key)
|
||||
evdevRecord(EV_KEY, 273, 1), // BTN_RIGHT down -> button-3
|
||||
evdevRecord(EV_KEY, 274, 1), // BTN_MIDDLE down-> button-2
|
||||
evdevRecord(EV_KEY, 0x110 + 8, 1), // a non-mouse key -> ignored
|
||||
]);
|
||||
const { presses, rest } = decodeEvdevButtonPresses(buf, 24);
|
||||
assert.deepEqual(presses, ['button-1', 'button-3', 'button-2']);
|
||||
assert.equal(rest.length, 0);
|
||||
});
|
||||
|
||||
test('evdev decoder keeps a trailing partial record for the next chunk', () => {
|
||||
const full = evdevRecord(0x01, 272, 1);
|
||||
const buf = Buffer.concat([full, full.subarray(0, 10)]); // 1.4 records
|
||||
const { presses, rest } = decodeEvdevButtonPresses(buf, 24);
|
||||
assert.deepEqual(presses, ['button-1']);
|
||||
assert.equal(rest.length, 10, 'the half record is returned to be completed later');
|
||||
});
|
||||
|
||||
// ---- watcher loss -----------------------------------------------------------------
|
||||
|
||||
test('losing the click watcher mid-session falls back to interval capture', () => {
|
||||
@@ -779,6 +877,49 @@ test('losing the click watcher mid-session falls back to interval capture', () =
|
||||
}
|
||||
});
|
||||
|
||||
test('losing the click watcher mid-session can fall back to hotkey-only', () => {
|
||||
const service = makeService();
|
||||
service.settings.get = (key) => {
|
||||
if (key === 'capture.fallbackTrigger') return 'hotkey';
|
||||
if (key === 'capture.autoIntervalSec') return 3;
|
||||
return null;
|
||||
};
|
||||
service.session = { guideId: 'guide-loss-hotkey', paused: false, count: 0, intervalSec: 0 };
|
||||
const states = [];
|
||||
service.notify = (channel, payload) => {
|
||||
states.push({ channel, payload });
|
||||
};
|
||||
|
||||
try {
|
||||
service.handleClickWatcherLoss('exited with code 1');
|
||||
|
||||
assert.equal(service.session.intervalSec, 0,
|
||||
'hotkey fallback must not silently start the 5-second timer');
|
||||
assert.equal(service.intervalTimer, null);
|
||||
assert.ok(states.some((s) => s.channel === 'capture:state'));
|
||||
} finally {
|
||||
service.finishSession();
|
||||
}
|
||||
});
|
||||
|
||||
test('starting a session with hotkey fallback keeps the timer off when clicks are unavailable', () => {
|
||||
const service = makeService();
|
||||
service.clickCaptureAvailable = () => false;
|
||||
service.settings.get = (key) => {
|
||||
if (key === 'capture.fallbackTrigger') return 'hotkey';
|
||||
if (key === 'capture.autoIntervalSec') return 7;
|
||||
if (key === 'capture.captureOutsideClicks') return false;
|
||||
return null;
|
||||
};
|
||||
|
||||
service.startSession('guide-hotkey-fallback');
|
||||
|
||||
assert.equal(service.session.intervalSec, 0);
|
||||
assert.equal(service.state().clickCaptureAvailable, false);
|
||||
assert.equal(service.state().clickCapture, false);
|
||||
service.finishSession();
|
||||
});
|
||||
|
||||
// ---- strict frame selection -----------------------------------------------------
|
||||
|
||||
test('a click is served instantly from the freshly buffered frame', async () => {
|
||||
@@ -1004,6 +1145,7 @@ test('clicks during an in-flight pre-click grab wait for the frame instead of be
|
||||
|
||||
test('click frames come from the stream backend when it is active', async () => {
|
||||
const service = makeService();
|
||||
service._wayland = false; // X11/Windows: click frame-requests are failable
|
||||
const clickAt = Date.now();
|
||||
service.session = { guideId: 'guide-stream', paused: false, count: 0, intervalSec: 0 };
|
||||
const requests = [];
|
||||
@@ -1036,8 +1178,8 @@ test('click frames come from the stream backend when it is active', async () =>
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(added, ['stream-frame']);
|
||||
assert.deepEqual(requests, [{ clickPos: { x: 10, y: 10 }, clickAt, strict: true, leadMs: 0 }],
|
||||
'the worker receives the hook-time click timestamp, strictness, and lead');
|
||||
assert.deepEqual(requests, [{ clickPos: { x: 10, y: 10 }, clickAt, strict: true, leadMs: 0, failable: true }],
|
||||
'the worker receives the hook-time click timestamp, strictness, lead, and failability');
|
||||
});
|
||||
|
||||
test('a stream backend with no qualifying frame falls through to the fresh-shot path', async () => {
|
||||
@@ -1080,6 +1222,9 @@ test('an unhealthy stream backend degrades to the in-process frame loop', () =>
|
||||
const service = makeService();
|
||||
service.session = { guideId: 'guide-degrade', paused: false, count: 0, intervalSec: 0 };
|
||||
service.streamBackend = { isActive: () => true, stop: () => {} };
|
||||
// Off Wayland the in-process loop is viable (getSources works), so an
|
||||
// unhealthy worker must degrade to the loop rather than silently stopping.
|
||||
service._wayland = false;
|
||||
let loopStarted = false;
|
||||
service.startFrameLoop = () => { loopStarted = true; };
|
||||
const states = [];
|
||||
@@ -1092,6 +1237,24 @@ test('an unhealthy stream backend degrades to the in-process frame loop', () =>
|
||||
assert.ok(states.includes('capture:state'));
|
||||
});
|
||||
|
||||
test('an unhealthy stream backend does NOT start the frame loop on Wayland', () => {
|
||||
// On Wayland the 200ms getSources() frame loop is broken (throws) and pops
|
||||
// the portal, so it's not a usable fallback — the portal stream is the only
|
||||
// capture path and the user must re-share if it stops.
|
||||
const service = makeService();
|
||||
service.session = { guideId: 'guide-degrade-wl', paused: false, count: 0, intervalSec: 0 };
|
||||
service.streamBackend = { isActive: () => true, stop: () => {} };
|
||||
service._wayland = true;
|
||||
let loopStarted = false;
|
||||
service.startFrameLoop = () => { loopStarted = true; };
|
||||
service.notify = () => {};
|
||||
|
||||
service.degradeToFrameLoop();
|
||||
|
||||
assert.equal(service.streamBackend, null);
|
||||
assert.equal(loopStarted, false, 'no frame loop on Wayland — getSources is unusable there');
|
||||
});
|
||||
|
||||
test('session state reports which frame recorder is serving clicks', () => {
|
||||
const service = makeService();
|
||||
service.session = { guideId: 'guide-state', paused: false, count: 0, intervalSec: 0 };
|
||||
@@ -1156,7 +1319,12 @@ test('a new session starts paused and does not hide the window until "Start reco
|
||||
isDestroyed() { return this.destroyed; },
|
||||
isVisible() { return this.visible; },
|
||||
isMinimized() { return this.minimized; },
|
||||
// The window is "tucked away" for recording either by hide() (with a tray)
|
||||
// or minimize() (Linux without a tray). Both make it not visible; assert on
|
||||
// visibility so the test is independent of which mechanism the platform picks.
|
||||
hide() { this.visible = false; this.hidden += 1; },
|
||||
minimize() { this.visible = false; this.minimized = true; },
|
||||
restore() { this.minimized = false; this.visible = true; },
|
||||
show() { this.visible = true; this.shown += 1; },
|
||||
showInactive() { this.visible = true; this.shown += 1; },
|
||||
focus() {},
|
||||
@@ -1171,15 +1339,15 @@ test('a new session starts paused and does not hide the window until "Start reco
|
||||
|
||||
assert.equal(service.session.paused, true, 'sessions start paused');
|
||||
assert.equal(service.state().paused, true);
|
||||
assert.equal(win.hidden, 0, 'window must stay visible until recording starts');
|
||||
assert.equal(win.visible, true, 'window must stay visible until recording starts');
|
||||
|
||||
// User clicks "Start recording" (the resume action).
|
||||
service.togglePause(false);
|
||||
assert.equal(service.session.paused, false);
|
||||
assert.equal(win.hidden, 0, 'hide is deferred until the resume path runs');
|
||||
assert.equal(win.visible, true, 'tuck-away is deferred until the resume path runs');
|
||||
|
||||
await new Promise((r) => setTimeout(r, 25));
|
||||
assert.equal(win.hidden, 1, 'window hides once recording actually starts');
|
||||
assert.equal(win.visible, false, 'window is tucked away once recording actually starts');
|
||||
} finally {
|
||||
service.finishSession();
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ const path = require('node:path');
|
||||
|
||||
const {
|
||||
buildMissingElectronError,
|
||||
linuxSandboxLaunchArgs,
|
||||
repairElectronInstall,
|
||||
resolveElectronBinary,
|
||||
} = require('../../scripts/electron-launcher');
|
||||
@@ -171,3 +172,21 @@ test('reports a helpful error when the runtime is missing', (t) => {
|
||||
assert.match(message, /Electron could not be started/);
|
||||
assert.match(message, /Expected the binary in:/);
|
||||
});
|
||||
|
||||
test('uses --no-sandbox when the Linux sandbox helper is not root-owned and setuid', () => {
|
||||
const args = linuxSandboxLaunchArgs({
|
||||
electronPath: '/tmp/stepforge/node_modules/electron/dist/electron',
|
||||
platform: 'linux',
|
||||
statSync: () => ({ uid: 1000, mode: 0o100755 }),
|
||||
});
|
||||
assert.deepEqual(args, ['--no-sandbox']);
|
||||
});
|
||||
|
||||
test('keeps the sandbox enabled when the Linux helper is root-owned and setuid', () => {
|
||||
const args = linuxSandboxLaunchArgs({
|
||||
electronPath: '/tmp/stepforge/node_modules/electron/dist/electron',
|
||||
platform: 'linux',
|
||||
statSync: () => ({ uid: 0, mode: 0o104755 }),
|
||||
});
|
||||
assert.deepEqual(args, []);
|
||||
});
|
||||
|
||||
@@ -21,6 +21,9 @@ test('Windows packaging uses an assisted NSIS installer', (t) => {
|
||||
assert.equal(config.nsis.createDesktopShortcut, true);
|
||||
assert.equal(config.nsis.createStartMenuShortcut, true);
|
||||
assert.equal(config.nsis.shortcutName, 'StepForge');
|
||||
assert.equal(config.buildVersion, '0.3.2.1');
|
||||
assert.equal(config.nsis.artifactName, '${productName} Setup ${buildVersion}.${ext}');
|
||||
assert.equal(config.nsis.include, 'build/installer.nsh');
|
||||
assert.equal(config.asar, true);
|
||||
assert.ok(config.files.includes('app/**/*'));
|
||||
assert.ok(config.files.includes('core/**/*'));
|
||||
@@ -34,7 +37,7 @@ test('Windows packaging uses an assisted NSIS installer', (t) => {
|
||||
const tmp = makeTmpDir('windows-installer');
|
||||
t.after(() => rmrf(tmp));
|
||||
fs.mkdirSync(path.join(tmp, 'nested', 'deeper'), { recursive: true });
|
||||
const installer = path.join(tmp, 'nested', 'deeper', 'StepForge Setup 0.2.0.exe');
|
||||
const installer = path.join(tmp, 'nested', 'deeper', 'StepForge Setup 0.3.2.1.exe');
|
||||
fs.writeFileSync(installer, Buffer.from('fake installer'));
|
||||
assert.equal(findInstallerExe(tmp), installer);
|
||||
});
|
||||
|
||||
@@ -56,13 +56,16 @@ test('settings persist, deep-merge with defaults, and store global placeholders'
|
||||
assert.equal(s1.get('appearance'), DEFAULT_SETTINGS.appearance);
|
||||
s1.set('appearance', 'dark');
|
||||
s1.set('capture.delayMs', 1500);
|
||||
s1.set('ai.ollama.model', 'qwen3:0.6b');
|
||||
s1.setGlobalPlaceholders({ Company: 'Acme', Author: 'Tyler' });
|
||||
|
||||
// A fresh instance reads back the changed values merged over defaults.
|
||||
const s2 = new Settings(dir);
|
||||
assert.equal(s2.get('appearance'), 'dark');
|
||||
assert.equal(s2.get('capture.delayMs'), 1500);
|
||||
assert.equal(s2.get('ai.ollama.model'), 'qwen3:0.6b');
|
||||
assert.equal(s2.get('capture.clickMarker'), DEFAULT_SETTINGS.capture.clickMarker);
|
||||
assert.equal(s2.get('capture.fallbackTrigger'), DEFAULT_SETTINGS.capture.fallbackTrigger);
|
||||
assert.deepEqual(s2.getGlobalPlaceholders(), { Company: 'Acme', Author: 'Tyler' });
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const { makeTmpDir, rmrf } = require('./helpers');
|
||||
const { stampVersion } = require('../../scripts/stamp-version');
|
||||
|
||||
test('stampVersion splits build labels into package and build versions', () => {
|
||||
const root = makeTmpDir('stamp-version');
|
||||
try {
|
||||
fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({
|
||||
name: 'stepforge',
|
||||
version: '0.1.0',
|
||||
private: true,
|
||||
buildVersion: '0.1.0',
|
||||
}, null, 2));
|
||||
|
||||
fs.writeFileSync(path.join(root, 'package-lock.json'), JSON.stringify({
|
||||
name: 'stepforge',
|
||||
version: '0.1.0',
|
||||
lockfileVersion: 3,
|
||||
requires: true,
|
||||
packages: {
|
||||
'': {
|
||||
name: 'stepforge',
|
||||
version: '0.1.0',
|
||||
},
|
||||
},
|
||||
}, null, 2));
|
||||
|
||||
stampVersion(root, '0.3.2.1');
|
||||
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
||||
const lock = JSON.parse(fs.readFileSync(path.join(root, 'package-lock.json'), 'utf8'));
|
||||
|
||||
assert.equal(pkg.version, '0.3.2');
|
||||
assert.equal(pkg.buildVersion, '0.3.2.1');
|
||||
assert.equal(lock.version, '0.3.2');
|
||||
assert.equal(lock.packages[''].version, '0.3.2');
|
||||
} finally {
|
||||
rmrf(root);
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
@@ -309,21 +311,112 @@ test('ollama connection test reports installed models', async (t) => {
|
||||
settings: makeSettings(),
|
||||
getWindow: () => null,
|
||||
dataDir: root,
|
||||
fetchImpl: async () => ({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{ name: 'llama3.2:1b' },
|
||||
{ name: 'qwen3:0.6b' },
|
||||
],
|
||||
}),
|
||||
}),
|
||||
fetchImpl: async (url) => {
|
||||
const pathname = new URL(url).pathname;
|
||||
if (pathname === '/api/tags') {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{ name: 'llama3.2:1b' },
|
||||
{ name: 'qwen3:0.6b' },
|
||||
],
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (pathname === '/api/show') {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
capabilities: ['completion', 'vision'],
|
||||
}),
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${pathname}`);
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.testAiConnection();
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.installed, true);
|
||||
assert.equal(result.model, 'llama3.2:1b');
|
||||
assert.equal(result.vision, true);
|
||||
});
|
||||
|
||||
test('vision-capable models receive the screenshot in the chat request', async (t) => {
|
||||
const root = makeTmpDir('text-intel-ai-vision');
|
||||
t.after(() => rmrf(root));
|
||||
const imagePath = path.join(root, 'step.png');
|
||||
fs.writeFileSync(imagePath, Buffer.from('fake screenshot bytes'));
|
||||
|
||||
const step = createStep({
|
||||
title: 'Old title',
|
||||
descriptionHtml: '<p>Old text</p>',
|
||||
image: {
|
||||
originalPath: 'original.png',
|
||||
workingPath: 'working.png',
|
||||
size: { width: 10, height: 10 },
|
||||
},
|
||||
captureMetadata: {
|
||||
windowTitle: 'Settings',
|
||||
appName: 'chrome',
|
||||
ocrText: 'Open settings',
|
||||
titleCandidate: 'Open settings',
|
||||
mode: 'fullscreen',
|
||||
},
|
||||
});
|
||||
const fetchCalls = [];
|
||||
const service = new TextIntelService({
|
||||
store: {
|
||||
settingsDir: root,
|
||||
getGuide: () => ({ guideId: 'g1', title: 'Guide', descriptionHtml: '', stepsOrder: ['s1'] }),
|
||||
getStep: () => step,
|
||||
stepImagePath: () => imagePath,
|
||||
saveStep: (_, next) => next,
|
||||
},
|
||||
settings: makeSettings(),
|
||||
getWindow: () => null,
|
||||
dataDir: root,
|
||||
fetchImpl: async (url, init = {}) => {
|
||||
const pathname = new URL(url).pathname;
|
||||
fetchCalls.push({ pathname, init });
|
||||
if (pathname === '/api/show') {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
capabilities: ['completion', 'vision'],
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (pathname === '/api/chat') {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
message: {
|
||||
content: JSON.stringify({
|
||||
title: 'Open settings',
|
||||
description: 'Use the AI tab.',
|
||||
}),
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${pathname}`);
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.generateStepPatch({
|
||||
guideId: 'g1',
|
||||
stepId: 's1',
|
||||
target: 'all',
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
const chatCall = fetchCalls.find((call) => call.pathname === '/api/chat');
|
||||
assert.ok(chatCall, 'expected an Ollama chat request');
|
||||
const body = JSON.parse(chatCall.init.body);
|
||||
assert.deepEqual(body.messages[1].images, [fs.readFileSync(imagePath).toString('base64')]);
|
||||
assert.match(body.messages[1].content, /Screenshot: attached/i);
|
||||
});
|
||||
|
||||
test('invalid ollama output fails safely without saving the step', async (t) => {
|
||||
|
||||
Reference in New Issue
Block a user