TylerandClaude Fable 5
ccbb9b03dc
Template tests / tests (pull_request) Failing after 34s
Phase 1 of the improvement plan (PR 3 of the sequence). The docs claimed "fully offline"/"never talks to the network," but text-intel makes HTTP requests to a configurable Ollama host that could be remote, with no timeout, no cancellation, and no size limit; the Windows hook logged raw keystrokes into capture metadata that could then be sent to that host. Privacy — raw keystroke capture: - New capture.captureTypedText setting, default false. With it off, printable characters are never buffered in JS and the Windows keyboard hook never even emits them across the process boundary (the flag is threaded into the C#). Shortcut/navigation detection (Ctrl+T, Enter, …) is unaffected. AI network hardening (app/text-intel.js): - Every Ollama call goes through fetchJson with an AbortController deadline (ai.timeoutMs, default 60s): a dead endpoint fails fast instead of leaving UI actions pending forever. - Cancellation: in-flight requests are tracked and cancelInflight(guideId) aborts them; new ai:cancel IPC + api.ai.cancel are called when the editor closes, and shutdown cancels everything. - Bounded concurrency (2) for AI network work. - Screenshots are only attached when allowed (ai.attachScreenshots), the model is vision-capable, and the image is within ai.maxImageBytes — no more unbounded base64-expanded 4K bodies. Local-first host policy (core/text-intel.js): - New isLoopbackHost + validateOllamaHost. By default only a loopback Ollama endpoint is contacted; a remote host is refused with a clear message unless ai.allowRemoteHost is explicitly enabled. Blocked hosts are never contacted. Honest documentation: - README, package.json, and the welcome screen drop "fully offline"/"never talks to the network"/"Electron is the only dependency" for an accurate local-first contract that discloses the optional AI path and the bundled Tesseract OCR dependency. - New docs/PRIVACY.md details exactly what is collected locally and the one outbound (opt-in, loopback-by-default) AI feature. Tests: loopback/remote host matrix, remote-blocked-without-opt-in (and never contacted), remote-allowed-with-opt-in, request timeout, explicit cancel vs timeout, typed-text off-by-default vs opted-in, shortcut detection still works, and a source guard that the C# CHAR emission stays behind the opt-in. 224 unit tests pass; startup smoke and workflow E2E pass. Co-Authored-By: Claude Fable 5 <[email protected]>
2073 lines
87 KiB
JavaScript
2073 lines
87 KiB
JavaScript
'use strict';
|
|
|
|
const path = require('node:path');
|
|
const fs = require('node:fs');
|
|
const { spawn, execFileSync } = require('node:child_process');
|
|
const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron');
|
|
const raster = require('../core/raster');
|
|
const { encodePng } = require('../core/png');
|
|
const {
|
|
selectFrameForClick,
|
|
frameUsableForClick,
|
|
pointInBounds,
|
|
DEFAULT_MAX_AGE_MS,
|
|
DEFAULT_START_SLACK_MS,
|
|
} = require('./click-frames');
|
|
const { physicalToDip } = require('./coords');
|
|
const { DEFAULT_CAPTURE_TITLES } = require('../core/text-intel');
|
|
|
|
/**
|
|
* Capture service: full-screen, active-window, and region capture, plus a
|
|
* click-marker annotation at the click position and a capture session
|
|
* (start/pause/resume/finish).
|
|
*
|
|
* A session captures continuously, with three triggers layered by what the
|
|
* platform supports:
|
|
* - click-capture via an OS adapter (xinput on X11, a low-level mouse hook
|
|
* on Windows),
|
|
* - a global hotkey (unreliable on some Wayland compositors),
|
|
* - interval auto-capture as the always-works fallback.
|
|
*
|
|
* Click captures are served from one of two frame recorders:
|
|
* - the stream backend (app/stream-backend.js): a hidden worker window
|
|
* samples a desktop media stream per display into a timestamped ring
|
|
* buffer, entirely off the main process. This is the preferred path —
|
|
* the main-process event loop stays free, so OS click events arrive on
|
|
* time, and the tight sampling cadence keeps a genuinely fresh pre-click
|
|
* frame available for every click;
|
|
* - the legacy in-process frame loop below, kept as the fallback when
|
|
* streams can't start (portal-less Wayland, exotic drivers).
|
|
*
|
|
* Either way the pairing rule is the same (click-frames.js): in strict mode
|
|
* a click only ever gets a frame captured at or before the click — never one
|
|
* whose grab started after it.
|
|
*
|
|
* Note: under Wayland/WSLg, screen capture may require portal support; all
|
|
* failures surface as { ok: false, reason } instead of crashing.
|
|
*/
|
|
|
|
// Leading-edge click debounce: the first click of a button is captured, and
|
|
// further clicks of that button within this window of the last *accepted*
|
|
// click are ignored. This collapses accidental fast / double clicks into one
|
|
// step, while any two deliberate clicks spaced more than the window apart
|
|
// each register. Tunable via capture.clickDebounceMs; this is only the
|
|
// default when the setting is absent.
|
|
const DEFAULT_CLICK_DEBOUNCE_MS = 200;
|
|
// How long a Linux raw button event waits for its regular twin (the
|
|
// representation that carries root coordinates) before firing without them.
|
|
const LINUX_CLICK_TWIN_MS = 25;
|
|
// Longest the window stays visible warming up the recorder at recording
|
|
// start. A slow capture-stream start (Windows can take several seconds,
|
|
// especially on battery) must not keep the window up and recording un-armed
|
|
// indefinitely — but the window is still visible during warmup, so the user
|
|
// hasn't begun their workflow yet, and the common case still proceeds the
|
|
// instant the stream is ready. The cap only bites when startup is slow, where
|
|
// a little more headroom buys a pre-click frame for the very first click
|
|
// instead of a post-click fresh shot.
|
|
const WARMUP_MAX_MS = 3000;
|
|
// Idle gap between legacy frame-loop grabs. Must stay well above zero:
|
|
// grabbing back-to-back starves the main-process event loop, which delays
|
|
// delivery of click events from the OS watcher by whole seconds. (The
|
|
// stream backend exists precisely because of this constraint.)
|
|
const FRAME_LOOP_IDLE_MS = 200;
|
|
// A buffered frame older than this is too stale to pass off as "the screen
|
|
// at the instant of the click". Shared with click-frames.js.
|
|
const CLICK_FRAME_MAX_AGE_MS = DEFAULT_MAX_AGE_MS;
|
|
// How long a click waits for the in-flight grab before falling back to a
|
|
// one-off fresh shot.
|
|
const CLICK_FRAME_WAIT_MS = 2000;
|
|
// Balanced (non-strict) mode only: a loop grab that started at most this
|
|
// long after the click is still accepted. Strict mode never does this.
|
|
const CLICK_FRAME_START_SLACK_MS = DEFAULT_START_SLACK_MS;
|
|
const CLICK_CAPTURE_HIDE_DELAY_MS = 25;
|
|
// Frames hold raw images (~20MB each at 2880x1800), so keep the history
|
|
// window wide enough to outlast any processing hiccup but the count low.
|
|
const RECENT_FRAME_RETENTION_MS = 4000;
|
|
const RECENT_FRAME_LIMIT = 4;
|
|
// The click that stops/pauses a session via the tray reaches the OS hook at
|
|
// almost the same instant the tray handler fires. We discard at most that
|
|
// one click — and only when it matches the recorded gesture in *both* time
|
|
// and position, so a fast workflow click that merely happens to land near
|
|
// the stop is never mistaken for the stop itself.
|
|
const SESSION_STOP_CLICK_WINDOW_MS = 700;
|
|
const SESSION_STOP_CLICK_RADIUS_PX = 8;
|
|
|
|
// Per-click diagnostics, enabled with STEPFORGE_CAPTURE_LOG=1. Cheap enough
|
|
// to leave in: one line per click/frame decision, nothing per frame-loop tick.
|
|
const CAPTURE_LOG = Boolean(process.env.STEPFORGE_CAPTURE_LOG);
|
|
function clog(...args) {
|
|
if (CAPTURE_LOG) console.log('[capture]', ...args);
|
|
}
|
|
|
|
function hasBinary(name) {
|
|
try {
|
|
execFileSync('which', [name], { stdio: 'pipe' });
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// On Wayland, xinput only sees XWayland (X11-bridge) events — native Wayland
|
|
// app clicks are delivered via the Wayland protocol and never reach xinput.
|
|
// Treating it as "available" would leave the session with no click capture AND
|
|
// no interval fallback, so zero steps get captured.
|
|
function isWayland() {
|
|
if (process.platform !== 'linux') return false;
|
|
// XDG_SESSION_TYPE is the authoritative session hint when present. Some
|
|
// desktops still export WAYLAND_DISPLAY even when the active session is X11,
|
|
// so only fall back to it when XDG_SESSION_TYPE is unavailable.
|
|
const sessionType = String(process.env.XDG_SESSION_TYPE || '').toLowerCase();
|
|
if (sessionType) return sessionType === 'wayland';
|
|
return Boolean(process.env.WAYLAND_DISPLAY);
|
|
}
|
|
|
|
// ---- evdev (Linux kernel input) click reader --------------------------------
|
|
// Reading /dev/input/event* directly sees mouse-button presses on BOTH X11 and
|
|
// Wayland, because it taps the kernel input layer below the display server —
|
|
// the one global-click source that survives Wayland's security model. It needs
|
|
// read access to the device nodes (the user must be in the `input` group).
|
|
//
|
|
// Each event is a fixed-size `struct input_event`: a timeval, then u16 type,
|
|
// u16 code, s32 value. The timeval is two `long`s, so the record is 24 bytes on
|
|
// 64-bit and 16 on 32-bit; type/code/value always sit in the last 8 bytes.
|
|
const EV_KEY = 0x01;
|
|
const EVDEV_PRESS = 1;
|
|
// BTN_LEFT/RIGHT/MIDDLE -> the same button-N naming the xinput path emits
|
|
// (1=left, 2=middle, 3=right) so downstream debounce/marker logic is identical.
|
|
const EVDEV_BUTTONS = { 272: 'button-1', 273: 'button-3', 274: 'button-2' };
|
|
const EVDEV_RECORD_SIZE = (process.arch === 'x64' || process.arch === 'arm64'
|
|
|| process.arch === 'ppc64' || process.arch === 's390x' || process.arch === 'loong64'
|
|
|| process.arch === 'riscv64') ? 24 : 16;
|
|
|
|
/**
|
|
* Decode a buffer of packed input_event records, returning the button presses
|
|
* found and any trailing partial record (a device read can split mid-record).
|
|
* Pure and size-parameterised so it is unit-testable without real devices.
|
|
*/
|
|
function decodeEvdevButtonPresses(buffer, recordSize = EVDEV_RECORD_SIZE) {
|
|
const presses = [];
|
|
let offset = 0;
|
|
while (buffer.length - offset >= recordSize) {
|
|
const type = buffer.readUInt16LE(offset + recordSize - 8);
|
|
const code = buffer.readUInt16LE(offset + recordSize - 6);
|
|
const value = buffer.readInt32LE(offset + recordSize - 4);
|
|
offset += recordSize;
|
|
if (type === EV_KEY && value === EVDEV_PRESS && EVDEV_BUTTONS[code]) {
|
|
presses.push(EVDEV_BUTTONS[code]);
|
|
}
|
|
}
|
|
return { presses, rest: buffer.subarray(offset) };
|
|
}
|
|
|
|
/**
|
|
* The /dev/input/event* nodes for pointing devices that are readable by this
|
|
* process. /proc/bus/input/devices lists every device with its Handlers line;
|
|
* a pointing device exposes a `mouseN` handler, and the matching `eventN` is
|
|
* the node to read. Unreadable nodes (no `input` group membership) are skipped.
|
|
*/
|
|
function readableEvdevMouseNodes() {
|
|
const nodes = [];
|
|
let table;
|
|
try {
|
|
table = fs.readFileSync('/proc/bus/input/devices', 'utf8');
|
|
} catch {
|
|
return nodes;
|
|
}
|
|
for (const block of table.split('\n\n')) {
|
|
const handlers = /H:\s*Handlers=([^\n]*)/.exec(block);
|
|
if (!handlers || !/\bmouse\d+\b/.test(handlers[1])) continue;
|
|
const event = /\bevent(\d+)\b/.exec(handlers[1]);
|
|
if (!event) continue;
|
|
const node = `/dev/input/event${event[1]}`;
|
|
try {
|
|
fs.accessSync(node, fs.constants.R_OK);
|
|
nodes.push(node);
|
|
} catch {
|
|
// Not readable — user is not in the `input` group for this node.
|
|
}
|
|
}
|
|
return nodes;
|
|
}
|
|
|
|
class CaptureService {
|
|
constructor({
|
|
store,
|
|
settings,
|
|
getWindow,
|
|
notify,
|
|
screenApi = screen,
|
|
textIntel = null,
|
|
}) {
|
|
this.store = store;
|
|
this.settings = settings;
|
|
this.getWindow = getWindow;
|
|
this.notify = notify;
|
|
// Injectable for tests; the click/coordinate paths must never reach for
|
|
// the global `screen` directly so coordinate handling stays testable.
|
|
this.screen = screenApi;
|
|
this.textIntel = textIntel;
|
|
// Cached display-server detection. A method (onWayland) reads this so tests
|
|
// can flip platform behavior without touching process.env.
|
|
this._wayland = isWayland();
|
|
this.session = null; // { guideId, paused, count, intervalSec }
|
|
this.intervalTimer = null;
|
|
this.clickWatcher = null;
|
|
this.frameLoopTimer = null;
|
|
this.frameLoopRunning = false;
|
|
this.frameWaiters = [];
|
|
this.latestFrame = null;
|
|
this.clickWatcherBuf = '';
|
|
this.clickWatcherErrTail = '';
|
|
this.linuxEvent = null; // event block currently being parsed
|
|
this.pendingRawClick = null; // raw press waiting for its coordinate twin
|
|
this.pendingClickOsPoint = null;
|
|
this._keyBuffer = ''; // printable chars typed since last capture
|
|
this._lastShortcut = ''; // last Ctrl+X / Alt+X combination
|
|
this._keyLastAt = 0; // timestamp of last key event
|
|
this._pendingWindowContext = null; // buffered CTX+ELEM from click watcher, consumed on CLICK
|
|
this.clickQueue = Promise.resolve();
|
|
this.frameLoopInFlight = false;
|
|
this.frameLoopGrabStartedAt = null;
|
|
this.recentFrames = [];
|
|
this.shooting = false;
|
|
this.lastAcceptedClickByButton = new Map();
|
|
this.streamBackend = null;
|
|
this.streamBackendStarting = false;
|
|
this.captureGen = 0; // bumped on stop to invalidate in-flight backend starts
|
|
// True only while a resume is warming up (window still visible, buffer
|
|
// not yet primed). Clicks are ignored until it clears — see armRecording.
|
|
this.warmingUp = false;
|
|
}
|
|
|
|
state() {
|
|
return this.session
|
|
? {
|
|
active: true,
|
|
paused: this.session.paused,
|
|
guideId: this.session.guideId,
|
|
count: this.session.count,
|
|
intervalSec: this.session.intervalSec || 0,
|
|
clickCapture: Boolean(this.clickWatcher),
|
|
clickCaptureAvailable: this.clickCaptureAvailable(),
|
|
clickFrameSource: this.streamBackend ? 'stream' : (this.frameLoopRunning ? 'loop' : 'idle'),
|
|
strictClickFrames: this.strictClickFrames(),
|
|
}
|
|
: { active: false, clickCaptureAvailable: this.clickCaptureAvailable() };
|
|
}
|
|
|
|
/**
|
|
* Strict is the default: a stored step must never show the screen *after*
|
|
* its click (a frame whose grab started post-click can already contain the
|
|
* click's effects). The setting exists as an explicit escape hatch for
|
|
* machines where capture is too slow to keep pre-click frames buffered —
|
|
* there, the legacy slack heuristics trade accuracy for fewer fresh-shot
|
|
* fallbacks.
|
|
*/
|
|
strictClickFrames() {
|
|
return this.settings.get('capture.strictClickFrames') !== false;
|
|
}
|
|
|
|
fallbackCaptureTrigger() {
|
|
const raw = String(this.settings.get('capture.fallbackTrigger') || 'interval').toLowerCase();
|
|
return raw === 'hotkey' ? 'hotkey' : 'interval';
|
|
}
|
|
|
|
fallbackIntervalSec() {
|
|
const raw = Number(this.settings.get('capture.autoIntervalSec'));
|
|
return Number.isFinite(raw) && raw > 0 ? raw : 5;
|
|
}
|
|
|
|
clickCaptureAvailable() {
|
|
if (this._clickAvail === undefined) {
|
|
// Three click sources, in order of fidelity:
|
|
// - Windows: the low-level mouse hook (position + timing);
|
|
// - X11: xinput test-xi2 (position + timing) — but it can't see native
|
|
// Wayland clicks, only XWayland ones, so it's gated to non-Wayland;
|
|
// - Linux evdev (/dev/input): button presses on X11 AND Wayland, but no
|
|
// cursor position on Wayland — used for per-click capture there (no
|
|
// marker). Requires the user to be in the `input` group.
|
|
this._clickAvail = process.platform === 'win32'
|
|
|| (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput'))
|
|
|| (process.platform === 'linux' && readableEvdevMouseNodes().length > 0);
|
|
}
|
|
return this._clickAvail;
|
|
}
|
|
|
|
/** Whether this is a Wayland session (cached; overridable in tests). */
|
|
onWayland() {
|
|
return this._wayland;
|
|
}
|
|
|
|
/**
|
|
* Whether the in-process frame loop is a usable fallback recorder. It grabs
|
|
* via desktopCapturer.getSources(), which on Wayland is broken (throws) and
|
|
* pops the portal — so the loop is viable only off Wayland. On Wayland the
|
|
* portal-backed stream backend is the sole capture path.
|
|
*/
|
|
canUseFrameLoop() {
|
|
return !this.onWayland();
|
|
}
|
|
|
|
startSession(guideId, { intervalSec = null } = {}) {
|
|
this.finishSession();
|
|
// Default trigger: clicks when the platform supports it, otherwise the
|
|
// user-selected fallback (timer or hotkey-only). That keeps Linux from
|
|
// silently dropping into an unwanted 5-second timer when click capture
|
|
// is unavailable.
|
|
let interval = intervalSec;
|
|
if (interval == null) {
|
|
if (this.clickCaptureAvailable()) {
|
|
interval = 0;
|
|
} else if (this.fallbackCaptureTrigger() === 'hotkey') {
|
|
interval = 0;
|
|
} else {
|
|
interval = this.fallbackIntervalSec();
|
|
}
|
|
}
|
|
// Sessions start paused: nothing hides and no capturing happens until
|
|
// the user explicitly presses "Start recording" in the capture bar, so
|
|
// New Capture never makes the window vanish out from under them.
|
|
this.session = { guideId, paused: true, count: 0, intervalSec: interval };
|
|
if (this.settings.get('capture.captureOutsideClicks') !== false) this.startClickWatcher();
|
|
this.applyInterval();
|
|
this.notify('capture:state', this.state());
|
|
|
|
// (Skipped for the dev screenshot hook, which needs a visible page.)
|
|
if (!process.env.STEPFORGE_SCREENSHOT) {
|
|
this.createSessionTray();
|
|
const win = this.getWindow();
|
|
// Remember whether the window was visible when the session was set
|
|
// up — that's what `togglePause` uses to decide whether to tuck the
|
|
// app away once the user actually starts recording.
|
|
this.hiddenForSession = Boolean(win && !win.isDestroyed() && win.isVisible());
|
|
}
|
|
}
|
|
|
|
/** Red-dot tray icon with session controls, shown while recording. */
|
|
createSessionTray() {
|
|
this.destroySessionTray();
|
|
try {
|
|
const img = raster.createImage(16, 16, [0, 0, 0, 0]);
|
|
raster.fillOval(img, 2, 2, 12, 12, [229, 72, 77, 255]);
|
|
this.tray = new Tray(nativeImage.createFromBuffer(encodePng(img)));
|
|
this.tray.setToolTip('StepForge — capture session running');
|
|
const rebuild = () => {
|
|
if (!this.tray || this.tray.isDestroyed()) return;
|
|
this.tray.setContextMenu(Menu.buildFromTemplate([
|
|
{ label: `Captured ${this.session ? this.session.count : 0} steps`, enabled: false },
|
|
{ type: 'separator' },
|
|
{ label: 'Capture now', click: () => this.sessionCapture('manual').then(rebuild).catch(() => {}) },
|
|
{
|
|
label: this.session && this.session.paused ? 'Resume capturing' : 'Pause capturing',
|
|
click: () => { this.noteUiStopGesture(); this.togglePause(); rebuild(); },
|
|
},
|
|
{
|
|
label: 'Open StepForge (pauses capture)',
|
|
click: () => {
|
|
this.noteUiStopGesture();
|
|
this.togglePause(true);
|
|
this.showWindow();
|
|
rebuild();
|
|
},
|
|
},
|
|
{ type: 'separator' },
|
|
{ label: 'Finish session', click: () => { this.noteUiStopGesture(); this.finishSession(); } },
|
|
]));
|
|
};
|
|
rebuild();
|
|
this.rebuildTrayMenu = rebuild;
|
|
this.tray.on('click', () => {
|
|
this.noteUiStopGesture();
|
|
this.togglePause(true);
|
|
this.showWindow();
|
|
rebuild();
|
|
});
|
|
} catch {
|
|
this.tray = null; // no tray on this desktop; cursor-over skip still protects clicks
|
|
}
|
|
}
|
|
|
|
destroySessionTray() {
|
|
if (this.tray && !this.tray.isDestroyed()) this.tray.destroy();
|
|
this.tray = null;
|
|
this.rebuildTrayMenu = null;
|
|
}
|
|
|
|
/**
|
|
* Record that the user just stopped/paused capture from StepForge's own UI
|
|
* (tray icon or its menu). The physical click that did so is also seen by
|
|
* the OS hook and would otherwise queue as a workflow step; isStopGesture
|
|
* uses this to discard exactly that one click — matched by position, not
|
|
* just time, so a real fast click elsewhere is never lost.
|
|
*/
|
|
noteUiStopGesture() {
|
|
let pos = null;
|
|
try { pos = this.screen.getCursorScreenPoint(); } catch { pos = null; }
|
|
this.uiStopGesture = { at: Date.now(), pos };
|
|
}
|
|
|
|
/** True when a queued click is the tray gesture that stopped the session. */
|
|
isStopGesture(clickPos, clickAt) {
|
|
const g = this.uiStopGesture;
|
|
if (!g) return false;
|
|
if (Math.abs((clickAt || Date.now()) - g.at) > SESSION_STOP_CLICK_WINDOW_MS) return false;
|
|
// No position to compare (e.g. cursor read failed): fall back to the
|
|
// time window alone, but only consume the gesture once.
|
|
if (!g.pos || !clickPos) {
|
|
this.uiStopGesture = null;
|
|
return true;
|
|
}
|
|
const near = Math.abs(clickPos.x - g.pos.x) <= SESSION_STOP_CLICK_RADIUS_PX
|
|
&& Math.abs(clickPos.y - g.pos.y) <= SESSION_STOP_CLICK_RADIUS_PX;
|
|
if (near) this.uiStopGesture = null; // one stop click per gesture
|
|
return near;
|
|
}
|
|
|
|
showWindow() {
|
|
const win = this.getWindow();
|
|
if (win && !win.isDestroyed()) {
|
|
if (win.isMinimized()) win.restore();
|
|
win.show();
|
|
win.focus();
|
|
}
|
|
}
|
|
|
|
setInterval(intervalSec) {
|
|
if (!this.session) return this.state();
|
|
this.session.intervalSec = Math.max(0, Number(intervalSec) || 0);
|
|
this.applyInterval();
|
|
this.notify('capture:state', this.state());
|
|
return this.state();
|
|
}
|
|
|
|
applyInterval() {
|
|
if (this.intervalTimer) {
|
|
clearInterval(this.intervalTimer);
|
|
this.intervalTimer = null;
|
|
}
|
|
const sec = this.session && this.session.intervalSec;
|
|
if (sec > 0) {
|
|
this.intervalTimer = setInterval(() => {
|
|
// Don't let a slow capture (e.g. a multi-second software PNG encode on
|
|
// a GPU-less host) overlap with the next tick — overlapping requests
|
|
// would pile up and could trip the backend's failure counter.
|
|
if (this.intervalCapturing) return;
|
|
this.intervalCapturing = true;
|
|
this.sessionCapture('interval')
|
|
.catch(() => {})
|
|
.finally(() => { this.intervalCapturing = false; });
|
|
}, sec * 1000);
|
|
}
|
|
}
|
|
|
|
togglePause(force) {
|
|
if (!this.session) return;
|
|
const wasPaused = this.session.paused;
|
|
this.session.paused = typeof force === 'boolean' ? force : !this.session.paused;
|
|
// Starting/resuming tucks the window away again for clean shots (after
|
|
// a brief delay so the user sees it happen) and starts the frame
|
|
// recorder that serves click captures. Pausing stops it and discards
|
|
// buffered frames, so a resume can never serve a pre-pause screen.
|
|
if (wasPaused && !this.session.paused) {
|
|
this.armRecording();
|
|
} else if (!wasPaused && this.session.paused) {
|
|
this.warmingUp = false; // cancel any in-flight warmup
|
|
this.stopFrameLoop();
|
|
this.stopClickFrameBackend();
|
|
}
|
|
if (this.rebuildTrayMenu) this.rebuildTrayMenu();
|
|
this.notify('capture:state', this.state());
|
|
}
|
|
|
|
/**
|
|
* Bring a session from paused to recording. The order matters for the
|
|
* first click: the frame recorder is warmed up *while the window is still
|
|
* visible*, then the window is hidden. Warming after the hide (the old
|
|
* order) left a ~1s gap where the worker had no buffered frame yet, so the
|
|
* first click fell back to a post-click fresh shot — "the first screenshot
|
|
* is late". By the time the window tucks away here, frames are already
|
|
* being buffered, so the first click is served a pre-click frame like
|
|
* every other.
|
|
*/
|
|
armRecording() {
|
|
const win = this.getWindow();
|
|
const wantHide = Boolean(this.hiddenForSession && win && !win.isDestroyed());
|
|
// Always start the frame recorder when stream capture is enabled — it
|
|
// buffers frames for click captures AND is used for interval/hotkey
|
|
// captures to avoid calling desktopCapturer.getSources() on every capture.
|
|
// On Linux/Wayland, each getSources() call goes through the XDG portal and
|
|
// shows a permission dialog; the stream backend eliminates that by keeping
|
|
// a live video stream open for the duration of the recording session.
|
|
const recorderWanted = this.settings.get('capture.streamCapture') !== false;
|
|
// Recording is not "live" until the window is hidden and the buffer is
|
|
// primed. While warming up, the window is still visible and over the
|
|
// user's work, so clicks in this period are ignored (onOsClick checks
|
|
// warmingUp) instead of being skipped erratically or shot post-click —
|
|
// the bug that made a restarted recording "stop after one click".
|
|
this.warmingUp = Boolean(wantHide || recorderWanted);
|
|
const settleMs = Number(this.settings.get('capture.postHideSettleMs'));
|
|
const run = async () => {
|
|
if (!this.session || this.session.paused) { this.warmingUp = false; return; }
|
|
if (recorderWanted) {
|
|
// Warm the recorder, but never let a slow backend start (it waits up
|
|
// to several seconds for the capture stream) keep the window visible
|
|
// and recording un-armed. Cap the wait; if it isn't ready by then,
|
|
// hide anyway and let the first click or two take the fresh-shot
|
|
// fallback while the stream finishes coming up in the background.
|
|
const warm = this.startClickFrameBackend().catch(() => {});
|
|
let capTimer = null;
|
|
const cap = new Promise((r) => { capTimer = setTimeout(r, WARMUP_MAX_MS); });
|
|
await Promise.race([warm, cap]);
|
|
if (capTimer) clearTimeout(capTimer);
|
|
if (!this.session || this.session.paused) { this.warmingUp = false; return; }
|
|
}
|
|
if (wantHide && win && !win.isDestroyed() && win.isVisible()) {
|
|
// On Linux, always minimize rather than hide. GNOME's system tray
|
|
// (StatusNotifier) is unreliable — it can fail or half-export over
|
|
// dbus — so a hidden window can be impossible to bring back, leaving
|
|
// the user unable to stop the recording. A minimized window is always
|
|
// restorable from the taskbar, and minimized windows aren't rendered
|
|
// so they still stay out of the fullscreen capture.
|
|
if (process.platform === 'linux') {
|
|
win.minimize();
|
|
} else {
|
|
win.hide();
|
|
}
|
|
// Let a couple of frames of the now-unobscured screen land before
|
|
// the user's first click, so that frame shows their work, not the
|
|
// app window that was just dismissed.
|
|
await new Promise((r) => setTimeout(r, Number.isFinite(settleMs) ? settleMs : 150));
|
|
}
|
|
this.warmingUp = false;
|
|
};
|
|
run().catch(() => { this.warmingUp = false; });
|
|
}
|
|
|
|
finishSession() {
|
|
if (this.intervalTimer) {
|
|
clearInterval(this.intervalTimer);
|
|
this.intervalTimer = null;
|
|
}
|
|
this.warmingUp = false;
|
|
this.stopClickWatcher();
|
|
this.stopFrameLoop();
|
|
this.stopClickFrameBackend();
|
|
this.destroySessionTray();
|
|
this.session = null;
|
|
if (this.hiddenForSession) {
|
|
this.hiddenForSession = false;
|
|
this.showWindow();
|
|
}
|
|
this.notify('capture:state', this.state());
|
|
}
|
|
|
|
/**
|
|
* True when the user is interacting with StepForge itself. Deliberately
|
|
* based on cursor position over the visible window, not isFocused():
|
|
* some compositors (WSLg) report focus as stuck-true, which would block
|
|
* every automatic capture forever.
|
|
*/
|
|
userIsInApp() {
|
|
const win = this.getWindow();
|
|
if (!win || win.isDestroyed() || !win.isVisible() || win.isMinimized()) return false;
|
|
const cur = this.screen.getCursorScreenPoint();
|
|
const b = win.getBounds();
|
|
return cur.x >= b.x && cur.x <= b.x + b.width && cur.y >= b.y && cur.y <= b.y + b.height;
|
|
}
|
|
|
|
/** One capture inside the active session (hotkey/click/interval/manual). */
|
|
async sessionCapture(trigger = 'hotkey', clickPos = null, clickMeta = null) {
|
|
// A click that was registered while recording carries its guide id
|
|
// (see enqueueClickCapture) and must become a step even if the session
|
|
// was paused or finished while it sat behind slower clicks in the
|
|
// queue. Dropping queued clicks at stop time is how "I clicked five
|
|
// times and only got two steps" happens on hosts with slow encodes.
|
|
const queuedClickGuide = trigger === 'click' && clickMeta && clickMeta.guideId
|
|
? clickMeta.guideId
|
|
: null;
|
|
if (!this.session || this.session.paused) {
|
|
if (!queuedClickGuide) return { ok: false, reason: 'no active capture session' };
|
|
} else if (trigger !== 'manual' && this.userIsInApp()) {
|
|
// Automatic triggers stand down while the user is in StepForge, so the
|
|
// app stays clickable mid-session and never screenshots itself.
|
|
return { ok: false, reason: 'skipped — StepForge is focused' };
|
|
}
|
|
|
|
// Clicks are served from the frame recorder: the chosen frame was
|
|
// captured at (or moments before) the click instant, so the background
|
|
// matches what the user clicked on. A click that lands while a grab is
|
|
// in flight waits for that frame instead of being dropped, so fast
|
|
// clicking still yields one step per click.
|
|
if (trigger === 'click') {
|
|
const clickAt = clickMeta && Number.isFinite(clickMeta.at) ? clickMeta.at : Date.now();
|
|
// Prefer the frame the click was paired with at event time (see
|
|
// enqueueClickCapture); ask now only when no eager pairing happened.
|
|
const frame = clickMeta && clickMeta.framePromise
|
|
? await clickMeta.framePromise
|
|
: await this.frameForClick(clickPos, clickAt);
|
|
const sessionLive = this.session && !this.session.paused;
|
|
const guideId = sessionLive ? this.session.guideId : queuedClickGuide;
|
|
if (!guideId) return { ok: false, reason: 'no active capture session' };
|
|
// The tray gesture that stopped the session is itself a hook click in
|
|
// the queue — storing it would append a junk step of the menu. Discard
|
|
// only that one click, matched by position so a fast workflow click is
|
|
// never collateral damage.
|
|
if (!sessionLive && this.isStopGesture(clickPos, clickAt)) {
|
|
clog('click@', clickAt, 'discarded — it triggered the session stop');
|
|
return { ok: false, reason: 'click stopped the session' };
|
|
}
|
|
if (frame) {
|
|
clog('click@', clickAt, 'frame', frame.source || 'loop',
|
|
'started', frame.startedAt - clickAt, 'ms, captured', frame.capturedAt - clickAt, 'ms rel. click');
|
|
const result = await this.storeFrameAsStep(guideId, frame.mode, frame, clickPos, clickMeta);
|
|
if (result.ok) this.noteStepAdded(result.step, trigger, guideId);
|
|
return result;
|
|
}
|
|
// No usable frame: fall through to a one-off fresh shot — but only
|
|
// while still recording. After a stop, a fresh shot would show
|
|
// whatever replaced the user's workflow on screen.
|
|
clog('click@', clickAt, 'no frame qualified — falling back to a fresh (post-click) shot');
|
|
if (!sessionLive) return { ok: false, reason: 'session ended before the fallback shot' };
|
|
}
|
|
|
|
// For non-click triggers (interval, hotkey, manual) pull the latest frame
|
|
// from the stream backend's ring buffer when available. This avoids a
|
|
// desktopCapturer.getSources() call per capture — on Linux/Wayland that
|
|
// call goes through the XDG portal and shows a dialog every time.
|
|
//
|
|
// No clickPos: a timed capture has no click position, and passing a cursor
|
|
// point here is actively harmful on Wayland — getCursorScreenPoint() can
|
|
// return a stale/out-of-bounds point, which makes the backend reject the
|
|
// frame (wrong display / out of bounds) and fall through to a getSources()
|
|
// shot, i.e. a portal dialog on every interval tick.
|
|
if (trigger !== 'click' && this.streamBackend && this.streamBackend.isActive()) {
|
|
const frame = await this.streamBackend.frameForClick({
|
|
clickPos: null,
|
|
clickAt: Date.now(),
|
|
strict: false, // no pre/post-click constraint for timed captures
|
|
leadMs: 0,
|
|
failable: false, // a slow timed-capture encode must not kill the stream
|
|
}).catch(() => null);
|
|
if (frame) {
|
|
const result = await this.storeFrameAsStep(this.session.guideId, 'fullscreen', frame);
|
|
if (result.ok) this.noteStepAdded(result.step, trigger);
|
|
clog(trigger, 'capture stored from stream; total', this.session && this.session.count);
|
|
return result;
|
|
}
|
|
clog(trigger, 'capture: no frame from stream this tick — will retry next tick');
|
|
} else if (trigger !== 'click' && this.onWayland()) {
|
|
clog(trigger, 'capture: stream backend not active (active=',
|
|
Boolean(this.streamBackend && this.streamBackend.isActive()), ')');
|
|
}
|
|
|
|
// On Wayland the only screen-grab fallback below is desktopCapturer
|
|
// .getSources(), which pops the XDG portal dialog every call. For the
|
|
// automatic timed triggers that would mean a dialog on every tick, so skip
|
|
// the fallback and wait for the open stream to deliver a frame on a later
|
|
// tick. Explicit captures (manual, and click on X11) still fall through —
|
|
// one dialog for one deliberate action.
|
|
if (this.onWayland() && (trigger === 'interval' || trigger === 'hotkey')) {
|
|
return { ok: false, reason: 'waiting for the screen-share stream' };
|
|
}
|
|
|
|
if (this.shooting) return { ok: false, reason: 'capture already in progress' };
|
|
this.shooting = true;
|
|
try {
|
|
const mode = this.settings.get('capture.mode') || 'fullscreen';
|
|
const grabMode = mode === 'region' ? 'fullscreen' : mode;
|
|
const finalResult = await this.shoot({
|
|
guideId: this.session.guideId,
|
|
mode: grabMode,
|
|
delayMs: 0,
|
|
hideWindowDelayMs: trigger === 'click' ? CLICK_CAPTURE_HIDE_DELAY_MS : null,
|
|
refocus: false, // don't steal focus from the app the user is documenting
|
|
clickPos,
|
|
});
|
|
if (finalResult.ok) this.noteStepAdded(finalResult.step, trigger);
|
|
return finalResult;
|
|
} finally {
|
|
this.shooting = false;
|
|
}
|
|
}
|
|
|
|
noteStepAdded(step, trigger, guideId = null) {
|
|
// Steps from queued clicks can land after the session object is gone.
|
|
if (this.session) this.session.count += 1;
|
|
this.notify('capture:added', {
|
|
guideId: guideId || (this.session && this.session.guideId),
|
|
step,
|
|
trigger,
|
|
});
|
|
this.notify('capture:state', this.state());
|
|
if (this.rebuildTrayMenu) this.rebuildTrayMenu(); // refresh step counter
|
|
}
|
|
|
|
hotkeyCapture() {
|
|
return this.sessionCapture('hotkey');
|
|
}
|
|
|
|
// ---- click-triggered capture --------------------------------------------
|
|
|
|
/**
|
|
* Fallback frame recorder: a continuous screen-grab loop in the main
|
|
* process, used only when the stream backend can't run. It keeps the most
|
|
* recent frames buffered so a click can be served from a frame grabbed at
|
|
* (or moments before) the instant of the click — a fresh grab started
|
|
* after the click would land hundreds of ms late and show the click's
|
|
* effects instead of what the user clicked on. Its cadence is capped at
|
|
* FRAME_LOOP_IDLE_MS because tighter grabbing here starves the event loop
|
|
* and delays the very click events it serves.
|
|
*/
|
|
startFrameLoop() {
|
|
if (this.frameLoopRunning) return;
|
|
this.frameLoopRunning = true;
|
|
const tick = async () => {
|
|
if (!this.frameLoopRunning) return;
|
|
if (!this.session || this.session.paused) {
|
|
this.frameLoopRunning = false;
|
|
this.frameLoopInFlight = false;
|
|
return;
|
|
}
|
|
try {
|
|
if (!this.shooting) {
|
|
this.frameLoopInFlight = true;
|
|
this.frameLoopGrabStartedAt = Date.now();
|
|
const mode = this.settings.get('capture.mode') || 'fullscreen';
|
|
const grabMode = mode === 'region' ? 'fullscreen' : mode;
|
|
const frame = await this.captureCurrentFrame(grabMode, null, this.frameLoopGrabStartedAt);
|
|
if (this.frameLoopRunning) this.acceptFrame(frame);
|
|
}
|
|
} catch {
|
|
// Grab failures are fine — clicks fall back to a one-off fresh shot.
|
|
} finally {
|
|
this.frameLoopInFlight = false;
|
|
this.frameLoopGrabStartedAt = null;
|
|
if (this.frameLoopRunning && this.session && !this.session.paused) {
|
|
this.frameLoopTimer = setTimeout(tick, FRAME_LOOP_IDLE_MS);
|
|
}
|
|
}
|
|
};
|
|
this.frameLoopTimer = setTimeout(tick, 0);
|
|
}
|
|
|
|
/** Store a grabbed frame and hand it to any clicks waiting on it. */
|
|
acceptFrame(frame) {
|
|
this.latestFrame = frame;
|
|
this.recentFrames.push(frame);
|
|
const cutoff = Date.now() - RECENT_FRAME_RETENTION_MS;
|
|
this.recentFrames = this.recentFrames
|
|
.filter((f) => f && f.capturedAt >= cutoff)
|
|
.slice(-RECENT_FRAME_LIMIT);
|
|
const waiters = this.frameWaiters;
|
|
this.frameWaiters = [];
|
|
for (const resolve of waiters) resolve(frame);
|
|
}
|
|
|
|
/** Resolves with the next frame the loop grabs (null on timeout/stop). */
|
|
nextFrame(timeoutMs) {
|
|
return new Promise((resolve) => {
|
|
const entry = (frame) => {
|
|
clearTimeout(timer);
|
|
resolve(frame);
|
|
};
|
|
const timer = setTimeout(() => {
|
|
this.frameWaiters = this.frameWaiters.filter((w) => w !== entry);
|
|
resolve(null);
|
|
}, timeoutMs);
|
|
this.frameWaiters.push(entry);
|
|
});
|
|
}
|
|
|
|
stopFrameLoop() {
|
|
if (this.frameLoopTimer) {
|
|
clearTimeout(this.frameLoopTimer);
|
|
this.frameLoopTimer = null;
|
|
}
|
|
this.frameLoopRunning = false;
|
|
this.frameLoopGrabStartedAt = null;
|
|
this.latestFrame = null;
|
|
this.recentFrames = [];
|
|
const waiters = this.frameWaiters;
|
|
this.frameWaiters = [];
|
|
for (const resolve of waiters) resolve(null);
|
|
}
|
|
|
|
/**
|
|
* Frame representing the screen at the instant of one click.
|
|
*
|
|
* Order of preference:
|
|
* 1. the stream backend's ring buffer (off-main-process, tight cadence);
|
|
* 2. the legacy loop's buffered frames;
|
|
* 3. waiting for the loop grab that was already in flight when the user
|
|
* clicked.
|
|
* Selection semantics live in click-frames.js. In strict mode every path
|
|
* obeys the same rule — never a frame whose grab started after the click —
|
|
* and when nothing qualifies this returns null so the caller takes the
|
|
* *explicit* fresh-shot fallback rather than silently passing a post-click
|
|
* frame off as the click-time screen.
|
|
*/
|
|
async frameForClick(clickPos = null, clickAt = Date.now()) {
|
|
const mode = this.settings.get('capture.mode') || 'fullscreen';
|
|
const grabMode = mode === 'region' ? 'fullscreen' : mode;
|
|
const clickTime = Number.isFinite(clickAt) ? clickAt : Date.now();
|
|
// Click lead: prefer a frame captured a little *before* the hook
|
|
// timestamp. The hook fires on button-down, but the visible UI often
|
|
// starts reacting within a frame or two (hover→press states, the cursor
|
|
// settling) and capture-stream pixels lag the real screen slightly, so a
|
|
// frame timestamped right at the click can still show the click's onset.
|
|
// The lead is a *preference*: selection falls back to any pre-click
|
|
// frame when none is old enough, so it never forces a post-click fresh
|
|
// shot. Tunable via capture.clickLeadMs.
|
|
const leadMs = Math.max(0, Number(this.settings.get('capture.clickLeadMs')) || 0);
|
|
const strict = this.strictClickFrames();
|
|
const opts = {
|
|
clickAt: clickTime,
|
|
leadMs,
|
|
clickPos,
|
|
mode: grabMode,
|
|
strict,
|
|
maxAgeMs: CLICK_FRAME_MAX_AGE_MS,
|
|
startSlackMs: CLICK_FRAME_START_SLACK_MS,
|
|
};
|
|
|
|
if (this.streamBackend && this.streamBackend.isActive() && grabMode === 'fullscreen') {
|
|
// On Wayland the stream is the only capture path (no frame-loop fallback),
|
|
// so a slow PNG encode must not let the 2-strikes rule tear it down.
|
|
const frame = await this.streamBackend.frameForClick({
|
|
clickPos, clickAt: clickTime, strict, leadMs, failable: !this.onWayland(),
|
|
});
|
|
if (frame) return frame;
|
|
// No qualifying frame (or the backend just went unhealthy): fall
|
|
// through to the loop buffer / fresh-shot fallbacks below.
|
|
}
|
|
|
|
const buffered = selectFrameForClick(
|
|
[...this.recentFrames, this.latestFrame].filter((f, i, arr) => f && arr.indexOf(f) === i),
|
|
opts,
|
|
);
|
|
if (buffered) return buffered;
|
|
if (!this.frameLoopRunning) return null;
|
|
|
|
if (strict) {
|
|
// Only a grab already in flight when the user clicked can still
|
|
// qualify: its pixels predate the click even though it completes
|
|
// after. Any grab starting later is post-click by definition, so
|
|
// don't wait around for one — return immediately and let the caller
|
|
// take the fresh-shot fallback.
|
|
const inFlightStartedBeforeClick = this.frameLoopInFlight
|
|
&& Number.isFinite(this.frameLoopGrabStartedAt)
|
|
&& this.frameLoopGrabStartedAt <= clickTime;
|
|
if (!inFlightStartedBeforeClick) return null;
|
|
const next = await this.nextFrame(CLICK_FRAME_WAIT_MS);
|
|
return frameUsableForClick(next, { ...opts, allowInFlight: true }) ? next : null;
|
|
}
|
|
|
|
// Balanced (legacy) mode: wait for the next loop frame and accept it if
|
|
// its grab started within the slack window after the click.
|
|
const deadline = Date.now() + CLICK_FRAME_WAIT_MS;
|
|
while (this.frameLoopRunning && Date.now() < deadline) {
|
|
const next = await this.nextFrame(Math.max(1, deadline - Date.now()));
|
|
if (frameUsableForClick(next, { ...opts, allowInFlight: true })) return next;
|
|
if (next && Number.isFinite(next.startedAt)
|
|
&& next.startedAt > clickTime + CLICK_FRAME_START_SLACK_MS) {
|
|
// Grabs only get later from here; let the fresh-shot path handle it.
|
|
return null;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// ---- click-frame backends -------------------------------------------------
|
|
|
|
/**
|
|
* Bring up the frame recorder for a recording run. The stream backend is
|
|
* the architecture path (capture entirely off the main process); the
|
|
* in-process frame loop is the fallback when streams can't start — and the
|
|
* automatic degradation target if the worker stops answering mid-session.
|
|
*/
|
|
async startClickFrameBackend() {
|
|
const mode = this.settings.get('capture.mode') || 'fullscreen';
|
|
// The worker streams screens; window-mode grabs need the loop's
|
|
// source-filtering logic. But the loop isn't viable on Wayland (getSources
|
|
// is broken/portal), so there we always take the stream backend regardless
|
|
// of the streamCapture/window settings.
|
|
if (this.canUseFrameLoop()
|
|
&& (this.settings.get('capture.streamCapture') === false || mode === 'window')) {
|
|
this.startFrameLoop();
|
|
return;
|
|
}
|
|
if (this.streamBackend || this.streamBackendStarting) return;
|
|
// Generation token: a stop/finish/pause bumps it. If it changes while
|
|
// this async start is in flight (e.g. the user finishes and restarts
|
|
// before a slow start resolves), the backend we built belongs to a dead
|
|
// session — discard it instead of installing it, and never leave
|
|
// streamBackendStarting stuck so the new session can start its own.
|
|
const gen = this.captureGen;
|
|
this.streamBackendStarting = true;
|
|
try {
|
|
// eslint-disable-next-line global-require
|
|
const { StreamCaptureBackend, createElectronHost } = require('./stream-backend');
|
|
const backend = new StreamCaptureBackend({
|
|
createHost: createElectronHost,
|
|
onUnhealthy: () => this.degradeToFrameLoop(),
|
|
});
|
|
const displays = this.screen.getAllDisplays();
|
|
// On Wayland, desktopCapturer.getSources() both fails to yield usable
|
|
// source ids AND pops the portal dialog. Skip it entirely and drive the
|
|
// worker through getDisplayMedia (the portal picker chooses the screen).
|
|
const useDisplayMedia = this.onWayland();
|
|
const sources = useDisplayMedia ? [] : await desktopCapturer.getSources({
|
|
types: ['screen'],
|
|
thumbnailSize: { width: 1, height: 1 }, // ids only — skip thumbnail work
|
|
});
|
|
const ok = await backend.start({
|
|
displays,
|
|
sources: sources.map((s) => ({ id: s.id, display_id: s.display_id })),
|
|
sampleMs: this.settings.get('capture.frameSampleMs') || 100,
|
|
useDisplayMedia,
|
|
});
|
|
const stale = gen !== this.captureGen;
|
|
if (!ok || stale || !this.session || this.session.paused) {
|
|
backend.stop();
|
|
if (!stale && this.session && !this.session.paused) {
|
|
if (this.canUseFrameLoop()) {
|
|
console.error('[stepforge] stream capture backend failed to start — using in-process frame loop');
|
|
this.startFrameLoop();
|
|
} else {
|
|
// On Wayland the frame loop would spam getSources() (portal) with
|
|
// nothing usable, so there's no fallback — the recording needs the
|
|
// portal stream. Tell the user how to recover.
|
|
console.error('[stepforge] screen-share stream did not start — pick a screen in the share dialog, or stop and start recording again');
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
this.streamBackend = backend;
|
|
// Visible in normal output (one line per recording): confirms the screen
|
|
// stream came up, so a "nothing records" report can be told apart from a
|
|
// stream that never started (which logs the failure paths above).
|
|
console.log(`[stepforge] screen-capture stream active (${useDisplayMedia ? 'getDisplayMedia/portal' : 'desktopCapturer'})`);
|
|
this.notify('capture:state', this.state());
|
|
} catch (err) {
|
|
if (gen === this.captureGen && this.session && !this.session.paused) {
|
|
if (this.canUseFrameLoop()) {
|
|
console.error(`[stepforge] stream capture backend error (${err && err.message}) — using in-process frame loop`);
|
|
this.startFrameLoop();
|
|
} else {
|
|
console.error(`[stepforge] screen-share stream error (${err && err.message}) — stop and start recording again`);
|
|
}
|
|
}
|
|
} finally {
|
|
if (gen === this.captureGen) this.streamBackendStarting = false;
|
|
}
|
|
}
|
|
|
|
stopClickFrameBackend() {
|
|
// Invalidate any in-flight start (see captureGen above) and free the
|
|
// guard so the next session can start a fresh backend immediately.
|
|
this.captureGen += 1;
|
|
this.streamBackendStarting = false;
|
|
if (!this.streamBackend) return;
|
|
const backend = this.streamBackend;
|
|
this.streamBackend = null;
|
|
backend.stop();
|
|
}
|
|
|
|
/**
|
|
* The worker stopped answering frame requests. Capture must not silently
|
|
* stop mid-session: drop the backend and run the in-process loop for the
|
|
* rest of the recording.
|
|
*/
|
|
degradeToFrameLoop() {
|
|
this.streamBackend = null;
|
|
if (this.canUseFrameLoop()) {
|
|
console.error('[stepforge] stream capture backend unhealthy — falling back to in-process frame loop');
|
|
if (this.session && !this.session.paused) this.startFrameLoop();
|
|
} else {
|
|
// On Wayland the frame loop isn't viable (getSources is broken/portal),
|
|
// so there's nothing to fall back to — the stream is the only path.
|
|
console.error('[stepforge] screen-share stream stopped — stop and start recording again to re-share');
|
|
}
|
|
this.notify('capture:state', this.state());
|
|
}
|
|
|
|
startClickWatcher() {
|
|
this.stopClickWatcher();
|
|
try {
|
|
this.clickWatcherBuf = '';
|
|
this.linuxEvent = null;
|
|
if (process.platform === 'linux' && !this.onWayland() && hasBinary('xinput')) {
|
|
// Stream raw button events from the X server; one capture per press.
|
|
// xinput block-buffers stdout when piped, so a press event can sit
|
|
// in its buffer until later motion events flush it — by then the
|
|
// cursor read in onOsClick lands where the mouse moved *after* the
|
|
// click. stdbuf -oL forces line-buffering so events (and the cursor
|
|
// read) line up with the actual click instant.
|
|
// (Skipped on Wayland: xinput only sees XWayland events — see isWayland.)
|
|
const argv = hasBinary('stdbuf')
|
|
? ['stdbuf', '-oL', 'xinput', 'test-xi2', '--root']
|
|
: ['xinput', 'test-xi2', '--root'];
|
|
this.clickWatcher = spawn(argv[0], argv.slice(1), { stdio: ['ignore', 'pipe', 'ignore'] });
|
|
this.clickWatcher.stdout.on('data', (chunk) => {
|
|
this.ingestClickWatcherChunk(chunk.toString(), 'linux');
|
|
});
|
|
} else if (process.platform === 'linux' && readableEvdevMouseNodes().length > 0) {
|
|
// Wayland (or X11 without xinput): read mouse buttons from the kernel
|
|
// input layer. This is the only global click source on Wayland, but it
|
|
// carries no cursor position — onOsClick gets a null point, so steps are
|
|
// captured per click without a marker. (X11 prefers the xinput branch
|
|
// above, which does carry root coordinates for the marker.)
|
|
this.startEvdevWatcher();
|
|
} else if (process.platform === 'win32') {
|
|
// Use a low-level Windows mouse hook instead of polling
|
|
// GetAsyncKeyState. The low bit from GetAsyncKeyState can be consumed
|
|
// by other processes and a polling loop can miss short clicks under
|
|
// load; WH_MOUSE_LL gives us one event for each button-down, with the
|
|
// hook-time cursor position and timestamp.
|
|
//
|
|
// Raw typed-text capture is a keylogging surface, so the hook only
|
|
// emits printable CHAR events when the user explicitly opted in; by
|
|
// default the characters never even cross the process boundary.
|
|
const captureTypedText = this.settings.get('capture.captureTypedText') ? 'true' : 'false';
|
|
const ps = `
|
|
$ErrorActionPreference = 'Stop'
|
|
Add-Type -TypeDefinition @'
|
|
using System;
|
|
using System.Collections.Concurrent;
|
|
using System.Runtime.InteropServices;
|
|
using System.Threading;
|
|
|
|
public static class SFHook {
|
|
private const int WH_MOUSE_LL = 14;
|
|
private const int WH_KEYBOARD_LL = 13;
|
|
private const int WM_LBUTTONDOWN = 0x0201;
|
|
private const int WM_RBUTTONDOWN = 0x0204;
|
|
private const int WM_MBUTTONDOWN = 0x0207;
|
|
private const int WM_XBUTTONDOWN = 0x020B;
|
|
private const int WM_KEYDOWN = 0x0100;
|
|
private const int WM_SYSKEYDOWN = 0x0104;
|
|
private const long UnixEpochMilliseconds = 62135596800000L;
|
|
// Opting this process out of Windows Power Throttling (EcoQoS). In a
|
|
// power-saving plan the OS CPU-starves background processes; a starved
|
|
// low-level mouse hook whose callback exceeds LowLevelHooksTimeout is
|
|
// silently skipped by Windows (events stop arriving) while the process
|
|
// stays alive, so clicks are missed with no error — the "only the first
|
|
// couple of clicks were captured" symptom.
|
|
private const int ProcessPowerThrottling = 4;
|
|
private const uint PROCESS_POWER_THROTTLING_CURRENT_VERSION = 1;
|
|
private const uint PROCESS_POWER_THROTTLING_EXECUTION_SPEED = 0x1;
|
|
private const uint HIGH_PRIORITY_CLASS = 0x00000080;
|
|
|
|
private static readonly bool CaptureTypedText = ${captureTypedText};
|
|
private static IntPtr hook = IntPtr.Zero;
|
|
private static IntPtr keyHook = IntPtr.Zero;
|
|
private static LowLevelMouseProc proc = MouseHookCallback;
|
|
private static LowLevelKeyboardProc keyProc = KeyboardHookCallback;
|
|
private static readonly ConcurrentQueue<string> queue = new ConcurrentQueue<string>();
|
|
private static readonly AutoResetEvent signal = new AutoResetEvent(false);
|
|
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
private struct POINT {
|
|
public int x;
|
|
public int y;
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
private struct MSLLHOOKSTRUCT {
|
|
public POINT pt;
|
|
public uint mouseData;
|
|
public uint flags;
|
|
public uint time;
|
|
public UIntPtr dwExtraInfo;
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
private struct MSG {
|
|
public IntPtr hwnd;
|
|
public uint message;
|
|
public UIntPtr wParam;
|
|
public IntPtr lParam;
|
|
public uint time;
|
|
public POINT pt;
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
private struct PROCESS_POWER_THROTTLING_STATE {
|
|
public uint Version;
|
|
public uint ControlMask;
|
|
public uint StateMask;
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
private struct KBDLLHOOKSTRUCT {
|
|
public uint vkCode;
|
|
public uint scanCode;
|
|
public uint flags;
|
|
public uint time;
|
|
public UIntPtr dwExtraInfo;
|
|
}
|
|
|
|
private delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam);
|
|
private delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);
|
|
|
|
[DllImport("user32.dll", SetLastError = true)]
|
|
private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId);
|
|
|
|
[DllImport("user32.dll", SetLastError = true)]
|
|
private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelKeyboardProc lpfn, IntPtr hMod, uint dwThreadId);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern short GetKeyState(int nVirtKey);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern bool GetKeyboardState([Out] byte[] lpKeyState);
|
|
|
|
[DllImport("user32.dll", CharSet = CharSet.Unicode)]
|
|
private static extern int ToUnicode(uint wVirtKey, uint wScanCode, byte[] lpKeyState,
|
|
System.Text.StringBuilder pwszBuff, int cchBuff, uint wFlags);
|
|
|
|
[DllImport("user32.dll", SetLastError = true)]
|
|
private static extern bool UnhookWindowsHookEx(IntPtr hhk);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam);
|
|
|
|
[DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
|
|
private static extern IntPtr GetModuleHandle(string lpModuleName);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern int GetMessage(out MSG lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern bool TranslateMessage(ref MSG lpMsg);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern IntPtr DispatchMessage(ref MSG lpMsg);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
|
|
|
[DllImport("kernel32.dll", SetLastError = true)]
|
|
private static extern bool SetProcessInformation(IntPtr hProcess, int ProcessInformationClass, ref PROCESS_POWER_THROTTLING_STATE ProcessInformation, uint ProcessInformationSize);
|
|
|
|
[DllImport("kernel32.dll")]
|
|
private static extern IntPtr GetCurrentProcess();
|
|
|
|
[DllImport("kernel32.dll")]
|
|
private static extern bool SetPriorityClass(IntPtr hProcess, uint dwPriorityClass);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern IntPtr GetForegroundWindow();
|
|
|
|
[DllImport("user32.dll", CharSet = CharSet.Unicode)]
|
|
private static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count);
|
|
|
|
[DllImport("user32.dll")]
|
|
private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId);
|
|
|
|
[DllImport("kernel32.dll", SetLastError = true)]
|
|
private static extern IntPtr OpenProcess(uint processAccess, bool bInheritHandle, uint processId);
|
|
|
|
[DllImport("kernel32.dll", SetLastError = true)]
|
|
private static extern bool CloseHandle(IntPtr hObject);
|
|
|
|
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
|
|
private static extern bool QueryFullProcessImageName(IntPtr hProcess, uint dwFlags, System.Text.StringBuilder lpExeName, ref uint lpdwSize);
|
|
|
|
private static string GetFwTitle() {
|
|
try {
|
|
IntPtr hwnd = GetForegroundWindow();
|
|
if (hwnd == IntPtr.Zero) return "";
|
|
var sb = new System.Text.StringBuilder(512);
|
|
GetWindowText(hwnd, sb, sb.Capacity);
|
|
return sb.ToString();
|
|
} catch { return ""; }
|
|
}
|
|
|
|
private static string GetFwApp() {
|
|
try {
|
|
IntPtr hwnd = GetForegroundWindow();
|
|
if (hwnd == IntPtr.Zero) return "";
|
|
uint pid = 0;
|
|
GetWindowThreadProcessId(hwnd, out pid);
|
|
if (pid == 0) return "";
|
|
IntPtr hProc = OpenProcess(0x1000u, false, pid);
|
|
if (hProc == IntPtr.Zero) return "";
|
|
var sb = new System.Text.StringBuilder(260);
|
|
uint sz = (uint)sb.Capacity;
|
|
QueryFullProcessImageName(hProc, 0u, sb, ref sz);
|
|
CloseHandle(hProc);
|
|
string path = sb.ToString();
|
|
return string.IsNullOrEmpty(path) ? "" : System.IO.Path.GetFileNameWithoutExtension(path);
|
|
} catch { return ""; }
|
|
}
|
|
|
|
// Base64-encodes a string for safe line-protocol transmission; "-" for empty.
|
|
private static string B64(string s) {
|
|
if (string.IsNullOrEmpty(s)) return "-";
|
|
try { return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(s)); } catch { return "-"; }
|
|
}
|
|
|
|
// Force this process to run at full CPU speed regardless of the power plan,
|
|
// so the mouse-hook callback never trips LowLevelHooksTimeout and clicks
|
|
// keep being delivered while the laptop is in eco / power-saving mode.
|
|
private static void KeepProcessResponsive() {
|
|
try {
|
|
PROCESS_POWER_THROTTLING_STATE state = new PROCESS_POWER_THROTTLING_STATE();
|
|
state.Version = PROCESS_POWER_THROTTLING_CURRENT_VERSION;
|
|
// Control EXECUTION_SPEED and set its state bit to 0 => throttling off
|
|
// (the documented way to opt a process out of EcoQoS).
|
|
state.ControlMask = PROCESS_POWER_THROTTLING_EXECUTION_SPEED;
|
|
state.StateMask = 0;
|
|
SetProcessInformation(GetCurrentProcess(), ProcessPowerThrottling, ref state, (uint)Marshal.SizeOf(state));
|
|
} catch { }
|
|
try { SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS); } catch { }
|
|
}
|
|
|
|
public static void Run() {
|
|
KeepProcessResponsive();
|
|
try { SetProcessDpiAwarenessContext(new IntPtr(-4)); } catch { }
|
|
|
|
Thread writer = new Thread(WriterLoop);
|
|
writer.IsBackground = true;
|
|
writer.Start();
|
|
|
|
hook = SetWindowsHookEx(WH_MOUSE_LL, proc, GetModuleHandle(null), 0);
|
|
if (hook == IntPtr.Zero) {
|
|
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
}
|
|
// Keyboard hook is optional — if it fails we still get click titles.
|
|
try { keyHook = SetWindowsHookEx(WH_KEYBOARD_LL, keyProc, GetModuleHandle(null), 0); } catch { }
|
|
|
|
Console.Out.WriteLine("READY");
|
|
Console.Out.Flush();
|
|
|
|
MSG msg;
|
|
while (GetMessage(out msg, IntPtr.Zero, 0, 0) > 0) {
|
|
TranslateMessage(ref msg);
|
|
DispatchMessage(ref msg);
|
|
}
|
|
|
|
UnhookWindowsHookEx(hook);
|
|
if (keyHook != IntPtr.Zero) UnhookWindowsHookEx(keyHook);
|
|
}
|
|
|
|
private static void WriterLoop() {
|
|
while (true) {
|
|
signal.WaitOne();
|
|
string line;
|
|
while (queue.TryDequeue(out line)) {
|
|
Console.Out.WriteLine(line);
|
|
}
|
|
Console.Out.Flush();
|
|
}
|
|
}
|
|
|
|
private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
|
|
if (nCode >= 0) {
|
|
int message = wParam.ToInt32();
|
|
string button = ButtonName(message, lParam);
|
|
if (button != null) {
|
|
MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT));
|
|
long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds;
|
|
// Capture window context while the user's app is still in foreground.
|
|
// GetForegroundWindow is synchronous and fast (no IPC overhead).
|
|
try {
|
|
string t = GetFwTitle(), a = GetFwApp();
|
|
queue.Enqueue("CTX " + B64(t) + " " + B64(a) + " " + unixMs);
|
|
} catch { }
|
|
queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs);
|
|
signal.Set();
|
|
}
|
|
}
|
|
return CallNextHookEx(hook, nCode, wParam, lParam);
|
|
}
|
|
|
|
private static IntPtr KeyboardHookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
|
|
if (nCode >= 0 && (wParam.ToInt32() == WM_KEYDOWN || wParam.ToInt32() == WM_SYSKEYDOWN)) {
|
|
try {
|
|
KBDLLHOOKSTRUCT kb = (KBDLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(KBDLLHOOKSTRUCT));
|
|
int vk = (int)kb.vkCode;
|
|
bool ctrl = (GetKeyState(0x11) & 0x8000) != 0;
|
|
bool alt = (GetKeyState(0x12) & 0x8000) != 0;
|
|
bool shift = (GetKeyState(0x10) & 0x8000) != 0;
|
|
// Skip standalone modifier keys.
|
|
bool isMod = vk == 0x10 || vk == 0x11 || vk == 0x12 || vk == 0x5B || vk == 0x5C;
|
|
if (!isMod) {
|
|
long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds;
|
|
if (ctrl || alt) {
|
|
// Emit shortcut: "KEY Ctrl+T 123456"
|
|
string name = VkName(vk);
|
|
if (name != null) {
|
|
string mods = (ctrl ? "Ctrl" : "") + (ctrl && (alt || shift) ? "+" : "") +
|
|
(alt ? "Alt" : "") + ((alt || ctrl) && shift ? "+" : "") +
|
|
(shift ? "Shift" : "");
|
|
queue.Enqueue("KEY " + mods + "+" + name + " " + unixMs);
|
|
signal.Set();
|
|
}
|
|
} else if (vk == 0x08) {
|
|
queue.Enqueue("KEY Backspace " + unixMs); signal.Set();
|
|
} else if (vk == 0x1B) {
|
|
queue.Enqueue("KEY Escape " + unixMs); signal.Set();
|
|
} else if (vk == 0x0D) {
|
|
queue.Enqueue("KEY Enter " + unixMs); signal.Set();
|
|
} else if (CaptureTypedText) {
|
|
// Map to Unicode character using current keyboard layout + shift
|
|
// state. Only reached when the user opted into typed-text capture;
|
|
// otherwise raw characters are never read or emitted.
|
|
byte[] ks = new byte[256];
|
|
GetKeyboardState(ks);
|
|
var sb = new System.Text.StringBuilder(4);
|
|
int res = ToUnicode(kb.vkCode, kb.scanCode, ks, sb, 4, 0);
|
|
if (res > 0) {
|
|
char ch = sb[0];
|
|
if (ch >= 0x20 && ch < 0x7F) {
|
|
queue.Enqueue("CHAR " + (int)ch + " " + unixMs);
|
|
signal.Set();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch { }
|
|
}
|
|
return CallNextHookEx(keyHook, nCode, wParam, lParam);
|
|
}
|
|
|
|
private static string VkName(int vk) {
|
|
if (vk >= 0x41 && vk <= 0x5A) return ((char)vk).ToString(); // A-Z
|
|
if (vk >= 0x30 && vk <= 0x39) return ((char)vk).ToString(); // 0-9
|
|
if (vk >= 0x70 && vk <= 0x87) return "F" + (vk - 0x6F); // F1-F24
|
|
switch (vk) {
|
|
case 0x09: return "Tab";
|
|
case 0x0D: return "Enter";
|
|
case 0x1B: return "Escape";
|
|
case 0x20: return "Space";
|
|
case 0x21: return "PageUp"; case 0x22: return "PageDown";
|
|
case 0x23: return "End"; case 0x24: return "Home";
|
|
case 0x25: return "Left"; case 0x26: return "Up";
|
|
case 0x27: return "Right"; case 0x28: return "Down";
|
|
case 0x2E: return "Delete";
|
|
case 0x6B: case 0xBB: return "Plus";
|
|
case 0x6D: case 0xBD: return "Minus";
|
|
case 0xBE: return "Period"; case 0xBF: return "Slash";
|
|
default: return null;
|
|
}
|
|
}
|
|
|
|
private static string ButtonName(int message, IntPtr lParam) {
|
|
if (message == WM_LBUTTONDOWN) return "left";
|
|
if (message == WM_RBUTTONDOWN) return "right";
|
|
if (message == WM_MBUTTONDOWN) return "middle";
|
|
if (message == WM_XBUTTONDOWN) {
|
|
MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT));
|
|
uint xButton = (data.mouseData >> 16) & 0xffff;
|
|
return xButton == 1 ? "x1" : "x2";
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
'@
|
|
[SFHook]::Run()
|
|
`;
|
|
this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', ps], {
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
windowsHide: true,
|
|
});
|
|
this.clickWatcher.stdout.on('data', (chunk) => {
|
|
this.ingestClickWatcherChunk(chunk.toString(), 'win32');
|
|
});
|
|
}
|
|
if (this.clickWatcher) {
|
|
const child = this.clickWatcher;
|
|
this.clickWatcherErrTail = '';
|
|
if (child.stderr) {
|
|
child.stderr.on('data', (chunk) => {
|
|
this.clickWatcherErrTail = String(chunk).slice(-400);
|
|
});
|
|
}
|
|
const lost = (reason) => {
|
|
if (this.clickWatcher !== child) return; // stopped deliberately
|
|
this.clickWatcher = null;
|
|
this.handleClickWatcherLoss(reason);
|
|
};
|
|
child.on('error', (err) => lost(err && err.message));
|
|
child.on('exit', (code) => lost(`exited with code ${code}`));
|
|
}
|
|
} catch {
|
|
this.clickWatcher = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The watcher process died mid-session (crashed X server, PowerShell
|
|
* blocked by policy, …). Captures must not silently stop: log why, switch
|
|
* the session to interval captures, and tell the UI.
|
|
*/
|
|
handleClickWatcherLoss(reason) {
|
|
this.linuxEvent = null;
|
|
this.discardPendingRawClick();
|
|
const detail = [reason, this.clickWatcherErrTail].filter(Boolean).join(' — ');
|
|
console.error(`[stepforge] click watcher stopped${detail ? `: ${detail}` : ''}`);
|
|
if (!this.session) return;
|
|
if (!this.session.intervalSec) {
|
|
this.session.intervalSec = this.fallbackCaptureTrigger() === 'hotkey'
|
|
? 0
|
|
: this.fallbackIntervalSec();
|
|
this.applyInterval();
|
|
}
|
|
this.notify('capture:state', this.state());
|
|
}
|
|
|
|
stopClickWatcher() {
|
|
if (this.clickWatcher) {
|
|
try { this.clickWatcher.kill(); } catch { /* already gone */ }
|
|
this.clickWatcher = null;
|
|
}
|
|
this.stopEvdevWatcher();
|
|
this.clickWatcherBuf = '';
|
|
this.linuxEvent = null;
|
|
this.discardPendingRawClick();
|
|
this.lastAcceptedClickByButton.clear();
|
|
}
|
|
|
|
/**
|
|
* Open every readable mouse device node and turn button-down events into
|
|
* onOsClick calls. One physical mouse is normally one node; the leading-edge
|
|
* debounce in onOsClick collapses any cross-node duplicates. Frames are still
|
|
* served by the stream backend; this only supplies the click *trigger*.
|
|
*/
|
|
startEvdevWatcher() {
|
|
const nodes = readableEvdevMouseNodes();
|
|
this.evdevStreams = [];
|
|
for (const node of nodes) {
|
|
try {
|
|
const stream = fs.createReadStream(node);
|
|
let buf = Buffer.alloc(0);
|
|
stream.on('data', (chunk) => {
|
|
buf = buf.length ? Buffer.concat([buf, chunk]) : chunk;
|
|
const { presses, rest } = decodeEvdevButtonPresses(buf);
|
|
buf = rest;
|
|
for (const button of presses) this.onOsClick(Date.now(), null, button);
|
|
});
|
|
// A device can disappear (unplugged); just drop that stream.
|
|
stream.on('error', () => {});
|
|
this.evdevStreams.push(stream);
|
|
} catch {
|
|
// Node became unreadable between enumeration and open — skip it.
|
|
}
|
|
}
|
|
if (!this.evdevStreams.length) {
|
|
console.error('[stepforge] no readable mouse input devices — add your user to the "input" group for per-click capture: sudo usermod -aG input "$USER" (then log out and back in)');
|
|
} else {
|
|
console.log(`[stepforge] per-click capture via evdev on ${this.evdevStreams.length} device(s)${this.onWayland() ? ' (Wayland: no click marker)' : ''}`);
|
|
}
|
|
}
|
|
|
|
stopEvdevWatcher() {
|
|
if (!this.evdevStreams) return;
|
|
for (const stream of this.evdevStreams) {
|
|
try { stream.destroy(); } catch { /* already closed */ }
|
|
}
|
|
this.evdevStreams = null;
|
|
}
|
|
|
|
/**
|
|
* Buffer stdout chunks and only parse complete lines: a chunk boundary
|
|
* can split an event line in half, which used to corrupt press/release
|
|
* parsing and swallow clicks.
|
|
*/
|
|
ingestClickWatcherChunk(chunk, platform = process.platform) {
|
|
this.clickWatcherBuf += String(chunk);
|
|
const cut = this.clickWatcherBuf.lastIndexOf('\n');
|
|
if (cut === -1) return;
|
|
const complete = this.clickWatcherBuf.slice(0, cut);
|
|
this.clickWatcherBuf = this.clickWatcherBuf.slice(cut + 1);
|
|
this.processClickWatcherData(complete, platform);
|
|
}
|
|
|
|
processClickWatcherData(text, platform = process.platform) {
|
|
const lines = String(text).split(/\r?\n/);
|
|
if (platform === 'linux') {
|
|
// xinput test-xi2 --root prints each event as a multi-line block:
|
|
//
|
|
// EVENT type 4 (ButtonPress) EVENT type 15 (RawButtonPress)
|
|
// device: 11 (10) device: 11 (11)
|
|
// detail: 1 detail: 1
|
|
// root: 644.52/343.55 valuators: …
|
|
//
|
|
// Regular (non-raw) blocks carry the event-time root coordinates —
|
|
// exactly what the click marker needs, because a cursor read at parse
|
|
// time drifts whenever delivery is delayed or the pointer keeps
|
|
// moving after the click. Raw blocks have no coordinates, but on many
|
|
// servers they are the only representation delivered for the root
|
|
// window, so both kinds must fire. One physical press can produce
|
|
// *both* representations; that duplication is resolved structurally
|
|
// in fireLinuxClick (raw press briefly waits for its regular twin and
|
|
// they merge into one click), never by a time-only debounce that
|
|
// could swallow legitimate fast clicks.
|
|
for (const line of lines) {
|
|
if (!line) continue;
|
|
const header = /EVENT type \d+ \(([A-Za-z]+)\)/.exec(line);
|
|
if (header) {
|
|
this.finishLinuxEvent();
|
|
const name = header[1];
|
|
this.linuxEvent = /ButtonPress$/.test(name)
|
|
? { name, raw: /^Raw/.test(name), button: null, at: Date.now(), fired: false }
|
|
: null;
|
|
continue;
|
|
}
|
|
const ev = this.linuxEvent;
|
|
if (!ev || ev.fired) continue;
|
|
const detail = /detail:\s*(\d+)/.exec(line);
|
|
if (detail) {
|
|
ev.button = Number(detail[1]);
|
|
if (ev.button >= 4 && ev.button <= 7) {
|
|
// Scroll-wheel ticks (X11 buttons 4-7) are not clicks.
|
|
this.linuxEvent = null;
|
|
} else if (ev.raw) {
|
|
// Raw blocks never carry coordinates; this one is complete.
|
|
ev.fired = true;
|
|
this.linuxEvent = null;
|
|
this.fireLinuxClick(ev.at, null, ev.button, { raw: true });
|
|
}
|
|
continue;
|
|
}
|
|
const root = /root:\s*(-?[\d.]+)\/(-?[\d.]+)/.exec(line);
|
|
if (root && !ev.raw && ev.button != null) {
|
|
ev.fired = true;
|
|
this.linuxEvent = null;
|
|
this.fireLinuxClick(ev.at, {
|
|
x: Math.round(parseFloat(root[1])),
|
|
y: Math.round(parseFloat(root[2])),
|
|
}, ev.button, { raw: false });
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
if (platform === 'win32') {
|
|
for (const line of lines) {
|
|
const trimmed = line.trim();
|
|
const clickM = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(trimmed);
|
|
if (clickM) {
|
|
const osPoint = clickM[1] === undefined ? null : { x: Number(clickM[1]), y: Number(clickM[2]) };
|
|
const eventAt = clickM[4] === undefined ? Date.now() : Number(clickM[4]);
|
|
this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, clickM[3] || 'mouse');
|
|
continue;
|
|
}
|
|
const keyM = /^KEY\s+(\S+)\s+(\d+)\s*$/.exec(trimmed);
|
|
if (keyM) {
|
|
this.onKeyboardEvent('KEY', keyM[1], Number(keyM[2]));
|
|
continue;
|
|
}
|
|
const charM = /^CHAR\s+(\d+)\s+(\d+)\s*$/.exec(trimmed);
|
|
if (charM) {
|
|
this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2]));
|
|
continue;
|
|
}
|
|
// CTX is emitted just before its paired CLICK from MouseHookCallback.
|
|
const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+\d+$/.exec(trimmed);
|
|
if (ctxM) {
|
|
const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8');
|
|
this._pendingWindowContext = {
|
|
windowTitle: decode(ctxM[1]),
|
|
appName: decode(ctxM[2]),
|
|
};
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A new event header arrived while a press block was still open: the block
|
|
* ended without the line we fire on. Old xinput builds sometimes omit
|
|
* detail lines entirely — treat such a press as a plain click rather than
|
|
* dropping it.
|
|
*/
|
|
finishLinuxEvent() {
|
|
const ev = this.linuxEvent;
|
|
this.linuxEvent = null;
|
|
if (!ev || ev.fired) return;
|
|
if (ev.button == null) {
|
|
this.onOsClick(ev.at, null, 'mouse');
|
|
} else if (!ev.raw) {
|
|
// Regular press whose root line never showed up — fire without
|
|
// coordinates; onOsClick falls back to a cursor read.
|
|
this.fireLinuxClick(ev.at, null, ev.button, { raw: false });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Funnel for parsed Linux button presses. Raw and regular blocks for the
|
|
* same physical press are merged here: a raw press (no coordinates) is
|
|
* held for LINUX_CLICK_TWIN_MS; if the regular twin (with root
|
|
* coordinates) arrives inside that window the pair fires once, with the
|
|
* raw block's earlier timestamp and the regular block's coordinates.
|
|
* Distinct presses always fire — there is no time-based dropping.
|
|
*/
|
|
fireLinuxClick(at, osPoint, button, { raw = false } = {}) {
|
|
const pending = this.pendingRawClick;
|
|
if (raw) {
|
|
// Two raw presses can't be one click — release the held one first.
|
|
this.flushPendingRawClick();
|
|
const entry = { button, at, timer: null };
|
|
entry.timer = setTimeout(() => {
|
|
if (this.pendingRawClick !== entry) return;
|
|
this.pendingRawClick = null;
|
|
this.onOsClick(entry.at, null, `button-${entry.button}`);
|
|
}, LINUX_CLICK_TWIN_MS);
|
|
if (entry.timer.unref) entry.timer.unref();
|
|
this.pendingRawClick = entry;
|
|
return;
|
|
}
|
|
if (pending && pending.button === button) {
|
|
// The regular twin of the held raw press: one physical click.
|
|
this.pendingRawClick = null;
|
|
clearTimeout(pending.timer);
|
|
this.onOsClick(Math.min(pending.at, at), osPoint, `button-${button}`);
|
|
return;
|
|
}
|
|
this.onOsClick(at, osPoint, `button-${button}`);
|
|
}
|
|
|
|
/** Fire the held raw press immediately (its twin is not coming). */
|
|
flushPendingRawClick() {
|
|
const pending = this.pendingRawClick;
|
|
if (!pending) return;
|
|
this.pendingRawClick = null;
|
|
clearTimeout(pending.timer);
|
|
this.onOsClick(pending.at, null, `button-${pending.button}`);
|
|
}
|
|
|
|
discardPendingRawClick() {
|
|
if (!this.pendingRawClick) return;
|
|
clearTimeout(this.pendingRawClick.timer);
|
|
this.pendingRawClick = null;
|
|
}
|
|
|
|
/** Debounce window in ms (capture.clickDebounceMs, default 200). */
|
|
clickDebounceMs() {
|
|
const raw = this.settings.get('capture.clickDebounceMs');
|
|
const v = Number(raw);
|
|
return raw != null && Number.isFinite(v) && v >= 0 ? v : DEFAULT_CLICK_DEBOUNCE_MS;
|
|
}
|
|
|
|
onOsClick(at = Date.now(), osPoint = null, button = 'mouse') {
|
|
if (!this.session || this.session.paused) return;
|
|
// Recording isn't live until the window is hidden and the buffer primed
|
|
// (see armRecording). Clicks during warmup land on the still-visible app
|
|
// window, not the user's work, so ignore them rather than capturing junk.
|
|
if (this.warmingUp) {
|
|
clog('click@', Number.isFinite(at) ? at : Date.now(), button, 'ignored — still warming up');
|
|
return;
|
|
}
|
|
const clickAt = Number.isFinite(at) ? at : Date.now();
|
|
// Leading-edge debounce: ignore a click that lands within the debounce
|
|
// window of the last accepted click of the same button. This makes fast
|
|
// / accidental repeat clicks register once, while two deliberate clicks
|
|
// spaced more than the window apart each register (one step per click).
|
|
if (this.isDebouncedClick(clickAt, button)) {
|
|
clog('click@', clickAt, button, 'debounced (within', this.clickDebounceMs(), 'ms of last accepted)');
|
|
return;
|
|
}
|
|
// Prefer the position the watcher sampled with the button-down event
|
|
// (physical px -> DIP); otherwise read the cursor synchronously, right
|
|
// now, so the marker lands where the user clicked even if the shot
|
|
// itself takes a moment to grab. (Clicks on StepForge itself are
|
|
// filtered by the cursor-position check in sessionCapture, not by
|
|
// window focus — WSLg reports focus unreliably.)
|
|
let clickPos = osPoint ? this.osPointToDip(osPoint) : null;
|
|
// Read the live cursor as a fallback only off Wayland: Wayland refuses to
|
|
// report the global pointer position (getCursorScreenPoint returns 0,0), so
|
|
// a fallback there would stamp every click marker in the top-left corner.
|
|
// Leaving clickPos null means the step is captured with no (wrong) marker.
|
|
if (!clickPos && !this.onWayland()) clickPos = this.screen.getCursorScreenPoint();
|
|
clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos);
|
|
this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y)
|
|
? { x: osPoint.x, y: osPoint.y }
|
|
: null;
|
|
this.enqueueClickCapture(clickPos, clickAt, button || 'mouse');
|
|
}
|
|
|
|
/**
|
|
* Whether this click should be dropped by the debounce. A click is dropped
|
|
* only when it follows the last *accepted* click of the same button by
|
|
* less than the debounce window — so the window is measured from accepted
|
|
* clicks, never from dropped ones, and a run of fast clicks can't push the
|
|
* next deliberate click out indefinitely. Accepting a click records it as
|
|
* the new reference point. Different buttons debounce independently.
|
|
*/
|
|
isDebouncedClick(at, button) {
|
|
const key = button || 'mouse';
|
|
const windowMs = this.clickDebounceMs();
|
|
const last = this.lastAcceptedClickByButton.get(key);
|
|
if (last != null && at >= last && at - last < windowMs) return true;
|
|
this.lastAcceptedClickByButton.set(key, at);
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Physical (OS event) pixels -> DIP. Windows exposes the canonical
|
|
* conversion; on Linux/X11 it is reconstructed from display geometry (see
|
|
* app/coords.js). Without this, the click marker drifts on any display
|
|
* scaled away from 100% and on secondary monitors.
|
|
*/
|
|
osPointToDip(osPoint) {
|
|
let geometryDip = null;
|
|
try {
|
|
const displays = this.screen && typeof this.screen.getAllDisplays === 'function'
|
|
? this.screen.getAllDisplays()
|
|
: [];
|
|
geometryDip = physicalToDip(osPoint, displays);
|
|
} catch {
|
|
geometryDip = null;
|
|
}
|
|
if (this.screen && typeof this.screen.screenToDipPoint === 'function') {
|
|
try {
|
|
const dip = this.screen.screenToDipPoint(osPoint);
|
|
if (dip && Number.isFinite(dip.x) && Number.isFinite(dip.y)) {
|
|
if (!geometryDip) return dip;
|
|
const offByX = Math.abs(dip.x - geometryDip.x);
|
|
const offByY = Math.abs(dip.y - geometryDip.y);
|
|
// Some Windows/Electron combinations have been observed to return a
|
|
// raw physical point here. That keeps the click marker off-screen on
|
|
// scaled displays, so trust the geometry path when the two disagree
|
|
// by more than a tiny rounding margin.
|
|
if (offByX <= 1 && offByY <= 1) return dip;
|
|
return geometryDip;
|
|
}
|
|
} catch { /* fall through to manual conversion */ }
|
|
}
|
|
return geometryDip || osPoint;
|
|
}
|
|
|
|
/**
|
|
* Serialize click captures: a click that lands while an earlier capture is
|
|
* still being stored queues behind it instead of being dropped by the
|
|
* "capture already in progress" guard. The marker position was already
|
|
* read at click time, so a queued step still circles the right spot.
|
|
*
|
|
* Crucially, only the *storing* is serialized. The click is paired with
|
|
* its frame right here, at event time: behind a slow store or PNG encode
|
|
* the queue can run seconds late, and a frame request issued that late
|
|
* could find the click-time frame already evicted from the ring buffer.
|
|
* Eager pairing keeps one-click-one-frame semantics intact no matter how
|
|
* fast the user clicks or how slow the encoder is.
|
|
*/
|
|
enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') {
|
|
const osPoint = this.pendingClickOsPoint && Number.isFinite(this.pendingClickOsPoint.x) && Number.isFinite(this.pendingClickOsPoint.y)
|
|
? this.pendingClickOsPoint
|
|
: null;
|
|
this.pendingClickOsPoint = null;
|
|
const clickMeta = {
|
|
at: Number.isFinite(clickAt) ? clickAt : Date.now(),
|
|
button: button || 'mouse',
|
|
};
|
|
if (osPoint) {
|
|
clickMeta.osPoint = { x: osPoint.x, y: osPoint.y };
|
|
}
|
|
// Snapshot keyboard context accumulated since the last capture.
|
|
clickMeta.keyContext = this.snapshotKeyContext();
|
|
// Attach the window + element context emitted by the click watcher.
|
|
if (this._pendingWindowContext) {
|
|
clickMeta.windowContext = this._pendingWindowContext;
|
|
this._pendingWindowContext = null;
|
|
}
|
|
if (this.session && !this.session.paused && !this.userIsInApp()) {
|
|
// The guide id pins the click to its recording so it can still be
|
|
// stored if the session stops while this click waits in the queue.
|
|
clickMeta.guideId = this.session.guideId;
|
|
clickMeta.framePromise = this.frameForClick(clickPos, clickMeta.at)
|
|
.catch(() => null);
|
|
}
|
|
this.clickQueue = this.clickQueue
|
|
.then(() => this.sessionCapture('click', clickPos, clickMeta))
|
|
.catch(() => {});
|
|
return this.clickQueue;
|
|
}
|
|
|
|
// --- Keyboard context tracking -------------------------------------------
|
|
|
|
onKeyboardEvent(type, data, eventAt) {
|
|
const now = Number.isFinite(eventAt) ? eventAt : Date.now();
|
|
// Discard stale typing that happened more than 8 seconds ago (user moved on).
|
|
if (now - this._keyLastAt > 8000) {
|
|
this._keyBuffer = '';
|
|
}
|
|
this._keyLastAt = now;
|
|
|
|
if (type === 'CHAR') {
|
|
// Raw typed-text capture is off by default: it can record passwords or
|
|
// other secrets. Only buffer printable characters when the user has
|
|
// explicitly opted in via capture.captureTypedText. Shortcut/navigation
|
|
// keys below are unaffected.
|
|
if (!this.settings.get('capture.captureTypedText')) return;
|
|
const ch = typeof data === 'number' ? String.fromCharCode(data) : String(data);
|
|
this._keyBuffer = (this._keyBuffer + ch).slice(-200);
|
|
} else if (type === 'KEY') {
|
|
const key = String(data);
|
|
if (key === 'Backspace') {
|
|
this._keyBuffer = this._keyBuffer.slice(0, -1);
|
|
} else if (key === 'Escape') {
|
|
this._keyBuffer = '';
|
|
} else if (key === 'Enter') {
|
|
// Keep typed text — Enter often submits what was typed.
|
|
} else {
|
|
// It's a modifier+key shortcut (Ctrl+T etc.).
|
|
this._lastShortcut = key;
|
|
this._keyBuffer = ''; // a shortcut resets the typed-text buffer
|
|
}
|
|
}
|
|
}
|
|
|
|
snapshotKeyContext() {
|
|
const ctx = {
|
|
recentTyped: this._keyBuffer.trim(),
|
|
recentShortcut: this._lastShortcut,
|
|
};
|
|
// Reset after snapshot so each step gets its own context.
|
|
this._keyBuffer = '';
|
|
this._lastShortcut = '';
|
|
return ctx;
|
|
}
|
|
|
|
async captureCurrentFrame(mode, capturePoint = null, startedAt = Date.now()) {
|
|
const grabbed = await this.grab(mode, capturePoint);
|
|
return {
|
|
mode,
|
|
// Keep the raw image and defer PNG encoding to storeFrameAsStep:
|
|
// toPNG() on a full-resolution frame blocks the main thread for
|
|
// hundreds of ms, and doing it every frame-loop tick starved the
|
|
// event loop so badly that click events arrived seconds late.
|
|
// Encoding once per *stored* step is cheap; encoding per grab is not.
|
|
image: grabbed.image,
|
|
size: grabbed.image.getSize(),
|
|
display: grabbed.display,
|
|
cursor: capturePoint || grabbed.cursor,
|
|
startedAt,
|
|
capturedAt: Date.now(),
|
|
};
|
|
}
|
|
|
|
async buildStepMeta(mode, frame, clickPos = null, clickMeta = null) {
|
|
try {
|
|
if (this.textIntel && typeof this.textIntel.buildCaptureContext === 'function') {
|
|
return await this.textIntel.buildCaptureContext({ mode, frame, clickPos, clickMeta });
|
|
}
|
|
} catch {
|
|
// fall back
|
|
}
|
|
return { title: this.autoTitle(mode), captureMetadata: null };
|
|
}
|
|
|
|
async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) {
|
|
if (!frame) return { ok: false, reason: 'no capture frame available' };
|
|
const annotations = [];
|
|
// The click position (DIP, read at event time) wins over the frame's
|
|
// grab-time cursor; stream-backend frames carry no cursor at all.
|
|
const cursor = clickPos || frame.cursor || null;
|
|
if (cursor && mode !== 'window' && this.settings.get('capture.clickMarker')) {
|
|
const fx = (cursor.x - frame.display.bounds.x) / frame.display.bounds.width;
|
|
const fy = (cursor.y - frame.display.bounds.y) / frame.display.bounds.height;
|
|
if (fx >= 0 && fx <= 1 && fy >= 0 && fy <= 1) {
|
|
const d = 0.035;
|
|
annotations.push({
|
|
type: 'oval',
|
|
x: fx - d / 2, y: fy - (d * frame.size.width / frame.size.height) / 2,
|
|
w: d, h: d * frame.size.width / frame.size.height,
|
|
style: {
|
|
stroke: this.settings.get('capture.clickMarkerColor') || '#E5484D',
|
|
strokeWidth: 4, fill: 'transparent',
|
|
},
|
|
});
|
|
}
|
|
}
|
|
|
|
const { title, captureMetadata } = await this.buildStepMeta(mode, frame, clickPos, clickMeta);
|
|
const step = this.store.addStep(guideId, {
|
|
title,
|
|
captureMetadata,
|
|
annotations,
|
|
focusedView: {
|
|
enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),
|
|
zoom: 1, panX: 0.5, panY: 0.5,
|
|
},
|
|
}, frame.png || frame.image.toPNG(), frame.size);
|
|
return { ok: true, step };
|
|
}
|
|
|
|
autoTitle(mode) {
|
|
return DEFAULT_CAPTURE_TITLES[mode] || 'Capture';
|
|
}
|
|
|
|
/** Grab the screen/window image as { image, display } or throw. */
|
|
async grab(mode, cursorPoint = null) {
|
|
const cursor = cursorPoint || this.screen.getCursorScreenPoint();
|
|
const display = this.screen.getDisplayNearestPoint(cursor);
|
|
const { width, height } = display.size;
|
|
const scale = display.scaleFactor || 1;
|
|
// Ask for both kinds: some compositors (WSLg/Wayland portals) expose no
|
|
// individual window sources, so window mode falls back to the screen.
|
|
const sources = await desktopCapturer.getSources({
|
|
types: mode === 'window' ? ['window', 'screen'] : ['screen'],
|
|
thumbnailSize: { width: Math.round(width * scale), height: Math.round(height * scale) },
|
|
});
|
|
if (!sources.length) throw new Error('no capture sources available (portal/permissions?)');
|
|
|
|
let source = null;
|
|
if (mode === 'window') {
|
|
const win = this.getWindow();
|
|
const ownTitle = win ? win.getTitle() : '';
|
|
const windows = sources.filter((s) => s.id.startsWith('window:'));
|
|
source = windows.find((s) => s.name && s.name !== ownTitle && !/stepforge/i.test(s.name))
|
|
|| windows[0]
|
|
|| sources.find((s) => s.id.startsWith('screen:'));
|
|
} else {
|
|
const screens = sources.filter((s) => s.id.startsWith('screen:'));
|
|
source = screens.find((s) => String(s.display_id) === String(display.id)) || screens[0] || sources[0];
|
|
}
|
|
if (!source) throw new Error('no capture source matched');
|
|
const image = source.thumbnail;
|
|
if (!image || image.isEmpty()) throw new Error('capture returned an empty image');
|
|
return { image, display, cursor };
|
|
}
|
|
|
|
/**
|
|
* Hide the app window while `fn` runs so screenshots show the user's work,
|
|
* not StepForge itself. Restores visibility afterwards.
|
|
*/
|
|
async withWindowHidden(fn, { refocus = true, pauseMs = 350 } = {}) {
|
|
const win = this.getWindow();
|
|
const wasVisible = win && !win.isDestroyed() && win.isVisible() && !win.isMinimized();
|
|
if (wasVisible) {
|
|
win.hide();
|
|
if (pauseMs > 0) {
|
|
await new Promise((r) => setTimeout(r, pauseMs)); // let the compositor repaint
|
|
}
|
|
}
|
|
try {
|
|
return await fn();
|
|
} finally {
|
|
if (wasVisible && win && !win.isDestroyed()) {
|
|
if (refocus) {
|
|
win.show();
|
|
win.focus();
|
|
} else {
|
|
win.showInactive();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Take a screenshot and append it to the guide as a new image step.
|
|
* Adds a click-marker annotation at the cursor position when enabled.
|
|
*/
|
|
async shoot({
|
|
guideId,
|
|
mode = 'fullscreen',
|
|
delayMs = null,
|
|
hideWindow = true,
|
|
refocus = true,
|
|
hideWindowDelayMs = null,
|
|
clickPos = null,
|
|
}) {
|
|
const delay = delayMs == null ? this.settings.get('capture.delayMs') || 0 : delayMs;
|
|
if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay));
|
|
let frame;
|
|
try {
|
|
frame = hideWindow
|
|
? await this.withWindowHidden(() => this.captureCurrentFrame(mode, clickPos), {
|
|
refocus,
|
|
pauseMs: hideWindowDelayMs == null ? 350 : hideWindowDelayMs,
|
|
})
|
|
: await this.captureCurrentFrame(mode, clickPos);
|
|
} catch (err) {
|
|
return { ok: false, reason: err.message };
|
|
}
|
|
return this.storeFrameAsStep(guideId, mode, frame, clickPos);
|
|
}
|
|
|
|
/**
|
|
* Region capture: shoot the full screen, then let the user drag a
|
|
* rectangle in a fullscreen overlay; the crop becomes the step image.
|
|
*/
|
|
async regionCapture(guideId) {
|
|
let grabbed;
|
|
try {
|
|
grabbed = await this.withWindowHidden(() => this.grab('fullscreen'));
|
|
} catch (err) {
|
|
return { ok: false, reason: err.message };
|
|
}
|
|
const { image, display } = grabbed;
|
|
const rect = await this.pickRegion(display, image);
|
|
if (!rect) return { ok: false, reason: 'selection cancelled' };
|
|
|
|
const cropped = image.crop(rect);
|
|
const size = cropped.getSize();
|
|
if (!size.width || !size.height) return { ok: false, reason: 'empty selection' };
|
|
const step = await this.storeFrameAsStep(guideId, 'region', {
|
|
image: cropped,
|
|
size,
|
|
display,
|
|
cursor: null,
|
|
}, null, null);
|
|
return { ok: true, step };
|
|
}
|
|
|
|
/** Fullscreen overlay window that resolves with a crop rect (image px). */
|
|
pickRegion(display, image) {
|
|
return new Promise((resolve) => {
|
|
const overlay = new BrowserWindow({
|
|
x: display.bounds.x,
|
|
y: display.bounds.y,
|
|
width: display.bounds.width,
|
|
height: display.bounds.height,
|
|
frame: false,
|
|
transparent: true,
|
|
alwaysOnTop: true,
|
|
fullscreen: true,
|
|
skipTaskbar: true,
|
|
webPreferences: {
|
|
preload: path.join(__dirname, 'region-preload.js'),
|
|
contextIsolation: true,
|
|
nodeIntegration: false,
|
|
sandbox: true,
|
|
},
|
|
});
|
|
// The overlay may only display region.html; deny navigation/popups.
|
|
require('./security').installWindowSecurity(overlay, 'region');
|
|
let settled = false;
|
|
const finish = (rect) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
if (!overlay.isDestroyed()) overlay.close();
|
|
resolve(rect);
|
|
};
|
|
const { ipcMain } = require('electron');
|
|
const onPick = (event, rect) => {
|
|
if (event.sender !== overlay.webContents) return;
|
|
ipcMain.removeListener('region:picked', onPick);
|
|
if (!rect) return finish(null);
|
|
const imgSize = image.getSize();
|
|
const sx = imgSize.width / display.bounds.width;
|
|
const sy = imgSize.height / display.bounds.height;
|
|
finish({
|
|
x: Math.round(rect.x * sx),
|
|
y: Math.round(rect.y * sy),
|
|
width: Math.round(rect.w * sx),
|
|
height: Math.round(rect.h * sy),
|
|
});
|
|
};
|
|
ipcMain.on('region:picked', onPick);
|
|
overlay.on('closed', () => finish(null));
|
|
overlay.loadFile(path.join(__dirname, 'renderer', 'region.html'));
|
|
});
|
|
}
|
|
}
|
|
|
|
module.exports = CaptureService;
|
|
// Exposed for unit tests (pure, no device access).
|
|
module.exports.decodeEvdevButtonPresses = decodeEvdevButtonPresses;
|