Fixed clicking location part 4
Template tests / tests (push) Successful in 1m52s

This commit is contained in:
Iisyourdad
2026-06-11 16:57:59 -05:00
parent 27439b475d
commit 6682cdae0f
17 changed files with 372 additions and 1006 deletions
+188 -58
View File
@@ -35,6 +35,8 @@ const CLICK_FRAME_MAX_AGE_MS = 600;
// one-off fresh shot.
const CLICK_FRAME_WAIT_MS = 2000;
const CLICK_CAPTURE_HIDE_DELAY_MS = 25;
const RECENT_FRAME_RETENTION_MS = 2000;
const RECENT_FRAME_LIMIT = 8;
function pointInBounds(point, bounds) {
if (!point || !bounds) return false;
@@ -66,13 +68,15 @@ class CaptureService {
this.frameLoopRunning = false;
this.frameWaiters = [];
this.latestFrame = null;
this.lastClickCapture = 0;
this.clickWatcherBuf = '';
this.clickWatcherPendingPress = false;
this.clickWatcherErrTail = '';
this.clickQueue = Promise.resolve();
this.frameLoopInFlight = false;
this.frameLoopGrabStartedAt = null;
this.recentFrames = [];
this.shooting = false;
this.lastClickCaptureByButton = new Map();
}
state() {
@@ -264,7 +268,7 @@ class CaptureService {
}
/** One capture inside the active session (hotkey/click/interval/manual). */
async sessionCapture(trigger = 'hotkey', clickPos = null) {
async sessionCapture(trigger = 'hotkey', clickPos = null, clickMeta = null) {
if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' };
// Automatic triggers stand down while the user is in StepForge, so the
// app stays clickable mid-session and never screenshots itself.
@@ -278,7 +282,8 @@ class CaptureService {
// flight waits for that frame instead of being dropped, so fast
// clicking still yields one step per click.
if (trigger === 'click') {
const frame = await this.frameForClick(clickPos);
const clickAt = clickMeta && Number.isFinite(clickMeta.at) ? clickMeta.at : Date.now();
const frame = await this.frameForClick(clickPos, clickAt);
if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' };
if (frame) {
const result = this.storeFrameAsStep(this.session.guideId, frame.mode, frame, clickPos);
@@ -342,15 +347,17 @@ class CaptureService {
try {
if (!this.shooting) {
this.frameLoopInFlight = true;
this.frameLoopGrabStartedAt = Date.now();
const mode = this.settings.get('capture.mode') || 'fullscreen';
const grabMode = mode === 'region' ? 'fullscreen' : mode;
const frame = await this.captureCurrentFrame(grabMode);
const frame = await this.captureCurrentFrame(grabMode, null, this.frameLoopGrabStartedAt);
if (this.frameLoopRunning) this.acceptFrame(frame);
}
} catch {
// Grab failures are fine — clicks fall back to a one-off fresh shot.
} finally {
this.frameLoopInFlight = false;
this.frameLoopGrabStartedAt = null;
if (this.frameLoopRunning && this.session && !this.session.paused) {
this.frameLoopTimer = setTimeout(tick, FRAME_LOOP_IDLE_MS);
}
@@ -362,6 +369,11 @@ class CaptureService {
/** Store a grabbed frame and hand it to any clicks waiting on it. */
acceptFrame(frame) {
this.latestFrame = frame;
this.recentFrames.push(frame);
const cutoff = Date.now() - RECENT_FRAME_RETENTION_MS;
this.recentFrames = this.recentFrames
.filter((f) => f && f.capturedAt >= cutoff)
.slice(-RECENT_FRAME_LIMIT);
const waiters = this.frameWaiters;
this.frameWaiters = [];
for (const resolve of waiters) resolve(frame);
@@ -388,7 +400,9 @@ class CaptureService {
this.frameLoopTimer = null;
}
this.frameLoopRunning = false;
this.frameLoopGrabStartedAt = null;
this.latestFrame = null;
this.recentFrames = [];
const waiters = this.frameWaiters;
this.frameWaiters = [];
for (const resolve of waiters) resolve(null);
@@ -399,24 +413,34 @@ class CaptureService {
* recent enough, otherwise the next frame the loop delivers. Null when the
* loop isn't running or can't deliver in time.
*/
async frameForClick(clickPos = null) {
async frameForClick(clickPos = null, clickAt = Date.now()) {
const mode = this.settings.get('capture.mode') || 'fullscreen';
const grabMode = mode === 'region' ? 'fullscreen' : mode;
const clickTime = Number.isFinite(clickAt) ? clickAt : Date.now();
// Fast clicks can move to another monitor before the buffered frame is
// consumed; only reuse frames from the clicked display.
const usable = (f) => {
const usable = (f, { allowInFlight = false } = {}) => {
const sameDisplay = !clickPos || pointInBounds(clickPos, f && f.display && f.display.bounds);
const startedAt = Number.isFinite(f && f.startedAt) ? f.startedAt : (f && f.capturedAt);
const completedBeforeClick = Number.isFinite(f && f.capturedAt) && f.capturedAt <= clickTime;
const startedBeforeClick = Number.isFinite(startedAt) && startedAt <= clickTime;
const timingMatches = completedBeforeClick
? clickTime - f.capturedAt <= CLICK_FRAME_MAX_AGE_MS
: allowInFlight && startedBeforeClick && clickTime - startedAt <= CLICK_FRAME_MAX_AGE_MS;
return Boolean(f)
&& f.mode === grabMode
&& Date.now() - f.capturedAt <= CLICK_FRAME_MAX_AGE_MS
&& timingMatches
&& sameDisplay;
};
if (usable(this.latestFrame)) return this.latestFrame;
if (!this.frameLoopRunning || !this.frameLoopInFlight) return null;
const buffered = [...this.recentFrames, this.latestFrame]
.filter((f, i, arr) => f && arr.indexOf(f) === i && usable(f))
.sort((a, b) => b.capturedAt - a.capturedAt)[0];
if (buffered) return buffered;
if (!this.frameLoopRunning || !this.frameLoopInFlight || this.frameLoopGrabStartedAt > clickTime) return null;
const deadline = Date.now() + CLICK_FRAME_WAIT_MS;
while (this.frameLoopRunning && Date.now() < deadline) {
const next = await this.nextFrame(Math.max(1, deadline - Date.now()));
if (usable(next)) return next;
if (usable(next, { allowInFlight: true })) return next;
}
return null;
}
@@ -441,49 +465,148 @@ class CaptureService {
this.ingestClickWatcherChunk(chunk.toString(), 'linux');
});
} else if (process.platform === 'win32') {
// Poll left/right/middle buttons via GetAsyncKeyState. Bit 0x8000 is
// "down right now"; bit 0x0001 is "was pressed since the previous
// poll", which catches clicks shorter than one poll interval. The
// cursor is read in the same poll iteration that sees the press and
// shipped with the event, so the marker position is sampled at the
// click instant (within one ~15ms poll tick) instead of after the
// event has crossed the pipe. Coordinates are physical pixels
// (per-monitor DPI aware); onOsClick converts them to DIPs.
// Use a low-level Windows mouse hook instead of polling
// GetAsyncKeyState. The low bit from GetAsyncKeyState can be consumed
// by other processes and a polling loop can miss short clicks under
// load; WH_MOUSE_LL gives us one event for each button-down, with the
// hook-time cursor position and timestamp.
const ps = `
$ErrorActionPreference = 'Stop'
Add-Type -TypeDefinition @'
using System;
using System.Collections.Concurrent;
using System.Runtime.InteropServices;
public struct SFPoint { public int X; public int Y; }
public static class SFMouse {
[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int vKey);
[DllImport("user32.dll")] public static extern bool GetCursorPos(out SFPoint p);
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
using System.Threading;
public static class SFMouseHook {
private const int WH_MOUSE_LL = 14;
private const int WM_LBUTTONDOWN = 0x0201;
private const int WM_RBUTTONDOWN = 0x0204;
private const int WM_MBUTTONDOWN = 0x0207;
private const int WM_XBUTTONDOWN = 0x020B;
private const long UnixEpochMilliseconds = 62135596800000L;
private static IntPtr hook = IntPtr.Zero;
private static LowLevelMouseProc proc = HookCallback;
private static readonly ConcurrentQueue<string> queue = new ConcurrentQueue<string>();
private static readonly AutoResetEvent signal = new AutoResetEvent(false);
[StructLayout(LayoutKind.Sequential)]
private struct POINT {
public int x;
public int y;
}
[StructLayout(LayoutKind.Sequential)]
private struct MSLLHOOKSTRUCT {
public POINT pt;
public uint mouseData;
public uint flags;
public uint time;
public UIntPtr dwExtraInfo;
}
[StructLayout(LayoutKind.Sequential)]
private struct MSG {
public IntPtr hwnd;
public uint message;
public UIntPtr wParam;
public IntPtr lParam;
public uint time;
public POINT pt;
}
private delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam);
[DllImport("user32.dll", SetLastError = true)]
private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId);
[DllImport("user32.dll", SetLastError = true)]
private static extern bool UnhookWindowsHookEx(IntPtr hhk);
[DllImport("user32.dll")]
private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam);
[DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
private static extern IntPtr GetModuleHandle(string lpModuleName);
[DllImport("user32.dll")]
private static extern int GetMessage(out MSG lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax);
[DllImport("user32.dll")]
private static extern bool TranslateMessage(ref MSG lpMsg);
[DllImport("user32.dll")]
private static extern IntPtr DispatchMessage(ref MSG lpMsg);
[DllImport("user32.dll")]
private static extern bool SetProcessDpiAwarenessContext(IntPtr value);
public static void Run() {
try { SetProcessDpiAwarenessContext(new IntPtr(-4)); } catch { }
Thread writer = new Thread(WriterLoop);
writer.IsBackground = true;
writer.Start();
hook = SetWindowsHookEx(WH_MOUSE_LL, proc, GetModuleHandle(null), 0);
if (hook == IntPtr.Zero) {
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
}
Console.Out.WriteLine("READY");
Console.Out.Flush();
MSG msg;
while (GetMessage(out msg, IntPtr.Zero, 0, 0) > 0) {
TranslateMessage(ref msg);
DispatchMessage(ref msg);
}
UnhookWindowsHookEx(hook);
}
private static void WriterLoop() {
while (true) {
signal.WaitOne();
string line;
while (queue.TryDequeue(out line)) {
Console.Out.WriteLine(line);
}
Console.Out.Flush();
}
}
private static IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
if (nCode >= 0) {
int message = wParam.ToInt32();
string button = ButtonName(message, lParam);
if (button != null) {
MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT));
long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds;
queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs);
signal.Set();
}
}
return CallNextHookEx(hook, nCode, wParam, lParam);
}
private static string ButtonName(int message, IntPtr lParam) {
if (message == WM_LBUTTONDOWN) return "left";
if (message == WM_RBUTTONDOWN) return "right";
if (message == WM_MBUTTONDOWN) return "middle";
if (message == WM_XBUTTONDOWN) {
MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT));
uint xButton = (data.mouseData >> 16) & 0xffff;
return xButton == 1 ? "x1" : "x2";
}
return null;
}
}
'@
try { [void][SFMouse]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch { }
$out = [Console]::Out
$out.WriteLine('READY')
$out.Flush()
$vks = 0x01, 0x02, 0x04
$down = @{}
foreach ($vk in $vks) { $down[$vk] = $false }
$pt = New-Object SFPoint
while ($true) {
foreach ($vk in $vks) {
$s = [SFMouse]::GetAsyncKeyState($vk)
$isDown = ($s -band 0x8000) -ne 0
$tapped = ($s -band 1) -ne 0
if (($isDown -or $tapped) -and -not $down[$vk]) {
[void][SFMouse]::GetCursorPos([ref]$pt)
$out.WriteLine('CLICK ' + $pt.X + ' ' + $pt.Y)
$out.Flush()
}
$down[$vk] = $isDown
}
Start-Sleep -Milliseconds 10
}`;
this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', ps], {
[SFMouseHook]::Run()
`;
this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', ps], {
stdio: ['ignore', 'pipe', 'pipe'],
windowsHide: true,
});
@@ -536,6 +659,7 @@ while ($true) {
}
this.clickWatcherBuf = '';
this.clickWatcherPendingPress = false;
this.lastClickCaptureByButton.clear();
}
/**
@@ -572,7 +696,7 @@ while ($true) {
if (detail) {
this.clickWatcherPendingPress = false;
const button = Number(detail[1]);
if (button < 4 || button > 7) this.onOsClick();
if (button < 4 || button > 7) this.onOsClick(Date.now(), null, `button-${button}`);
} else if (line.includes('EVENT type')) {
// Next event arrived without a detail line in between — treat the
// pending press as a plain click rather than dropping it.
@@ -584,21 +708,25 @@ while ($true) {
}
if (platform === 'win32') {
for (const line of lines) {
const m = /CLICK(?:\s+(-?\d+)\s+(-?\d+))?/.exec(line);
const m = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(line.trim());
if (m) {
const osPoint = m[1] === undefined ? null : { x: Number(m[1]), y: Number(m[2]) };
this.onOsClick(Date.now(), osPoint);
const eventAt = m[4] === undefined ? Date.now() : Number(m[4]);
this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, m[3] || 'mouse');
}
}
}
}
onOsClick(at = Date.now(), osPoint = null) {
onOsClick(at = Date.now(), osPoint = null, button = 'mouse') {
if (!this.session || this.session.paused) return;
if (at - this.lastClickCapture < CLICK_DEBOUNCE_MS) return;
this.lastClickCapture = at;
// Prefer the position the watcher sampled in the same poll that saw the
// press (physical px → DIP); otherwise read the cursor synchronously,
const clickAt = Number.isFinite(at) ? at : Date.now();
const debounceKey = button || 'mouse';
const last = this.lastClickCaptureByButton.get(debounceKey) || 0;
if (clickAt >= last && clickAt - last < CLICK_DEBOUNCE_MS) return;
this.lastClickCaptureByButton.set(debounceKey, clickAt);
// Prefer the position the watcher sampled with the button-down event
// (physical px -> DIP); otherwise read the cursor synchronously,
// right now, so the marker lands where the user clicked even if the
// shot itself takes a moment to grab. (Clicks on StepForge itself are
// filtered by the cursor-position check in sessionCapture, not by
@@ -610,7 +738,7 @@ while ($true) {
: osPoint;
}
if (!clickPos) clickPos = screen.getCursorScreenPoint();
this.enqueueClickCapture(clickPos);
this.enqueueClickCapture(clickPos, clickAt, debounceKey);
}
/**
@@ -619,14 +747,15 @@ while ($true) {
* "capture already in progress" guard. The marker position was already
* read at click time, so a queued step still circles the right spot.
*/
enqueueClickCapture(clickPos) {
enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') {
const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' };
this.clickQueue = this.clickQueue
.then(() => this.sessionCapture('click', clickPos))
.then(() => this.sessionCapture('click', clickPos, clickMeta))
.catch(() => {});
return this.clickQueue;
}
async captureCurrentFrame(mode, capturePoint = null) {
async captureCurrentFrame(mode, capturePoint = null, startedAt = Date.now()) {
const grabbed = await this.grab(mode, capturePoint);
return {
mode,
@@ -634,6 +763,7 @@ while ($true) {
size: grabbed.image.getSize(),
display: grabbed.display,
cursor: capturePoint || grabbed.cursor,
startedAt,
capturedAt: Date.now(),
};
}
+76 -76
View File
@@ -1,99 +1,99 @@
{
"format": "stepforge-artifacts-manifest",
"version": 1,
"generatedAt": "2026-06-11T02:38:15.736Z",
"generatedAt": "2026-06-11T21:54:13.294Z",
"packageVersion": "0.1.0",
"files": [
{
"kind": "artifact",
"path": "artifacts/stepforge_0.1.0_amd64.deb",
"size": 103669070,
"sha256": "445758cd92c80c4fc2c5bec5929e62ab33dd83583da859bcba66bda9390ba077"
"size": 103691640,
"sha256": "320faa345f5997905fdc831c045dbe243490a7b4328680d105934f2aec1b4ffb"
},
{
"kind": "artifact",
"path": "artifacts/stepforge_0.1.0_linux-x64.tar.gz",
"size": 139348957,
"sha256": "beb78b999ae40faf7c65d6ccf9bba23f7955b2d7fdc4214e0e9a94e4e3faec74"
"size": 139378628,
"sha256": "32971595d4df40b429cb41ba644e57b84351ec1239b4bbe8d5b82fe3b01a4cc9"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/guide.json",
"size": 928,
"sha256": "62cfedbb54089f92b8f52493255fab8b37a201cd12746dc8eff2e40be6978dda"
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/guide.json",
"size": 843,
"sha256": "0d7760246d96a85a4d79c15c1ab1a7e481bd79e1d8ff29d177ac6d35ffe6cde6"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-277488ad-bf0c-488d-a6a2-e54be3e41241/original.png",
"size": 13602,
"sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-277488ad-bf0c-488d-a6a2-e54be3e41241/step.json",
"size": 1975,
"sha256": "d5a7f45ef230d080cca8ecf13b20bfefc7294342eed375db2f0f7d576df6ef1b"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-277488ad-bf0c-488d-a6a2-e54be3e41241/working.png",
"size": 13602,
"sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-2a243bf5-6b4f-406c-ba43-b66ba11aea16/step.json",
"size": 547,
"sha256": "e3b3ce474ec4234f083deac7304cb4d665f3f2ccdcf39c60a78ecc71312ee170"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-87ca03b4-fccd-46c1-97f8-6c1f7176a569/step.json",
"size": 784,
"sha256": "f6e96cefa9861a0926ae65af738138d6b0881872c92ca5a352788c6ac8c921f0"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-b4f37096-7cf4-4e10-b7a8-05b7c3f517ea/original.png",
"size": 14031,
"sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-b4f37096-7cf4-4e10-b7a8-05b7c3f517ea/step.json",
"size": 1886,
"sha256": "27884ea4829c4269e25e742d2bdce3da417f6ef391f48a4fa932c06131663447"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-b4f37096-7cf4-4e10-b7a8-05b7c3f517ea/working.png",
"size": 14031,
"sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-da491a7e-52b7-437a-baa6-c9aa520b3b3f/step.json",
"size": 521,
"sha256": "343e0e6344b7f0724bd1eea6d38da9e97e0da24b8f86707b2eec14e99e0ef0bf"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-e3373723-c536-4bc9-b8b4-55cd9b8ba525/original.png",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/original.png",
"size": 13643,
"sha256": "09e12f935511bb6fabb5637501aa7743516b96d990adfab62ccfa311a7b60606"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-e3373723-c536-4bc9-b8b4-55cd9b8ba525/step.json",
"size": 1609,
"sha256": "a6932c7b03650a3b8cc1b5b46f3759855baf3495d5ee87ac6a09995a91b6629b"
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/step.json",
"size": 1593,
"sha256": "36deb8a8b51952a668e871a03b1c7fba2aaf72b1722fd2d88c7e5d5cbbd8da91"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-e3373723-c536-4bc9-b8b4-55cd9b8ba525/working.png",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/working.png",
"size": 13643,
"sha256": "09e12f935511bb6fabb5637501aa7743516b96d990adfab62ccfa311a7b60606"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/original.png",
"size": 14031,
"sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/step.json",
"size": 1873,
"sha256": "4b0b3b74f851d034a2769c69df2e9d0428373ae15009fb3e2100afb5c10f7ebb"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/working.png",
"size": 14031,
"sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02a-permission-prompt/step.json",
"size": 763,
"sha256": "e8539addbe730e3a1baaa6898bf9779d2ce80564aecb3f196619d8c7ff67cbae"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/original.png",
"size": 13602,
"sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/step.json",
"size": 1968,
"sha256": "6f1cbb9d2ed32b89dec3d221822c6e973dbba42c11e93c2b35499179e8481532"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/working.png",
"size": 13602,
"sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-04-legacy-note/step.json",
"size": 506,
"sha256": "c6f78405f86f4183612f5865820b2454471cbc47975e9c15e055e7bf742b1eba"
},
{
"kind": "sample",
"path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-05-deprecated-flow/step.json",
"size": 536,
"sha256": "709887c0a5debddc851216920ee3f6f2766e986059b21ac399e2533611ab5aed"
},
{
"kind": "sample",
"path": "../examples/sample-exports/docx/reset-a-password-in-admin-portal.docx",
@@ -109,20 +109,20 @@
{
"kind": "sample",
"path": "../examples/sample-exports/html-rich/reset-a-password-in-admin-portal-rich.html",
"size": 149943,
"sha256": "e3c636d463c4db277f89a5330559661b6c9db193f94de9c4ad6801a88e3a7ff4"
"size": 149884,
"sha256": "70695d55c37d69abb191e65675f726eaa7aea6686a15a2e7d387195441e3ca8c"
},
{
"kind": "sample",
"path": "../examples/sample-exports/html-simple/reset-a-password-in-admin-portal.html",
"size": 146646,
"sha256": "2bf28616ed9ce47527e6e6da950b495e914b25a04376fb90e40e787ad0f4819a"
"sha256": "860c774b9b8bfd821b22deadc21f8021dc66bc9cacf43702eecbf55e944c6a9a"
},
{
"kind": "sample",
"path": "../examples/sample-exports/image-bundle/reset-a-password-in-admin-portal-bundle.json",
"size": 779,
"sha256": "38bdbf55d25f1620eb824192472fcf8faa65f38f4ef340bcd98e0668a66bdefe"
"sha256": "0108a63331925ea1fdd03326dd37dce1af3642a02d167e583d0d67b50fce27fa"
},
{
"kind": "sample",
@@ -146,7 +146,7 @@
"kind": "sample",
"path": "../examples/sample-exports/json/reset-a-password-in-admin-portal.json",
"size": 6740,
"sha256": "434d5d41df7983686da8057a9939a57739fa9089dd2957862ccb810963f66532"
"sha256": "4a27376e75c3bb33fa8e1c4117c3314a04ec4766d67a9b5c729180d116d384d1"
},
{
"kind": "sample",
@@ -205,14 +205,14 @@
{
"kind": "sample",
"path": "../examples/sample-guide.sfgz",
"size": 88968,
"sha256": "d041d4994f8d1a2f41a1c818ea706f64bb789eb4a3b903f67f86d6305339e412"
"size": 88427,
"sha256": "313b88f48e53e5ad7fb4e0a8189700ba0f6be642ec2311b1a2fe7ea4f3dd0481"
},
{
"kind": "sample",
"path": "../examples/sample-manifest.json",
"size": 1186,
"sha256": "c88b283799b932cb3b23e096288222eed84beaaf6bf8ce0fecdf05de89cc2c1b"
"size": 1163,
"sha256": "cb2920e7500758074f1f54867db1ed187d3304c8badbceb06fd611057bd6fe5d"
}
]
}
+1 -1
View File
@@ -1,7 +1,7 @@
# StepForge Build Report
Version: 0.1.0
Generated: 2026-06-11T02:38:15.730Z
Generated: 2026-06-11T21:54:13.292Z
Host: linux x64 (node v20.20.2)
## Outputs
+1 -840
View File
@@ -1,842 +1,3 @@
'use strict';
const path = require('node:path');
const { spawn, execFileSync } = require('node:child_process');
const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu, Notification } = require('electron');
const { expandPlaceholders } = require('../core/placeholders');
const raster = require('../core/raster');
const { encodePng } = require('../core/png');
/**
* Capture service: full-screen, active-window, and region capture via
* Electron's desktopCapturer, plus a click-marker annotation at the cursor
* position and a capture session (start/pause/resume/finish).
*
* A session captures continuously, with three triggers layered by what the
* platform supports:
* - click-capture via an OS adapter (xinput on X11, PowerShell on Windows),
* - a global hotkey (unreliable on some Wayland compositors),
* - interval auto-capture as the always-works fallback.
*
* Note: under Wayland/WSLg, screen capture may require portal support; all
* failures surface as { ok: false, reason } instead of crashing.
*/
// Dedupe duplicate watcher events for one physical click while still
// allowing intentionally fast clicking.
const CLICK_DEBOUNCE_MS = 40;
// Idle gap between frame-loop grabs. Keep this at zero so the buffered
// frame stays as close to real time as possible while recording.
const FRAME_LOOP_IDLE_MS = 0;
// A buffered frame older than this is too stale to pass off as "the screen
// at the instant of the click".
const CLICK_FRAME_MAX_AGE_MS = 600;
// How long a click waits for the in-flight grab before falling back to a
// one-off fresh shot.
const CLICK_FRAME_WAIT_MS = 2000;
const CLICK_CAPTURE_HIDE_DELAY_MS = 25;
function pointInBounds(point, bounds) {
if (!point || !bounds) return false;
return point.x >= bounds.x
&& point.x <= bounds.x + bounds.width
&& point.y >= bounds.y
&& point.y <= bounds.y + bounds.height;
}
function hasBinary(name) {
try {
execFileSync('which', [name], { stdio: 'pipe' });
return true;
} catch {
return false;
}
}
class CaptureService {
constructor({ store, settings, getWindow, notify }) {
this.store = store;
this.settings = settings;
this.getWindow = getWindow;
this.notify = notify;
this.session = null; // { guideId, paused, count, intervalSec }
this.intervalTimer = null;
this.clickWatcher = null;
this.frameLoopTimer = null;
this.frameLoopRunning = false;
this.frameWaiters = [];
this.latestFrame = null;
this.lastClickCapture = 0;
this.clickWatcherBuf = '';
this.clickWatcherPendingPress = false;
this.clickWatcherErrTail = '';
this.clickQueue = Promise.resolve();
this.frameLoopInFlight = false;
this.shooting = false;
}
state() {
return this.session
? {
active: true,
paused: this.session.paused,
guideId: this.session.guideId,
count: this.session.count,
intervalSec: this.session.intervalSec || 0,
clickCapture: Boolean(this.clickWatcher),
clickCaptureAvailable: this.clickCaptureAvailable(),
}
: { active: false, clickCaptureAvailable: this.clickCaptureAvailable() };
}
clickCaptureAvailable() {
if (this._clickAvail === undefined) {
this._clickAvail = process.platform === 'win32' || (process.platform === 'linux' && hasBinary('xinput'));
}
return this._clickAvail;
}
startSession(guideId, { intervalSec = null } = {}) {
this.finishSession();
// Default trigger: clicks when the platform supports it, otherwise an
// interval so a session always produces steps even if the global hotkey
// never fires (common under Wayland/WSLg).
let interval = intervalSec;
if (interval == null) {
interval = this.clickCaptureAvailable() ? 0 : (this.settings.get('capture.autoIntervalSec') || 5);
}
// Sessions start paused: nothing hides and no capturing happens until
// the user explicitly presses "Start recording" in the capture bar, so
// New Capture never makes the window vanish out from under them.
this.session = { guideId, paused: true, count: 0, intervalSec: interval };
if (this.settings.get('capture.captureOutsideClicks') !== false) this.startClickWatcher();
this.applyInterval();
this.notify('capture:state', this.state());
// (Skipped for the dev screenshot hook, which needs a visible page.)
if (!process.env.STEPFORGE_SCREENSHOT) {
this.createSessionTray();
const win = this.getWindow();
// Remember whether the window was visible when the session was set
// up — that's what `togglePause` uses to decide whether to tuck the
// app away once the user actually starts recording.
this.hiddenForSession = Boolean(win && !win.isDestroyed() && win.isVisible());
try {
new Notification({
title: 'StepForge is ready to capture',
body: 'Click "Start recording" in the red capture bar when youre ready. The window tucks away and the red tray icon takes over.',
}).show();
} catch { /* notifications unavailable on this desktop */ }
}
}
/** Red-dot tray icon with session controls, shown while recording. */
createSessionTray() {
this.destroySessionTray();
try {
const img = raster.createImage(16, 16, [0, 0, 0, 0]);
raster.fillOval(img, 2, 2, 12, 12, [229, 72, 77, 255]);
this.tray = new Tray(nativeImage.createFromBuffer(encodePng(img)));
this.tray.setToolTip('StepForge — capture session running');
const rebuild = () => {
if (!this.tray || this.tray.isDestroyed()) return;
this.tray.setContextMenu(Menu.buildFromTemplate([
{ label: `Captured ${this.session ? this.session.count : 0} steps`, enabled: false },
{ type: 'separator' },
{ label: 'Capture now', click: () => this.sessionCapture('manual').then(rebuild).catch(() => {}) },
{
label: this.session && this.session.paused ? 'Resume capturing' : 'Pause capturing',
click: () => { this.togglePause(); rebuild(); },
},
{
label: 'Open StepForge (pauses capture)',
click: () => {
this.togglePause(true);
this.showWindow();
rebuild();
},
},
{ type: 'separator' },
{ label: 'Finish session', click: () => this.finishSession() },
]));
};
rebuild();
this.rebuildTrayMenu = rebuild;
this.tray.on('click', () => {
this.togglePause(true);
this.showWindow();
rebuild();
});
} catch {
this.tray = null; // no tray on this desktop; cursor-over skip still protects clicks
}
}
destroySessionTray() {
if (this.tray && !this.tray.isDestroyed()) this.tray.destroy();
this.tray = null;
this.rebuildTrayMenu = null;
}
showWindow() {
const win = this.getWindow();
if (win && !win.isDestroyed()) {
win.show();
win.focus();
}
}
setInterval(intervalSec) {
if (!this.session) return this.state();
this.session.intervalSec = Math.max(0, Number(intervalSec) || 0);
this.applyInterval();
this.notify('capture:state', this.state());
return this.state();
}
applyInterval() {
if (this.intervalTimer) {
clearInterval(this.intervalTimer);
this.intervalTimer = null;
}
const sec = this.session && this.session.intervalSec;
if (sec > 0) {
this.intervalTimer = setInterval(() => {
this.sessionCapture('interval').catch(() => {});
}, sec * 1000);
}
}
togglePause(force) {
if (!this.session) return;
const wasPaused = this.session.paused;
this.session.paused = typeof force === 'boolean' ? force : !this.session.paused;
// Starting/resuming tucks the window away again for clean shots (after
// a brief delay so the user sees it happen) and starts the frame loop
// that serves click captures. Pausing stops the loop and discards the
// buffered frame, so a resume can never serve a pre-pause screen.
if (wasPaused && !this.session.paused) {
const win = this.getWindow();
const arm = () => {
if (!this.session || this.session.paused) return;
if (this.hiddenForSession && win && !win.isDestroyed() && win.isVisible()) win.hide();
if (this.settings.get('capture.captureOutsideClicks') !== false && this.clickCaptureAvailable()) {
this.startFrameLoop();
}
};
if (this.hiddenForSession && win && !win.isDestroyed()) setTimeout(arm, 400);
else arm();
} else if (!wasPaused && this.session.paused) {
this.stopFrameLoop();
}
if (this.rebuildTrayMenu) this.rebuildTrayMenu();
this.notify('capture:state', this.state());
}
finishSession() {
if (this.intervalTimer) {
clearInterval(this.intervalTimer);
this.intervalTimer = null;
}
this.stopClickWatcher();
this.stopFrameLoop();
this.destroySessionTray();
this.session = null;
if (this.hiddenForSession) {
this.hiddenForSession = false;
this.showWindow();
}
this.notify('capture:state', this.state());
}
/**
* True when the user is interacting with StepForge itself. Deliberately
* based on cursor position over the visible window, not isFocused():
* some compositors (WSLg) report focus as stuck-true, which would block
* every automatic capture forever.
*/
userIsInApp() {
const win = this.getWindow();
if (!win || win.isDestroyed() || !win.isVisible() || win.isMinimized()) return false;
const cur = screen.getCursorScreenPoint();
const b = win.getBounds();
return cur.x >= b.x && cur.x <= b.x + b.width && cur.y >= b.y && cur.y <= b.y + b.height;
}
/** One capture inside the active session (hotkey/click/interval/manual). */
async sessionCapture(trigger = 'hotkey', clickPos = null) {
if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' };
// Automatic triggers stand down while the user is in StepForge, so the
// app stays clickable mid-session and never screenshots itself.
if (trigger !== 'manual' && this.userIsInApp()) {
return { ok: false, reason: 'skipped — StepForge is focused' };
}
// Clicks are served from the frame loop: the buffered frame was grabbed
// at (or moments before) the click instant, so the background matches
// what the user clicked on. A click that lands while a grab is in
// flight waits for that frame instead of being dropped, so fast
// clicking still yields one step per click.
if (trigger === 'click') {
const frame = await this.frameForClick(clickPos);
if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' };
if (frame) {
const result = this.storeFrameAsStep(this.session.guideId, frame.mode, frame, clickPos);
if (result.ok) this.noteStepAdded(result.step, trigger);
return result;
}
// No usable frame (loop not running or grab failing): fall through
// to a one-off fresh shot.
}
if (this.shooting) return { ok: false, reason: 'capture already in progress' };
this.shooting = true;
try {
const mode = this.settings.get('capture.mode') || 'fullscreen';
const grabMode = mode === 'region' ? 'fullscreen' : mode;
const finalResult = await this.shoot({
guideId: this.session.guideId,
mode: grabMode,
delayMs: 0,
hideWindowDelayMs: trigger === 'click' ? CLICK_CAPTURE_HIDE_DELAY_MS : null,
refocus: false, // don't steal focus from the app the user is documenting
clickPos,
});
if (finalResult.ok) this.noteStepAdded(finalResult.step, trigger);
return finalResult;
} finally {
this.shooting = false;
}
}
noteStepAdded(step, trigger) {
this.session.count += 1;
this.notify('capture:added', { guideId: this.session.guideId, step, trigger });
this.notify('capture:state', this.state());
if (this.rebuildTrayMenu) this.rebuildTrayMenu(); // refresh step counter
}
hotkeyCapture() {
return this.sessionCapture('hotkey');
}
// ---- click-triggered capture --------------------------------------------
/**
* Continuous screen-grab loop that runs while recording. It keeps the most
* recent frame in `latestFrame` so a click can be served from a frame
* grabbed at (or moments before) the instant of the click — a fresh grab
* started after the click would land hundreds of ms late and show the
* click's effects instead of what the user clicked on.
*/
startFrameLoop() {
if (this.frameLoopRunning) return;
this.frameLoopRunning = true;
const tick = async () => {
if (!this.frameLoopRunning) return;
if (!this.session || this.session.paused) {
this.frameLoopRunning = false;
this.frameLoopInFlight = false;
return;
}
try {
if (!this.shooting) {
this.frameLoopInFlight = true;
const mode = this.settings.get('capture.mode') || 'fullscreen';
const grabMode = mode === 'region' ? 'fullscreen' : mode;
const frame = await this.captureCurrentFrame(grabMode);
if (this.frameLoopRunning) this.acceptFrame(frame);
}
} catch {
// Grab failures are fine — clicks fall back to a one-off fresh shot.
} finally {
this.frameLoopInFlight = false;
if (this.frameLoopRunning && this.session && !this.session.paused) {
this.frameLoopTimer = setTimeout(tick, FRAME_LOOP_IDLE_MS);
}
}
};
this.frameLoopTimer = setTimeout(tick, 0);
}
/** Store a grabbed frame and hand it to any clicks waiting on it. */
acceptFrame(frame) {
this.latestFrame = frame;
const waiters = this.frameWaiters;
this.frameWaiters = [];
for (const resolve of waiters) resolve(frame);
}
/** Resolves with the next frame the loop grabs (null on timeout/stop). */
nextFrame(timeoutMs) {
return new Promise((resolve) => {
const entry = (frame) => {
clearTimeout(timer);
resolve(frame);
};
const timer = setTimeout(() => {
this.frameWaiters = this.frameWaiters.filter((w) => w !== entry);
resolve(null);
}, timeoutMs);
this.frameWaiters.push(entry);
});
}
stopFrameLoop() {
if (this.frameLoopTimer) {
clearTimeout(this.frameLoopTimer);
this.frameLoopTimer = null;
}
this.frameLoopRunning = false;
this.latestFrame = null;
const waiters = this.frameWaiters;
this.frameWaiters = [];
for (const resolve of waiters) resolve(null);
}
/**
* Freshest frame usable for a click capture: the buffered frame when it's
* recent enough, otherwise the next frame the loop delivers. Null when the
* loop isn't running or can't deliver in time.
*/
async frameForClick(clickPos = null) {
const mode = this.settings.get('capture.mode') || 'fullscreen';
const grabMode = mode === 'region' ? 'fullscreen' : mode;
// Fast clicks can move to another monitor before the buffered frame is
// consumed; only reuse frames from the clicked display.
const usable = (f) => {
const sameDisplay = !clickPos || pointInBounds(clickPos, f && f.display && f.display.bounds);
return Boolean(f)
&& f.mode === grabMode
&& Date.now() - f.capturedAt <= CLICK_FRAME_MAX_AGE_MS
&& sameDisplay;
};
if (usable(this.latestFrame)) return this.latestFrame;
if (!this.frameLoopRunning || !this.frameLoopInFlight) return null;
const deadline = Date.now() + CLICK_FRAME_WAIT_MS;
while (this.frameLoopRunning && Date.now() < deadline) {
const next = await this.nextFrame(Math.max(1, deadline - Date.now()));
if (usable(next)) return next;
}
return null;
}
startClickWatcher() {
this.stopClickWatcher();
try {
this.clickWatcherBuf = '';
this.clickWatcherPendingPress = false;
if (process.platform === 'linux' && hasBinary('xinput')) {
// Stream raw button events from the X server; one capture per press.
// xinput block-buffers stdout when piped, so a press event can sit
// in its buffer until later motion events flush it — by then the
// cursor read in onOsClick lands where the mouse moved *after* the
// click. stdbuf -oL forces line-buffering so events (and the cursor
// read) line up with the actual click instant.
const argv = hasBinary('stdbuf')
? ['stdbuf', '-oL', 'xinput', 'test-xi2', '--root']
: ['xinput', 'test-xi2', '--root'];
this.clickWatcher = spawn(argv[0], argv.slice(1), { stdio: ['ignore', 'pipe', 'ignore'] });
this.clickWatcher.stdout.on('data', (chunk) => {
this.ingestClickWatcherChunk(chunk.toString(), 'linux');
});
} else if (process.platform === 'win32') {
// Poll left/right/middle buttons via GetAsyncKeyState. Bit 0x8000 is
// "down right now"; bit 0x0001 is "was pressed since the previous
// poll", which catches clicks shorter than one poll interval. The
// cursor is read in the same poll iteration that sees the press and
// shipped with the event, so the marker position is sampled at the
// click instant (within one ~15ms poll tick) instead of after the
// event has crossed the pipe. Coordinates are physical pixels
// (per-monitor DPI aware); onOsClick converts them to DIPs.
const ps = `
$ErrorActionPreference = 'Stop'
Add-Type -TypeDefinition @'
using System;
using System.Runtime.InteropServices;
public struct SFPoint { public int X; public int Y; }
public static class SFMouse {
[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int vKey);
[DllImport("user32.dll")] public static extern bool GetCursorPos(out SFPoint p);
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
}
'@
try { [void][SFMouse]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch { }
$out = [Console]::Out
$out.WriteLine('READY')
$out.Flush()
$vks = 0x01, 0x02, 0x04
$down = @{}
foreach ($vk in $vks) { $down[$vk] = $false }
$pt = New-Object SFPoint
while ($true) {
foreach ($vk in $vks) {
$s = [SFMouse]::GetAsyncKeyState($vk)
$isDown = ($s -band 0x8000) -ne 0
$tapped = ($s -band 1) -ne 0
if (($isDown -or $tapped) -and -not $down[$vk]) {
[void][SFMouse]::GetCursorPos([ref]$pt)
$out.WriteLine('CLICK ' + $pt.X + ' ' + $pt.Y)
$out.Flush()
}
$down[$vk] = $isDown
}
Start-Sleep -Milliseconds 10
}`;
this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', ps], {
stdio: ['ignore', 'pipe', 'pipe'],
windowsHide: true,
});
this.clickWatcher.stdout.on('data', (chunk) => {
this.ingestClickWatcherChunk(chunk.toString(), 'win32');
});
}
if (this.clickWatcher) {
const child = this.clickWatcher;
this.clickWatcherErrTail = '';
if (child.stderr) {
child.stderr.on('data', (chunk) => {
this.clickWatcherErrTail = String(chunk).slice(-400);
});
}
const lost = (reason) => {
if (this.clickWatcher !== child) return; // stopped deliberately
this.clickWatcher = null;
this.handleClickWatcherLoss(reason);
};
child.on('error', (err) => lost(err && err.message));
child.on('exit', (code) => lost(`exited with code ${code}`));
}
} catch {
this.clickWatcher = null;
}
}
/**
* The watcher process died mid-session (crashed X server, PowerShell
* blocked by policy, …). Captures must not silently stop: log why, switch
* the session to interval captures, and tell the UI.
*/
handleClickWatcherLoss(reason) {
this.clickWatcherPendingPress = false;
const detail = [reason, this.clickWatcherErrTail].filter(Boolean).join(' — ');
console.error(`[stepforge] click watcher stopped${detail ? `: ${detail}` : ''}`);
if (!this.session) return;
if (!this.session.intervalSec) {
this.session.intervalSec = this.settings.get('capture.autoIntervalSec') || 5;
this.applyInterval();
}
this.notify('capture:state', this.state());
}
stopClickWatcher() {
if (this.clickWatcher) {
try { this.clickWatcher.kill(); } catch { /* already gone */ }
this.clickWatcher = null;
}
this.clickWatcherBuf = '';
this.clickWatcherPendingPress = false;
}
/**
* Buffer stdout chunks and only parse complete lines: a chunk boundary
* can split an event line in half, which used to corrupt press/release
* parsing and swallow clicks.
*/
ingestClickWatcherChunk(chunk, platform = process.platform) {
this.clickWatcherBuf += String(chunk);
const cut = this.clickWatcherBuf.lastIndexOf('\n');
if (cut === -1) return;
const complete = this.clickWatcherBuf.slice(0, cut);
this.clickWatcherBuf = this.clickWatcherBuf.slice(cut + 1);
this.processClickWatcherData(complete, platform);
}
processClickWatcherData(text, platform = process.platform) {
const lines = String(text).split(/\r?\n/);
if (platform === 'linux') {
// xinput prints each event as a multi-line block: an "EVENT type …
// (RawButtonPress)" header followed by a "detail: N" line carrying the
// button number. Fire on the detail line so scroll-wheel ticks (X11
// reports them as buttons 4-7) neither create steps nor debounce away
// the real clicks that follow them.
for (const line of lines) {
if (!line) continue;
if (/RawButtonPress|ButtonPress/.test(line)) {
if (this.clickWatcherPendingPress) this.onOsClick();
this.clickWatcherPendingPress = true;
continue;
}
if (!this.clickWatcherPendingPress) continue;
const detail = line.match(/detail:\s*(\d+)/);
if (detail) {
this.clickWatcherPendingPress = false;
const button = Number(detail[1]);
if (button < 4 || button > 7) this.onOsClick();
} else if (line.includes('EVENT type')) {
// Next event arrived without a detail line in between — treat the
// pending press as a plain click rather than dropping it.
this.clickWatcherPendingPress = false;
this.onOsClick();
}
}
return;
}
if (platform === 'win32') {
for (const line of lines) {
const m = /CLICK(?:\s+(-?\d+)\s+(-?\d+))?/.exec(line);
if (m) {
const osPoint = m[1] === undefined ? null : { x: Number(m[1]), y: Number(m[2]) };
this.onOsClick(Date.now(), osPoint);
}
}
}
}
onOsClick(at = Date.now(), osPoint = null) {
if (!this.session || this.session.paused) return;
if (at - this.lastClickCapture < CLICK_DEBOUNCE_MS) return;
this.lastClickCapture = at;
// Prefer the position the watcher sampled in the same poll that saw the
// press (physical px → DIP); otherwise read the cursor synchronously,
// right now, so the marker lands where the user clicked even if the
// shot itself takes a moment to grab. (Clicks on StepForge itself are
// filtered by the cursor-position check in sessionCapture, not by
// window focus — WSLg reports focus unreliably.)
let clickPos = null;
if (osPoint) {
clickPos = typeof screen.screenToDipPoint === 'function'
? screen.screenToDipPoint(osPoint)
: osPoint;
}
if (!clickPos) clickPos = screen.getCursorScreenPoint();
this.enqueueClickCapture(clickPos);
}
/**
* Serialize click captures: a click that lands while an earlier capture is
* still being stored queues behind it instead of being dropped by the
* "capture already in progress" guard. The marker position was already
* read at click time, so a queued step still circles the right spot.
*/
enqueueClickCapture(clickPos) {
this.clickQueue = this.clickQueue
.then(() => this.sessionCapture('click', clickPos))
.catch(() => {});
return this.clickQueue;
}
async captureCurrentFrame(mode, capturePoint = null) {
const grabbed = await this.grab(mode, capturePoint);
return {
mode,
png: grabbed.image.toPNG(),
size: grabbed.image.getSize(),
display: grabbed.display,
cursor: capturePoint || grabbed.cursor,
capturedAt: Date.now(),
};
}
storeFrameAsStep(guideId, mode, frame, clickPos = null) {
if (!frame) return { ok: false, reason: 'no capture frame available' };
const annotations = [];
const cursor = clickPos || frame.cursor;
if (mode !== 'window' && this.settings.get('capture.clickMarker')) {
const fx = (cursor.x - frame.display.bounds.x) / frame.display.bounds.width;
const fy = (cursor.y - frame.display.bounds.y) / frame.display.bounds.height;
if (fx >= 0 && fx <= 1 && fy >= 0 && fy <= 1) {
const d = 0.035;
annotations.push({
type: 'oval',
x: fx - d / 2, y: fy - (d * frame.size.width / frame.size.height) / 2,
w: d, h: d * frame.size.width / frame.size.height,
style: {
stroke: this.settings.get('capture.clickMarkerColor') || '#E5484D',
strokeWidth: 4, fill: 'transparent',
},
});
}
}
const step = this.store.addStep(guideId, {
title: this.autoTitle(mode),
annotations,
focusedView: {
enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),
zoom: 1, panX: 0.5, panY: 0.5,
},
}, frame.png, frame.size);
return { ok: true, step };
}
autoTitle(mode) {
const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]';
const now = new Date();
const pad = (n) => String(n).padStart(2, '0');
return expandPlaceholders(tplStr, {
Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen',
Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`,
Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`,
});
}
/** Grab the screen/window image as { image, display } or throw. */
async grab(mode, cursorPoint = null) {
const cursor = cursorPoint || screen.getCursorScreenPoint();
const display = screen.getDisplayNearestPoint(cursor);
const { width, height } = display.size;
const scale = display.scaleFactor || 1;
// Ask for both kinds: some compositors (WSLg/Wayland portals) expose no
// individual window sources, so window mode falls back to the screen.
const sources = await desktopCapturer.getSources({
types: mode === 'window' ? ['window', 'screen'] : ['screen'],
thumbnailSize: { width: Math.round(width * scale), height: Math.round(height * scale) },
});
if (!sources.length) throw new Error('no capture sources available (portal/permissions?)');
let source = null;
if (mode === 'window') {
const win = this.getWindow();
const ownTitle = win ? win.getTitle() : '';
const windows = sources.filter((s) => s.id.startsWith('window:'));
source = windows.find((s) => s.name && s.name !== ownTitle && !/stepforge/i.test(s.name))
|| windows[0]
|| sources.find((s) => s.id.startsWith('screen:'));
} else {
const screens = sources.filter((s) => s.id.startsWith('screen:'));
source = screens.find((s) => String(s.display_id) === String(display.id)) || screens[0] || sources[0];
}
if (!source) throw new Error('no capture source matched');
const image = source.thumbnail;
if (!image || image.isEmpty()) throw new Error('capture returned an empty image');
return { image, display, cursor };
}
/**
* Hide the app window while `fn` runs so screenshots show the user's work,
* not StepForge itself. Restores visibility afterwards.
*/
async withWindowHidden(fn, { refocus = true, pauseMs = 350 } = {}) {
const win = this.getWindow();
const wasVisible = win && !win.isDestroyed() && win.isVisible() && !win.isMinimized();
if (wasVisible) {
win.hide();
if (pauseMs > 0) {
await new Promise((r) => setTimeout(r, pauseMs)); // let the compositor repaint
}
}
try {
return await fn();
} finally {
if (wasVisible && win && !win.isDestroyed()) {
if (refocus) {
win.show();
win.focus();
} else {
win.showInactive();
}
}
}
}
/**
* Take a screenshot and append it to the guide as a new image step.
* Adds a click-marker annotation at the cursor position when enabled.
*/
async shoot({
guideId,
mode = 'fullscreen',
delayMs = null,
hideWindow = true,
refocus = true,
hideWindowDelayMs = null,
clickPos = null,
}) {
const delay = delayMs == null ? this.settings.get('capture.delayMs') || 0 : delayMs;
if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay));
let frame;
try {
frame = hideWindow
? await this.withWindowHidden(() => this.captureCurrentFrame(mode, clickPos), {
refocus,
pauseMs: hideWindowDelayMs == null ? 350 : hideWindowDelayMs,
})
: await this.captureCurrentFrame(mode, clickPos);
} catch (err) {
return { ok: false, reason: err.message };
}
return this.storeFrameAsStep(guideId, mode, frame, clickPos);
}
/**
* Region capture: shoot the full screen, then let the user drag a
* rectangle in a fullscreen overlay; the crop becomes the step image.
*/
async regionCapture(guideId) {
let grabbed;
try {
grabbed = await this.withWindowHidden(() => this.grab('fullscreen'));
} catch (err) {
return { ok: false, reason: err.message };
}
const { image, display } = grabbed;
const rect = await this.pickRegion(display, image);
if (!rect) return { ok: false, reason: 'selection cancelled' };
const cropped = image.crop(rect);
const size = cropped.getSize();
if (!size.width || !size.height) return { ok: false, reason: 'empty selection' };
const step = this.store.addStep(guideId, { title: this.autoTitle('region') },
cropped.toPNG(), size);
return { ok: true, step };
}
/** Fullscreen overlay window that resolves with a crop rect (image px). */
pickRegion(display, image) {
return new Promise((resolve) => {
const overlay = new BrowserWindow({
x: display.bounds.x,
y: display.bounds.y,
width: display.bounds.width,
height: display.bounds.height,
frame: false,
transparent: true,
alwaysOnTop: true,
fullscreen: true,
skipTaskbar: true,
webPreferences: {
preload: path.join(__dirname, 'region-preload.js'),
contextIsolation: true,
},
});
let settled = false;
const finish = (rect) => {
if (settled) return;
settled = true;
if (!overlay.isDestroyed()) overlay.close();
resolve(rect);
};
const { ipcMain } = require('electron');
const onPick = (event, rect) => {
if (event.sender !== overlay.webContents) return;
ipcMain.removeListener('region:picked', onPick);
if (!rect) return finish(null);
const imgSize = image.getSize();
const sx = imgSize.width / display.bounds.width;
const sy = imgSize.height / display.bounds.height;
finish({
x: Math.round(rect.x * sx),
y: Math.round(rect.y * sy),
width: Math.round(rect.w * sx),
height: Math.round(rect.h * sy),
});
};
ipcMain.on('region:picked', onPick);
overlay.on('closed', () => finish(null));
overlay.loadFile(path.join(__dirname, 'renderer', 'region.html'));
});
}
}
module.exports = CaptureService;
module.exports = require('./app/capture');
+11
View File
@@ -3,6 +3,17 @@
All notable user-visible changes are recorded here. The format follows
Keep-a-Changelog conventions; versions follow semver.
## [Unreleased]
### Fixed
- Windows continuous click capture now uses a low-level mouse hook instead
of timer polling, so normal left-clicks are not missed when the app or
target system is under load. Click captures also preserve the original
click timestamp through the queue and choose a buffered frame from before
the click when one is available, keeping the marker aligned with the
click-time cursor position.
## [0.1.0] - 2026-06-10
Initial release.
@@ -13,8 +13,8 @@
"hideSkippedStepsInExports": true
},
"themeOverride": "system",
"createdAt": "2026-06-11T14:16:39Z",
"updatedAt": "2026-06-11T14:16:39Z",
"createdAt": "2026-06-11T21:52:50Z",
"updatedAt": "2026-06-11T21:52:50Z",
"stepsOrder": [
"step-sample-01-open-users",
"step-sample-02-enable-policy",
@@ -25,7 +25,7 @@
"extraImages": [],
"annotations": [
{
"id": "ann-80c0a4c6-8b7a-4ba9-8d38-35be0f890ccb",
"id": "ann-def495f6-e1e8-4d71-95cc-a7e5be6cc80a",
"type": "rect",
"x": 0.275,
"y": 0.18,
@@ -41,7 +41,7 @@
}
},
{
"id": "ann-403e9d28-8b61-43e8-8a8c-74e35bf8b0ff",
"id": "ann-835f60f6-d649-428c-bdf9-4ab27c0b57be",
"type": "number",
"x": 0.3,
"y": 0.08,
@@ -60,7 +60,7 @@
],
"textBlocks": [
{
"id": "tb-9f433853-ccc5-450a-98ed-5dcbde7463ac",
"id": "tb-feafdd76-8d10-4d42-8fba-46cb9d2e8dae",
"position": "after-description",
"level": "info",
"title": "Tip",
@@ -25,7 +25,7 @@
"extraImages": [],
"annotations": [
{
"id": "ann-d2a29aa5-f083-45b6-8018-bd1e66a878fd",
"id": "ann-1cd13ad1-1723-406b-8c1a-c7d010b314c3",
"type": "arrow",
"x": 0.47,
"y": 0.24,
@@ -41,7 +41,7 @@
}
},
{
"id": "ann-d4d4807d-87f4-4b9b-9aac-fe106edafa56",
"id": "ann-abdee8b7-3325-4dd0-b1e3-c75bffe6fba5",
"type": "tooltip",
"x": 0.53,
"y": 0.13,
@@ -58,7 +58,7 @@
}
},
{
"id": "ann-a34d3ef1-f227-43ac-a59e-a433dfb7f8d5",
"id": "ann-140206cb-5e66-4027-87cf-2e8898fd6c9b",
"type": "number",
"x": 0.31,
"y": 0.08,
@@ -19,7 +19,7 @@
"annotations": [],
"textBlocks": [
{
"id": "tb-5ef09593-6f95-475d-b0c6-6144d855bed7",
"id": "tb-3d70c3b9-3d42-42a1-a7b9-a063b83c1c49",
"position": "after-description",
"level": "warn",
"title": "Access",
@@ -25,7 +25,7 @@
"extraImages": [],
"annotations": [
{
"id": "ann-cab1dba5-f903-47c3-a4ec-efff6f70cc3f",
"id": "ann-e650d25f-0c72-432d-afca-26e1f4c7ea6a",
"type": "blur",
"x": 0.49,
"y": 0.32,
@@ -42,7 +42,7 @@
"radius": 12
},
{
"id": "ann-f6e54195-1e03-4455-83cf-8ecfafde5993",
"id": "ann-51f325e5-2522-47ad-a0de-b53318cdcd00",
"type": "highlight",
"x": 0.47,
"y": 0.24,
@@ -58,7 +58,7 @@
}
},
{
"id": "ann-0c74b069-aad1-4d83-bf96-f902009905d6",
"id": "ann-f120f280-f3cc-46c2-9f3e-ef2dac1a9d27",
"type": "number",
"x": 0.31,
"y": 0.08,
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -3,7 +3,7 @@
"version": 1,
"guide": {
"title": "Reset a password in Admin Portal",
"generatedAt": "2026-06-11T14:16:39.204Z"
"generatedAt": "2026-06-11T21:52:50.346Z"
},
"steps": [
{
@@ -1,12 +1,12 @@
{
"format": "stepforge-guide",
"version": 1,
"generatedAt": "2026-06-11T14:16:39.204Z",
"generatedAt": "2026-06-11T21:52:50.346Z",
"guide": {
"title": "Reset a password in Admin Portal",
"descriptionHtml": "<p>Offline sample guide showing capture, annotations, rich text, and exports.</p>",
"createdAt": "2026-06-11T14:16:39Z",
"updatedAt": "2026-06-11T14:16:39Z"
"createdAt": "2026-06-11T21:52:50Z",
"updatedAt": "2026-06-11T21:52:50Z"
},
"steps": [
{
@@ -23,7 +23,7 @@
},
"annotations": [
{
"id": "ann-80c0a4c6-8b7a-4ba9-8d38-35be0f890ccb",
"id": "ann-def495f6-e1e8-4d71-95cc-a7e5be6cc80a",
"type": "rect",
"x": 0.275,
"y": 0.18,
@@ -39,7 +39,7 @@
}
},
{
"id": "ann-403e9d28-8b61-43e8-8a8c-74e35bf8b0ff",
"id": "ann-835f60f6-d649-428c-bdf9-4ab27c0b57be",
"type": "number",
"x": 0.3,
"y": 0.08,
@@ -82,7 +82,7 @@
},
"annotations": [
{
"id": "ann-d2a29aa5-f083-45b6-8018-bd1e66a878fd",
"id": "ann-1cd13ad1-1723-406b-8c1a-c7d010b314c3",
"type": "arrow",
"x": 0.47,
"y": 0.24,
@@ -98,7 +98,7 @@
}
},
{
"id": "ann-d4d4807d-87f4-4b9b-9aac-fe106edafa56",
"id": "ann-abdee8b7-3325-4dd0-b1e3-c75bffe6fba5",
"type": "tooltip",
"x": 0.53,
"y": 0.13,
@@ -115,7 +115,7 @@
}
},
{
"id": "ann-a34d3ef1-f227-43ac-a59e-a433dfb7f8d5",
"id": "ann-140206cb-5e66-4027-87cf-2e8898fd6c9b",
"type": "number",
"x": 0.31,
"y": 0.08,
@@ -178,7 +178,7 @@
},
"annotations": [
{
"id": "ann-cab1dba5-f903-47c3-a4ec-efff6f70cc3f",
"id": "ann-e650d25f-0c72-432d-afca-26e1f4c7ea6a",
"type": "blur",
"x": 0.49,
"y": 0.32,
@@ -195,7 +195,7 @@
"radius": 12
},
{
"id": "ann-f6e54195-1e03-4455-83cf-8ecfafde5993",
"id": "ann-51f325e5-2522-47ad-a0de-b53318cdcd00",
"type": "highlight",
"x": 0.47,
"y": 0.24,
@@ -211,7 +211,7 @@
}
},
{
"id": "ann-0c74b069-aad1-4d83-bf96-f902009905d6",
"id": "ann-f120f280-f3cc-46c2-9f3e-ef2dac1a9d27",
"type": "number",
"x": 0.31,
"y": 0.08,
Binary file not shown.
+1 -1
View File
@@ -1,7 +1,7 @@
{
"format": "stepforge-sample-manifest",
"version": 1,
"generatedAt": "2026-06-11T14:16:39.204Z",
"generatedAt": "2026-06-11T21:52:50.344Z",
"guideId": "guide-sample-reset-password",
"title": "Reset a password in Admin Portal",
"dataDir": "sample-data",
+68 -4
View File
@@ -168,7 +168,7 @@ test('windows click watcher output is counted line by line', () => {
assert.equal(clicks, 2);
});
test('windows click lines carry the poll-time cursor position', () => {
test('windows click lines carry the click-time cursor position', () => {
const service = makeService();
const seen = [];
service.onOsClick = (at, osPoint) => {
@@ -181,6 +181,22 @@ test('windows click lines carry the poll-time cursor position', () => {
'coordinates ride along with the event; bare CLICK still works');
});
test('windows hook click lines carry button and event timestamp', () => {
const service = makeService();
const seen = [];
service.onOsClick = (at, osPoint, button) => {
seen.push({ at, osPoint, button });
};
service.processClickWatcherData('READY\r\nCLICK 321 -9 left 1770000000123\r\n', 'win32');
assert.deepEqual(seen, [{
at: 1770000000123,
osPoint: { x: 321, y: -9 },
button: 'left',
}]);
});
test('losing the click watcher mid-session falls back to interval capture', () => {
const service = makeService();
service.settings.get = (key) => (key === 'capture.autoIntervalSec' ? 3 : null);
@@ -221,6 +237,50 @@ test('a click is served instantly from the freshly buffered frame', async () =>
assert.equal(service.session.count, 1);
});
test('click capture uses the newest frame completed before the click time', async () => {
const service = makeService();
const clickAt = Date.now();
service.session = { guideId: 'guide-history', paused: false, count: 0, intervalSec: 0 };
const before = makeFrame('before-click');
before.startedAt = clickAt - 40;
before.capturedAt = clickAt - 30;
const after = makeFrame('after-click');
after.startedAt = clickAt + 5;
after.capturedAt = clickAt + 15;
service.recentFrames = [before, after];
service.latestFrame = after;
service.shoot = async () => {
throw new Error('a matching pre-click frame should be used');
};
const added = [];
service.store.addStep = (guideId, fields, png) => {
added.push(png.toString());
return { stepId: 'step-history' };
};
const result = await service.sessionCapture('click', { x: 10, y: 10 }, { at: clickAt });
assert.equal(result.ok, true);
assert.deepEqual(added, ['before-click']);
});
test('queued click captures preserve the original event time and button', async () => {
const service = makeService();
const seen = [];
service.sessionCapture = async (trigger, clickPos, clickMeta) => {
seen.push({ trigger, clickPos, clickMeta });
return { ok: true };
};
await service.enqueueClickCapture({ x: 7, y: 8 }, 1770000000456, 'left');
assert.deepEqual(seen, [{
trigger: 'click',
clickPos: { x: 7, y: 8 },
clickMeta: { at: 1770000000456, button: 'left' },
}]);
});
test('a buffered frame from a different display is ignored for click capture', async () => {
const service = makeService();
service.session = { guideId: 'guide-display', paused: false, count: 0, intervalSec: 0 };
@@ -298,6 +358,8 @@ test('clicks during an in-flight grab wait for the frame instead of being droppe
service.session = { guideId: 'guide-fast', paused: false, count: 0, intervalSec: 0 };
service.frameLoopRunning = true; // a grab is in flight, no frame buffered yet
service.frameLoopInFlight = true;
const clickAt = Date.now();
service.frameLoopGrabStartedAt = clickAt - 10;
service.shoot = async () => {
throw new Error('waiting clicks must use the loop frame, not a competing shot');
};
@@ -308,9 +370,11 @@ test('clicks during an in-flight grab wait for the frame instead of being droppe
};
// Two rapid clicks land before the grab completes.
const first = service.sessionCapture('click', { x: 1, y: 1 });
const second = service.sessionCapture('click', { x: 2, y: 2 });
service.acceptFrame(makeFrame('loop-frame'));
const first = service.sessionCapture('click', { x: 1, y: 1 }, { at: clickAt });
const second = service.sessionCapture('click', { x: 2, y: 2 }, { at: clickAt });
const loopFrame = makeFrame('loop-frame');
loopFrame.startedAt = clickAt - 10;
service.acceptFrame(loopFrame);
const [r1, r2] = await Promise.all([first, second]);
assert.equal(r1.ok, true);