From 6682cdae0fccdfb5de70eadfe91e5e3e726d807a Mon Sep 17 00:00:00 2001 From: Iisyourdad Date: Thu, 11 Jun 2026 16:57:59 -0500 Subject: [PATCH] Fixed clicking location part 4 --- app/capture.js | 246 +++-- build/artifacts_manifest.json | 152 ++-- build/build_report.md | 2 +- capture.js | 841 +----------------- docs/CHANGELOG.md | 11 + .../guide-sample-reset-password/guide.json | 4 +- .../steps/step-sample-01-open-users/step.json | 6 +- .../step-sample-02-enable-policy/step.json | 6 +- .../step.json | 2 +- .../step.json | 6 +- ...reset-a-password-in-admin-portal-rich.html | 2 +- .../reset-a-password-in-admin-portal.html | 2 +- ...set-a-password-in-admin-portal-bundle.json | 2 +- .../reset-a-password-in-admin-portal.json | 22 +- examples/sample-guide.sfgz | Bin 88423 -> 88427 bytes examples/sample-manifest.json | 2 +- tests/unit/capture.test.js | 72 +- 17 files changed, 372 insertions(+), 1006 deletions(-) diff --git a/app/capture.js b/app/capture.js index 283d108..f5d4f78 100644 --- a/app/capture.js +++ b/app/capture.js @@ -35,6 +35,8 @@ const CLICK_FRAME_MAX_AGE_MS = 600; // one-off fresh shot. const CLICK_FRAME_WAIT_MS = 2000; const CLICK_CAPTURE_HIDE_DELAY_MS = 25; +const RECENT_FRAME_RETENTION_MS = 2000; +const RECENT_FRAME_LIMIT = 8; function pointInBounds(point, bounds) { if (!point || !bounds) return false; @@ -66,13 +68,15 @@ class CaptureService { this.frameLoopRunning = false; this.frameWaiters = []; this.latestFrame = null; - this.lastClickCapture = 0; this.clickWatcherBuf = ''; this.clickWatcherPendingPress = false; this.clickWatcherErrTail = ''; this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; + this.frameLoopGrabStartedAt = null; + this.recentFrames = []; this.shooting = false; + this.lastClickCaptureByButton = new Map(); } state() { @@ -264,7 +268,7 @@ class CaptureService { } /** One capture inside the active session (hotkey/click/interval/manual). */ - async sessionCapture(trigger = 'hotkey', clickPos = null) { + async sessionCapture(trigger = 'hotkey', clickPos = null, clickMeta = null) { if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' }; // Automatic triggers stand down while the user is in StepForge, so the // app stays clickable mid-session and never screenshots itself. @@ -278,7 +282,8 @@ class CaptureService { // flight waits for that frame instead of being dropped, so fast // clicking still yields one step per click. if (trigger === 'click') { - const frame = await this.frameForClick(clickPos); + const clickAt = clickMeta && Number.isFinite(clickMeta.at) ? clickMeta.at : Date.now(); + const frame = await this.frameForClick(clickPos, clickAt); if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' }; if (frame) { const result = this.storeFrameAsStep(this.session.guideId, frame.mode, frame, clickPos); @@ -342,15 +347,17 @@ class CaptureService { try { if (!this.shooting) { this.frameLoopInFlight = true; + this.frameLoopGrabStartedAt = Date.now(); const mode = this.settings.get('capture.mode') || 'fullscreen'; const grabMode = mode === 'region' ? 'fullscreen' : mode; - const frame = await this.captureCurrentFrame(grabMode); + const frame = await this.captureCurrentFrame(grabMode, null, this.frameLoopGrabStartedAt); if (this.frameLoopRunning) this.acceptFrame(frame); } } catch { // Grab failures are fine — clicks fall back to a one-off fresh shot. } finally { this.frameLoopInFlight = false; + this.frameLoopGrabStartedAt = null; if (this.frameLoopRunning && this.session && !this.session.paused) { this.frameLoopTimer = setTimeout(tick, FRAME_LOOP_IDLE_MS); } @@ -362,6 +369,11 @@ class CaptureService { /** Store a grabbed frame and hand it to any clicks waiting on it. */ acceptFrame(frame) { this.latestFrame = frame; + this.recentFrames.push(frame); + const cutoff = Date.now() - RECENT_FRAME_RETENTION_MS; + this.recentFrames = this.recentFrames + .filter((f) => f && f.capturedAt >= cutoff) + .slice(-RECENT_FRAME_LIMIT); const waiters = this.frameWaiters; this.frameWaiters = []; for (const resolve of waiters) resolve(frame); @@ -388,7 +400,9 @@ class CaptureService { this.frameLoopTimer = null; } this.frameLoopRunning = false; + this.frameLoopGrabStartedAt = null; this.latestFrame = null; + this.recentFrames = []; const waiters = this.frameWaiters; this.frameWaiters = []; for (const resolve of waiters) resolve(null); @@ -399,24 +413,34 @@ class CaptureService { * recent enough, otherwise the next frame the loop delivers. Null when the * loop isn't running or can't deliver in time. */ - async frameForClick(clickPos = null) { + async frameForClick(clickPos = null, clickAt = Date.now()) { const mode = this.settings.get('capture.mode') || 'fullscreen'; const grabMode = mode === 'region' ? 'fullscreen' : mode; + const clickTime = Number.isFinite(clickAt) ? clickAt : Date.now(); // Fast clicks can move to another monitor before the buffered frame is // consumed; only reuse frames from the clicked display. - const usable = (f) => { + const usable = (f, { allowInFlight = false } = {}) => { const sameDisplay = !clickPos || pointInBounds(clickPos, f && f.display && f.display.bounds); + const startedAt = Number.isFinite(f && f.startedAt) ? f.startedAt : (f && f.capturedAt); + const completedBeforeClick = Number.isFinite(f && f.capturedAt) && f.capturedAt <= clickTime; + const startedBeforeClick = Number.isFinite(startedAt) && startedAt <= clickTime; + const timingMatches = completedBeforeClick + ? clickTime - f.capturedAt <= CLICK_FRAME_MAX_AGE_MS + : allowInFlight && startedBeforeClick && clickTime - startedAt <= CLICK_FRAME_MAX_AGE_MS; return Boolean(f) && f.mode === grabMode - && Date.now() - f.capturedAt <= CLICK_FRAME_MAX_AGE_MS + && timingMatches && sameDisplay; }; - if (usable(this.latestFrame)) return this.latestFrame; - if (!this.frameLoopRunning || !this.frameLoopInFlight) return null; + const buffered = [...this.recentFrames, this.latestFrame] + .filter((f, i, arr) => f && arr.indexOf(f) === i && usable(f)) + .sort((a, b) => b.capturedAt - a.capturedAt)[0]; + if (buffered) return buffered; + if (!this.frameLoopRunning || !this.frameLoopInFlight || this.frameLoopGrabStartedAt > clickTime) return null; const deadline = Date.now() + CLICK_FRAME_WAIT_MS; while (this.frameLoopRunning && Date.now() < deadline) { const next = await this.nextFrame(Math.max(1, deadline - Date.now())); - if (usable(next)) return next; + if (usable(next, { allowInFlight: true })) return next; } return null; } @@ -441,49 +465,148 @@ class CaptureService { this.ingestClickWatcherChunk(chunk.toString(), 'linux'); }); } else if (process.platform === 'win32') { - // Poll left/right/middle buttons via GetAsyncKeyState. Bit 0x8000 is - // "down right now"; bit 0x0001 is "was pressed since the previous - // poll", which catches clicks shorter than one poll interval. The - // cursor is read in the same poll iteration that sees the press and - // shipped with the event, so the marker position is sampled at the - // click instant (within one ~15ms poll tick) instead of after the - // event has crossed the pipe. Coordinates are physical pixels - // (per-monitor DPI aware); onOsClick converts them to DIPs. + // Use a low-level Windows mouse hook instead of polling + // GetAsyncKeyState. The low bit from GetAsyncKeyState can be consumed + // by other processes and a polling loop can miss short clicks under + // load; WH_MOUSE_LL gives us one event for each button-down, with the + // hook-time cursor position and timestamp. const ps = ` $ErrorActionPreference = 'Stop' Add-Type -TypeDefinition @' using System; +using System.Collections.Concurrent; using System.Runtime.InteropServices; -public struct SFPoint { public int X; public int Y; } -public static class SFMouse { - [DllImport("user32.dll")] public static extern short GetAsyncKeyState(int vKey); - [DllImport("user32.dll")] public static extern bool GetCursorPos(out SFPoint p); - [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); +using System.Threading; + +public static class SFMouseHook { + private const int WH_MOUSE_LL = 14; + private const int WM_LBUTTONDOWN = 0x0201; + private const int WM_RBUTTONDOWN = 0x0204; + private const int WM_MBUTTONDOWN = 0x0207; + private const int WM_XBUTTONDOWN = 0x020B; + private const long UnixEpochMilliseconds = 62135596800000L; + + private static IntPtr hook = IntPtr.Zero; + private static LowLevelMouseProc proc = HookCallback; + private static readonly ConcurrentQueue queue = new ConcurrentQueue(); + private static readonly AutoResetEvent signal = new AutoResetEvent(false); + + [StructLayout(LayoutKind.Sequential)] + private struct POINT { + public int x; + public int y; + } + + [StructLayout(LayoutKind.Sequential)] + private struct MSLLHOOKSTRUCT { + public POINT pt; + public uint mouseData; + public uint flags; + public uint time; + public UIntPtr dwExtraInfo; + } + + [StructLayout(LayoutKind.Sequential)] + private struct MSG { + public IntPtr hwnd; + public uint message; + public UIntPtr wParam; + public IntPtr lParam; + public uint time; + public POINT pt; + } + + private delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll", SetLastError = true)] + private static extern bool UnhookWindowsHookEx(IntPtr hhk); + + [DllImport("user32.dll")] + private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + private static extern IntPtr GetModuleHandle(string lpModuleName); + + [DllImport("user32.dll")] + private static extern int GetMessage(out MSG lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax); + + [DllImport("user32.dll")] + private static extern bool TranslateMessage(ref MSG lpMsg); + + [DllImport("user32.dll")] + private static extern IntPtr DispatchMessage(ref MSG lpMsg); + + [DllImport("user32.dll")] + private static extern bool SetProcessDpiAwarenessContext(IntPtr value); + + public static void Run() { + try { SetProcessDpiAwarenessContext(new IntPtr(-4)); } catch { } + + Thread writer = new Thread(WriterLoop); + writer.IsBackground = true; + writer.Start(); + + hook = SetWindowsHookEx(WH_MOUSE_LL, proc, GetModuleHandle(null), 0); + if (hook == IntPtr.Zero) { + throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); + } + + Console.Out.WriteLine("READY"); + Console.Out.Flush(); + + MSG msg; + while (GetMessage(out msg, IntPtr.Zero, 0, 0) > 0) { + TranslateMessage(ref msg); + DispatchMessage(ref msg); + } + + UnhookWindowsHookEx(hook); + } + + private static void WriterLoop() { + while (true) { + signal.WaitOne(); + string line; + while (queue.TryDequeue(out line)) { + Console.Out.WriteLine(line); + } + Console.Out.Flush(); + } + } + + private static IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0) { + int message = wParam.ToInt32(); + string button = ButtonName(message, lParam); + if (button != null) { + MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT)); + long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds; + queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs); + signal.Set(); + } + } + return CallNextHookEx(hook, nCode, wParam, lParam); + } + + private static string ButtonName(int message, IntPtr lParam) { + if (message == WM_LBUTTONDOWN) return "left"; + if (message == WM_RBUTTONDOWN) return "right"; + if (message == WM_MBUTTONDOWN) return "middle"; + if (message == WM_XBUTTONDOWN) { + MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT)); + uint xButton = (data.mouseData >> 16) & 0xffff; + return xButton == 1 ? "x1" : "x2"; + } + return null; + } } '@ -try { [void][SFMouse]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch { } -$out = [Console]::Out -$out.WriteLine('READY') -$out.Flush() -$vks = 0x01, 0x02, 0x04 -$down = @{} -foreach ($vk in $vks) { $down[$vk] = $false } -$pt = New-Object SFPoint -while ($true) { - foreach ($vk in $vks) { - $s = [SFMouse]::GetAsyncKeyState($vk) - $isDown = ($s -band 0x8000) -ne 0 - $tapped = ($s -band 1) -ne 0 - if (($isDown -or $tapped) -and -not $down[$vk]) { - [void][SFMouse]::GetCursorPos([ref]$pt) - $out.WriteLine('CLICK ' + $pt.X + ' ' + $pt.Y) - $out.Flush() - } - $down[$vk] = $isDown - } - Start-Sleep -Milliseconds 10 -}`; - this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', ps], { +[SFMouseHook]::Run() +`; + this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', ps], { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, }); @@ -536,6 +659,7 @@ while ($true) { } this.clickWatcherBuf = ''; this.clickWatcherPendingPress = false; + this.lastClickCaptureByButton.clear(); } /** @@ -572,7 +696,7 @@ while ($true) { if (detail) { this.clickWatcherPendingPress = false; const button = Number(detail[1]); - if (button < 4 || button > 7) this.onOsClick(); + if (button < 4 || button > 7) this.onOsClick(Date.now(), null, `button-${button}`); } else if (line.includes('EVENT type')) { // Next event arrived without a detail line in between — treat the // pending press as a plain click rather than dropping it. @@ -584,21 +708,25 @@ while ($true) { } if (platform === 'win32') { for (const line of lines) { - const m = /CLICK(?:\s+(-?\d+)\s+(-?\d+))?/.exec(line); + const m = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(line.trim()); if (m) { const osPoint = m[1] === undefined ? null : { x: Number(m[1]), y: Number(m[2]) }; - this.onOsClick(Date.now(), osPoint); + const eventAt = m[4] === undefined ? Date.now() : Number(m[4]); + this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, m[3] || 'mouse'); } } } } - onOsClick(at = Date.now(), osPoint = null) { + onOsClick(at = Date.now(), osPoint = null, button = 'mouse') { if (!this.session || this.session.paused) return; - if (at - this.lastClickCapture < CLICK_DEBOUNCE_MS) return; - this.lastClickCapture = at; - // Prefer the position the watcher sampled in the same poll that saw the - // press (physical px → DIP); otherwise read the cursor synchronously, + const clickAt = Number.isFinite(at) ? at : Date.now(); + const debounceKey = button || 'mouse'; + const last = this.lastClickCaptureByButton.get(debounceKey) || 0; + if (clickAt >= last && clickAt - last < CLICK_DEBOUNCE_MS) return; + this.lastClickCaptureByButton.set(debounceKey, clickAt); + // Prefer the position the watcher sampled with the button-down event + // (physical px -> DIP); otherwise read the cursor synchronously, // right now, so the marker lands where the user clicked even if the // shot itself takes a moment to grab. (Clicks on StepForge itself are // filtered by the cursor-position check in sessionCapture, not by @@ -610,7 +738,7 @@ while ($true) { : osPoint; } if (!clickPos) clickPos = screen.getCursorScreenPoint(); - this.enqueueClickCapture(clickPos); + this.enqueueClickCapture(clickPos, clickAt, debounceKey); } /** @@ -619,14 +747,15 @@ while ($true) { * "capture already in progress" guard. The marker position was already * read at click time, so a queued step still circles the right spot. */ - enqueueClickCapture(clickPos) { + enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') { + const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' }; this.clickQueue = this.clickQueue - .then(() => this.sessionCapture('click', clickPos)) + .then(() => this.sessionCapture('click', clickPos, clickMeta)) .catch(() => {}); return this.clickQueue; } - async captureCurrentFrame(mode, capturePoint = null) { + async captureCurrentFrame(mode, capturePoint = null, startedAt = Date.now()) { const grabbed = await this.grab(mode, capturePoint); return { mode, @@ -634,6 +763,7 @@ while ($true) { size: grabbed.image.getSize(), display: grabbed.display, cursor: capturePoint || grabbed.cursor, + startedAt, capturedAt: Date.now(), }; } diff --git a/build/artifacts_manifest.json b/build/artifacts_manifest.json index f2e197c..d0a829f 100644 --- a/build/artifacts_manifest.json +++ b/build/artifacts_manifest.json @@ -1,99 +1,99 @@ { "format": "stepforge-artifacts-manifest", "version": 1, - "generatedAt": "2026-06-11T02:38:15.736Z", + "generatedAt": "2026-06-11T21:54:13.294Z", "packageVersion": "0.1.0", "files": [ { "kind": "artifact", "path": "artifacts/stepforge_0.1.0_amd64.deb", - "size": 103669070, - "sha256": "445758cd92c80c4fc2c5bec5929e62ab33dd83583da859bcba66bda9390ba077" + "size": 103691640, + "sha256": "320faa345f5997905fdc831c045dbe243490a7b4328680d105934f2aec1b4ffb" }, { "kind": "artifact", "path": "artifacts/stepforge_0.1.0_linux-x64.tar.gz", - "size": 139348957, - "sha256": "beb78b999ae40faf7c65d6ccf9bba23f7955b2d7fdc4214e0e9a94e4e3faec74" + "size": 139378628, + "sha256": "32971595d4df40b429cb41ba644e57b84351ec1239b4bbe8d5b82fe3b01a4cc9" }, { "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/guide.json", - "size": 928, - "sha256": "62cfedbb54089f92b8f52493255fab8b37a201cd12746dc8eff2e40be6978dda" + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/guide.json", + "size": 843, + "sha256": "0d7760246d96a85a4d79c15c1ab1a7e481bd79e1d8ff29d177ac6d35ffe6cde6" }, { "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-277488ad-bf0c-488d-a6a2-e54be3e41241/original.png", - "size": 13602, - "sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-277488ad-bf0c-488d-a6a2-e54be3e41241/step.json", - "size": 1975, - "sha256": "d5a7f45ef230d080cca8ecf13b20bfefc7294342eed375db2f0f7d576df6ef1b" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-277488ad-bf0c-488d-a6a2-e54be3e41241/working.png", - "size": 13602, - "sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-2a243bf5-6b4f-406c-ba43-b66ba11aea16/step.json", - "size": 547, - "sha256": "e3b3ce474ec4234f083deac7304cb4d665f3f2ccdcf39c60a78ecc71312ee170" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-87ca03b4-fccd-46c1-97f8-6c1f7176a569/step.json", - "size": 784, - "sha256": "f6e96cefa9861a0926ae65af738138d6b0881872c92ca5a352788c6ac8c921f0" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-b4f37096-7cf4-4e10-b7a8-05b7c3f517ea/original.png", - "size": 14031, - "sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-b4f37096-7cf4-4e10-b7a8-05b7c3f517ea/step.json", - "size": 1886, - "sha256": "27884ea4829c4269e25e742d2bdce3da417f6ef391f48a4fa932c06131663447" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-b4f37096-7cf4-4e10-b7a8-05b7c3f517ea/working.png", - "size": 14031, - "sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-da491a7e-52b7-437a-baa6-c9aa520b3b3f/step.json", - "size": 521, - "sha256": "343e0e6344b7f0724bd1eea6d38da9e97e0da24b8f86707b2eec14e99e0ef0bf" - }, - { - "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-e3373723-c536-4bc9-b8b4-55cd9b8ba525/original.png", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/original.png", "size": 13643, "sha256": "09e12f935511bb6fabb5637501aa7743516b96d990adfab62ccfa311a7b60606" }, { "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-e3373723-c536-4bc9-b8b4-55cd9b8ba525/step.json", - "size": 1609, - "sha256": "a6932c7b03650a3b8cc1b5b46f3759855baf3495d5ee87ac6a09995a91b6629b" + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/step.json", + "size": 1593, + "sha256": "36deb8a8b51952a668e871a03b1c7fba2aaf72b1722fd2d88c7e5d5cbbd8da91" }, { "kind": "sample", - "path": "../examples/sample-data/library/guides/guide-132037ac-1dcb-4ec0-8276-58f566a35489/steps/step-e3373723-c536-4bc9-b8b4-55cd9b8ba525/working.png", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/working.png", "size": 13643, "sha256": "09e12f935511bb6fabb5637501aa7743516b96d990adfab62ccfa311a7b60606" }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/original.png", + "size": 14031, + "sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/step.json", + "size": 1873, + "sha256": "4b0b3b74f851d034a2769c69df2e9d0428373ae15009fb3e2100afb5c10f7ebb" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/working.png", + "size": 14031, + "sha256": "b5e93a0ee74e2bdbbdf0871e901726dfbdc8b45dd648c959743520f92b02e7a2" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02a-permission-prompt/step.json", + "size": 763, + "sha256": "e8539addbe730e3a1baaa6898bf9779d2ce80564aecb3f196619d8c7ff67cbae" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/original.png", + "size": 13602, + "sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/step.json", + "size": 1968, + "sha256": "6f1cbb9d2ed32b89dec3d221822c6e973dbba42c11e93c2b35499179e8481532" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/working.png", + "size": 13602, + "sha256": "c96eedffdc5fd2eb9b63942cc00f1c8a91d01a0c5c2316c1d12d750b9b49e3d0" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-04-legacy-note/step.json", + "size": 506, + "sha256": "c6f78405f86f4183612f5865820b2454471cbc47975e9c15e055e7bf742b1eba" + }, + { + "kind": "sample", + "path": "../examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-05-deprecated-flow/step.json", + "size": 536, + "sha256": "709887c0a5debddc851216920ee3f6f2766e986059b21ac399e2533611ab5aed" + }, { "kind": "sample", "path": "../examples/sample-exports/docx/reset-a-password-in-admin-portal.docx", @@ -109,20 +109,20 @@ { "kind": "sample", "path": "../examples/sample-exports/html-rich/reset-a-password-in-admin-portal-rich.html", - "size": 149943, - "sha256": "e3c636d463c4db277f89a5330559661b6c9db193f94de9c4ad6801a88e3a7ff4" + "size": 149884, + "sha256": "70695d55c37d69abb191e65675f726eaa7aea6686a15a2e7d387195441e3ca8c" }, { "kind": "sample", "path": "../examples/sample-exports/html-simple/reset-a-password-in-admin-portal.html", "size": 146646, - "sha256": "2bf28616ed9ce47527e6e6da950b495e914b25a04376fb90e40e787ad0f4819a" + "sha256": "860c774b9b8bfd821b22deadc21f8021dc66bc9cacf43702eecbf55e944c6a9a" }, { "kind": "sample", "path": "../examples/sample-exports/image-bundle/reset-a-password-in-admin-portal-bundle.json", "size": 779, - "sha256": "38bdbf55d25f1620eb824192472fcf8faa65f38f4ef340bcd98e0668a66bdefe" + "sha256": "0108a63331925ea1fdd03326dd37dce1af3642a02d167e583d0d67b50fce27fa" }, { "kind": "sample", @@ -146,7 +146,7 @@ "kind": "sample", "path": "../examples/sample-exports/json/reset-a-password-in-admin-portal.json", "size": 6740, - "sha256": "434d5d41df7983686da8057a9939a57739fa9089dd2957862ccb810963f66532" + "sha256": "4a27376e75c3bb33fa8e1c4117c3314a04ec4766d67a9b5c729180d116d384d1" }, { "kind": "sample", @@ -205,14 +205,14 @@ { "kind": "sample", "path": "../examples/sample-guide.sfgz", - "size": 88968, - "sha256": "d041d4994f8d1a2f41a1c818ea706f64bb789eb4a3b903f67f86d6305339e412" + "size": 88427, + "sha256": "313b88f48e53e5ad7fb4e0a8189700ba0f6be642ec2311b1a2fe7ea4f3dd0481" }, { "kind": "sample", "path": "../examples/sample-manifest.json", - "size": 1186, - "sha256": "c88b283799b932cb3b23e096288222eed84beaaf6bf8ce0fecdf05de89cc2c1b" + "size": 1163, + "sha256": "cb2920e7500758074f1f54867db1ed187d3304c8badbceb06fd611057bd6fe5d" } ] } diff --git a/build/build_report.md b/build/build_report.md index a6e9a4d..9d10760 100644 --- a/build/build_report.md +++ b/build/build_report.md @@ -1,7 +1,7 @@ # StepForge Build Report Version: 0.1.0 -Generated: 2026-06-11T02:38:15.730Z +Generated: 2026-06-11T21:54:13.292Z Host: linux x64 (node v20.20.2) ## Outputs diff --git a/capture.js b/capture.js index 283d108..b826372 100644 --- a/capture.js +++ b/capture.js @@ -1,842 +1,3 @@ 'use strict'; -const path = require('node:path'); -const { spawn, execFileSync } = require('node:child_process'); -const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu, Notification } = require('electron'); -const { expandPlaceholders } = require('../core/placeholders'); -const raster = require('../core/raster'); -const { encodePng } = require('../core/png'); - -/** - * Capture service: full-screen, active-window, and region capture via - * Electron's desktopCapturer, plus a click-marker annotation at the cursor - * position and a capture session (start/pause/resume/finish). - * - * A session captures continuously, with three triggers layered by what the - * platform supports: - * - click-capture via an OS adapter (xinput on X11, PowerShell on Windows), - * - a global hotkey (unreliable on some Wayland compositors), - * - interval auto-capture as the always-works fallback. - * - * Note: under Wayland/WSLg, screen capture may require portal support; all - * failures surface as { ok: false, reason } instead of crashing. - */ - -// Dedupe duplicate watcher events for one physical click while still -// allowing intentionally fast clicking. -const CLICK_DEBOUNCE_MS = 40; -// Idle gap between frame-loop grabs. Keep this at zero so the buffered -// frame stays as close to real time as possible while recording. -const FRAME_LOOP_IDLE_MS = 0; -// A buffered frame older than this is too stale to pass off as "the screen -// at the instant of the click". -const CLICK_FRAME_MAX_AGE_MS = 600; -// How long a click waits for the in-flight grab before falling back to a -// one-off fresh shot. -const CLICK_FRAME_WAIT_MS = 2000; -const CLICK_CAPTURE_HIDE_DELAY_MS = 25; - -function pointInBounds(point, bounds) { - if (!point || !bounds) return false; - return point.x >= bounds.x - && point.x <= bounds.x + bounds.width - && point.y >= bounds.y - && point.y <= bounds.y + bounds.height; -} - -function hasBinary(name) { - try { - execFileSync('which', [name], { stdio: 'pipe' }); - return true; - } catch { - return false; - } -} - -class CaptureService { - constructor({ store, settings, getWindow, notify }) { - this.store = store; - this.settings = settings; - this.getWindow = getWindow; - this.notify = notify; - this.session = null; // { guideId, paused, count, intervalSec } - this.intervalTimer = null; - this.clickWatcher = null; - this.frameLoopTimer = null; - this.frameLoopRunning = false; - this.frameWaiters = []; - this.latestFrame = null; - this.lastClickCapture = 0; - this.clickWatcherBuf = ''; - this.clickWatcherPendingPress = false; - this.clickWatcherErrTail = ''; - this.clickQueue = Promise.resolve(); - this.frameLoopInFlight = false; - this.shooting = false; - } - - state() { - return this.session - ? { - active: true, - paused: this.session.paused, - guideId: this.session.guideId, - count: this.session.count, - intervalSec: this.session.intervalSec || 0, - clickCapture: Boolean(this.clickWatcher), - clickCaptureAvailable: this.clickCaptureAvailable(), - } - : { active: false, clickCaptureAvailable: this.clickCaptureAvailable() }; - } - - clickCaptureAvailable() { - if (this._clickAvail === undefined) { - this._clickAvail = process.platform === 'win32' || (process.platform === 'linux' && hasBinary('xinput')); - } - return this._clickAvail; - } - - startSession(guideId, { intervalSec = null } = {}) { - this.finishSession(); - // Default trigger: clicks when the platform supports it, otherwise an - // interval so a session always produces steps even if the global hotkey - // never fires (common under Wayland/WSLg). - let interval = intervalSec; - if (interval == null) { - interval = this.clickCaptureAvailable() ? 0 : (this.settings.get('capture.autoIntervalSec') || 5); - } - // Sessions start paused: nothing hides and no capturing happens until - // the user explicitly presses "Start recording" in the capture bar, so - // New Capture never makes the window vanish out from under them. - this.session = { guideId, paused: true, count: 0, intervalSec: interval }; - if (this.settings.get('capture.captureOutsideClicks') !== false) this.startClickWatcher(); - this.applyInterval(); - this.notify('capture:state', this.state()); - - // (Skipped for the dev screenshot hook, which needs a visible page.) - if (!process.env.STEPFORGE_SCREENSHOT) { - this.createSessionTray(); - const win = this.getWindow(); - // Remember whether the window was visible when the session was set - // up — that's what `togglePause` uses to decide whether to tuck the - // app away once the user actually starts recording. - this.hiddenForSession = Boolean(win && !win.isDestroyed() && win.isVisible()); - try { - new Notification({ - title: 'StepForge is ready to capture', - body: 'Click "Start recording" in the red capture bar when you’re ready. The window tucks away and the red tray icon takes over.', - }).show(); - } catch { /* notifications unavailable on this desktop */ } - } - } - - /** Red-dot tray icon with session controls, shown while recording. */ - createSessionTray() { - this.destroySessionTray(); - try { - const img = raster.createImage(16, 16, [0, 0, 0, 0]); - raster.fillOval(img, 2, 2, 12, 12, [229, 72, 77, 255]); - this.tray = new Tray(nativeImage.createFromBuffer(encodePng(img))); - this.tray.setToolTip('StepForge — capture session running'); - const rebuild = () => { - if (!this.tray || this.tray.isDestroyed()) return; - this.tray.setContextMenu(Menu.buildFromTemplate([ - { label: `Captured ${this.session ? this.session.count : 0} steps`, enabled: false }, - { type: 'separator' }, - { label: 'Capture now', click: () => this.sessionCapture('manual').then(rebuild).catch(() => {}) }, - { - label: this.session && this.session.paused ? 'Resume capturing' : 'Pause capturing', - click: () => { this.togglePause(); rebuild(); }, - }, - { - label: 'Open StepForge (pauses capture)', - click: () => { - this.togglePause(true); - this.showWindow(); - rebuild(); - }, - }, - { type: 'separator' }, - { label: 'Finish session', click: () => this.finishSession() }, - ])); - }; - rebuild(); - this.rebuildTrayMenu = rebuild; - this.tray.on('click', () => { - this.togglePause(true); - this.showWindow(); - rebuild(); - }); - } catch { - this.tray = null; // no tray on this desktop; cursor-over skip still protects clicks - } - } - - destroySessionTray() { - if (this.tray && !this.tray.isDestroyed()) this.tray.destroy(); - this.tray = null; - this.rebuildTrayMenu = null; - } - - showWindow() { - const win = this.getWindow(); - if (win && !win.isDestroyed()) { - win.show(); - win.focus(); - } - } - - setInterval(intervalSec) { - if (!this.session) return this.state(); - this.session.intervalSec = Math.max(0, Number(intervalSec) || 0); - this.applyInterval(); - this.notify('capture:state', this.state()); - return this.state(); - } - - applyInterval() { - if (this.intervalTimer) { - clearInterval(this.intervalTimer); - this.intervalTimer = null; - } - const sec = this.session && this.session.intervalSec; - if (sec > 0) { - this.intervalTimer = setInterval(() => { - this.sessionCapture('interval').catch(() => {}); - }, sec * 1000); - } - } - - togglePause(force) { - if (!this.session) return; - const wasPaused = this.session.paused; - this.session.paused = typeof force === 'boolean' ? force : !this.session.paused; - // Starting/resuming tucks the window away again for clean shots (after - // a brief delay so the user sees it happen) and starts the frame loop - // that serves click captures. Pausing stops the loop and discards the - // buffered frame, so a resume can never serve a pre-pause screen. - if (wasPaused && !this.session.paused) { - const win = this.getWindow(); - const arm = () => { - if (!this.session || this.session.paused) return; - if (this.hiddenForSession && win && !win.isDestroyed() && win.isVisible()) win.hide(); - if (this.settings.get('capture.captureOutsideClicks') !== false && this.clickCaptureAvailable()) { - this.startFrameLoop(); - } - }; - if (this.hiddenForSession && win && !win.isDestroyed()) setTimeout(arm, 400); - else arm(); - } else if (!wasPaused && this.session.paused) { - this.stopFrameLoop(); - } - if (this.rebuildTrayMenu) this.rebuildTrayMenu(); - this.notify('capture:state', this.state()); - } - - finishSession() { - if (this.intervalTimer) { - clearInterval(this.intervalTimer); - this.intervalTimer = null; - } - this.stopClickWatcher(); - this.stopFrameLoop(); - this.destroySessionTray(); - this.session = null; - if (this.hiddenForSession) { - this.hiddenForSession = false; - this.showWindow(); - } - this.notify('capture:state', this.state()); - } - - /** - * True when the user is interacting with StepForge itself. Deliberately - * based on cursor position over the visible window, not isFocused(): - * some compositors (WSLg) report focus as stuck-true, which would block - * every automatic capture forever. - */ - userIsInApp() { - const win = this.getWindow(); - if (!win || win.isDestroyed() || !win.isVisible() || win.isMinimized()) return false; - const cur = screen.getCursorScreenPoint(); - const b = win.getBounds(); - return cur.x >= b.x && cur.x <= b.x + b.width && cur.y >= b.y && cur.y <= b.y + b.height; - } - - /** One capture inside the active session (hotkey/click/interval/manual). */ - async sessionCapture(trigger = 'hotkey', clickPos = null) { - if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' }; - // Automatic triggers stand down while the user is in StepForge, so the - // app stays clickable mid-session and never screenshots itself. - if (trigger !== 'manual' && this.userIsInApp()) { - return { ok: false, reason: 'skipped — StepForge is focused' }; - } - - // Clicks are served from the frame loop: the buffered frame was grabbed - // at (or moments before) the click instant, so the background matches - // what the user clicked on. A click that lands while a grab is in - // flight waits for that frame instead of being dropped, so fast - // clicking still yields one step per click. - if (trigger === 'click') { - const frame = await this.frameForClick(clickPos); - if (!this.session || this.session.paused) return { ok: false, reason: 'no active capture session' }; - if (frame) { - const result = this.storeFrameAsStep(this.session.guideId, frame.mode, frame, clickPos); - if (result.ok) this.noteStepAdded(result.step, trigger); - return result; - } - // No usable frame (loop not running or grab failing): fall through - // to a one-off fresh shot. - } - - if (this.shooting) return { ok: false, reason: 'capture already in progress' }; - this.shooting = true; - try { - const mode = this.settings.get('capture.mode') || 'fullscreen'; - const grabMode = mode === 'region' ? 'fullscreen' : mode; - const finalResult = await this.shoot({ - guideId: this.session.guideId, - mode: grabMode, - delayMs: 0, - hideWindowDelayMs: trigger === 'click' ? CLICK_CAPTURE_HIDE_DELAY_MS : null, - refocus: false, // don't steal focus from the app the user is documenting - clickPos, - }); - if (finalResult.ok) this.noteStepAdded(finalResult.step, trigger); - return finalResult; - } finally { - this.shooting = false; - } - } - - noteStepAdded(step, trigger) { - this.session.count += 1; - this.notify('capture:added', { guideId: this.session.guideId, step, trigger }); - this.notify('capture:state', this.state()); - if (this.rebuildTrayMenu) this.rebuildTrayMenu(); // refresh step counter - } - - hotkeyCapture() { - return this.sessionCapture('hotkey'); - } - - // ---- click-triggered capture -------------------------------------------- - - /** - * Continuous screen-grab loop that runs while recording. It keeps the most - * recent frame in `latestFrame` so a click can be served from a frame - * grabbed at (or moments before) the instant of the click — a fresh grab - * started after the click would land hundreds of ms late and show the - * click's effects instead of what the user clicked on. - */ - startFrameLoop() { - if (this.frameLoopRunning) return; - this.frameLoopRunning = true; - const tick = async () => { - if (!this.frameLoopRunning) return; - if (!this.session || this.session.paused) { - this.frameLoopRunning = false; - this.frameLoopInFlight = false; - return; - } - try { - if (!this.shooting) { - this.frameLoopInFlight = true; - const mode = this.settings.get('capture.mode') || 'fullscreen'; - const grabMode = mode === 'region' ? 'fullscreen' : mode; - const frame = await this.captureCurrentFrame(grabMode); - if (this.frameLoopRunning) this.acceptFrame(frame); - } - } catch { - // Grab failures are fine — clicks fall back to a one-off fresh shot. - } finally { - this.frameLoopInFlight = false; - if (this.frameLoopRunning && this.session && !this.session.paused) { - this.frameLoopTimer = setTimeout(tick, FRAME_LOOP_IDLE_MS); - } - } - }; - this.frameLoopTimer = setTimeout(tick, 0); - } - - /** Store a grabbed frame and hand it to any clicks waiting on it. */ - acceptFrame(frame) { - this.latestFrame = frame; - const waiters = this.frameWaiters; - this.frameWaiters = []; - for (const resolve of waiters) resolve(frame); - } - - /** Resolves with the next frame the loop grabs (null on timeout/stop). */ - nextFrame(timeoutMs) { - return new Promise((resolve) => { - const entry = (frame) => { - clearTimeout(timer); - resolve(frame); - }; - const timer = setTimeout(() => { - this.frameWaiters = this.frameWaiters.filter((w) => w !== entry); - resolve(null); - }, timeoutMs); - this.frameWaiters.push(entry); - }); - } - - stopFrameLoop() { - if (this.frameLoopTimer) { - clearTimeout(this.frameLoopTimer); - this.frameLoopTimer = null; - } - this.frameLoopRunning = false; - this.latestFrame = null; - const waiters = this.frameWaiters; - this.frameWaiters = []; - for (const resolve of waiters) resolve(null); - } - - /** - * Freshest frame usable for a click capture: the buffered frame when it's - * recent enough, otherwise the next frame the loop delivers. Null when the - * loop isn't running or can't deliver in time. - */ - async frameForClick(clickPos = null) { - const mode = this.settings.get('capture.mode') || 'fullscreen'; - const grabMode = mode === 'region' ? 'fullscreen' : mode; - // Fast clicks can move to another monitor before the buffered frame is - // consumed; only reuse frames from the clicked display. - const usable = (f) => { - const sameDisplay = !clickPos || pointInBounds(clickPos, f && f.display && f.display.bounds); - return Boolean(f) - && f.mode === grabMode - && Date.now() - f.capturedAt <= CLICK_FRAME_MAX_AGE_MS - && sameDisplay; - }; - if (usable(this.latestFrame)) return this.latestFrame; - if (!this.frameLoopRunning || !this.frameLoopInFlight) return null; - const deadline = Date.now() + CLICK_FRAME_WAIT_MS; - while (this.frameLoopRunning && Date.now() < deadline) { - const next = await this.nextFrame(Math.max(1, deadline - Date.now())); - if (usable(next)) return next; - } - return null; - } - - startClickWatcher() { - this.stopClickWatcher(); - try { - this.clickWatcherBuf = ''; - this.clickWatcherPendingPress = false; - if (process.platform === 'linux' && hasBinary('xinput')) { - // Stream raw button events from the X server; one capture per press. - // xinput block-buffers stdout when piped, so a press event can sit - // in its buffer until later motion events flush it — by then the - // cursor read in onOsClick lands where the mouse moved *after* the - // click. stdbuf -oL forces line-buffering so events (and the cursor - // read) line up with the actual click instant. - const argv = hasBinary('stdbuf') - ? ['stdbuf', '-oL', 'xinput', 'test-xi2', '--root'] - : ['xinput', 'test-xi2', '--root']; - this.clickWatcher = spawn(argv[0], argv.slice(1), { stdio: ['ignore', 'pipe', 'ignore'] }); - this.clickWatcher.stdout.on('data', (chunk) => { - this.ingestClickWatcherChunk(chunk.toString(), 'linux'); - }); - } else if (process.platform === 'win32') { - // Poll left/right/middle buttons via GetAsyncKeyState. Bit 0x8000 is - // "down right now"; bit 0x0001 is "was pressed since the previous - // poll", which catches clicks shorter than one poll interval. The - // cursor is read in the same poll iteration that sees the press and - // shipped with the event, so the marker position is sampled at the - // click instant (within one ~15ms poll tick) instead of after the - // event has crossed the pipe. Coordinates are physical pixels - // (per-monitor DPI aware); onOsClick converts them to DIPs. - const ps = ` -$ErrorActionPreference = 'Stop' -Add-Type -TypeDefinition @' -using System; -using System.Runtime.InteropServices; -public struct SFPoint { public int X; public int Y; } -public static class SFMouse { - [DllImport("user32.dll")] public static extern short GetAsyncKeyState(int vKey); - [DllImport("user32.dll")] public static extern bool GetCursorPos(out SFPoint p); - [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); -} -'@ -try { [void][SFMouse]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch { } -$out = [Console]::Out -$out.WriteLine('READY') -$out.Flush() -$vks = 0x01, 0x02, 0x04 -$down = @{} -foreach ($vk in $vks) { $down[$vk] = $false } -$pt = New-Object SFPoint -while ($true) { - foreach ($vk in $vks) { - $s = [SFMouse]::GetAsyncKeyState($vk) - $isDown = ($s -band 0x8000) -ne 0 - $tapped = ($s -band 1) -ne 0 - if (($isDown -or $tapped) -and -not $down[$vk]) { - [void][SFMouse]::GetCursorPos([ref]$pt) - $out.WriteLine('CLICK ' + $pt.X + ' ' + $pt.Y) - $out.Flush() - } - $down[$vk] = $isDown - } - Start-Sleep -Milliseconds 10 -}`; - this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', ps], { - stdio: ['ignore', 'pipe', 'pipe'], - windowsHide: true, - }); - this.clickWatcher.stdout.on('data', (chunk) => { - this.ingestClickWatcherChunk(chunk.toString(), 'win32'); - }); - } - if (this.clickWatcher) { - const child = this.clickWatcher; - this.clickWatcherErrTail = ''; - if (child.stderr) { - child.stderr.on('data', (chunk) => { - this.clickWatcherErrTail = String(chunk).slice(-400); - }); - } - const lost = (reason) => { - if (this.clickWatcher !== child) return; // stopped deliberately - this.clickWatcher = null; - this.handleClickWatcherLoss(reason); - }; - child.on('error', (err) => lost(err && err.message)); - child.on('exit', (code) => lost(`exited with code ${code}`)); - } - } catch { - this.clickWatcher = null; - } - } - - /** - * The watcher process died mid-session (crashed X server, PowerShell - * blocked by policy, …). Captures must not silently stop: log why, switch - * the session to interval captures, and tell the UI. - */ - handleClickWatcherLoss(reason) { - this.clickWatcherPendingPress = false; - const detail = [reason, this.clickWatcherErrTail].filter(Boolean).join(' — '); - console.error(`[stepforge] click watcher stopped${detail ? `: ${detail}` : ''}`); - if (!this.session) return; - if (!this.session.intervalSec) { - this.session.intervalSec = this.settings.get('capture.autoIntervalSec') || 5; - this.applyInterval(); - } - this.notify('capture:state', this.state()); - } - - stopClickWatcher() { - if (this.clickWatcher) { - try { this.clickWatcher.kill(); } catch { /* already gone */ } - this.clickWatcher = null; - } - this.clickWatcherBuf = ''; - this.clickWatcherPendingPress = false; - } - - /** - * Buffer stdout chunks and only parse complete lines: a chunk boundary - * can split an event line in half, which used to corrupt press/release - * parsing and swallow clicks. - */ - ingestClickWatcherChunk(chunk, platform = process.platform) { - this.clickWatcherBuf += String(chunk); - const cut = this.clickWatcherBuf.lastIndexOf('\n'); - if (cut === -1) return; - const complete = this.clickWatcherBuf.slice(0, cut); - this.clickWatcherBuf = this.clickWatcherBuf.slice(cut + 1); - this.processClickWatcherData(complete, platform); - } - - processClickWatcherData(text, platform = process.platform) { - const lines = String(text).split(/\r?\n/); - if (platform === 'linux') { - // xinput prints each event as a multi-line block: an "EVENT type … - // (RawButtonPress)" header followed by a "detail: N" line carrying the - // button number. Fire on the detail line so scroll-wheel ticks (X11 - // reports them as buttons 4-7) neither create steps nor debounce away - // the real clicks that follow them. - for (const line of lines) { - if (!line) continue; - if (/RawButtonPress|ButtonPress/.test(line)) { - if (this.clickWatcherPendingPress) this.onOsClick(); - this.clickWatcherPendingPress = true; - continue; - } - if (!this.clickWatcherPendingPress) continue; - const detail = line.match(/detail:\s*(\d+)/); - if (detail) { - this.clickWatcherPendingPress = false; - const button = Number(detail[1]); - if (button < 4 || button > 7) this.onOsClick(); - } else if (line.includes('EVENT type')) { - // Next event arrived without a detail line in between — treat the - // pending press as a plain click rather than dropping it. - this.clickWatcherPendingPress = false; - this.onOsClick(); - } - } - return; - } - if (platform === 'win32') { - for (const line of lines) { - const m = /CLICK(?:\s+(-?\d+)\s+(-?\d+))?/.exec(line); - if (m) { - const osPoint = m[1] === undefined ? null : { x: Number(m[1]), y: Number(m[2]) }; - this.onOsClick(Date.now(), osPoint); - } - } - } - } - - onOsClick(at = Date.now(), osPoint = null) { - if (!this.session || this.session.paused) return; - if (at - this.lastClickCapture < CLICK_DEBOUNCE_MS) return; - this.lastClickCapture = at; - // Prefer the position the watcher sampled in the same poll that saw the - // press (physical px → DIP); otherwise read the cursor synchronously, - // right now, so the marker lands where the user clicked even if the - // shot itself takes a moment to grab. (Clicks on StepForge itself are - // filtered by the cursor-position check in sessionCapture, not by - // window focus — WSLg reports focus unreliably.) - let clickPos = null; - if (osPoint) { - clickPos = typeof screen.screenToDipPoint === 'function' - ? screen.screenToDipPoint(osPoint) - : osPoint; - } - if (!clickPos) clickPos = screen.getCursorScreenPoint(); - this.enqueueClickCapture(clickPos); - } - - /** - * Serialize click captures: a click that lands while an earlier capture is - * still being stored queues behind it instead of being dropped by the - * "capture already in progress" guard. The marker position was already - * read at click time, so a queued step still circles the right spot. - */ - enqueueClickCapture(clickPos) { - this.clickQueue = this.clickQueue - .then(() => this.sessionCapture('click', clickPos)) - .catch(() => {}); - return this.clickQueue; - } - - async captureCurrentFrame(mode, capturePoint = null) { - const grabbed = await this.grab(mode, capturePoint); - return { - mode, - png: grabbed.image.toPNG(), - size: grabbed.image.getSize(), - display: grabbed.display, - cursor: capturePoint || grabbed.cursor, - capturedAt: Date.now(), - }; - } - - storeFrameAsStep(guideId, mode, frame, clickPos = null) { - if (!frame) return { ok: false, reason: 'no capture frame available' }; - const annotations = []; - const cursor = clickPos || frame.cursor; - if (mode !== 'window' && this.settings.get('capture.clickMarker')) { - const fx = (cursor.x - frame.display.bounds.x) / frame.display.bounds.width; - const fy = (cursor.y - frame.display.bounds.y) / frame.display.bounds.height; - if (fx >= 0 && fx <= 1 && fy >= 0 && fy <= 1) { - const d = 0.035; - annotations.push({ - type: 'oval', - x: fx - d / 2, y: fy - (d * frame.size.width / frame.size.height) / 2, - w: d, h: d * frame.size.width / frame.size.height, - style: { - stroke: this.settings.get('capture.clickMarkerColor') || '#E5484D', - strokeWidth: 4, fill: 'transparent', - }, - }); - } - } - - const step = this.store.addStep(guideId, { - title: this.autoTitle(mode), - annotations, - focusedView: { - enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')), - zoom: 1, panX: 0.5, panY: 0.5, - }, - }, frame.png, frame.size); - return { ok: true, step }; - } - - autoTitle(mode) { - const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]'; - const now = new Date(); - const pad = (n) => String(n).padStart(2, '0'); - return expandPlaceholders(tplStr, { - Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen', - Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`, - Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`, - }); - } - - /** Grab the screen/window image as { image, display } or throw. */ - async grab(mode, cursorPoint = null) { - const cursor = cursorPoint || screen.getCursorScreenPoint(); - const display = screen.getDisplayNearestPoint(cursor); - const { width, height } = display.size; - const scale = display.scaleFactor || 1; - // Ask for both kinds: some compositors (WSLg/Wayland portals) expose no - // individual window sources, so window mode falls back to the screen. - const sources = await desktopCapturer.getSources({ - types: mode === 'window' ? ['window', 'screen'] : ['screen'], - thumbnailSize: { width: Math.round(width * scale), height: Math.round(height * scale) }, - }); - if (!sources.length) throw new Error('no capture sources available (portal/permissions?)'); - - let source = null; - if (mode === 'window') { - const win = this.getWindow(); - const ownTitle = win ? win.getTitle() : ''; - const windows = sources.filter((s) => s.id.startsWith('window:')); - source = windows.find((s) => s.name && s.name !== ownTitle && !/stepforge/i.test(s.name)) - || windows[0] - || sources.find((s) => s.id.startsWith('screen:')); - } else { - const screens = sources.filter((s) => s.id.startsWith('screen:')); - source = screens.find((s) => String(s.display_id) === String(display.id)) || screens[0] || sources[0]; - } - if (!source) throw new Error('no capture source matched'); - const image = source.thumbnail; - if (!image || image.isEmpty()) throw new Error('capture returned an empty image'); - return { image, display, cursor }; - } - - /** - * Hide the app window while `fn` runs so screenshots show the user's work, - * not StepForge itself. Restores visibility afterwards. - */ - async withWindowHidden(fn, { refocus = true, pauseMs = 350 } = {}) { - const win = this.getWindow(); - const wasVisible = win && !win.isDestroyed() && win.isVisible() && !win.isMinimized(); - if (wasVisible) { - win.hide(); - if (pauseMs > 0) { - await new Promise((r) => setTimeout(r, pauseMs)); // let the compositor repaint - } - } - try { - return await fn(); - } finally { - if (wasVisible && win && !win.isDestroyed()) { - if (refocus) { - win.show(); - win.focus(); - } else { - win.showInactive(); - } - } - } - } - - /** - * Take a screenshot and append it to the guide as a new image step. - * Adds a click-marker annotation at the cursor position when enabled. - */ - async shoot({ - guideId, - mode = 'fullscreen', - delayMs = null, - hideWindow = true, - refocus = true, - hideWindowDelayMs = null, - clickPos = null, - }) { - const delay = delayMs == null ? this.settings.get('capture.delayMs') || 0 : delayMs; - if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay)); - let frame; - try { - frame = hideWindow - ? await this.withWindowHidden(() => this.captureCurrentFrame(mode, clickPos), { - refocus, - pauseMs: hideWindowDelayMs == null ? 350 : hideWindowDelayMs, - }) - : await this.captureCurrentFrame(mode, clickPos); - } catch (err) { - return { ok: false, reason: err.message }; - } - return this.storeFrameAsStep(guideId, mode, frame, clickPos); - } - - /** - * Region capture: shoot the full screen, then let the user drag a - * rectangle in a fullscreen overlay; the crop becomes the step image. - */ - async regionCapture(guideId) { - let grabbed; - try { - grabbed = await this.withWindowHidden(() => this.grab('fullscreen')); - } catch (err) { - return { ok: false, reason: err.message }; - } - const { image, display } = grabbed; - const rect = await this.pickRegion(display, image); - if (!rect) return { ok: false, reason: 'selection cancelled' }; - - const cropped = image.crop(rect); - const size = cropped.getSize(); - if (!size.width || !size.height) return { ok: false, reason: 'empty selection' }; - const step = this.store.addStep(guideId, { title: this.autoTitle('region') }, - cropped.toPNG(), size); - return { ok: true, step }; - } - - /** Fullscreen overlay window that resolves with a crop rect (image px). */ - pickRegion(display, image) { - return new Promise((resolve) => { - const overlay = new BrowserWindow({ - x: display.bounds.x, - y: display.bounds.y, - width: display.bounds.width, - height: display.bounds.height, - frame: false, - transparent: true, - alwaysOnTop: true, - fullscreen: true, - skipTaskbar: true, - webPreferences: { - preload: path.join(__dirname, 'region-preload.js'), - contextIsolation: true, - }, - }); - let settled = false; - const finish = (rect) => { - if (settled) return; - settled = true; - if (!overlay.isDestroyed()) overlay.close(); - resolve(rect); - }; - const { ipcMain } = require('electron'); - const onPick = (event, rect) => { - if (event.sender !== overlay.webContents) return; - ipcMain.removeListener('region:picked', onPick); - if (!rect) return finish(null); - const imgSize = image.getSize(); - const sx = imgSize.width / display.bounds.width; - const sy = imgSize.height / display.bounds.height; - finish({ - x: Math.round(rect.x * sx), - y: Math.round(rect.y * sy), - width: Math.round(rect.w * sx), - height: Math.round(rect.h * sy), - }); - }; - ipcMain.on('region:picked', onPick); - overlay.on('closed', () => finish(null)); - overlay.loadFile(path.join(__dirname, 'renderer', 'region.html')); - }); - } -} - -module.exports = CaptureService; +module.exports = require('./app/capture'); diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e320f2b..b554ee4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,6 +3,17 @@ All notable user-visible changes are recorded here. The format follows Keep-a-Changelog conventions; versions follow semver. +## [Unreleased] + +### Fixed + +- Windows continuous click capture now uses a low-level mouse hook instead + of timer polling, so normal left-clicks are not missed when the app or + target system is under load. Click captures also preserve the original + click timestamp through the queue and choose a buffered frame from before + the click when one is available, keeping the marker aligned with the + click-time cursor position. + ## [0.1.0] - 2026-06-10 Initial release. diff --git a/examples/sample-data/library/guides/guide-sample-reset-password/guide.json b/examples/sample-data/library/guides/guide-sample-reset-password/guide.json index ae38be0..9723a17 100644 --- a/examples/sample-data/library/guides/guide-sample-reset-password/guide.json +++ b/examples/sample-data/library/guides/guide-sample-reset-password/guide.json @@ -13,8 +13,8 @@ "hideSkippedStepsInExports": true }, "themeOverride": "system", - "createdAt": "2026-06-11T14:16:39Z", - "updatedAt": "2026-06-11T14:16:39Z", + "createdAt": "2026-06-11T21:52:50Z", + "updatedAt": "2026-06-11T21:52:50Z", "stepsOrder": [ "step-sample-01-open-users", "step-sample-02-enable-policy", diff --git a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/step.json b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/step.json index bd2fcc8..90302db 100644 --- a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/step.json +++ b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-01-open-users/step.json @@ -25,7 +25,7 @@ "extraImages": [], "annotations": [ { - "id": "ann-80c0a4c6-8b7a-4ba9-8d38-35be0f890ccb", + "id": "ann-def495f6-e1e8-4d71-95cc-a7e5be6cc80a", "type": "rect", "x": 0.275, "y": 0.18, @@ -41,7 +41,7 @@ } }, { - "id": "ann-403e9d28-8b61-43e8-8a8c-74e35bf8b0ff", + "id": "ann-835f60f6-d649-428c-bdf9-4ab27c0b57be", "type": "number", "x": 0.3, "y": 0.08, @@ -60,7 +60,7 @@ ], "textBlocks": [ { - "id": "tb-9f433853-ccc5-450a-98ed-5dcbde7463ac", + "id": "tb-feafdd76-8d10-4d42-8fba-46cb9d2e8dae", "position": "after-description", "level": "info", "title": "Tip", diff --git a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/step.json b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/step.json index 960aa27..d38f753 100644 --- a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/step.json +++ b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02-enable-policy/step.json @@ -25,7 +25,7 @@ "extraImages": [], "annotations": [ { - "id": "ann-d2a29aa5-f083-45b6-8018-bd1e66a878fd", + "id": "ann-1cd13ad1-1723-406b-8c1a-c7d010b314c3", "type": "arrow", "x": 0.47, "y": 0.24, @@ -41,7 +41,7 @@ } }, { - "id": "ann-d4d4807d-87f4-4b9b-9aac-fe106edafa56", + "id": "ann-abdee8b7-3325-4dd0-b1e3-c75bffe6fba5", "type": "tooltip", "x": 0.53, "y": 0.13, @@ -58,7 +58,7 @@ } }, { - "id": "ann-a34d3ef1-f227-43ac-a59e-a433dfb7f8d5", + "id": "ann-140206cb-5e66-4027-87cf-2e8898fd6c9b", "type": "number", "x": 0.31, "y": 0.08, diff --git a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02a-permission-prompt/step.json b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02a-permission-prompt/step.json index f71c3c6..d9018ee 100644 --- a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02a-permission-prompt/step.json +++ b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-02a-permission-prompt/step.json @@ -19,7 +19,7 @@ "annotations": [], "textBlocks": [ { - "id": "tb-5ef09593-6f95-475d-b0c6-6144d855bed7", + "id": "tb-3d70c3b9-3d42-42a1-a7b9-a063b83c1c49", "position": "after-description", "level": "warn", "title": "Access", diff --git a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/step.json b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/step.json index 02daa37..35b53ba 100644 --- a/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/step.json +++ b/examples/sample-data/library/guides/guide-sample-reset-password/steps/step-sample-03-review-confirmation/step.json @@ -25,7 +25,7 @@ "extraImages": [], "annotations": [ { - "id": "ann-cab1dba5-f903-47c3-a4ec-efff6f70cc3f", + "id": "ann-e650d25f-0c72-432d-afca-26e1f4c7ea6a", "type": "blur", "x": 0.49, "y": 0.32, @@ -42,7 +42,7 @@ "radius": 12 }, { - "id": "ann-f6e54195-1e03-4455-83cf-8ecfafde5993", + "id": "ann-51f325e5-2522-47ad-a0de-b53318cdcd00", "type": "highlight", "x": 0.47, "y": 0.24, @@ -58,7 +58,7 @@ } }, { - "id": "ann-0c74b069-aad1-4d83-bf96-f902009905d6", + "id": "ann-f120f280-f3cc-46c2-9f3e-ef2dac1a9d27", "type": "number", "x": 0.31, "y": 0.08, diff --git a/examples/sample-exports/html-rich/reset-a-password-in-admin-portal-rich.html b/examples/sample-exports/html-rich/reset-a-password-in-admin-portal-rich.html index a768c8c..81f30c6 100644 --- a/examples/sample-exports/html-rich/reset-a-password-in-admin-portal-rich.html +++ b/examples/sample-exports/html-rich/reset-a-password-in-admin-portal-rich.html @@ -95,7 +95,7 @@ Step 3
FieldValue
TitleAdmin Portal
OwnerSupport
-
Generated by StepForge on 2026-06-11T14:16:39.204Z — 4 steps
+
Generated by StepForge on 2026-06-11T21:52:50.346Z — 4 steps