Tyler/StepForge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: capture window title from click watcher instead of spawning PowerShell

Browse Source 
Root cause of all-"Screen capture" titles: collectWindowsWindowContext
called execFileSync('powershell.exe') with a 1200 ms timeout, but
PowerShell cold-start on typical Windows systems takes 1-3 seconds.
Every capture timed out silently, returned empty metadata, and fell
back to "Screen capture".

Fix:
- The click watcher C# process (already running, already compiled)
  now emits a CTX event immediately before each CLICK event using
  GetForegroundWindow → GetWindowText + QueryFullProcessImageName.
  These are synchronous Win32 calls, sub-millisecond, no startup cost.
- CTX payload is base64-encoded to survive the line protocol safely:
    CTX <b64title> <b64app> <unixMs>
- processClickWatcherData parses CTX lines and stores the decoded
  strings in this._lastWindowContext.
- enqueueClickCapture attaches it as clickMeta.windowContext.
- buildCaptureContext uses it directly (Promise.resolve) when present,
  bypassing the PowerShell spawn entirely.

Fallback (manual captures without a session click watcher running):
- collectWindowsWindowContext is now async and uses execFile with a
  4 s timeout instead of execFileSync at 1200 ms, so it no longer
  blocks the event loop and has room to succeed on slower machines.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Tyler
2026-06-24 10:16:11 -05:00
co-authored by Claude Sonnet 4.6
parent 13b5f67de8
commit 879434f14f
2 changed files with 91 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/capture.js
+72
View File
@@ -139,6 +139,7 @@ class CaptureService {
this._keyBuffer = ''; // printable chars typed since last capture
this._lastShortcut = ''; // last Ctrl+X / Alt+X combination
this._keyLastAt = 0; // timestamp of last key event
this._lastWindowContext = null; // last CTX event from the click watcher
this.clickQueue = Promise.resolve();
this.frameLoopInFlight = false;
this.frameLoopGrabStartedAt = null;
@@ -920,6 +921,58 @@ public static class SFHook {
[DllImport("kernel32.dll")]
private static extern bool SetPriorityClass(IntPtr hProcess, uint dwPriorityClass);
[DllImport("user32.dll")]
private static extern IntPtr GetForegroundWindow();
[DllImport("user32.dll", CharSet = CharSet.Unicode)]
private static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count);
[DllImport("user32.dll")]
private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern IntPtr OpenProcess(uint processAccess, bool bInheritHandle, uint processId);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool CloseHandle(IntPtr hObject);
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
private static extern bool QueryFullProcessImageName(IntPtr hProcess, uint dwFlags, System.Text.StringBuilder lpExeName, ref uint lpdwSize);
private static string GetFwTitle() {
try {
IntPtr hwnd = GetForegroundWindow();
if (hwnd == IntPtr.Zero) return "";
var sb = new System.Text.StringBuilder(512);
GetWindowText(hwnd, sb, sb.Capacity);
return sb.ToString();
} catch { return ""; }
}
private static string GetFwApp() {
try {
IntPtr hwnd = GetForegroundWindow();
if (hwnd == IntPtr.Zero) return "";
uint pid = 0;
GetWindowThreadProcessId(hwnd, out pid);
if (pid == 0) return "";
IntPtr hProc = OpenProcess(0x1000u, false, pid);
if (hProc == IntPtr.Zero) return "";
var sb = new System.Text.StringBuilder(260);
uint sz = (uint)sb.Capacity;
QueryFullProcessImageName(hProc, 0u, sb, ref sz);
CloseHandle(hProc);
string path = sb.ToString();
return string.IsNullOrEmpty(path) ? "" : System.IO.Path.GetFileNameWithoutExtension(path);
} catch { return ""; }
}
// Base64-encodes a string for safe line-protocol transmission; "-" for empty.
private static string B64(string s) {
if (string.IsNullOrEmpty(s)) return "-";
try { return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(s)); } catch { return "-"; }
}
// Force this process to run at full CPU speed regardless of the power plan,
// so the mouse-hook callback never trips LowLevelHooksTimeout and clicks
// keep being delivered while the laptop is in eco / power-saving mode.
@@ -982,6 +1035,12 @@ public static class SFHook {
if (button != null) {
MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT));
long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds;
// Capture window context while the user's app is still in foreground.
// GetForegroundWindow is synchronous and fast (no IPC overhead).
try {
string t = GetFwTitle(), a = GetFwApp();
queue.Enqueue("CTX " + B64(t) + " " + B64(a) + " " + unixMs);
} catch { }
queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs);
signal.Set();
}
@@ -1222,6 +1281,15 @@ public static class SFHook {
const charM = /^CHAR\s+(\d+)\s+(\d+)\s*$/.exec(trimmed);
if (charM) {
this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2]));
continue;
}
const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+\d+$/.exec(trimmed);
if (ctxM) {
const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8');
this._lastWindowContext = {
windowTitle: decode(ctxM[1]),
appName: decode(ctxM[2]),
};
}
}
}
@@ -1401,6 +1469,10 @@ public static class SFHook {
}
// Snapshot keyboard context accumulated since the last capture.
clickMeta.keyContext = this.snapshotKeyContext();
// Attach the window context emitted by the click watcher alongside the click.
if (this._lastWindowContext) {
clickMeta.windowContext = this._lastWindowContext;
}
if (this.session && !this.session.paused && !this.userIsInApp()) {
// The guide id pins the click to its recording so it can still be
// stored if the session stops while this click waits in the queue.
app/text-intel.js
+19 -9
View File
@@ -2,7 +2,7 @@
const fs = require('node:fs');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const { execFileSync, execFile } = require('node:child_process');
const {
DEFAULT_CAPTURE_TITLES,
@@ -170,7 +170,7 @@ class TextIntelService {
return { appName: '', windowTitle: '' };
}
collectWindowsWindowContext(osPoint = null) {
async collectWindowsWindowContext(osPoint = null) {
const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y);
const clickX = hasPoint ? Number(osPoint.x) : 0;
const clickY = hasPoint ? Number(osPoint.y) : 0;
@@ -231,12 +231,17 @@ public static class Win32 {
};
$out | ConvertTo-Json -Compress;
`;
const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
}).trim();
return JSON.parse(result || '{}');
return new Promise(resolve => {
execFile('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], {
encoding: 'utf8',
timeout: 4000,
windowsHide: true,
}, (err, stdout) => {
if (err) { resolve({}); return; }
try { resolve(JSON.parse(stdout.trim() || '{}')); }
catch { resolve({}); }
});
});
}
collectMacWindowContext() {
@@ -295,8 +300,13 @@ public static class Win32 {
const keyContext = clickMeta?.keyContext || {};
const recentTyped = keyContext.recentTyped || '';
const recentShortcut = keyContext.recentShortcut || '';
// Use window context pre-captured by the click watcher when available.
// This avoids a costly PowerShell cold-start (1–3 s) on every capture.
const fastContext = clickMeta?.windowContext || null;
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
fastContext
? Promise.resolve(fastContext)
: this.collectForegroundWindowContext(clickMeta?.osPoint || null),
this.ocrAroundClick(frame, clickPos),
]);
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text, recentTyped, recentShortcut });