add keyboard tracking and app-qualified smart titles
**Keyboard hook (Windows)**
- Extends the existing C# WH_MOUSE_LL process to also install
WH_KEYBOARD_LL alongside it (keyboard hook is optional — failure
does not break mouse capture).
- Emits CHAR <code> <ts> for printable keystrokes, KEY <name> <ts>
for modifier combos (Ctrl+T) and special keys (Backspace, Enter).
**Text accumulation in capture session**
- CaptureService tracks _keyBuffer (typed chars since last step) and
_lastShortcut (last modifier combo) using the new onKeyboardEvent()
method.
- snapshotKeyContext() is called at enqueueClickCapture time so each
step's clickMeta.keyContext carries { recentTyped, recentShortcut }.
- Buffer resets after each snapshot; stale input (>8s gap) is dropped.
**UIAutomation element value**
- collectWindowsWindowContext now reads ValuePattern.Current.Value
from the clicked element — giving us what's actually typed in a
search box or text field without needing the keyboard buffer.
**Smart title generation (core/text-intel.js)**
- Priority chain: keyboard shortcut → element value → typed text
→ OCR → element label → page title → app name.
- SHORTCUT_TITLES maps 50+ common shortcuts (Ctrl+T, Ctrl+S, F5 …)
to natural language descriptions: "Open new tab", "Save", etc.
- qualifyTitleWithApp() appends "in Chrome / VS Code / Terminal / …"
to every title when the app is known: "Click Save in VS Code",
"Search for oracle in Chrome", "Open new tab in Chrome".
- APP_DISPLAY_NAMES covers browsers, editors, terminals, office apps.
Six new unit tests cover shortcuts, typed-text search, element value,
and app-qualified OCR titles. Capture test updated for keyContext.
Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Claude Sonnet 4.6
+165
-9
@@ -136,6 +136,9 @@ class CaptureService {
|
||||
this.linuxEvent = null; // event block currently being parsed
|
||||
this.pendingRawClick = null; // raw press waiting for its coordinate twin
|
||||
this.pendingClickOsPoint = null;
|
||||
this._keyBuffer = ''; // printable chars typed since last capture
|
||||
this._lastShortcut = ''; // last Ctrl+X / Alt+X combination
|
||||
this._keyLastAt = 0; // timestamp of last key event
|
||||
this.clickQueue = Promise.resolve();
|
||||
this.frameLoopInFlight = false;
|
||||
this.frameLoopGrabStartedAt = null;
|
||||
@@ -799,12 +802,15 @@ using System.Collections.Concurrent;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
public static class SFMouseHook {
|
||||
public static class SFHook {
|
||||
private const int WH_MOUSE_LL = 14;
|
||||
private const int WH_KEYBOARD_LL = 13;
|
||||
private const int WM_LBUTTONDOWN = 0x0201;
|
||||
private const int WM_RBUTTONDOWN = 0x0204;
|
||||
private const int WM_MBUTTONDOWN = 0x0207;
|
||||
private const int WM_XBUTTONDOWN = 0x020B;
|
||||
private const int WM_KEYDOWN = 0x0100;
|
||||
private const int WM_SYSKEYDOWN = 0x0104;
|
||||
private const long UnixEpochMilliseconds = 62135596800000L;
|
||||
// Opting this process out of Windows Power Throttling (EcoQoS). In a
|
||||
// power-saving plan the OS CPU-starves background processes; a starved
|
||||
@@ -818,7 +824,9 @@ public static class SFMouseHook {
|
||||
private const uint HIGH_PRIORITY_CLASS = 0x00000080;
|
||||
|
||||
private static IntPtr hook = IntPtr.Zero;
|
||||
private static LowLevelMouseProc proc = HookCallback;
|
||||
private static IntPtr keyHook = IntPtr.Zero;
|
||||
private static LowLevelMouseProc proc = MouseHookCallback;
|
||||
private static LowLevelKeyboardProc keyProc = KeyboardHookCallback;
|
||||
private static readonly ConcurrentQueue<string> queue = new ConcurrentQueue<string>();
|
||||
private static readonly AutoResetEvent signal = new AutoResetEvent(false);
|
||||
|
||||
@@ -854,11 +862,34 @@ public static class SFMouseHook {
|
||||
public uint StateMask;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct KBDLLHOOKSTRUCT {
|
||||
public uint vkCode;
|
||||
public uint scanCode;
|
||||
public uint flags;
|
||||
public uint time;
|
||||
public UIntPtr dwExtraInfo;
|
||||
}
|
||||
|
||||
private delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam);
|
||||
private delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelKeyboardProc lpfn, IntPtr hMod, uint dwThreadId);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern short GetKeyState(int nVirtKey);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern bool GetKeyboardState([Out] byte[] lpKeyState);
|
||||
|
||||
[DllImport("user32.dll", CharSet = CharSet.Unicode)]
|
||||
private static extern int ToUnicode(uint wVirtKey, uint wScanCode, byte[] lpKeyState,
|
||||
System.Text.StringBuilder pwszBuff, int cchBuff, uint wFlags);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
private static extern bool UnhookWindowsHookEx(IntPtr hhk);
|
||||
|
||||
@@ -917,6 +948,8 @@ public static class SFMouseHook {
|
||||
if (hook == IntPtr.Zero) {
|
||||
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
||||
}
|
||||
// Keyboard hook is optional — if it fails we still get click titles.
|
||||
try { keyHook = SetWindowsHookEx(WH_KEYBOARD_LL, keyProc, GetModuleHandle(null), 0); } catch { }
|
||||
|
||||
Console.Out.WriteLine("READY");
|
||||
Console.Out.Flush();
|
||||
@@ -928,6 +961,7 @@ public static class SFMouseHook {
|
||||
}
|
||||
|
||||
UnhookWindowsHookEx(hook);
|
||||
if (keyHook != IntPtr.Zero) UnhookWindowsHookEx(keyHook);
|
||||
}
|
||||
|
||||
private static void WriterLoop() {
|
||||
@@ -941,7 +975,7 @@ public static class SFMouseHook {
|
||||
}
|
||||
}
|
||||
|
||||
private static IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
|
||||
private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
|
||||
if (nCode >= 0) {
|
||||
int message = wParam.ToInt32();
|
||||
string button = ButtonName(message, lParam);
|
||||
@@ -955,6 +989,75 @@ public static class SFMouseHook {
|
||||
return CallNextHookEx(hook, nCode, wParam, lParam);
|
||||
}
|
||||
|
||||
private static IntPtr KeyboardHookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
|
||||
if (nCode >= 0 && (wParam.ToInt32() == WM_KEYDOWN || wParam.ToInt32() == WM_SYSKEYDOWN)) {
|
||||
try {
|
||||
KBDLLHOOKSTRUCT kb = (KBDLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(KBDLLHOOKSTRUCT));
|
||||
int vk = (int)kb.vkCode;
|
||||
bool ctrl = (GetKeyState(0x11) & 0x8000) != 0;
|
||||
bool alt = (GetKeyState(0x12) & 0x8000) != 0;
|
||||
bool shift = (GetKeyState(0x10) & 0x8000) != 0;
|
||||
// Skip standalone modifier keys.
|
||||
bool isMod = vk == 0x10 || vk == 0x11 || vk == 0x12 || vk == 0x5B || vk == 0x5C;
|
||||
if (!isMod) {
|
||||
long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds;
|
||||
if (ctrl || alt) {
|
||||
// Emit shortcut: "KEY Ctrl+T 123456"
|
||||
string name = VkName(vk);
|
||||
if (name != null) {
|
||||
string mods = (ctrl ? "Ctrl" : "") + (ctrl && (alt || shift) ? "+" : "") +
|
||||
(alt ? "Alt" : "") + ((alt || ctrl) && shift ? "+" : "") +
|
||||
(shift ? "Shift" : "");
|
||||
queue.Enqueue("KEY " + mods + "+" + name + " " + unixMs);
|
||||
signal.Set();
|
||||
}
|
||||
} else if (vk == 0x08) {
|
||||
queue.Enqueue("KEY Backspace " + unixMs); signal.Set();
|
||||
} else if (vk == 0x1B) {
|
||||
queue.Enqueue("KEY Escape " + unixMs); signal.Set();
|
||||
} else if (vk == 0x0D) {
|
||||
queue.Enqueue("KEY Enter " + unixMs); signal.Set();
|
||||
} else {
|
||||
// Map to Unicode character using current keyboard layout + shift state.
|
||||
byte[] ks = new byte[256];
|
||||
GetKeyboardState(ks);
|
||||
var sb = new System.Text.StringBuilder(4);
|
||||
int res = ToUnicode(kb.vkCode, kb.scanCode, ks, sb, 4, 0);
|
||||
if (res > 0) {
|
||||
char ch = sb[0];
|
||||
if (ch >= 0x20 && ch < 0x7F) {
|
||||
queue.Enqueue("CHAR " + (int)ch + " " + unixMs);
|
||||
signal.Set();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
return CallNextHookEx(keyHook, nCode, wParam, lParam);
|
||||
}
|
||||
|
||||
private static string VkName(int vk) {
|
||||
if (vk >= 0x41 && vk <= 0x5A) return ((char)vk).ToString(); // A-Z
|
||||
if (vk >= 0x30 && vk <= 0x39) return ((char)vk).ToString(); // 0-9
|
||||
if (vk >= 0x70 && vk <= 0x87) return "F" + (vk - 0x6F); // F1-F24
|
||||
switch (vk) {
|
||||
case 0x09: return "Tab";
|
||||
case 0x0D: return "Enter";
|
||||
case 0x1B: return "Escape";
|
||||
case 0x20: return "Space";
|
||||
case 0x21: return "PageUp"; case 0x22: return "PageDown";
|
||||
case 0x23: return "End"; case 0x24: return "Home";
|
||||
case 0x25: return "Left"; case 0x26: return "Up";
|
||||
case 0x27: return "Right"; case 0x28: return "Down";
|
||||
case 0x2E: return "Delete";
|
||||
case 0x6B: case 0xBB: return "Plus";
|
||||
case 0x6D: case 0xBD: return "Minus";
|
||||
case 0xBE: return "Period"; case 0xBF: return "Slash";
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string ButtonName(int message, IntPtr lParam) {
|
||||
if (message == WM_LBUTTONDOWN) return "left";
|
||||
if (message == WM_RBUTTONDOWN) return "right";
|
||||
@@ -968,7 +1071,7 @@ public static class SFMouseHook {
|
||||
}
|
||||
}
|
||||
'@
|
||||
[SFMouseHook]::Run()
|
||||
[SFHook]::Run()
|
||||
`;
|
||||
this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', ps], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
@@ -1103,11 +1206,22 @@ public static class SFMouseHook {
|
||||
}
|
||||
if (platform === 'win32') {
|
||||
for (const line of lines) {
|
||||
const m = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(line.trim());
|
||||
if (m) {
|
||||
const osPoint = m[1] === undefined ? null : { x: Number(m[1]), y: Number(m[2]) };
|
||||
const eventAt = m[4] === undefined ? Date.now() : Number(m[4]);
|
||||
this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, m[3] || 'mouse');
|
||||
const trimmed = line.trim();
|
||||
const clickM = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(trimmed);
|
||||
if (clickM) {
|
||||
const osPoint = clickM[1] === undefined ? null : { x: Number(clickM[1]), y: Number(clickM[2]) };
|
||||
const eventAt = clickM[4] === undefined ? Date.now() : Number(clickM[4]);
|
||||
this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, clickM[3] || 'mouse');
|
||||
continue;
|
||||
}
|
||||
const keyM = /^KEY\s+(\S+)\s+(\d+)\s*$/.exec(trimmed);
|
||||
if (keyM) {
|
||||
this.onKeyboardEvent('KEY', keyM[1], Number(keyM[2]));
|
||||
continue;
|
||||
}
|
||||
const charM = /^CHAR\s+(\d+)\s+(\d+)\s*$/.exec(trimmed);
|
||||
if (charM) {
|
||||
this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2]));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1285,6 +1399,8 @@ public static class SFMouseHook {
|
||||
if (osPoint) {
|
||||
clickMeta.osPoint = { x: osPoint.x, y: osPoint.y };
|
||||
}
|
||||
// Snapshot keyboard context accumulated since the last capture.
|
||||
clickMeta.keyContext = this.snapshotKeyContext();
|
||||
if (this.session && !this.session.paused && !this.userIsInApp()) {
|
||||
// The guide id pins the click to its recording so it can still be
|
||||
// stored if the session stops while this click waits in the queue.
|
||||
@@ -1298,6 +1414,46 @@ public static class SFMouseHook {
|
||||
return this.clickQueue;
|
||||
}
|
||||
|
||||
// --- Keyboard context tracking -------------------------------------------
|
||||
|
||||
onKeyboardEvent(type, data, eventAt) {
|
||||
const now = Number.isFinite(eventAt) ? eventAt : Date.now();
|
||||
// Discard stale typing that happened more than 8 seconds ago (user moved on).
|
||||
if (now - this._keyLastAt > 8000) {
|
||||
this._keyBuffer = '';
|
||||
}
|
||||
this._keyLastAt = now;
|
||||
|
||||
if (type === 'CHAR') {
|
||||
const ch = typeof data === 'number' ? String.fromCharCode(data) : String(data);
|
||||
this._keyBuffer = (this._keyBuffer + ch).slice(-200);
|
||||
} else if (type === 'KEY') {
|
||||
const key = String(data);
|
||||
if (key === 'Backspace') {
|
||||
this._keyBuffer = this._keyBuffer.slice(0, -1);
|
||||
} else if (key === 'Escape') {
|
||||
this._keyBuffer = '';
|
||||
} else if (key === 'Enter') {
|
||||
// Keep typed text — Enter often submits what was typed.
|
||||
} else {
|
||||
// It's a modifier+key shortcut (Ctrl+T etc.).
|
||||
this._lastShortcut = key;
|
||||
this._keyBuffer = ''; // a shortcut resets the typed-text buffer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
snapshotKeyContext() {
|
||||
const ctx = {
|
||||
recentTyped: this._keyBuffer.trim(),
|
||||
recentShortcut: this._lastShortcut,
|
||||
};
|
||||
// Reset after snapshot so each step gets its own context.
|
||||
this._keyBuffer = '';
|
||||
this._lastShortcut = '';
|
||||
return ctx;
|
||||
}
|
||||
|
||||
async captureCurrentFrame(mode, capturePoint = null, startedAt = Date.now()) {
|
||||
const grabbed = await this.grab(mode, capturePoint);
|
||||
return {
|
||||
|
||||
+18
-1
@@ -181,6 +181,7 @@ class TextIntelService {
|
||||
$elementRole = '';
|
||||
$elementClass = '';
|
||||
$elementProcessId = 0;
|
||||
$elementValue = '';
|
||||
if (${hasPoint ? '$true' : '$false'}) {
|
||||
try {
|
||||
Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null
|
||||
@@ -192,6 +193,12 @@ class TextIntelService {
|
||||
$elementRole = $current.LocalizedControlType;
|
||||
$elementClass = $current.ClassName;
|
||||
$elementProcessId = $current.ProcessId;
|
||||
try {
|
||||
$valPattern = [System.Windows.Automation.ValuePattern]::Pattern;
|
||||
if ($element.GetSupportedPatterns() -contains $valPattern) {
|
||||
$elementValue = $element.GetCurrentPattern($valPattern).Current.Value;
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
@@ -218,6 +225,7 @@ public static class Win32 {
|
||||
elementLabel = $elementLabel;
|
||||
elementRole = $elementRole;
|
||||
elementClass = $elementClass;
|
||||
elementValue = $elementValue;
|
||||
elementProcessId = $elementProcessId;
|
||||
pid = $pid;
|
||||
};
|
||||
@@ -284,11 +292,14 @@ public static class Win32 {
|
||||
}
|
||||
|
||||
async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
|
||||
const keyContext = clickMeta?.keyContext || {};
|
||||
const recentTyped = keyContext.recentTyped || '';
|
||||
const recentShortcut = keyContext.recentShortcut || '';
|
||||
const [metadata, ocr] = await Promise.all([
|
||||
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
|
||||
this.ocrAroundClick(frame, clickPos),
|
||||
]);
|
||||
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text });
|
||||
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text, recentTyped, recentShortcut });
|
||||
return {
|
||||
title,
|
||||
captureMetadata: {
|
||||
@@ -297,6 +308,9 @@ public static class Win32 {
|
||||
appName: metadata.appName || '',
|
||||
elementLabel: metadata.elementLabel || '',
|
||||
elementRole: metadata.elementRole || '',
|
||||
elementValue: metadata.elementValue || '',
|
||||
recentTyped,
|
||||
recentShortcut,
|
||||
mode,
|
||||
},
|
||||
};
|
||||
@@ -440,8 +454,11 @@ public static class Win32 {
|
||||
appName: step.captureMetadata.appName,
|
||||
elementLabel: step.captureMetadata.elementLabel,
|
||||
elementRole: step.captureMetadata.elementRole,
|
||||
elementValue: step.captureMetadata.elementValue,
|
||||
},
|
||||
ocrText: step.captureMetadata.ocrText,
|
||||
recentTyped: step.captureMetadata.recentTyped,
|
||||
recentShortcut: step.captureMetadata.recentShortcut,
|
||||
});
|
||||
captureContext = {
|
||||
...step.captureMetadata,
|
||||
|
||||
Reference in New Issue
Block a user