add keyboard tracking and app-qualified smart titles
**Keyboard hook (Windows)**
- Extends the existing C# WH_MOUSE_LL process to also install
WH_KEYBOARD_LL alongside it (keyboard hook is optional — failure
does not break mouse capture).
- Emits CHAR <code> <ts> for printable keystrokes, KEY <name> <ts>
for modifier combos (Ctrl+T) and special keys (Backspace, Enter).
**Text accumulation in capture session**
- CaptureService tracks _keyBuffer (typed chars since last step) and
_lastShortcut (last modifier combo) using the new onKeyboardEvent()
method.
- snapshotKeyContext() is called at enqueueClickCapture time so each
step's clickMeta.keyContext carries { recentTyped, recentShortcut }.
- Buffer resets after each snapshot; stale input (>8s gap) is dropped.
**UIAutomation element value**
- collectWindowsWindowContext now reads ValuePattern.Current.Value
from the clicked element — giving us what's actually typed in a
search box or text field without needing the keyboard buffer.
**Smart title generation (core/text-intel.js)**
- Priority chain: keyboard shortcut → element value → typed text
→ OCR → element label → page title → app name.
- SHORTCUT_TITLES maps 50+ common shortcuts (Ctrl+T, Ctrl+S, F5 …)
to natural language descriptions: "Open new tab", "Save", etc.
- qualifyTitleWithApp() appends "in Chrome / VS Code / Terminal / …"
to every title when the app is known: "Click Save in VS Code",
"Search for oracle in Chrome", "Open new tab in Chrome".
- APP_DISPLAY_NAMES covers browsers, editors, terminals, office apps.
Six new unit tests cover shortcuts, typed-text search, element value,
and app-qualified OCR titles. Capture test updated for keyContext.
Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
+18
-1
@@ -181,6 +181,7 @@ class TextIntelService {
|
||||
$elementRole = '';
|
||||
$elementClass = '';
|
||||
$elementProcessId = 0;
|
||||
$elementValue = '';
|
||||
if (${hasPoint ? '$true' : '$false'}) {
|
||||
try {
|
||||
Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null
|
||||
@@ -192,6 +193,12 @@ class TextIntelService {
|
||||
$elementRole = $current.LocalizedControlType;
|
||||
$elementClass = $current.ClassName;
|
||||
$elementProcessId = $current.ProcessId;
|
||||
try {
|
||||
$valPattern = [System.Windows.Automation.ValuePattern]::Pattern;
|
||||
if ($element.GetSupportedPatterns() -contains $valPattern) {
|
||||
$elementValue = $element.GetCurrentPattern($valPattern).Current.Value;
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
@@ -218,6 +225,7 @@ public static class Win32 {
|
||||
elementLabel = $elementLabel;
|
||||
elementRole = $elementRole;
|
||||
elementClass = $elementClass;
|
||||
elementValue = $elementValue;
|
||||
elementProcessId = $elementProcessId;
|
||||
pid = $pid;
|
||||
};
|
||||
@@ -284,11 +292,14 @@ public static class Win32 {
|
||||
}
|
||||
|
||||
async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
|
||||
const keyContext = clickMeta?.keyContext || {};
|
||||
const recentTyped = keyContext.recentTyped || '';
|
||||
const recentShortcut = keyContext.recentShortcut || '';
|
||||
const [metadata, ocr] = await Promise.all([
|
||||
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
|
||||
this.ocrAroundClick(frame, clickPos),
|
||||
]);
|
||||
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text });
|
||||
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text, recentTyped, recentShortcut });
|
||||
return {
|
||||
title,
|
||||
captureMetadata: {
|
||||
@@ -297,6 +308,9 @@ public static class Win32 {
|
||||
appName: metadata.appName || '',
|
||||
elementLabel: metadata.elementLabel || '',
|
||||
elementRole: metadata.elementRole || '',
|
||||
elementValue: metadata.elementValue || '',
|
||||
recentTyped,
|
||||
recentShortcut,
|
||||
mode,
|
||||
},
|
||||
};
|
||||
@@ -440,8 +454,11 @@ public static class Win32 {
|
||||
appName: step.captureMetadata.appName,
|
||||
elementLabel: step.captureMetadata.elementLabel,
|
||||
elementRole: step.captureMetadata.elementRole,
|
||||
elementValue: step.captureMetadata.elementValue,
|
||||
},
|
||||
ocrText: step.captureMetadata.ocrText,
|
||||
recentTyped: step.captureMetadata.recentTyped,
|
||||
recentShortcut: step.captureMetadata.recentShortcut,
|
||||
});
|
||||
captureContext = {
|
||||
...step.captureMetadata,
|
||||
|
||||
Reference in New Issue
Block a user