Tyler/StepForge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix step-behind titles: UIAutomation element label from click watcher + smarter search fallback

Browse Source 
**Root cause**
When OCR fails, the title fell back to the browser window title.  For a
click on a search-results page, the window title reflects the *previous*
search query ("oracle - Google Search"), producing "Search for Oracle"
even though the user is clicking a link *on* that page.

**Fix 1: UIAutomation element label from the click watcher**
The C# click-watcher hook now enriches each click in a background thread
(ClickProcessorLoop) rather than in the hook callback:
- MouseHookCallback captures window title synchronously (fast Win32),
  then queues a PendingClick and returns immediately.
- ClickProcessorLoop calls AutomationElement.FromPoint() via reflection
  (no compile-time assembly reference → no startup failure if UIA is
  absent).  Wrapped in a 300ms timeout thread so slow UIA calls don't
  delay the click event past the frame buffer window.
- Emits CTX + ELEM (label/role/value) + CLICK as an atomic batch.

Node.js:
- Parses ELEM events, merges element info into _pendingWindowContext.
- clickMeta.windowContext now carries elementLabel/elementRole/elementValue
  in addition to windowTitle/appName.
- buildCaptureTitle priority-5 (element label) now fires from click-watcher
  data, giving "Select Oracle | Cloud Applications…" without OCR.

**Fix 2: Wider OCR crop**
ocrAroundClick now uses a full-display-width × 100px horizontal strip at
the click height.  The previous 420 px crop cropped through long link text
(e.g. "Oracle | Cloud Applications and Cloud Platform"), causing fragments
to be scored lower than the complete text.

**Fix 3: Search-results window title fallback**
extractSearchQuery now only produces "Search for Oracle" when recentTyped
is non-empty (the user was actually typing a query).  For a pure click on
the search-results page (no recent typing), the fallback is "Select a
Oracle result in Chrome" — honest about what we know without implying the
user performed the search in this step.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Tyler
2026-06-24 11:05:34 -05:00
co-authored by Claude Sonnet 4.6
parent dfa65c894b
commit d0b1816175
4 changed files with 154 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/capture.js
+124 -14
View File
@@ -139,7 +139,7 @@ class CaptureService {
this._keyBuffer = ''; // printable chars typed since last capture
this._lastShortcut = ''; // last Ctrl+X / Alt+X combination
this._keyLastAt = 0; // timestamp of last key event
this._lastWindowContext = null; // last CTX event from the click watcher
this._pendingWindowContext = null; // buffered CTX+ELEM from click watcher, consumed on CLICK
this.clickQueue = Promise.resolve();
this.frameLoopInFlight = false;
this.frameLoopGrabStartedAt = null;
@@ -831,6 +831,99 @@ public static class SFHook {
private static readonly ConcurrentQueue<string> queue = new ConcurrentQueue<string>();
private static readonly AutoResetEvent signal = new AutoResetEvent(false);
// Pending click queue for background UIAutomation enrichment.
private struct PendingClick {
public int x, y;
public string button;
public long ts;
public string wTitle;
public string wApp;
}
private static readonly ConcurrentQueue<PendingClick> pendingClicks = new ConcurrentQueue<PendingClick>();
private static readonly AutoResetEvent clickSignal = new AutoResetEvent(false);
// UIAutomation state — loaded once via reflection so the C# code has no
// compile-time assembly reference (avoids breaking the whole hook if UIA is absent).
private static bool uiaLoadAttempted = false;
private static System.Reflection.MethodInfo uiaFromPoint = null;
private static System.Reflection.PropertyInfo uiaCurrent = null;
private static System.Type uiaPointType = null;
private static System.Reflection.Assembly uiaClientAssembly = null;
private static void TryLoadUia() {
if (uiaLoadAttempted) return;
uiaLoadAttempted = true;
try {
uiaClientAssembly = System.Reflection.Assembly.Load("UIAutomationClient");
var wbase = System.Reflection.Assembly.Load("WindowsBase");
var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement");
uiaFromPoint = aeType.GetMethod("FromPoint");
uiaCurrent = aeType.GetProperty("Current");
uiaPointType = wbase.GetType("System.Windows.Point");
} catch { }
}
private static void GetUiaElement(int x, int y, out string label, out string role, out string value) {
label = ""; role = ""; value = "";
if (uiaFromPoint == null || uiaPointType == null) return;
try {
var pt = Activator.CreateInstance(uiaPointType, (double)x, (double)y);
var element = uiaFromPoint.Invoke(null, new object[] { pt });
if (element == null) return;
var current = uiaCurrent.GetValue(element);
var ct = current.GetType();
label = ct.GetProperty("Name")?.GetValue(current) as string ?? "";
role = ct.GetProperty("LocalizedControlType")?.GetValue(current) as string ?? "";
// Try to read the element's current value (text box contents etc.).
try {
var vpType = uiaClientAssembly.GetType("System.Windows.Automation.ValuePattern");
var vpPatternId = vpType?.GetField("Pattern")?.GetValue(null);
if (vpPatternId != null) {
var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement");
var tryGet = aeType.GetMethod("TryGetCurrentPattern",
new System.Type[] { vpPatternId.GetType(), typeof(object).MakeByRefType() });
if (tryGet != null) {
object[] args = new object[] { vpPatternId, null };
if ((bool)tryGet.Invoke(element, args) && args[1] != null) {
var curr = args[1].GetType().GetProperty("Current")?.GetValue(args[1]);
value = curr?.GetType().GetProperty("Value")?.GetValue(curr) as string ?? "";
}
}
}
} catch { }
} catch { }
}
// Background thread: enriches each click with UIAutomation element info, then
// emits CTX + ELEM (optional) + CLICK as an atomic batch.
private static void ClickProcessorLoop() {
TryLoadUia();
while (true) {
clickSignal.WaitOne();
PendingClick click;
while (pendingClicks.TryDequeue(out click)) {
string label = "", role = "", val = "";
// Run UIAutomation on a separate thread so we can impose a timeout.
// 300 ms is enough for most apps; if UIA is slow we skip element enrichment
// but still emit the click (no steps are lost).
string[] r = new string[] { "", "", "" };
var t = new Thread(() => {
GetUiaElement(click.x, click.y, out r[0], out r[1], out r[2]);
});
t.IsBackground = true;
t.Start();
if (t.Join(300)) { label = r[0]; role = r[1]; val = r[2]; }
// Emit window context (always), element info (when available), then click.
queue.Enqueue("CTX " + B64(click.wTitle) + " " + B64(click.wApp) + " " + click.ts);
if (label.Length > 0 || role.Length > 0) {
queue.Enqueue("ELEM " + B64(label) + " " + B64(role) + " " + B64(val) + " " + click.ts);
}
queue.Enqueue("CLICK " + click.x + " " + click.y + " " + click.button + " " + click.ts);
signal.Set();
}
}
}
[StructLayout(LayoutKind.Sequential)]
private struct POINT {
public int x;
@@ -997,6 +1090,10 @@ public static class SFHook {
writer.IsBackground = true;
writer.Start();
Thread clickProcessor = new Thread(ClickProcessorLoop);
clickProcessor.IsBackground = true;
clickProcessor.Start();
hook = SetWindowsHookEx(WH_MOUSE_LL, proc, GetModuleHandle(null), 0);
if (hook == IntPtr.Zero) {
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
@@ -1035,14 +1132,12 @@ public static class SFHook {
if (button != null) {
MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT));
long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds;
// Capture window context while the user's app is still in foreground.
// GetForegroundWindow is synchronous and fast (no IPC overhead).
try {
string t = GetFwTitle(), a = GetFwApp();
queue.Enqueue("CTX " + B64(t) + " " + B64(a) + " " + unixMs);
} catch { }
queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs);
signal.Set();
// Capture window title synchronously (fast Win32, safe in hook callback).
// UIAutomation element lookup runs on the background ClickProcessorLoop thread.
string wTitle = "", wApp = "";
try { wTitle = GetFwTitle(); wApp = GetFwApp(); } catch { }
pendingClicks.Enqueue(new PendingClick { x = data.pt.x, y = data.pt.y, button = button, ts = unixMs, wTitle = wTitle, wApp = wApp });
clickSignal.Set();
}
}
return CallNextHookEx(hook, nCode, wParam, lParam);
@@ -1283,13 +1378,27 @@ public static class SFHook {
this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2]));
continue;
}
const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+\d+$/.exec(trimmed);
// CTX always arrives just before its paired ELEM+CLICK from the same background thread batch.
const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed);
if (ctxM) {
const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8');
this._lastWindowContext = {
this._pendingWindowContext = {
windowTitle: decode(ctxM[1]),
appName: decode(ctxM[2]),
ts: Number(ctxM[3]),
};
continue;
}
// ELEM carries UIAutomation element info for the same click as the preceding CTX.
const elemM = /^ELEM\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed);
if (elemM) {
const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8');
if (this._pendingWindowContext && this._pendingWindowContext.ts === Number(elemM[4])) {
this._pendingWindowContext.elementLabel = decode(elemM[1]);
this._pendingWindowContext.elementRole = decode(elemM[2]);
this._pendingWindowContext.elementValue = decode(elemM[3]);
}
continue;
}
}
}
@@ -1469,9 +1578,10 @@ public static class SFHook {
}
// Snapshot keyboard context accumulated since the last capture.
clickMeta.keyContext = this.snapshotKeyContext();
// Attach the window context emitted by the click watcher alongside the click.
if (this._lastWindowContext) {
clickMeta.windowContext = this._lastWindowContext;
// Attach the window + element context emitted by the click watcher.
if (this._pendingWindowContext) {
clickMeta.windowContext = this._pendingWindowContext;
this._pendingWindowContext = null;
}
if (this.session && !this.session.paused && !this.userIsInApp()) {
// The guide id pins the click to its recording so it can still be
app/text-intel.js
+8 -1
View File
@@ -151,7 +151,14 @@ class TextIntelService {
async ocrAroundClick(frame, clickPos) {
if (!frame || !frame.image) return { text: '', confidence: null };
const rect = this.cropRectForPoint(frame, clickPos);
// Use a full-width horizontal strip at the click height. This preserves complete
// link text (e.g. "Oracle | Cloud Applications and Cloud Platform") rather than
// cropping through it when the element spans more than the 420 px default width.
const bounds = frame.display?.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height };
const rect = this.cropRectForPoint(frame, clickPos, {
width: bounds.width, // full display width → full image width after DPI scaling
height: 100, // ~2 lines tall, enough context without too much noise
});
try {
return await this.recognizeCrop(frame.image, rect);
} catch {
core/text-intel.js
+9 -1
View File
@@ -500,7 +500,15 @@ function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '', r
if (strippedWindowTitle) {
const searchQuery = extractSearchQuery(strippedWindowTitle);
if (searchQuery) {
const base = `Search for ${sentenceCase(searchQuery)}`;
// Only claim this step IS the search action when the user was actually typing
// (recentTyped). Without typing context, the search page title is from the
// PREVIOUS step — the current step is a click ON the search results page.
if (recentTyped) {
const base = `Search for ${sentenceCase(searchQuery)}`;
return app ? qualifyTitleWithApp(base, metadata.appName) : base;
}
// User is clicking something on the search results page — don't claim they searched.
const base = `Select a ${sentenceCase(searchQuery)} result`;
return app ? qualifyTitleWithApp(base, metadata.appName) : base;
}
const windowPhrase = pickBestTitleFragment(strippedWindowTitle, { source: 'window', metadata });
tests/unit/text-intel.test.js
+13 -5
View File
@@ -102,18 +102,26 @@ test('browser window title strips browser name and falls back to page title', ()
assert.ok(title.toLowerCase().includes('oracle') || title.toLowerCase().includes('cloud'), `Expected oracle/cloud in title, got: ${title}`);
});
test('search query is extracted from browser window title pattern', () => {
test('search query is extracted when user was typing (search step)', () => {
const title = buildCaptureTitle({
mode: 'fullscreen',
metadata: {
windowTitle: 'oracle - Google Search - Google Chrome',
appName: 'chrome',
},
metadata: { windowTitle: 'oracle - Google Search - Google Chrome', appName: 'chrome' },
ocrText: '',
recentTyped: 'oracle', // user was typing → this IS the search step
});
assert.equal(title, 'Search for Oracle in Chrome');
});
test('search results window title produces select-result title when no typing (click on results page)', () => {
const title = buildCaptureTitle({
mode: 'fullscreen',
metadata: { windowTitle: 'oracle - Google Search - Google Chrome', appName: 'chrome' },
ocrText: '',
recentTyped: '', // no recent typing → user is clicking a result, not searching
});
assert.equal(title, 'Select a Oracle result in Chrome');
});
test('full link text with pipe separator is preserved in OCR phrases', () => {
const title = buildCaptureTitle({
mode: 'fullscreen',