From d0b181617588b5cb9dbeba6b4998a4bd64c09d91 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 11:05:34 -0500 Subject: [PATCH] fix step-behind titles: UIAutomation element label from click watcher + smarter search fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Root cause** When OCR fails, the title fell back to the browser window title. For a click on a search-results page, the window title reflects the *previous* search query ("oracle - Google Search"), producing "Search for Oracle" even though the user is clicking a link *on* that page. **Fix 1: UIAutomation element label from the click watcher** The C# click-watcher hook now enriches each click in a background thread (ClickProcessorLoop) rather than in the hook callback: - MouseHookCallback captures window title synchronously (fast Win32), then queues a PendingClick and returns immediately. - ClickProcessorLoop calls AutomationElement.FromPoint() via reflection (no compile-time assembly reference → no startup failure if UIA is absent). Wrapped in a 300ms timeout thread so slow UIA calls don't delay the click event past the frame buffer window. - Emits CTX + ELEM (label/role/value) + CLICK as an atomic batch. Node.js: - Parses ELEM events, merges element info into _pendingWindowContext. - clickMeta.windowContext now carries elementLabel/elementRole/elementValue in addition to windowTitle/appName. - buildCaptureTitle priority-5 (element label) now fires from click-watcher data, giving "Select Oracle | Cloud Applications…" without OCR. **Fix 2: Wider OCR crop** ocrAroundClick now uses a full-display-width × 100px horizontal strip at the click height. The previous 420 px crop cropped through long link text (e.g. "Oracle | Cloud Applications and Cloud Platform"), causing fragments to be scored lower than the complete text. **Fix 3: Search-results window title fallback** extractSearchQuery now only produces "Search for Oracle" when recentTyped is non-empty (the user was actually typing a query). For a pure click on the search-results page (no recent typing), the fallback is "Select a Oracle result in Chrome" — honest about what we know without implying the user performed the search in this step. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 138 ++++++++++++++++++++++++++++++---- app/text-intel.js | 9 ++- core/text-intel.js | 10 ++- tests/unit/text-intel.test.js | 18 +++-- 4 files changed, 154 insertions(+), 21 deletions(-) diff --git a/app/capture.js b/app/capture.js index a3a2350..b1114fc 100644 --- a/app/capture.js +++ b/app/capture.js @@ -139,7 +139,7 @@ class CaptureService { this._keyBuffer = ''; // printable chars typed since last capture this._lastShortcut = ''; // last Ctrl+X / Alt+X combination this._keyLastAt = 0; // timestamp of last key event - this._lastWindowContext = null; // last CTX event from the click watcher + this._pendingWindowContext = null; // buffered CTX+ELEM from click watcher, consumed on CLICK this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -831,6 +831,99 @@ public static class SFHook { private static readonly ConcurrentQueue queue = new ConcurrentQueue(); private static readonly AutoResetEvent signal = new AutoResetEvent(false); + // Pending click queue for background UIAutomation enrichment. + private struct PendingClick { + public int x, y; + public string button; + public long ts; + public string wTitle; + public string wApp; + } + private static readonly ConcurrentQueue pendingClicks = new ConcurrentQueue(); + private static readonly AutoResetEvent clickSignal = new AutoResetEvent(false); + + // UIAutomation state — loaded once via reflection so the C# code has no + // compile-time assembly reference (avoids breaking the whole hook if UIA is absent). + private static bool uiaLoadAttempted = false; + private static System.Reflection.MethodInfo uiaFromPoint = null; + private static System.Reflection.PropertyInfo uiaCurrent = null; + private static System.Type uiaPointType = null; + private static System.Reflection.Assembly uiaClientAssembly = null; + + private static void TryLoadUia() { + if (uiaLoadAttempted) return; + uiaLoadAttempted = true; + try { + uiaClientAssembly = System.Reflection.Assembly.Load("UIAutomationClient"); + var wbase = System.Reflection.Assembly.Load("WindowsBase"); + var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement"); + uiaFromPoint = aeType.GetMethod("FromPoint"); + uiaCurrent = aeType.GetProperty("Current"); + uiaPointType = wbase.GetType("System.Windows.Point"); + } catch { } + } + + private static void GetUiaElement(int x, int y, out string label, out string role, out string value) { + label = ""; role = ""; value = ""; + if (uiaFromPoint == null || uiaPointType == null) return; + try { + var pt = Activator.CreateInstance(uiaPointType, (double)x, (double)y); + var element = uiaFromPoint.Invoke(null, new object[] { pt }); + if (element == null) return; + var current = uiaCurrent.GetValue(element); + var ct = current.GetType(); + label = ct.GetProperty("Name")?.GetValue(current) as string ?? ""; + role = ct.GetProperty("LocalizedControlType")?.GetValue(current) as string ?? ""; + // Try to read the element's current value (text box contents etc.). + try { + var vpType = uiaClientAssembly.GetType("System.Windows.Automation.ValuePattern"); + var vpPatternId = vpType?.GetField("Pattern")?.GetValue(null); + if (vpPatternId != null) { + var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement"); + var tryGet = aeType.GetMethod("TryGetCurrentPattern", + new System.Type[] { vpPatternId.GetType(), typeof(object).MakeByRefType() }); + if (tryGet != null) { + object[] args = new object[] { vpPatternId, null }; + if ((bool)tryGet.Invoke(element, args) && args[1] != null) { + var curr = args[1].GetType().GetProperty("Current")?.GetValue(args[1]); + value = curr?.GetType().GetProperty("Value")?.GetValue(curr) as string ?? ""; + } + } + } + } catch { } + } catch { } + } + + // Background thread: enriches each click with UIAutomation element info, then + // emits CTX + ELEM (optional) + CLICK as an atomic batch. + private static void ClickProcessorLoop() { + TryLoadUia(); + while (true) { + clickSignal.WaitOne(); + PendingClick click; + while (pendingClicks.TryDequeue(out click)) { + string label = "", role = "", val = ""; + // Run UIAutomation on a separate thread so we can impose a timeout. + // 300 ms is enough for most apps; if UIA is slow we skip element enrichment + // but still emit the click (no steps are lost). + string[] r = new string[] { "", "", "" }; + var t = new Thread(() => { + GetUiaElement(click.x, click.y, out r[0], out r[1], out r[2]); + }); + t.IsBackground = true; + t.Start(); + if (t.Join(300)) { label = r[0]; role = r[1]; val = r[2]; } + // Emit window context (always), element info (when available), then click. + queue.Enqueue("CTX " + B64(click.wTitle) + " " + B64(click.wApp) + " " + click.ts); + if (label.Length > 0 || role.Length > 0) { + queue.Enqueue("ELEM " + B64(label) + " " + B64(role) + " " + B64(val) + " " + click.ts); + } + queue.Enqueue("CLICK " + click.x + " " + click.y + " " + click.button + " " + click.ts); + signal.Set(); + } + } + } + [StructLayout(LayoutKind.Sequential)] private struct POINT { public int x; @@ -997,6 +1090,10 @@ public static class SFHook { writer.IsBackground = true; writer.Start(); + Thread clickProcessor = new Thread(ClickProcessorLoop); + clickProcessor.IsBackground = true; + clickProcessor.Start(); + hook = SetWindowsHookEx(WH_MOUSE_LL, proc, GetModuleHandle(null), 0); if (hook == IntPtr.Zero) { throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); @@ -1035,14 +1132,12 @@ public static class SFHook { if (button != null) { MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT)); long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds; - // Capture window context while the user's app is still in foreground. - // GetForegroundWindow is synchronous and fast (no IPC overhead). - try { - string t = GetFwTitle(), a = GetFwApp(); - queue.Enqueue("CTX " + B64(t) + " " + B64(a) + " " + unixMs); - } catch { } - queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs); - signal.Set(); + // Capture window title synchronously (fast Win32, safe in hook callback). + // UIAutomation element lookup runs on the background ClickProcessorLoop thread. + string wTitle = "", wApp = ""; + try { wTitle = GetFwTitle(); wApp = GetFwApp(); } catch { } + pendingClicks.Enqueue(new PendingClick { x = data.pt.x, y = data.pt.y, button = button, ts = unixMs, wTitle = wTitle, wApp = wApp }); + clickSignal.Set(); } } return CallNextHookEx(hook, nCode, wParam, lParam); @@ -1283,13 +1378,27 @@ public static class SFHook { this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2])); continue; } - const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+\d+$/.exec(trimmed); + // CTX always arrives just before its paired ELEM+CLICK from the same background thread batch. + const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed); if (ctxM) { const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8'); - this._lastWindowContext = { + this._pendingWindowContext = { windowTitle: decode(ctxM[1]), appName: decode(ctxM[2]), + ts: Number(ctxM[3]), }; + continue; + } + // ELEM carries UIAutomation element info for the same click as the preceding CTX. + const elemM = /^ELEM\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed); + if (elemM) { + const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8'); + if (this._pendingWindowContext && this._pendingWindowContext.ts === Number(elemM[4])) { + this._pendingWindowContext.elementLabel = decode(elemM[1]); + this._pendingWindowContext.elementRole = decode(elemM[2]); + this._pendingWindowContext.elementValue = decode(elemM[3]); + } + continue; } } } @@ -1469,9 +1578,10 @@ public static class SFHook { } // Snapshot keyboard context accumulated since the last capture. clickMeta.keyContext = this.snapshotKeyContext(); - // Attach the window context emitted by the click watcher alongside the click. - if (this._lastWindowContext) { - clickMeta.windowContext = this._lastWindowContext; + // Attach the window + element context emitted by the click watcher. + if (this._pendingWindowContext) { + clickMeta.windowContext = this._pendingWindowContext; + this._pendingWindowContext = null; } if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be diff --git a/app/text-intel.js b/app/text-intel.js index 2695bda..961db39 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -151,7 +151,14 @@ class TextIntelService { async ocrAroundClick(frame, clickPos) { if (!frame || !frame.image) return { text: '', confidence: null }; - const rect = this.cropRectForPoint(frame, clickPos); + // Use a full-width horizontal strip at the click height. This preserves complete + // link text (e.g. "Oracle | Cloud Applications and Cloud Platform") rather than + // cropping through it when the element spans more than the 420 px default width. + const bounds = frame.display?.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height }; + const rect = this.cropRectForPoint(frame, clickPos, { + width: bounds.width, // full display width → full image width after DPI scaling + height: 100, // ~2 lines tall, enough context without too much noise + }); try { return await this.recognizeCrop(frame.image, rect); } catch { diff --git a/core/text-intel.js b/core/text-intel.js index ada8a47..5fe0a42 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -500,7 +500,15 @@ function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '', r if (strippedWindowTitle) { const searchQuery = extractSearchQuery(strippedWindowTitle); if (searchQuery) { - const base = `Search for ${sentenceCase(searchQuery)}`; + // Only claim this step IS the search action when the user was actually typing + // (recentTyped). Without typing context, the search page title is from the + // PREVIOUS step — the current step is a click ON the search results page. + if (recentTyped) { + const base = `Search for ${sentenceCase(searchQuery)}`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + // User is clicking something on the search results page — don't claim they searched. + const base = `Select a ${sentenceCase(searchQuery)} result`; return app ? qualifyTitleWithApp(base, metadata.appName) : base; } const windowPhrase = pickBestTitleFragment(strippedWindowTitle, { source: 'window', metadata }); diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 4c7a7ce..86cd3ff 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -102,18 +102,26 @@ test('browser window title strips browser name and falls back to page title', () assert.ok(title.toLowerCase().includes('oracle') || title.toLowerCase().includes('cloud'), `Expected oracle/cloud in title, got: ${title}`); }); -test('search query is extracted from browser window title pattern', () => { +test('search query is extracted when user was typing (search step)', () => { const title = buildCaptureTitle({ mode: 'fullscreen', - metadata: { - windowTitle: 'oracle - Google Search - Google Chrome', - appName: 'chrome', - }, + metadata: { windowTitle: 'oracle - Google Search - Google Chrome', appName: 'chrome' }, ocrText: '', + recentTyped: 'oracle', // user was typing → this IS the search step }); assert.equal(title, 'Search for Oracle in Chrome'); }); +test('search results window title produces select-result title when no typing (click on results page)', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { windowTitle: 'oracle - Google Search - Google Chrome', appName: 'chrome' }, + ocrText: '', + recentTyped: '', // no recent typing → user is clicking a result, not searching + }); + assert.equal(title, 'Select a Oracle result in Chrome'); +}); + test('full link text with pipe separator is preserved in OCR phrases', () => { const title = buildCaptureTitle({ mode: 'fullscreen',