From 13b5f67de83991c4f14e80f48c394c314783c4d4 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 10:04:34 -0500 Subject: [PATCH] add keyboard tracking and app-qualified smart titles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Keyboard hook (Windows)** - Extends the existing C# WH_MOUSE_LL process to also install WH_KEYBOARD_LL alongside it (keyboard hook is optional — failure does not break mouse capture). - Emits CHAR for printable keystrokes, KEY for modifier combos (Ctrl+T) and special keys (Backspace, Enter). **Text accumulation in capture session** - CaptureService tracks _keyBuffer (typed chars since last step) and _lastShortcut (last modifier combo) using the new onKeyboardEvent() method. - snapshotKeyContext() is called at enqueueClickCapture time so each step's clickMeta.keyContext carries { recentTyped, recentShortcut }. - Buffer resets after each snapshot; stale input (>8s gap) is dropped. **UIAutomation element value** - collectWindowsWindowContext now reads ValuePattern.Current.Value from the clicked element — giving us what's actually typed in a search box or text field without needing the keyboard buffer. **Smart title generation (core/text-intel.js)** - Priority chain: keyboard shortcut → element value → typed text → OCR → element label → page title → app name. - SHORTCUT_TITLES maps 50+ common shortcuts (Ctrl+T, Ctrl+S, F5 …) to natural language descriptions: "Open new tab", "Save", etc. - qualifyTitleWithApp() appends "in Chrome / VS Code / Terminal / …" to every title when the app is known: "Click Save in VS Code", "Search for oracle in Chrome", "Open new tab in Chrome". - APP_DISPLAY_NAMES covers browsers, editors, terminals, office apps. Six new unit tests cover shortcuts, typed-text search, element value, and app-qualified OCR titles. Capture test updated for keyContext. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 174 ++++++++++++++++++++++++++++++-- app/text-intel.js | 19 +++- core/text-intel.js | 183 ++++++++++++++++++++++++++++++++-- tests/unit/capture.test.js | 2 +- tests/unit/text-intel.test.js | 55 +++++++++- 5 files changed, 409 insertions(+), 24 deletions(-) diff --git a/app/capture.js b/app/capture.js index db918a8..d7f8243 100644 --- a/app/capture.js +++ b/app/capture.js @@ -136,6 +136,9 @@ class CaptureService { this.linuxEvent = null; // event block currently being parsed this.pendingRawClick = null; // raw press waiting for its coordinate twin this.pendingClickOsPoint = null; + this._keyBuffer = ''; // printable chars typed since last capture + this._lastShortcut = ''; // last Ctrl+X / Alt+X combination + this._keyLastAt = 0; // timestamp of last key event this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -799,12 +802,15 @@ using System.Collections.Concurrent; using System.Runtime.InteropServices; using System.Threading; -public static class SFMouseHook { +public static class SFHook { private const int WH_MOUSE_LL = 14; + private const int WH_KEYBOARD_LL = 13; private const int WM_LBUTTONDOWN = 0x0201; private const int WM_RBUTTONDOWN = 0x0204; private const int WM_MBUTTONDOWN = 0x0207; private const int WM_XBUTTONDOWN = 0x020B; + private const int WM_KEYDOWN = 0x0100; + private const int WM_SYSKEYDOWN = 0x0104; private const long UnixEpochMilliseconds = 62135596800000L; // Opting this process out of Windows Power Throttling (EcoQoS). In a // power-saving plan the OS CPU-starves background processes; a starved @@ -818,7 +824,9 @@ public static class SFMouseHook { private const uint HIGH_PRIORITY_CLASS = 0x00000080; private static IntPtr hook = IntPtr.Zero; - private static LowLevelMouseProc proc = HookCallback; + private static IntPtr keyHook = IntPtr.Zero; + private static LowLevelMouseProc proc = MouseHookCallback; + private static LowLevelKeyboardProc keyProc = KeyboardHookCallback; private static readonly ConcurrentQueue queue = new ConcurrentQueue(); private static readonly AutoResetEvent signal = new AutoResetEvent(false); @@ -854,11 +862,34 @@ public static class SFMouseHook { public uint StateMask; } + [StructLayout(LayoutKind.Sequential)] + private struct KBDLLHOOKSTRUCT { + public uint vkCode; + public uint scanCode; + public uint flags; + public uint time; + public UIntPtr dwExtraInfo; + } + private delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + private delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam); [DllImport("user32.dll", SetLastError = true)] private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelKeyboardProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll")] + private static extern short GetKeyState(int nVirtKey); + + [DllImport("user32.dll")] + private static extern bool GetKeyboardState([Out] byte[] lpKeyState); + + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern int ToUnicode(uint wVirtKey, uint wScanCode, byte[] lpKeyState, + System.Text.StringBuilder pwszBuff, int cchBuff, uint wFlags); + [DllImport("user32.dll", SetLastError = true)] private static extern bool UnhookWindowsHookEx(IntPtr hhk); @@ -917,6 +948,8 @@ public static class SFMouseHook { if (hook == IntPtr.Zero) { throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); } + // Keyboard hook is optional — if it fails we still get click titles. + try { keyHook = SetWindowsHookEx(WH_KEYBOARD_LL, keyProc, GetModuleHandle(null), 0); } catch { } Console.Out.WriteLine("READY"); Console.Out.Flush(); @@ -928,6 +961,7 @@ public static class SFMouseHook { } UnhookWindowsHookEx(hook); + if (keyHook != IntPtr.Zero) UnhookWindowsHookEx(keyHook); } private static void WriterLoop() { @@ -941,7 +975,7 @@ public static class SFMouseHook { } } - private static IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { if (nCode >= 0) { int message = wParam.ToInt32(); string button = ButtonName(message, lParam); @@ -955,6 +989,75 @@ public static class SFMouseHook { return CallNextHookEx(hook, nCode, wParam, lParam); } + private static IntPtr KeyboardHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0 && (wParam.ToInt32() == WM_KEYDOWN || wParam.ToInt32() == WM_SYSKEYDOWN)) { + try { + KBDLLHOOKSTRUCT kb = (KBDLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(KBDLLHOOKSTRUCT)); + int vk = (int)kb.vkCode; + bool ctrl = (GetKeyState(0x11) & 0x8000) != 0; + bool alt = (GetKeyState(0x12) & 0x8000) != 0; + bool shift = (GetKeyState(0x10) & 0x8000) != 0; + // Skip standalone modifier keys. + bool isMod = vk == 0x10 || vk == 0x11 || vk == 0x12 || vk == 0x5B || vk == 0x5C; + if (!isMod) { + long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds; + if (ctrl || alt) { + // Emit shortcut: "KEY Ctrl+T 123456" + string name = VkName(vk); + if (name != null) { + string mods = (ctrl ? "Ctrl" : "") + (ctrl && (alt || shift) ? "+" : "") + + (alt ? "Alt" : "") + ((alt || ctrl) && shift ? "+" : "") + + (shift ? "Shift" : ""); + queue.Enqueue("KEY " + mods + "+" + name + " " + unixMs); + signal.Set(); + } + } else if (vk == 0x08) { + queue.Enqueue("KEY Backspace " + unixMs); signal.Set(); + } else if (vk == 0x1B) { + queue.Enqueue("KEY Escape " + unixMs); signal.Set(); + } else if (vk == 0x0D) { + queue.Enqueue("KEY Enter " + unixMs); signal.Set(); + } else { + // Map to Unicode character using current keyboard layout + shift state. + byte[] ks = new byte[256]; + GetKeyboardState(ks); + var sb = new System.Text.StringBuilder(4); + int res = ToUnicode(kb.vkCode, kb.scanCode, ks, sb, 4, 0); + if (res > 0) { + char ch = sb[0]; + if (ch >= 0x20 && ch < 0x7F) { + queue.Enqueue("CHAR " + (int)ch + " " + unixMs); + signal.Set(); + } + } + } + } + } catch { } + } + return CallNextHookEx(keyHook, nCode, wParam, lParam); + } + + private static string VkName(int vk) { + if (vk >= 0x41 && vk <= 0x5A) return ((char)vk).ToString(); // A-Z + if (vk >= 0x30 && vk <= 0x39) return ((char)vk).ToString(); // 0-9 + if (vk >= 0x70 && vk <= 0x87) return "F" + (vk - 0x6F); // F1-F24 + switch (vk) { + case 0x09: return "Tab"; + case 0x0D: return "Enter"; + case 0x1B: return "Escape"; + case 0x20: return "Space"; + case 0x21: return "PageUp"; case 0x22: return "PageDown"; + case 0x23: return "End"; case 0x24: return "Home"; + case 0x25: return "Left"; case 0x26: return "Up"; + case 0x27: return "Right"; case 0x28: return "Down"; + case 0x2E: return "Delete"; + case 0x6B: case 0xBB: return "Plus"; + case 0x6D: case 0xBD: return "Minus"; + case 0xBE: return "Period"; case 0xBF: return "Slash"; + default: return null; + } + } + private static string ButtonName(int message, IntPtr lParam) { if (message == WM_LBUTTONDOWN) return "left"; if (message == WM_RBUTTONDOWN) return "right"; @@ -968,7 +1071,7 @@ public static class SFMouseHook { } } '@ -[SFMouseHook]::Run() +[SFHook]::Run() `; this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', ps], { stdio: ['ignore', 'pipe', 'pipe'], @@ -1103,11 +1206,22 @@ public static class SFMouseHook { } if (platform === 'win32') { for (const line of lines) { - const m = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(line.trim()); - if (m) { - const osPoint = m[1] === undefined ? null : { x: Number(m[1]), y: Number(m[2]) }; - const eventAt = m[4] === undefined ? Date.now() : Number(m[4]); - this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, m[3] || 'mouse'); + const trimmed = line.trim(); + const clickM = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(trimmed); + if (clickM) { + const osPoint = clickM[1] === undefined ? null : { x: Number(clickM[1]), y: Number(clickM[2]) }; + const eventAt = clickM[4] === undefined ? Date.now() : Number(clickM[4]); + this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, clickM[3] || 'mouse'); + continue; + } + const keyM = /^KEY\s+(\S+)\s+(\d+)\s*$/.exec(trimmed); + if (keyM) { + this.onKeyboardEvent('KEY', keyM[1], Number(keyM[2])); + continue; + } + const charM = /^CHAR\s+(\d+)\s+(\d+)\s*$/.exec(trimmed); + if (charM) { + this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2])); } } } @@ -1285,6 +1399,8 @@ public static class SFMouseHook { if (osPoint) { clickMeta.osPoint = { x: osPoint.x, y: osPoint.y }; } + // Snapshot keyboard context accumulated since the last capture. + clickMeta.keyContext = this.snapshotKeyContext(); if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be // stored if the session stops while this click waits in the queue. @@ -1298,6 +1414,46 @@ public static class SFMouseHook { return this.clickQueue; } + // --- Keyboard context tracking ------------------------------------------- + + onKeyboardEvent(type, data, eventAt) { + const now = Number.isFinite(eventAt) ? eventAt : Date.now(); + // Discard stale typing that happened more than 8 seconds ago (user moved on). + if (now - this._keyLastAt > 8000) { + this._keyBuffer = ''; + } + this._keyLastAt = now; + + if (type === 'CHAR') { + const ch = typeof data === 'number' ? String.fromCharCode(data) : String(data); + this._keyBuffer = (this._keyBuffer + ch).slice(-200); + } else if (type === 'KEY') { + const key = String(data); + if (key === 'Backspace') { + this._keyBuffer = this._keyBuffer.slice(0, -1); + } else if (key === 'Escape') { + this._keyBuffer = ''; + } else if (key === 'Enter') { + // Keep typed text — Enter often submits what was typed. + } else { + // It's a modifier+key shortcut (Ctrl+T etc.). + this._lastShortcut = key; + this._keyBuffer = ''; // a shortcut resets the typed-text buffer + } + } + } + + snapshotKeyContext() { + const ctx = { + recentTyped: this._keyBuffer.trim(), + recentShortcut: this._lastShortcut, + }; + // Reset after snapshot so each step gets its own context. + this._keyBuffer = ''; + this._lastShortcut = ''; + return ctx; + } + async captureCurrentFrame(mode, capturePoint = null, startedAt = Date.now()) { const grabbed = await this.grab(mode, capturePoint); return { diff --git a/app/text-intel.js b/app/text-intel.js index 3206670..1e5ff39 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -181,6 +181,7 @@ class TextIntelService { $elementRole = ''; $elementClass = ''; $elementProcessId = 0; + $elementValue = ''; if (${hasPoint ? '$true' : '$false'}) { try { Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null @@ -192,6 +193,12 @@ class TextIntelService { $elementRole = $current.LocalizedControlType; $elementClass = $current.ClassName; $elementProcessId = $current.ProcessId; + try { + $valPattern = [System.Windows.Automation.ValuePattern]::Pattern; + if ($element.GetSupportedPatterns() -contains $valPattern) { + $elementValue = $element.GetCurrentPattern($valPattern).Current.Value; + } + } catch { } } } catch { } } @@ -218,6 +225,7 @@ public static class Win32 { elementLabel = $elementLabel; elementRole = $elementRole; elementClass = $elementClass; + elementValue = $elementValue; elementProcessId = $elementProcessId; pid = $pid; }; @@ -284,11 +292,14 @@ public static class Win32 { } async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) { + const keyContext = clickMeta?.keyContext || {}; + const recentTyped = keyContext.recentTyped || ''; + const recentShortcut = keyContext.recentShortcut || ''; const [metadata, ocr] = await Promise.all([ this.collectForegroundWindowContext(clickMeta?.osPoint || null), this.ocrAroundClick(frame, clickPos), ]); - const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text }); + const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text, recentTyped, recentShortcut }); return { title, captureMetadata: { @@ -297,6 +308,9 @@ public static class Win32 { appName: metadata.appName || '', elementLabel: metadata.elementLabel || '', elementRole: metadata.elementRole || '', + elementValue: metadata.elementValue || '', + recentTyped, + recentShortcut, mode, }, }; @@ -440,8 +454,11 @@ public static class Win32 { appName: step.captureMetadata.appName, elementLabel: step.captureMetadata.elementLabel, elementRole: step.captureMetadata.elementRole, + elementValue: step.captureMetadata.elementValue, }, ocrText: step.captureMetadata.ocrText, + recentTyped: step.captureMetadata.recentTyped, + recentShortcut: step.captureMetadata.recentShortcut, }); captureContext = { ...step.captureMetadata, diff --git a/core/text-intel.js b/core/text-intel.js index c235c94..7bf1d08 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -70,6 +70,121 @@ const SEARCH_ENGINE_PAGE_NAMES = new Set([ 'brave search', ]); +// Common keyboard shortcuts → short action descriptions used as step titles. +const SHORTCUT_TITLES = { + 'Ctrl+T': 'Open new tab', + 'Ctrl+N': 'Open new window', + 'Ctrl+W': 'Close tab', + 'Ctrl+Shift+T': 'Reopen closed tab', + 'Ctrl+Shift+N': 'Open incognito window', + 'Ctrl+S': 'Save', + 'Ctrl+Shift+S': 'Save as', + 'Ctrl+Z': 'Undo', + 'Ctrl+Y': 'Redo', + 'Ctrl+Shift+Z': 'Redo', + 'Ctrl+C': 'Copy selection', + 'Ctrl+V': 'Paste', + 'Ctrl+X': 'Cut selection', + 'Ctrl+A': 'Select all', + 'Ctrl+F': 'Open Find', + 'Ctrl+H': 'Open Find and Replace', + 'Ctrl+R': 'Reload page', + 'Ctrl+Shift+R': 'Hard reload page', + 'Ctrl+L': 'Focus address bar', + 'Ctrl+D': 'Bookmark page', + 'Ctrl+Tab': 'Switch to next tab', + 'Ctrl+Shift+Tab': 'Switch to previous tab', + 'Ctrl+Plus': 'Zoom in', + 'Ctrl+Minus': 'Zoom out', + 'Ctrl+0': 'Reset zoom', + 'Ctrl+P': 'Print', + 'Ctrl+O': 'Open file', + 'Ctrl+E': 'Focus search bar', + 'Ctrl+K': 'Focus search bar', + 'Ctrl+G': 'Go to line', + 'Ctrl+B': 'Toggle sidebar', + 'Ctrl+Shift+P': 'Open command palette', + 'Ctrl+Shift+E': 'Show file explorer', + 'Ctrl+Shift+G': 'Show source control', + 'Ctrl+Shift+D': 'Show debug panel', + 'Ctrl+Shift+X': 'Show extensions', + 'Alt+F4': 'Close window', + 'Alt+Left': 'Go back', + 'Alt+Right': 'Go forward', + 'Alt+Tab': 'Switch application', + 'F2': 'Rename', + 'F3': 'Find next', + 'F4': 'Open address bar', + 'F5': 'Reload page', + 'F11': 'Toggle fullscreen', + 'F12': 'Open developer tools', +}; + +// Process name → human-readable display name (used to append "in Chrome" etc. to titles). +const APP_DISPLAY_NAMES = { + chrome: 'Chrome', + msedge: 'Edge', + firefox: 'Firefox', + safari: 'Safari', + opera: 'Opera', + brave: 'Brave', + vivaldi: 'Vivaldi', + code: 'VS Code', + cursor: 'Cursor', + 'sublime_text': 'Sublime Text', + atom: 'Atom', + notepad: 'Notepad', + 'notepad++': 'Notepad++', + winword: 'Word', + excel: 'Excel', + powerpnt: 'PowerPoint', + outlook: 'Outlook', + teams: 'Teams', + slack: 'Slack', + discord: 'Discord', + zoom: 'Zoom', + figma: 'Figma', + postman: 'Postman', + insomnia: 'Insomnia', + notion: 'Notion', + obsidian: 'Obsidian', + spotify: 'Spotify', + terminal: 'Terminal', + cmd: 'Command Prompt', + powershell: 'PowerShell', + windowsterminal: 'Windows Terminal', + wt: 'Windows Terminal', + iterm2: 'iTerm', + wezterm: 'WezTerm', + alacritty: 'Alacritty', + kitty: 'Kitty', + 'gnome-terminal': 'Terminal', + konsole: 'Konsole', + xterm: 'Terminal', + xfce4terminal: 'Terminal', + bash: 'Terminal', + zsh: 'Terminal', + fish: 'Terminal', + finder: 'Finder', + explorer: 'File Explorer', + 'files-uwp': 'File Explorer', + steam: 'Steam', + 'steamwebhelper': 'Steam', +}; + +function cleanAppName(rawName) { + if (!rawName) return ''; + const key = normalizeWhitespace(rawName).toLowerCase().replace(/\.exe$/i, ''); + return APP_DISPLAY_NAMES[key] || sentenceCase(rawName.replace(/\.exe$/i, '')); +} + +function qualifyTitleWithApp(title, appName) { + const app = cleanAppName(appName); + if (!app) return title; + if (new RegExp(`\\b${app.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i').test(title)) return title; + return `${title} in ${app}`; +} + const ACTION_PREFIXES = [ 'click', 'select', @@ -310,27 +425,70 @@ function pickBestTitleFragment(text, { source = 'window', metadata = {} } = {}) return best ? formatCaptureTitle(best, { source, metadata }) : ''; } -function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) { +function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '', recentTyped = '', recentShortcut = '' } = {}) { + const app = cleanAppName(metadata.appName); + + // 1. Keyboard shortcut → most reliable signal for "what action did the user take". + if (recentShortcut && SHORTCUT_TITLES[recentShortcut]) { + const base = SHORTCUT_TITLES[recentShortcut]; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + + // 2. UIAutomation element value — what's actually typed inside the clicked field. + const elementValue = normalizeWhitespace(metadata.elementValue || ''); + if (elementValue) { + const roleLower = normalizeWhitespace(metadata.elementRole || '').toLowerCase(); + const labelLower = normalizeWhitespace(metadata.elementLabel || '').toLowerCase(); + const looksLikeSearch = /(search|find|query|omnibox|address bar)/.test(roleLower + ' ' + labelLower); + const action = looksLikeSearch ? 'Search for' : 'Type'; + const base = `${action} "${elementValue}"`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + + // 3. Keyboard-buffer text (typed between captures) + input role context. + const typed = normalizeWhitespace(recentTyped || ''); + if (typed) { + const roleLower = normalizeWhitespace(metadata.elementRole || '').toLowerCase(); + const labelLower = normalizeWhitespace(metadata.elementLabel || '').toLowerCase(); + const isSearchRole = /(search box|searchbox|search field|search bar|search input)/.test(roleLower); + const looksLikeSearch = isSearchRole || /(search|find|query|omnibox|address bar)/.test(roleLower + ' ' + labelLower); + const isAnyInput = /(text field|edit|input|field|combo box|textbox|text box)/.test(roleLower); + if (looksLikeSearch) { + const base = `Search for "${typed}"`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + if (isAnyInput) { + const base = `Type "${typed}"`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + } + + // 4. OCR text around the click — link text, button labels, menu items. const ocrPhrase = pickBestOcrPhrase(ocrText); - if (ocrPhrase) return formatCaptureTitle(ocrPhrase, { source: 'ocr', metadata }); + if (ocrPhrase) { + const title = formatCaptureTitle(ocrPhrase, { source: 'ocr', metadata }); + return app ? qualifyTitleWithApp(title, metadata.appName) : title; + } + // 5. UIAutomation element label. const elementPhrase = pickBestTitleFragment(metadata.elementLabel, { source: 'element', metadata }); - if (elementPhrase) return elementPhrase; + if (elementPhrase) { + return app ? qualifyTitleWithApp(elementPhrase, metadata.appName) : elementPhrase; + } - // Strip the browser name suffix before processing window titles so that - // "oracle - Google Search - Google Chrome" becomes "oracle - Google Search" - // and "Oracle | Cloud Applications - Google Chrome" becomes just the page title. - const rawWindowTitle = metadata.windowTitle || ''; - const strippedWindowTitle = stripBrowserNameSuffix(rawWindowTitle); + // 6. Browser window title (suffix stripped) → page title or search query. + const strippedWindowTitle = stripBrowserNameSuffix(metadata.windowTitle || ''); if (strippedWindowTitle) { - // Detect "[query] - Google Search" → "Search for oracle" const searchQuery = extractSearchQuery(strippedWindowTitle); - if (searchQuery) return `Search for ${sentenceCase(searchQuery)}`; - // Use the page title (now free of browser name noise). + if (searchQuery) { + const base = `Search for ${sentenceCase(searchQuery)}`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } const windowPhrase = pickBestTitleFragment(strippedWindowTitle, { source: 'window', metadata }); if (windowPhrase) return windowPhrase; } + // 7. App name alone as last resort. const appPhrase = pickBestTitleFragment(metadata.appName, { source: 'app', metadata }); if (appPhrase) return appPhrase; @@ -516,6 +674,9 @@ function buildAiPrompt({ captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null, captureContext.appName ? `App: ${captureContext.appName}` : null, captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null, + captureContext.elementValue ? `Element content (what was typed): ${captureContext.elementValue}` : null, + captureContext.recentTyped ? `Keyboard input before this step: ${captureContext.recentTyped}` : null, + captureContext.recentShortcut ? `Keyboard shortcut used: ${captureContext.recentShortcut}` : null, captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null, captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null, ].filter(Boolean) : []; diff --git a/tests/unit/capture.test.js b/tests/unit/capture.test.js index 51323ca..5e0414f 100644 --- a/tests/unit/capture.test.js +++ b/tests/unit/capture.test.js @@ -375,7 +375,7 @@ test('queued click captures preserve the original event time and button', async assert.deepEqual(seen, [{ trigger: 'click', clickPos: { x: 7, y: 8 }, - clickMeta: { at: 1770000000456, button: 'left' }, + clickMeta: { at: 1770000000456, button: 'left', keyContext: { recentTyped: '', recentShortcut: '' } }, }]); }); diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 9471f1e..4c7a7ce 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -61,7 +61,8 @@ test('capture titles ignore browser chrome noise in favor of OCR', () => { }, ocrText: 'New tab', }); - assert.equal(title, 'Click New tab'); + // OCR wins over the noisy window title; app name is appended for context. + assert.equal(title, 'Click New tab in Google Chrome'); }); test('tab-like roles use select when OCR identifies a tab label', () => { @@ -110,7 +111,7 @@ test('search query is extracted from browser window title pattern', () => { }, ocrText: '', }); - assert.equal(title, 'Search for Oracle'); + assert.equal(title, 'Search for Oracle in Chrome'); }); test('full link text with pipe separator is preserved in OCR phrases', () => { @@ -140,6 +141,56 @@ test('search box element role uses Search for verb', () => { assert.equal(title, 'Search for Oracle'); }); +test('keyboard shortcut produces action title qualified with app', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { appName: 'chrome' }, + ocrText: '', + recentShortcut: 'Ctrl+T', + }); + assert.equal(title, 'Open new tab in Chrome'); +}); + +test('keyboard shortcut title without app name', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: {}, + ocrText: '', + recentShortcut: 'Ctrl+S', + }); + assert.equal(title, 'Save'); +}); + +test('typed text with search input role produces Search for title', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { elementRole: 'search box', appName: 'chrome' }, + ocrText: '', + recentTyped: 'oracle', + }); + assert.equal(title, 'Search for "oracle" in Chrome'); +}); + +test('UIAutomation element value takes priority over keyboard buffer', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { elementRole: 'edit', elementValue: 'oracle', appName: 'chrome' }, + ocrText: '', + recentTyped: 'ignored', + }); + // elementValue (from UIAutomation) wins over the keyboard buffer + assert.ok(title.includes('oracle'), `expected oracle in title, got: ${title}`); +}); + +test('app-qualified OCR title includes app name', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { appName: 'code' }, + ocrText: 'Save', + }); + assert.equal(title, 'Click Save in VS Code'); +}); + test('ai prompts include the deterministic OCR-backed title candidate', () => { const { prompt } = buildAiPrompt({ captureContext: {