From 33eb0f745e1b1abe866c44d2c83f994a9260b281 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Tue, 23 Jun 2026 17:46:33 -0500 Subject: [PATCH 01/19] semantic titles ai assist --- app/capture.js | 54 ++-- app/main.js | 29 +++ app/preload.js | 4 + app/renderer/app.js | 3 + app/renderer/dialogs.js | 66 +++++ app/renderer/editor.js | 109 +++++++- app/renderer/style.css | 11 + app/text-intel.js | 438 ++++++++++++++++++++++++++++++++ core/settings.js | 7 + core/text-intel.js | 416 ++++++++++++++++++++++++++++++ docs/getting_started_with_ai.md | 75 ++++++ package-lock.json | 124 +++++++++ package.json | 4 + tests/unit/text-intel.test.js | 195 ++++++++++++++ 14 files changed, 1518 insertions(+), 17 deletions(-) create mode 100644 app/text-intel.js create mode 100644 core/text-intel.js create mode 100644 docs/getting_started_with_ai.md create mode 100644 tests/unit/text-intel.test.js diff --git a/app/capture.js b/app/capture.js index 00b1ded..aefbe36 100644 --- a/app/capture.js +++ b/app/capture.js @@ -3,7 +3,6 @@ const path = require('node:path'); const { spawn, execFileSync } = require('node:child_process'); const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron'); -const { expandPlaceholders } = require('../core/placeholders'); const raster = require('../core/raster'); const { encodePng } = require('../core/png'); const { @@ -14,6 +13,7 @@ const { DEFAULT_START_SLACK_MS, } = require('./click-frames'); const { physicalToDip } = require('./coords'); +const { DEFAULT_CAPTURE_TITLES } = require('../core/text-intel'); /** * Capture service: full-screen, active-window, and region capture, plus a @@ -114,6 +114,7 @@ class CaptureService { getWindow, notify, screenApi = screen, + textIntel = null, }) { this.store = store; this.settings = settings; @@ -122,6 +123,7 @@ class CaptureService { // Injectable for tests; the click/coordinate paths must never reach for // the global `screen` directly so coordinate handling stays testable. this.screen = screenApi; + this.textIntel = textIntel; this.session = null; // { guideId, paused, count, intervalSec } this.intervalTimer = null; this.clickWatcher = null; @@ -133,6 +135,7 @@ class CaptureService { this.clickWatcherErrTail = ''; this.linuxEvent = null; // event block currently being parsed this.pendingRawClick = null; // raw press waiting for its coordinate twin + this.pendingClickOsPoint = null; this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -463,7 +466,7 @@ class CaptureService { if (frame) { clog('click@', clickAt, 'frame', frame.source || 'loop', 'started', frame.startedAt - clickAt, 'ms, captured', frame.capturedAt - clickAt, 'ms rel. click'); - const result = this.storeFrameAsStep(guideId, frame.mode, frame, clickPos); + const result = await this.storeFrameAsStep(guideId, frame.mode, frame, clickPos, clickMeta); if (result.ok) this.noteStepAdded(result.step, trigger, guideId); return result; } @@ -1211,6 +1214,9 @@ public static class SFMouseHook { let clickPos = osPoint ? this.osPointToDip(osPoint) : null; if (!clickPos) clickPos = this.screen.getCursorScreenPoint(); clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos); + this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y) + ? { x: osPoint.x, y: osPoint.y } + : null; this.enqueueClickCapture(clickPos, clickAt, button || 'mouse'); } @@ -1268,7 +1274,17 @@ public static class SFMouseHook { * fast the user clicks or how slow the encoder is. */ enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') { - const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' }; + const osPoint = this.pendingClickOsPoint && Number.isFinite(this.pendingClickOsPoint.x) && Number.isFinite(this.pendingClickOsPoint.y) + ? this.pendingClickOsPoint + : null; + this.pendingClickOsPoint = null; + const clickMeta = { + at: Number.isFinite(clickAt) ? clickAt : Date.now(), + button: button || 'mouse', + }; + if (osPoint) { + clickMeta.osPoint = { x: osPoint.x, y: osPoint.y }; + } if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be // stored if the session stops while this click waits in the queue. @@ -1300,7 +1316,18 @@ public static class SFMouseHook { }; } - storeFrameAsStep(guideId, mode, frame, clickPos = null) { + async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) { + try { + if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') { + return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta }); + } + } catch { + // fall back to the local semantic title below + } + return this.autoTitle(mode); + } + + async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) { if (!frame) return { ok: false, reason: 'no capture frame available' }; const annotations = []; // The click position (DIP, read at event time) wins over the frame's @@ -1324,7 +1351,7 @@ public static class SFMouseHook { } const step = this.store.addStep(guideId, { - title: this.autoTitle(mode), + title: await this.buildStepTitle(mode, frame, clickPos, clickMeta), annotations, focusedView: { enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')), @@ -1335,14 +1362,7 @@ public static class SFMouseHook { } autoTitle(mode) { - const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]'; - const now = new Date(); - const pad = (n) => String(n).padStart(2, '0'); - return expandPlaceholders(tplStr, { - Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen', - Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`, - Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`, - }); + return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; } /** Grab the screen/window image as { image, display } or throw. */ @@ -1451,8 +1471,12 @@ public static class SFMouseHook { const cropped = image.crop(rect); const size = cropped.getSize(); if (!size.width || !size.height) return { ok: false, reason: 'empty selection' }; - const step = this.store.addStep(guideId, { title: this.autoTitle('region') }, - cropped.toPNG(), size); + const step = await this.storeFrameAsStep(guideId, 'region', { + image: cropped, + size, + display, + cursor: null, + }, null, null); return { ok: true, step }; } diff --git a/app/main.js b/app/main.js index c89abc0..6a87840 100644 --- a/app/main.js +++ b/app/main.js @@ -19,6 +19,7 @@ const { exportGuideArchive, importGuideArchive, saveLinkedGuide, readArchive } = const { createSnapshot, listSnapshots, restoreSnapshot } = require('../core/snapshots'); const { readLock } = require('../core/locks'); const CaptureService = require('./capture'); +const { TextIntelService } = require('./text-intel'); const { keepProcessesResponsive } = require('./win-power'); const APP_ID = 'com.stepforge.app'; @@ -59,6 +60,7 @@ let settings; let searchIndex; let templates; let capture; +let textIntel; let mainWindow; function reindex(guideId) { @@ -503,6 +505,22 @@ function setupIpc() { if (keyPath.startsWith('capture.hotkey')) registerHotkeys(); return settings.data; }); + h('ai:test', async ({ enabled = null, ollama = null } = {}) => { + return textIntel.testAiConnection({ + enabled, + ollama, + }); + }); + h('ai:fillStep', async ({ guideId, stepId, target = 'all', blockId = null } = {}) => { + const result = await textIntel.generateStepPatch({ + guideId, + stepId, + target, + blockId, + }); + if (result.ok) reindex(guideId); + return result; + }); h('placeholders:globals:get', () => settings.getGlobalPlaceholders()); h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values)); @@ -739,6 +757,13 @@ if (!gotLock) { settings = new Settings(store.settingsDir); searchIndex = new SearchIndex(store.indexDir); templates = new TemplateManager(store.templatesDir); + textIntel = new TextIntelService({ + store, + settings, + getWindow: () => mainWindow, + dataDir, + screenApi: screen, + }); // Bringing up the desktop-capture stream spawns/upgrades Chromium's GPU // and screen-capture utility processes — which can be born after a session // already started, so the start-time EcoQoS opt-out misses them. Re-apply @@ -759,6 +784,7 @@ if (!gotLock) { settings, getWindow: () => mainWindow, notify: captureNotify, + textIntel, }); applyTheme(); @@ -778,6 +804,9 @@ if (!gotLock) { capture.stopClickWatcher(); capture.destroySessionTray(); } + if (textIntel) { + textIntel.shutdown().catch(() => {}); + } // clean preview temp files on close try { for (const entry of fs.readdirSync(store.tempDir)) { diff --git a/app/preload.js b/app/preload.js index 432e31b..71f60b8 100644 --- a/app/preload.js +++ b/app/preload.js @@ -52,6 +52,10 @@ const api = { globalPlaceholders: invoke('placeholders:globals:get'), setGlobalPlaceholders: invoke('placeholders:globals:set'), }, + ai: { + test: invoke('ai:test'), + fillStep: invoke('ai:fillStep'), + }, capture: { shoot: invoke('capture:shoot'), region: invoke('capture:region'), diff --git a/app/renderer/app.js b/app/renderer/app.js index bbd99c4..8baae92 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -117,6 +117,7 @@ class StepForgeApp { guideFolders: library.folders?.guideFolders || {}, }; this.state.trash = trash; + this.editor.setSettings(settings); } async refreshLibrary({ keepFilter = true } = {}) { @@ -320,6 +321,7 @@ class StepForgeApp { { label: 'Guide information…', action: () => this.editor.openGuideInfo() }, { label: 'Guide placeholders…', action: () => this.editor.openGuidePlaceholders() }, { label: 'Backups & snapshots…', action: () => this.editor.openBackupsDialog() }, + { label: 'Generate all text fields with AI', action: () => this.editor.generateAllTextFieldsWithAi() }, { label: guide && guide.linkedSource ? 'Linked guide…' : 'Linked guide (not linked)', action: () => this.editor.openLinkedGuide() }, 'sep', { label: 'Keyboard shortcuts…', action: () => this.editor.openShortcutsHelp() }, @@ -866,6 +868,7 @@ class StepForgeApp { await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck }); await api.settings.set({ keyPath: 'capture', value: next.capture }); await api.settings.set({ keyPath: 'editor', value: next.editor }); + await api.settings.set({ keyPath: 'ai', value: next.ai }); await api.settings.set({ keyPath: 'exports', value: next.exports }); await api.settings.set({ keyPath: 'backups', value: next.backups }); await api.settings.setGlobalPlaceholders(next.placeholders || {}); diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index 272c811..8ce8a84 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -313,6 +313,42 @@ function showSettingsDialog({ const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) }); const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) }); const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 }); + const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) }); + const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434'); + const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b'); + const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.'); + const testAiBtn = el('button', { type: 'button' }, 'Test connection'); + + const updateAiStatus = (message, { error = false } = {}) => { + aiStatus.textContent = message; + aiStatus.classList.toggle('error', Boolean(error)); + }; + + const testAiConnection = async () => { + setButtonLoading(testAiBtn, true, 'Testing…'); + updateAiStatus('Checking Ollama at the configured host…'); + try { + const result = await api.ai.test({ + ollama: { + host: ollamaHost.value.trim(), + model: ollamaModel.value.trim(), + }, + }); + if (!result.ok) { + updateAiStatus(result.reason || 'Could not connect to Ollama.', { error: true }); + return; + } + if (result.installed) { + updateAiStatus(`Connected to ${result.host} with ${result.model}.`); + } else { + updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true }); + } + } catch (err) { + updateAiStatus(err.message || 'Could not connect to Ollama.', { error: true }); + } finally { + setButtonLoading(testAiBtn, false); + } + }; const placeholderRows = el('div', { className: 'placeholder-rows' }); const rows = []; @@ -369,6 +405,19 @@ function showSettingsDialog({ el('legend', {}, 'Backups'), labeledRow('Keep last snapshots', keepLast), ), + el('fieldset', {}, + el('legend', {}, 'AI'), + labeledRow('Enable AI text filling', aiEnabled), + labeledRow('Ollama host', ollamaHost), + labeledRow('Ollama model', ollamaModel), + el('div.row', { style: { justifyContent: 'space-between' } }, + aiStatus, + testAiBtn, + ), + el('div.muted', {}, + 'AI generation is manual only. Captures stay deterministic until you click a generate button.', + ), + ), el('fieldset', {}, el('legend', {}, 'Global placeholders'), placeholderRows, @@ -412,6 +461,15 @@ function showSettingsDialog({ ...settings.backups, keepLast: Number(keepLast.value || 0), }, + ai: { + ...settings.ai, + enabled: aiEnabled.checked, + ollama: { + ...(settings.ai?.ollama || {}), + host: ollamaHost.value.trim(), + model: ollamaModel.value.trim(), + }, + }, placeholders: rows.reduce((acc, row) => { const inputs = row.querySelectorAll('input'); const key = inputs[0].value.trim(); @@ -430,6 +488,14 @@ function showSettingsDialog({ }); form.addEventListener('submit', (e) => e.preventDefault()); + testAiBtn.addEventListener('click', testAiConnection); + aiEnabled.addEventListener('change', () => { + updateAiStatus( + aiEnabled.checked + ? 'AI generation will be available once Ollama is reachable.' + : 'AI generation is disabled. The settings are still saved for later.', + ); + }); }); } diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 840f252..dd51028 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -135,6 +135,7 @@ class GuideEditor { this.descriptionDirty = false; this.titleDirty = false; this.active = true; + this.settings = {}; this.saveStepDebounced = debounce(() => this.flushStep(), 180); this.saveGuideDebounced = debounce(() => this.flushGuide(), 180); @@ -152,6 +153,80 @@ class GuideEditor { this.active = Boolean(active); } + setSettings(settings) { + this.settings = settings || {}; + this.updateAiButtonState(); + } + + isAiEnabled() { + return Boolean(this.settings?.ai?.enabled); + } + + updateAiButtonState() { + const enabled = this.isAiEnabled(); + const buttons = [ + this.dom?.titleAiBtn, + this.dom?.descAiBtn, + ...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []), + ].filter(Boolean); + for (const button of buttons) { + button.disabled = !enabled; + button.title = enabled + ? button.dataset.aiTitle || 'Generate with AI' + : 'Enable AI in Settings first.'; + } + } + + async runAiGeneration(target, { blockId = null, button = null } = {}) { + if (!this.currentStep) { + this.onToast('Select a step first.', { error: true }); + return null; + } + if (!this.isAiEnabled()) { + this.onToast('Enable AI in Settings first.', { error: true }); + return null; + } + if (this.pendingSave) await this.flushStep(); + if (button) setButtonLoading(button, true, 'Generating…'); + try { + const result = await api.ai.fillStep({ + guideId: this.guideId, + stepId: this.currentStep.stepId, + target, + blockId, + }); + if (!result || !result.ok) { + this.onToast(result?.reason || 'AI generation failed.', { error: true }); + return null; + } + await this.reload(result.step.stepId); + this.onToast('AI text filled.'); + return result.step; + } catch (err) { + this.onToast(err.message || 'AI generation failed.', { error: true }); + return null; + } finally { + if (button) setButtonLoading(button, false); + } + } + + async generateTitleWithAi(button = null) { + return this.runAiGeneration('title', { button }); + } + + async generateDescriptionWithAi(button = null) { + return this.runAiGeneration('description', { button }); + } + + async generateAllTextFieldsWithAi(button = null) { + return this.runAiGeneration('all', { button }); + } + + async generateBlockWithAi(kind, block, button = null) { + if (!block) return null; + return this.runAiGeneration('block', { blockId: block.id, button }); + } + get currentStep() { return this.stepMap.get(this.selectedStepId) || null; } @@ -271,7 +346,14 @@ class GuideEditor { el('aside.pane-props', {}, el('section', {}, el('h3', {}, 'Step'), - this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title' }), + el('div.row', {}, + this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title', style: { flex: 1 } }), + this.dom.titleAiBtn = el('button.ai', { + type: 'button', + title: 'Generate the step title with AI', + dataset: { aiAction: 'title', aiTitle: 'Generate the step title with AI' }, + }, 'AI'), + ), this.dom.statusSelect = makeSelect('todo', [ { value: 'todo', label: 'Todo' }, { value: 'in-progress', label: 'In progress' }, @@ -296,7 +378,14 @@ class GuideEditor { ), ), el('section', {}, - el('h3', {}, 'Description'), + el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } }, + el('h3', { style: { margin: 0 } }, 'Description'), + this.dom.descAiBtn = el('button.ai', { + type: 'button', + title: 'Generate the description with AI', + dataset: { aiAction: 'description', aiTitle: 'Generate the description with AI' }, + }, 'AI'), + ), this.dom.richToolbar = el('div.rich-toolbar', {}, this.toolbarBtn('bold', 'Bold'), this.toolbarBtn('italic', 'Italic'), @@ -416,6 +505,7 @@ class GuideEditor { this.renderStepList(); this.emitMeta(); }); + this.dom.titleAiBtn.addEventListener('click', () => this.generateTitleWithAi(this.dom.titleAiBtn)); this.dom.statusSelect.addEventListener('change', () => { if (!this.currentStep) return; @@ -491,6 +581,7 @@ class GuideEditor { document.addEventListener('selectionchange', () => { if (document.activeElement === this.dom.descEditor) this.updateToolbarState(); }); + this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn)); this.dom.descEditor.addEventListener('paste', (e) => { // Keep pasted text simple; backend sanitization will handle the rest. @@ -504,6 +595,8 @@ class GuideEditor { if (!item) return; this.canvas.select(item.dataset.annId); }); + + this.updateAiButtonState(); } renderAll() { @@ -513,6 +606,7 @@ class GuideEditor { this.renderCanvas(); this.renderAnnotationPanel(); this.renderBlocksPanel(); + this.updateAiButtonState(); this.emitMeta(); } @@ -670,6 +764,15 @@ class GuideEditor { el('span.spacer'), el('button.icon', { type: 'button', title: 'Move block up', disabled: !canMoveUp, onClick: moveUp, dataset: { blockMove: 'up' } }, '↑'), el('button.icon', { type: 'button', title: 'Move block down', disabled: !canMoveDown, onClick: moveDown, dataset: { blockMove: 'down' } }, '↓'), + (() => { + const aiBtn = el('button.ai', { + type: 'button', + title: 'Generate this block with AI', + dataset: { aiAction: 'block', aiTitle: 'Generate this block with AI' }, + onClick: () => this.generateBlockWithAi(kind, block, aiBtn), + }, 'AI'); + return aiBtn; + })(), removeBtn(() => { if (kind === 'text') step.textBlocks = (step.textBlocks || []).filter((b) => b !== block); else if (kind === 'code') step.codeBlocks = (step.codeBlocks || []).filter((b) => b !== block); @@ -761,6 +864,7 @@ class GuideEditor { if (!blocks.length) { this.dom.blocksList.append(el('div.muted', {}, 'Informational text, code, and table blocks can be reordered with drag handles or arrows.')); } + this.updateAiButtonState(); } renderStepList() { @@ -1658,6 +1762,7 @@ class GuideEditor { await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck }); await api.settings.set({ keyPath: 'capture', value: next.capture }); await api.settings.set({ keyPath: 'editor', value: next.editor }); + await api.settings.set({ keyPath: 'ai', value: next.ai }); await api.settings.set({ keyPath: 'exports', value: next.exports }); await api.settings.set({ keyPath: 'backups', value: next.backups }); await api.settings.setGlobalPlaceholders(next.placeholders || {}); diff --git a/app/renderer/style.css b/app/renderer/style.css index 6842b29..5ea2b5a 100644 --- a/app/renderer/style.css +++ b/app/renderer/style.css @@ -90,6 +90,16 @@ button.tool.active { border-color: var(--accent); color: var(--accent-fg); } +button.ai { + padding: 5px 10px; + border-color: color-mix(in srgb, var(--accent) 38%, var(--border)); + background: color-mix(in srgb, var(--accent) 10%, var(--panel-solid)); + color: var(--accent-strong); + font-weight: 650; + letter-spacing: 0.02em; +} +button.ai:hover { background: color-mix(in srgb, var(--accent) 16%, var(--panel-2)); } +button.ai:disabled { opacity: 0.6; } button:disabled { opacity: 0.6; cursor: default; } button.loading { display: inline-flex; @@ -751,6 +761,7 @@ fieldset legend { color: var(--muted); line-height: 1.5; } +.ai-status.error { color: var(--danger); } #modal-root:not(:empty) { position: fixed; diff --git a/app/text-intel.js b/app/text-intel.js new file mode 100644 index 0000000..0d305a4 --- /dev/null +++ b/app/text-intel.js @@ -0,0 +1,438 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +const { createWorker } = require('tesseract.js'); +const { + buildCaptureTitle, + normalizeOllamaHost, + normalizeAiPatch, + buildAiPrompt, + applyAiPatchToStep, + displayText, + normalizeWhitespace, +} = require('../core/text-intel'); + +const OCR_CROP = { + width: 420, + height: 220, +}; + +function hasBinary(name) { + try { + execFileSync('which', [name], { stdio: 'pipe' }); + return true; + } catch { + return false; + } +} + +function clamp(v, min, max) { + return Math.min(max, Math.max(min, v)); +} + +class TextIntelService { + constructor({ + store, + settings, + getWindow = () => null, + dataDir, + fetchImpl = global.fetch, + screenApi = null, + }) { + this.store = store; + this.settings = settings; + this.getWindow = getWindow; + this.dataDir = dataDir; + this.fetch = fetchImpl; + this.screen = screenApi; + this.worker = null; + this.workerPromise = null; + this.workerQueue = Promise.resolve(); + this.ocrDataDir = path.join(dataDir, 'ocr', 'eng'); + } + + async shutdown() { + if (this.worker) { + try { + await this.worker.terminate(); + } catch { + // best effort + } + this.worker = null; + this.workerPromise = null; + } + } + + ensureLangData() { + const packageDir = path.dirname(require.resolve('@tesseract.js-data/eng/package.json')); + const source = path.join(packageDir, '4.0.0_best_int', 'eng.traineddata.gz'); + const targetDir = this.ocrDataDir; + const target = path.join(targetDir, 'eng.traineddata.gz'); + if (!fs.existsSync(target)) { + fs.mkdirSync(targetDir, { recursive: true }); + fs.copyFileSync(source, target); + } + return targetDir; + } + + async getWorker() { + if (this.workerPromise) return this.workerPromise; + this.workerPromise = (async () => { + const langPath = this.ensureLangData(); + const worker = await createWorker('eng', 1, { + langPath, + }); + await worker.setParameters({ + preserve_interword_spaces: '1', + }); + this.worker = worker; + return worker; + })(); + return this.workerPromise; + } + + async recognizeCrop(image, rect = null) { + const worker = await this.getWorker(); + const cropped = rect ? image.crop(rect) : image; + const buffer = cropped.toPNG(); + const result = await worker.recognize(buffer); + const text = String(result?.data?.text || '').trim(); + return { + text, + confidence: Number.isFinite(result?.data?.confidence) ? result.data.confidence : null, + raw: result, + }; + } + + cropRectForPoint(frame, clickPos, { width = OCR_CROP.width, height = OCR_CROP.height } = {}) { + if (!frame || !frame.size) return null; + const bounds = frame.display.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height }; + const scaleX = frame.size.width / bounds.width; + const scaleY = frame.size.height / bounds.height; + const point = clickPos || { + x: bounds.x + bounds.width / 2, + y: bounds.y + bounds.height / 2, + }; + const centerX = (point.x - bounds.x) * scaleX; + const centerY = (point.y - bounds.y) * scaleY; + const rectW = Math.max(1, Math.round(width * scaleX)); + const rectH = Math.max(1, Math.round(height * scaleY)); + const rect = { + x: Math.round(centerX - rectW / 2), + y: Math.round(centerY - rectH / 2), + width: rectW, + height: rectH, + }; + rect.x = clamp(rect.x, 0, Math.max(0, frame.size.width - rect.width)); + rect.y = clamp(rect.y, 0, Math.max(0, frame.size.height - rect.height)); + rect.width = clamp(rect.width, 1, frame.size.width); + rect.height = clamp(rect.height, 1, frame.size.height); + return rect; + } + + async ocrAroundClick(frame, clickPos) { + if (!frame || !frame.image) return { text: '', confidence: null }; + const rect = this.cropRectForPoint(frame, clickPos); + try { + return await this.recognizeCrop(frame.image, rect); + } catch { + return { text: '', confidence: null }; + } + } + + async collectForegroundWindowContext(osPoint = null) { + try { + if (process.platform === 'win32') return this.collectWindowsWindowContext(osPoint); + if (process.platform === 'darwin') return this.collectMacWindowContext(); + if (process.platform === 'linux') return this.collectLinuxWindowContext(); + } catch { + // best effort only + } + return { appName: '', windowTitle: '' }; + } + + collectWindowsWindowContext(osPoint = null) { + const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y); + const clickX = hasPoint ? Number(osPoint.x) : 0; + const clickY = hasPoint ? Number(osPoint.y) : 0; + const script = ` + $clickX = ${clickX}; + $clickY = ${clickY}; + $elementLabel = ''; + $elementRole = ''; + $elementClass = ''; + $elementProcessId = 0; + if (${hasPoint ? '$true' : '$false'}) { + try { + Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null + $point = New-Object System.Windows.Point($clickX, $clickY); + $element = [System.Windows.Automation.AutomationElement]::FromPoint($point); + if ($element) { + $current = $element.Current; + $elementLabel = $current.Name; + $elementRole = $current.LocalizedControlType; + $elementClass = $current.ClassName; + $elementProcessId = $current.ProcessId; + } + } catch { } + } + Add-Type @" +using System; +using System.Runtime.InteropServices; +using System.Text; +public static class Win32 { + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count); + [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); +} +"@; + $hWnd = [Win32]::GetForegroundWindow(); + $sb = New-Object System.Text.StringBuilder 512; + [void][Win32]::GetWindowText($hWnd, $sb, $sb.Capacity); + $pid = 0; + [void][Win32]::GetWindowThreadProcessId($hWnd, [ref]$pid); + $proc = Get-Process -Id $pid -ErrorAction SilentlyContinue | Select-Object -First 1; + $out = [ordered]@{ + appName = if ($proc) { $proc.ProcessName } else { '' }; + windowTitle = $sb.ToString(); + elementLabel = $elementLabel; + elementRole = $elementRole; + elementClass = $elementClass; + elementProcessId = $elementProcessId; + pid = $pid; + }; + $out | ConvertTo-Json -Compress; + `; + const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }).trim(); + return JSON.parse(result || '{}'); + } + + collectMacWindowContext() { + const script = ` + set appName to "" + set windowTitle to "" + tell application "System Events" + try + set frontApp to first application process whose frontmost is true + set appName to name of frontApp + try + set windowTitle to name of front window of frontApp + end try + end try + end tell + return appName & linefeed & windowTitle + `; + const result = execFileSync('osascript', ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }).trimEnd(); + const [appName = '', windowTitle = ''] = result.split(/\r?\n/); + return { appName, windowTitle }; + } + + collectLinuxWindowContext() { + if (!hasBinary('xprop')) return { appName: '', windowTitle: '' }; + const active = execFileSync('xprop', ['-root', '_NET_ACTIVE_WINDOW'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }); + const activeMatch = active.match(/window id # (0x[0-9a-fA-F]+)/); + if (!activeMatch) return { appName: '', windowTitle: '' }; + const winId = activeMatch[1]; + const details = execFileSync('xprop', ['-id', winId, '_NET_WM_NAME', 'WM_NAME', 'WM_CLASS'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }); + const titleMatch = details.match(/(?:_NET_WM_NAME\(UTF8_STRING\)|WM_NAME\(STRING\)|WM_NAME\(UTF8_STRING\)) = "([^"]*)"/); + const classMatch = details.match(/WM_CLASS\(STRING\) = "([^"]*)"(?:, "([^"]*)")?/); + return { + appName: classMatch ? (classMatch[2] || classMatch[1] || '') : '', + windowTitle: titleMatch ? titleMatch[1] : '', + }; + } + + async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) { + const [metadata, ocr] = await Promise.all([ + this.collectForegroundWindowContext(clickMeta?.osPoint || null), + this.ocrAroundClick(frame, clickPos), + ]); + return buildCaptureTitle({ + mode, + metadata, + ocrText: ocr.text, + }); + } + + aiEnabled() { + return Boolean(this.settings.get('ai.enabled')); + } + + aiConfig(override = null) { + const stored = this.settings.get('ai') || {}; + const merged = override ? { + ...stored, + ...override, + ollama: { + ...(stored.ollama || {}), + ...(override.ollama || {}), + }, + } : stored; + return { + ...merged, + enabled: override && Object.prototype.hasOwnProperty.call(override, 'enabled') + ? Boolean(override.enabled) + : Boolean(stored.enabled), + ollama: { + host: normalizeOllamaHost(merged.ollama?.host || ''), + model: normalizeWhitespace(merged.ollama?.model || ''), + }, + }; + } + + async testAiConnection(override = null) { + const config = this.aiConfig(override); + if (!config.ollama.host) { + return { ok: false, reason: 'Set an Ollama host first.' }; + } + const tagsUrl = new URL('/api/tags', `${config.ollama.host.replace(/\/+$/, '')}/`); + const res = await this.fetch(tagsUrl, { method: 'GET' }); + if (!res.ok) { + return { ok: false, reason: `Ollama check failed (${res.status})` }; + } + const data = await res.json(); + const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : []; + const installed = config.ollama.model ? models.includes(config.ollama.model) : false; + return { + ok: true, + installed, + models, + host: config.ollama.host, + model: config.ollama.model, + }; + } + + async callOllama({ host, model, prompt, systemPrompt }) { + const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); + const response = await this.fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + stream: false, + format: 'json', + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: prompt }, + ], + options: { + temperature: 0.2, + }, + }), + }); + if (!response.ok) { + throw new Error(`Ollama request failed (${response.status})`); + } + const payload = await response.json(); + const content = payload?.message?.content; + if (typeof content !== 'string' || !content.trim()) { + throw new Error('Ollama returned an empty response'); + } + return content; + } + + async generateStepPatch({ + guideId, + stepId, + target = 'all', + blockId = null, + }) { + try { + const config = this.aiConfig(); + if (!config.enabled) { + return { ok: false, reason: 'Enable AI in settings first.' }; + } + if (!config.ollama.host || !config.ollama.model) { + return { ok: false, reason: 'Configure Ollama host and model in Settings.' }; + } + + const guide = this.store.getGuide(guideId); + const step = this.store.getStep(guideId, stepId); + if (!guide || !step) { + return { ok: false, reason: 'Guide or step not found.' }; + } + + const currentBlock = blockId + ? [...(step.textBlocks || []), ...(step.codeBlocks || []), ...(step.tableBlocks || [])].find((b) => b.id === blockId) || null + : null; + if (blockId && target === 'block' && !currentBlock) { + return { ok: false, reason: 'Block not found.' }; + } + + let captureContext = null; + if (step.image) { + const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original'); + if (imagePath && fs.existsSync(imagePath)) { + const { nativeImage } = require('electron'); + const image = nativeImage.createFromPath(imagePath); + if (!image.isEmpty()) { + const clickPoint = this.clickPointFromStep(step, image); + const [metadata, ocr] = await Promise.all([ + this.collectForegroundWindowContext(), + this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint), + ]); + captureContext = { + ...metadata, + ocrText: ocr.text, + mode: step.kind === 'image' ? 'fullscreen' : 'content', + }; + } + } + } + + const { systemPrompt, prompt } = buildAiPrompt({ + target, + guide, + step, + captureContext, + block: currentBlock, + }); + + const raw = await this.callOllama({ + host: config.ollama.host, + model: config.ollama.model, + prompt, + systemPrompt, + }); + const patch = normalizeAiPatch(raw); + const updated = applyAiPatchToStep(step, patch, { target, blockId }); + const saved = this.store.saveStep(guideId, updated); + return { ok: true, step: saved, patch }; + } catch (err) { + return { ok: false, reason: err && err.message ? err.message : 'AI generation failed.' }; + } + } + + clickPointFromStep(step, image = null) { + const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h)); + if (!marker) return null; + const size = image ? image.getSize() : step.image?.size || { width: 0, height: 0 }; + if (!size.width || !size.height) return null; + return { + x: Math.round((marker.x + marker.w / 2) * size.width), + y: Math.round((marker.y + marker.h / 2) * size.height), + }; + } +} + +module.exports = { TextIntelService }; diff --git a/core/settings.js b/core/settings.js index 62aa672..02b1552 100644 --- a/core/settings.js +++ b/core/settings.js @@ -47,6 +47,13 @@ const DEFAULT_SETTINGS = { focusedViewDefaultForNewSteps: false, autoTitleTemplate: '[[Mode]] capture [[Time]]', }, + ai: { + enabled: false, + ollama: { + host: 'http://127.0.0.1:11434', + model: 'llama3.2:1b', + }, + }, exports: { previewStepCount: 3, openFolderAfterExport: true, diff --git a/core/text-intel.js b/core/text-intel.js new file mode 100644 index 0000000..78c7e6a --- /dev/null +++ b/core/text-intel.js @@ -0,0 +1,416 @@ +'use strict'; +const { deepClone, htmlToText, escapeHtml } = require('./util'); +const { sanitizeHtml } = require('./sanitize'); +const { + TEXTBLOCK_LEVELS, + TEXTBLOCK_POSITIONS, + normalizeTextBlock, + normalizeCodeBlock, + normalizeTableBlock, +} = require('./schema'); + +const DEFAULT_CAPTURE_TITLES = { + fullscreen: 'Screen capture', + window: 'Window capture', + region: 'Region capture', +}; + +const AI_LEVEL_ALIASES = new Map([ + ['note', 'info'], + ['info', 'info'], + ['tip', 'success'], + ['success', 'success'], + ['warning', 'warn'], + ['warn', 'warn'], + ['important', 'error'], + ['error', 'error'], +]); + +const GENERIC_OCR_PHRASES = new Set([ + 'button', + 'click', + 'double click', + 'menu', + 'item', + 'field', + 'text field', + 'search', + 'submit', + 'cancel', + 'ok', + 'open', + 'select', + 'enter', + 'type', +]); + +function normalizeWhitespace(text) { + return String(text == null ? '' : text) + .replace(/\s+/g, ' ') + .trim(); +} + +function titleCaseWord(word) { + if (!word) return word; + if (/^[A-Z0-9]{2,}$/.test(word)) return word; + if (/^\d+$/.test(word)) return word; + return word[0].toUpperCase() + word.slice(1).toLowerCase(); +} + +function displayText(text) { + const clean = normalizeWhitespace(text) + .replace(/^[\s"'`([{<]+|[\s"'`)}\]>.,;:!?]+$/g, '') + .trim(); + if (!clean) return ''; + if (clean === clean.toUpperCase()) { + return clean.split(/\s+/).map(titleCaseWord).join(' '); + } + return clean.replace(/\s+/g, ' '); +} + +function candidateWords(text) { + const clean = normalizeWhitespace(text); + if (!clean) return []; + return clean.split(/\s+/).filter(Boolean); +} + +function scoreCandidate(text, { source = 'ocr' } = {}) { + const clean = displayText(text); + if (!clean) return -Infinity; + const words = candidateWords(clean); + if (!words.length) return -Infinity; + let score = 0; + score += source === 'element' ? 110 : source === 'window' ? 95 : source === 'app' ? 80 : 100; + score += Math.min(words.length, 6) * 12; + score -= Math.max(0, words.length - 6) * 10; + score -= Math.max(0, clean.length - 56) * 0.6; + if (GENERIC_OCR_PHRASES.has(clean.toLowerCase())) score -= 50; + if (clean.length <= 24) score += 10; + if (/^(click|open|enter|type|choose|select|save|cancel|confirm)\b/i.test(clean)) score += 8; + if (/^[\p{P}\p{S}0-9]+$/u.test(clean)) score -= 100; + return score; +} + +function pickBestOcrPhrase(ocrText) { + const text = normalizeWhitespace(ocrText); + if (!text) return ''; + const parts = text + .split(/\n+/) + .map((line) => displayText(line)) + .filter(Boolean) + .filter((line) => !GENERIC_OCR_PHRASES.has(line.toLowerCase())); + if (!parts.length) return ''; + let best = ''; + let bestScore = -Infinity; + for (const part of parts) { + const score = scoreCandidate(part, { source: 'ocr' }); + if (score > bestScore) { + best = part; + bestScore = score; + } + } + return best; +} + +function isShortUiLabel(text) { + const words = candidateWords(text); + return words.length > 0 && words.length <= 2 && text.length <= 24; +} + +function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) { + const candidates = []; + if (metadata.elementLabel) candidates.push({ text: metadata.elementLabel, source: 'element' }); + if (metadata.windowTitle) candidates.push({ text: metadata.windowTitle, source: 'window' }); + if (metadata.appName && metadata.appName !== metadata.windowTitle) { + candidates.push({ text: metadata.appName, source: 'app' }); + } + const ocrPhrase = pickBestOcrPhrase(ocrText); + if (ocrPhrase) candidates.push({ text: ocrPhrase, source: 'ocr' }); + + let winner = null; + let winnerScore = -Infinity; + for (const candidate of candidates) { + const score = scoreCandidate(candidate.text, { source: candidate.source }); + if (score > winnerScore) { + winner = candidate; + winnerScore = score; + } + } + + const cleanedWinner = winner ? displayText(winner.text) : ''; + if (!cleanedWinner) return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; + + if (winner.source === 'window' || winner.source === 'app') { + return `Open ${cleanedWinner}`; + } + if (winner.source === 'element') { + return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; + } + if (winner.source === 'ocr') { + return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; + } + return cleanedWinner; +} + +function plainTextToHtml(text) { + const trimmed = normalizeWhitespace(text); + if (!trimmed) return ''; + return trimmed + .split(/\n{2,}/) + .map((para) => `

${escapeHtml(para).replace(/\n/g, '
')}

`) + .join(''); +} + +function normalizeOllamaHost(host) { + const raw = normalizeWhitespace(host); + if (!raw) return ''; + if (/^https?:\/\//i.test(raw)) return raw.replace(/\/+$/, ''); + return `http://${raw.replace(/\/+$/, '')}`; +} + +function normalizeAiLevel(level) { + const key = normalizeWhitespace(level).toLowerCase(); + return AI_LEVEL_ALIASES.get(key) || (TEXTBLOCK_LEVELS.includes(key) ? key : 'info'); +} + +function normalizeAiPosition(position) { + const key = normalizeWhitespace(position).toLowerCase(); + return TEXTBLOCK_POSITIONS.includes(key) ? key : 'after-description'; +} + +function normalizeAiBlock(block) { + if (!block || typeof block !== 'object') return null; + const kind = normalizeWhitespace(block.kind).toLowerCase(); + if (kind === 'text') { + const normalized = normalizeTextBlock({ + id: block.id, + order: Number.isFinite(block.order) ? block.order : null, + position: normalizeAiPosition(block.position), + level: normalizeAiLevel(block.level), + title: displayText(block.title), + descriptionHtml: plainTextToHtml(block.body ?? block.description ?? block.text ?? ''), + }, Number.isFinite(block.order) ? block.order : null); + return { ...normalized, kind: 'text' }; + } + if (kind === 'code') { + return { + ...normalizeCodeBlock({ + id: block.id, + order: Number.isFinite(block.order) ? block.order : null, + language: displayText(block.language).toLowerCase(), + code: String(block.code ?? ''), + }, Number.isFinite(block.order) ? block.order : null), + kind: 'code', + }; + } + if (kind === 'table') { + const rows = Array.isArray(block.rows) + ? block.rows.map((row) => (Array.isArray(row) ? row.map((cell) => displayText(cell)) : [])) + : []; + return { + ...normalizeTableBlock({ + id: block.id, + order: Number.isFinite(block.order) ? block.order : null, + rows, + }, Number.isFinite(block.order) ? block.order : null), + kind: 'table', + }; + } + return null; +} + +function normalizeAiPatch(raw) { + let data = raw; + if (typeof raw === 'string') { + const trimmed = raw.trim().replace(/^```(?:json)?\s*|\s*```$/g, ''); + const start = trimmed.indexOf('{'); + const end = trimmed.lastIndexOf('}'); + const jsonText = start >= 0 && end > start ? trimmed.slice(start, end + 1) : trimmed; + data = JSON.parse(jsonText); + } + if (!data || typeof data !== 'object' || Array.isArray(data)) { + throw new Error('AI response must be a JSON object'); + } + const out = { + title: displayText(data.title), + descriptionHtml: plainTextToHtml(data.description ?? data.descriptionText ?? ''), + blocks: Array.isArray(data.blocks) + ? data.blocks.map((block) => normalizeAiBlock(block)).filter(Boolean) + : [], + }; + return out; +} + +function summarizeBlocks(step = {}) { + const parts = []; + for (const block of step.textBlocks || []) { + const body = htmlToText(block.descriptionHtml || ''); + parts.push(`- Text (${block.level || 'info'}, ${block.position || 'after-description'}): ${block.title || ''}${body ? ` — ${body}` : ''}`.trim()); + } + for (const block of step.codeBlocks || []) { + const code = String(block.code || '').trim(); + parts.push(`- Code (${block.language || 'plain'}):\n${code || '(empty)'}`); + } + for (const block of step.tableBlocks || []) { + const rows = Array.isArray(block.rows) ? block.rows.length : 0; + const cols = rows > 0 && Array.isArray(block.rows[0]) ? block.rows[0].length : 0; + parts.push(`- Table (${rows}x${cols})`); + } + return parts.length ? parts.join('\n') : '(none)'; +} + +function summarizeStepForAi(step = {}) { + return [ + `Step title: ${step.title || '(empty)'}`, + `Step description: ${htmlToText(step.descriptionHtml || '') || '(empty)'}`, + `Step status: ${step.status || 'todo'}`, + `Blocks:\n${summarizeBlocks(step)}`, + ].join('\n'); +} + +function summarizeGuideForAi(guide = {}) { + return [ + `Guide title: ${guide.title || '(untitled)'}`, + `Guide description: ${htmlToText(guide.descriptionHtml || '') || '(empty)'}`, + ].join('\n'); +} + +function buildAiPrompt({ + target = 'all', + guide = null, + step = null, + captureContext = null, + block = null, +} = {}) { + const targetText = { + title: 'rewrite only the step title', + description: 'rewrite only the step description', + block: 'rewrite only one block', + all: 'rewrite the step title, description, and any useful blocks', + }[target] || 'rewrite the step'; + + const allowedBlockNote = [ + 'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.', + 'Use block.kind = "code" for code snippets.', + 'Use block.kind = "table" for tables, with rows as arrays of strings.', + 'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].', + ].join(' '); + + const prompt = [ + 'You write concise, human-sounding step-by-step documentation for an offline desktop app.', + 'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.', + 'Schema:', + '{', + ' "title": string,', + ' "description": string,', + ' "blocks": [{', + ' "kind": "text" | "code" | "table",', + ' "position"?: "before-title" | "after-title" | "before-image" | "after-image" | "before-description" | "after-description",', + ' "level"?: "info" | "warn" | "error" | "success",', + ' "title"?: string,', + ' "body"?: string,', + ' "language"?: string,', + ' "code"?: string,', + ' "rows"?: string[][]', + ' }]', + '}', + '', + `Target: ${targetText}.`, + allowedBlockNote, + '', + guide ? summarizeGuideForAi(guide) : 'Guide: (not provided)', + '', + step ? summarizeStepForAi(step) : 'Step: (not provided)', + '', + captureContext ? [ + `Active window title: ${captureContext.windowTitle || '(unknown)'}`, + `Active app: ${captureContext.appName || '(unknown)'}`, + `OCR text near click: ${captureContext.ocrText || '(none)'}`, + `Capture mode: ${captureContext.mode || '(unknown)'}`, + ].join('\n') : 'Capture context: (not provided)', + '', + block ? `Target block:\n${JSON.stringify(block, null, 2)}` : 'Target block: (not provided)', + '', + 'Rules:', + '- Keep the wording natural and specific.', + '- Prefer short titles.', + '- Only return blocks that add value.', + '- Do not invent details that are not supported by the capture context.', + '- If the target is one block, only rewrite that block.', + ].join('\n'); + + return { + systemPrompt: 'You are a documentation assistant that only emits JSON.', + prompt, + }; +} + +function applyAiPatchToStep(step, patch, { target = 'all', blockId = null } = {}) { + const next = deepClone(step); + if ((target === 'all' || target === 'title') && patch.title) { + next.title = displayText(patch.title); + } + if ((target === 'all' || target === 'description') && patch.descriptionHtml) { + next.descriptionHtml = sanitizeHtml(patch.descriptionHtml); + } + + if (target === 'all' && Array.isArray(patch.blocks) && patch.blocks.length) { + const textBlocks = []; + const codeBlocks = []; + const tableBlocks = []; + let nextOrder = 1; + for (const block of patch.blocks) { + const clone = deepClone(block); + clone.order = nextOrder++; + if (clone.kind === 'text') textBlocks.push(clone); + else if (clone.kind === 'code') codeBlocks.push(clone); + else if (clone.kind === 'table') tableBlocks.push(clone); + } + next.textBlocks = textBlocks; + next.codeBlocks = codeBlocks; + next.tableBlocks = tableBlocks; + } else if (target === 'block' && blockId && Array.isArray(patch.blocks) && patch.blocks.length) { + const replacement = patch.blocks[0]; + const textBlock = (next.textBlocks || []).find((block) => block.id === blockId); + const codeBlock = (next.codeBlocks || []).find((block) => block.id === blockId); + const tableBlock = (next.tableBlocks || []).find((block) => block.id === blockId); + if (textBlock && replacement.kind === 'text') { + if (replacement.position) textBlock.position = replacement.position; + if (replacement.level) textBlock.level = replacement.level; + if (replacement.title) textBlock.title = replacement.title; + if (replacement.descriptionHtml) textBlock.descriptionHtml = sanitizeHtml(replacement.descriptionHtml); + } else if (codeBlock && replacement.kind === 'code') { + if (replacement.language) codeBlock.language = replacement.language; + if (replacement.code) codeBlock.code = replacement.code; + } else if (tableBlock && replacement.kind === 'table') { + if (replacement.rows) tableBlock.rows = replacement.rows; + } + } + if (!next.image) { + const hasBody = Boolean( + next.title || + htmlToText(next.descriptionHtml || '') || + (next.textBlocks || []).length || + (next.codeBlocks || []).length || + (next.tableBlocks || []).length, + ); + if (hasBody) next.kind = 'content'; + } + return next; +} + +module.exports = { + DEFAULT_CAPTURE_TITLES, + buildCaptureTitle, + plainTextToHtml, + normalizeOllamaHost, + normalizeAiPatch, + buildAiPrompt, + applyAiPatchToStep, + summarizeStepForAi, + summarizeGuideForAi, + displayText, + normalizeWhitespace, + scoreCandidate, + pickBestOcrPhrase, +}; diff --git a/docs/getting_started_with_ai.md b/docs/getting_started_with_ai.md new file mode 100644 index 0000000..193218e --- /dev/null +++ b/docs/getting_started_with_ai.md @@ -0,0 +1,75 @@ +# Getting Started With AI + +StepForge keeps AI local. It talks to your own Ollama server on your machine and does not send guide content to the cloud. + +## 1. Install Ollama + +Install Ollama from https://ollama.com and make sure the service is running. + +On most systems you can verify it with: + +```bash +ollama --version +``` + +## 2. Pull a lightweight model + +The recommended default is: + +```bash +ollama pull llama3.2:1b +``` + +That model is small enough to feel responsive on modest hardware, but still good enough for human-sounding titles and short text blocks. + +If you need something even smaller, try: + +```bash +ollama pull qwen3:0.6b +``` + +or: + +```bash +ollama pull gemma3:270m +``` + +Those are lighter, but they are usually weaker at writing polished step text. + +## 3. Open StepForge settings + +In StepForge, open `Settings` and find the `AI` section. + +Set: + +* `Enable AI text filling` to on +* `Ollama host` to your local Ollama server +* `Ollama model` to `llama3.2:1b` or the smaller model you pulled + +The default host is: + +```text +http://127.0.0.1:11434 +``` + +## 4. Test the connection + +Use the `Test connection` button in the AI settings section. + +If the model is installed, StepForge should confirm the host and model. + +## 5. Use AI manually + +AI is never automatic. After capture, use the `AI` button next to: + +* the step title +* the step description +* each text, code, and table block + +You can also use `More -> Generate all text fields with AI` to fill the whole step in one pass. + +## Notes + +* Capture titles are still generated automatically without AI. +* AI generation only works when `Enable AI text filling` is turned on. +* The app always uses local OCR around the click area first, then local AI only when you ask for it. diff --git a/package-lock.json b/package-lock.json index 49e5bb5..9cc3468 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,10 @@ "name": "stepforge", "version": "0.1.0", "license": "MPL-2.0", + "dependencies": { + "@tesseract.js-data/eng": "^1.0.0", + "tesseract.js": "^7.0.0" + }, "devDependencies": { "electron": "^41.7.1", "electron-builder": "^26.15.2" @@ -624,6 +628,12 @@ "node": ">=10" } }, + "node_modules/@tesseract.js-data/eng": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@tesseract.js-data/eng/-/eng-1.0.0.tgz", + "integrity": "sha512-mbTumm6KQPUHyzTPQaF3ObXYnx0SqqfV2nabqFVQBwD6Kl7PhGSLSzOlfFTWy0P3BjghaSKA2W9GB19Jk+ZcTg==", + "license": "MIT" + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -1057,6 +1067,12 @@ "dev": true, "license": "MIT" }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/boolean": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", @@ -2512,6 +2528,12 @@ "node": ">= 14" } }, + "node_modules/idb-keyval": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.5.tgz", + "integrity": "sha512-eKQkTnS0relYsSOYomx8ozIbmdsQCKUdhyuIaQ2DZgKuaxtyQQMkyD/wlnQN32pO3yutN1b1L8uqwcDKaJd7/Q==", + "license": "Apache-2.0" + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -2541,6 +2563,12 @@ "node": ">=8" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -2903,6 +2931,26 @@ "node": ">=10" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.4.0.tgz", @@ -3024,6 +3072,15 @@ "wrappy": "1" } }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/p-cancelable": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", @@ -3314,6 +3371,12 @@ "util-deprecate": "~1.0.1" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3728,6 +3791,30 @@ "node": ">= 10.0.0" } }, + "node_modules/tesseract.js": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz", + "integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^7.0.0", + "wasm-feature-detect": "^1.8.0", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz", + "integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==", + "license": "Apache-2.0" + }, "node_modules/tiny-async-pool": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.3.0.tgz", @@ -3785,6 +3872,12 @@ "tmp": "^0.2.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/truncate-utf8-bytes": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", @@ -3909,6 +4002,12 @@ "dev": true, "license": "MIT" }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, "node_modules/webcrypto-core": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/webcrypto-core/-/webcrypto-core-1.9.2.tgz", @@ -3923,6 +4022,22 @@ "tslib": "^2.8.1" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/which/-/which-5.0.0.tgz", @@ -4043,6 +4158,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } } } } diff --git a/package.json b/package.json index 8e44b59..41fa808 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,10 @@ "verify": "bash scripts/verify.sh", "bootstrap": "bash scripts/bootstrap-offline.sh" }, + "dependencies": { + "@tesseract.js-data/eng": "^1.0.0", + "tesseract.js": "^7.0.0" + }, "devDependencies": { "electron": "^41.7.1", "electron-builder": "^26.15.2" diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js new file mode 100644 index 0000000..861e322 --- /dev/null +++ b/tests/unit/text-intel.test.js @@ -0,0 +1,195 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { makeTmpDir, rmrf } = require('./helpers'); +const { createStep } = require('../../core/schema'); +const { + buildCaptureTitle, + normalizeAiPatch, + applyAiPatchToStep, +} = require('../../core/text-intel'); +const { TextIntelService } = require('../../app/text-intel'); + +function makeSettings(values = {}) { + const data = { + ai: { + enabled: true, + ollama: { + host: 'http://127.0.0.1:11434', + model: 'llama3.2:1b', + }, + }, + ...values, + }; + return { + get(key) { + return key.split('.').reduce((acc, part) => (acc == null ? undefined : acc[part]), data); + }, + }; +} + +test('capture titles prefer semantic metadata before OCR fallback', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { windowTitle: 'Reset user password in admin portal' }, + ocrText: 'Save', + }); + assert.equal(title, 'Open Reset user password in admin portal'); +}); + +test('capture titles prefer element metadata before window chrome and OCR', () => { + const title = buildCaptureTitle({ + mode: 'window', + metadata: { + elementLabel: 'Open advanced settings', + windowTitle: 'Preferences', + }, + ocrText: 'Cancel', + }); + assert.equal(title, 'Open advanced settings'); +}); + +test('capture titles fall back to OCR when metadata is absent', () => { + const title = buildCaptureTitle({ + mode: 'window', + metadata: {}, + ocrText: 'Save changes', + }); + assert.equal(title, 'Click Save changes'); +}); + +test('ocr crop rectangles clamp to the image bounds', (t) => { + const root = makeTmpDir('text-intel-crop'); + t.after(() => rmrf(root)); + const service = new TextIntelService({ + store: { settingsDir: root }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: global.fetch, + }); + + const frame = { + size: { width: 1000, height: 500 }, + display: { bounds: { x: 0, y: 0, width: 1000, height: 500 } }, + }; + + const topLeft = service.cropRectForPoint(frame, { x: 5, y: 5 }); + assert.deepEqual(topLeft, { x: 0, y: 0, width: 420, height: 220 }); + + const bottomRight = service.cropRectForPoint(frame, { x: 995, y: 495 }); + assert.deepEqual(bottomRight, { x: 580, y: 280, width: 420, height: 220 }); +}); + +test('ai response normalization and application keeps fields structured', () => { + const patch = normalizeAiPatch(JSON.stringify({ + title: 'Open settings', + description: 'Pick the AI tab.', + blocks: [ + { + kind: 'text', + position: 'after-description', + level: 'tip', + title: 'Tip', + body: 'Use the local Ollama model.', + }, + { + kind: 'code', + language: 'bash', + code: 'ollama pull llama3.2:1b', + }, + { + kind: 'table', + rows: [['Name', 'Value'], ['Host', '127.0.0.1']], + }, + ], + })); + + const step = createStep({ + title: 'Old title', + descriptionHtml: '

Old text

', + textBlocks: [{ id: 'tb1', order: 1, position: 'after-description', level: 'info', title: 'Old tip', descriptionHtml: '

Old body

' }], + codeBlocks: [{ id: 'cb1', order: 2, language: 'text', code: 'old' }], + tableBlocks: [{ id: 'tbl1', order: 3, rows: [['x']] }], + }); + + const updated = applyAiPatchToStep(step, patch, { target: 'all' }); + assert.equal(updated.title, 'Open settings'); + assert.equal(updated.descriptionHtml, '

Pick the AI tab.

'); + assert.equal(updated.textBlocks.length, 1); + assert.equal(updated.textBlocks[0].level, 'success'); + assert.equal(updated.textBlocks[0].descriptionHtml, '

Use the local Ollama model.

'); + assert.equal(updated.codeBlocks[0].code, 'ollama pull llama3.2:1b'); + assert.deepEqual(updated.tableBlocks[0].rows, [['Name', 'Value'], ['Host', '127.0.0.1']]); +}); + +test('ollama connection test reports installed models', async (t) => { + const root = makeTmpDir('text-intel-ai'); + t.after(() => rmrf(root)); + const service = new TextIntelService({ + store: { settingsDir: root }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: async () => ({ + ok: true, + json: async () => ({ + models: [ + { name: 'llama3.2:1b' }, + { name: 'qwen3:0.6b' }, + ], + }), + }), + }); + + const result = await service.testAiConnection(); + assert.equal(result.ok, true); + assert.equal(result.installed, true); + assert.equal(result.model, 'llama3.2:1b'); +}); + +test('invalid ollama output fails safely without saving the step', async (t) => { + const root = makeTmpDir('text-intel-ai-invalid'); + t.after(() => rmrf(root)); + let saveCalls = 0; + const step = createStep({ + title: 'Old title', + descriptionHtml: '

Old text

', + }); + const service = new TextIntelService({ + store: { + settingsDir: root, + getGuide: () => ({ guideId: 'g1', title: 'Guide', descriptionHtml: '', stepsOrder: ['s1'] }), + getStep: () => step, + stepImagePath: () => null, + saveStep: () => { + saveCalls += 1; + throw new Error('save should not be called'); + }, + }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: async () => ({ + ok: true, + json: async () => ({ + message: { + content: 'not json at all', + }, + }), + }), + }); + + const result = await service.generateStepPatch({ + guideId: 'g1', + stepId: 's1', + target: 'all', + }); + + assert.equal(result.ok, false); + assert.match(result.reason, /JSON/i); + assert.equal(saveCalls, 0); + assert.equal(step.title, 'Old title'); +}); From bd10f7d1471038b0869471c36fbb31f6882be04e Mon Sep 17 00:00:00 2001 From: Twest2 Date: Tue, 23 Jun 2026 18:02:24 -0500 Subject: [PATCH 02/19] lazy load ocr dependency --- app/text-intel.js | 17 +++++++++++++++-- tests/unit/text-intel.test.js | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/app/text-intel.js b/app/text-intel.js index 0d305a4..2e1e08d 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -4,7 +4,6 @@ const fs = require('node:fs'); const path = require('node:path'); const { execFileSync } = require('node:child_process'); -const { createWorker } = require('tesseract.js'); const { buildCaptureTitle, normalizeOllamaHost, @@ -33,6 +32,16 @@ function clamp(v, min, max) { return Math.min(max, Math.max(min, v)); } +let createWorkerImpl = null; +function loadCreateWorker() { + if (createWorkerImpl) return createWorkerImpl; + // OCR is optional at startup; lazy-load it so the app can still boot when + // the dependency has not been installed yet. + // eslint-disable-next-line global-require + ({ createWorker: createWorkerImpl } = require('tesseract.js')); + return createWorkerImpl; +} + class TextIntelService { constructor({ store, @@ -81,8 +90,9 @@ class TextIntelService { async getWorker() { if (this.workerPromise) return this.workerPromise; this.workerPromise = (async () => { + const workerFactory = loadCreateWorker(); const langPath = this.ensureLangData(); - const worker = await createWorker('eng', 1, { + const worker = await workerFactory('eng', 1, { langPath, }); await worker.setParameters({ @@ -91,6 +101,9 @@ class TextIntelService { this.worker = worker; return worker; })(); + this.workerPromise.catch(() => { + this.workerPromise = null; + }); return this.workerPromise; } diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 861e322..63f254e 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -83,6 +83,29 @@ test('ocr crop rectangles clamp to the image bounds', (t) => { assert.deepEqual(bottomRight, { x: 580, y: 280, width: 420, height: 220 }); }); +test('ocr failures fall back to empty text instead of crashing', async (t) => { + const root = makeTmpDir('text-intel-ocr-fallback'); + t.after(() => rmrf(root)); + const service = new TextIntelService({ + store: { settingsDir: root }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: global.fetch, + }); + service.getWorker = async () => { + throw new Error('tesseract missing'); + }; + + const result = await service.ocrAroundClick({ + image: {}, + size: { width: 100, height: 100 }, + display: { bounds: { x: 0, y: 0, width: 100, height: 100 } }, + }, { x: 50, y: 50 }); + + assert.deepEqual(result, { text: '', confidence: null }); +}); + test('ai response normalization and application keeps fields structured', () => { const patch = normalizeAiPatch(JSON.stringify({ title: 'Open settings', From 0bd326d8b2482a2da54f78cc880f29bf4b9fa4fe Mon Sep 17 00:00:00 2001 From: Twest2 Date: Tue, 23 Jun 2026 18:10:35 -0500 Subject: [PATCH 03/19] harden Electron startup repair --- scripts/electron-launcher.js | 171 ++++++++++++++++++++------- tests/unit/electron-launcher.test.js | 40 +++++++ 2 files changed, 171 insertions(+), 40 deletions(-) diff --git a/scripts/electron-launcher.js b/scripts/electron-launcher.js index 6164685..a23a993 100644 --- a/scripts/electron-launcher.js +++ b/scripts/electron-launcher.js @@ -10,6 +10,11 @@ const ELECTRON_SKIP_ENV_KEYS = [ 'NPM_CONFIG_ELECTRON_SKIP_BINARY_DOWNLOAD', ]; +const NPM_IGNORE_SCRIPTS_ENV_KEYS = [ + 'npm_config_ignore_scripts', + 'NPM_CONFIG_IGNORE_SCRIPTS', +]; + function resolveElectronPackageRoot() { try { return path.dirname(require.resolve('electron/package.json')); @@ -48,6 +53,9 @@ function sanitizeElectronEnv(baseEnv = process.env) { for (const key of ELECTRON_SKIP_ENV_KEYS) { delete env[key]; } + for (const key of NPM_IGNORE_SCRIPTS_ENV_KEYS) { + delete env[key]; + } return env; } @@ -67,8 +75,10 @@ function electronBinaryCandidates({ packageRoot, distDir, platform }) { return candidatePaths; } -function runNpmRebuild({ +function runNpmCommand({ packageRoot, + npmArgs, + errorLabel, npmExecPath = process.env.npm_execpath || null, npmNodeExecPath = process.env.npm_node_execpath || process.execPath, }) { @@ -76,31 +86,63 @@ function runNpmRebuild({ return false; } - const result = spawnSync( - npmNodeExecPath, - [npmExecPath, 'rebuild', 'electron', '--force', '--foreground-scripts'], - { - cwd: packageRoot, - env: sanitizeElectronEnv(), - stdio: 'inherit', - } - ); + const result = spawnSync(npmNodeExecPath, [npmExecPath, ...npmArgs], { + cwd: packageRoot, + env: sanitizeElectronEnv(), + stdio: 'inherit', + }); if (result.error) { throw result.error; } if (result.signal) { - throw new Error(`Electron repair was interrupted by ${result.signal}`); + throw new Error(`${errorLabel} was interrupted by ${result.signal}`); } if (result.status !== 0) { - throw new Error(`Electron rebuild failed with exit code ${result.status ?? 1}`); + throw new Error(`${errorLabel} failed with exit code ${result.status ?? 1}`); } return true; } +function runNpmRebuild({ + packageRoot, + npmExecPath = process.env.npm_execpath || null, + npmNodeExecPath = process.env.npm_node_execpath || process.execPath, +}) { + return runNpmCommand({ + packageRoot, + npmArgs: ['rebuild', 'electron', '--force', '--foreground-scripts'], + errorLabel: 'Electron rebuild', + npmExecPath, + npmNodeExecPath, + }); +} + +function runNpmInstall({ + packageRoot, + npmExecPath = process.env.npm_execpath || null, + npmNodeExecPath = process.env.npm_node_execpath || process.execPath, +}) { + return runNpmCommand({ + packageRoot, + npmArgs: [ + 'install', + '--include=dev', + '--ignore-scripts=false', + '--foreground-scripts', + '--no-audit', + '--no-fund', + '--package-lock=false', + ], + errorLabel: 'Electron dependency install', + npmExecPath, + npmNodeExecPath, + }); +} + function repairElectronInstall({ packageRoot, }) { @@ -153,45 +195,93 @@ function buildMissingElectronError({ packageRoot, distDir, candidatePaths }) { function resolveElectronBinary({ packageRoot = resolveElectronPackageRoot(), + projectRoot = process.cwd(), platform = process.platform, overrideDistPath = process.env.ELECTRON_OVERRIDE_DIST_PATH || null, } = {}) { - if (!packageRoot && !overrideDistPath) { + const repairErrors = []; + + function resolveCurrentPackageRoot() { + if (packageRoot) return packageRoot; + const conventionalRoot = path.join(projectRoot, 'node_modules', 'electron'); + if (fs.existsSync(path.join(conventionalRoot, 'package.json'))) { + packageRoot = conventionalRoot; + return packageRoot; + } + packageRoot = resolveElectronPackageRoot(); + return packageRoot; + } + + function tryRepair(label, repairFn) { + try { + if (!repairFn()) { + return null; + } + } catch (error) { + repairErrors.push(`${label}: ${error && error.message ? error.message : String(error)}`); + return null; + } + + const currentPackageRoot = resolveCurrentPackageRoot(); + if (!currentPackageRoot && !overrideDistPath) { + return null; + } + + const distDir = overrideDistPath || path.join(currentPackageRoot, 'dist'); + return electronBinaryCandidates({ packageRoot: currentPackageRoot, distDir, platform }).find((candidate) => + fs.existsSync(candidate) + ); + } + + let currentPackageRoot = resolveCurrentPackageRoot(); + if (!currentPackageRoot && !overrideDistPath) { + const installed = tryRepair('Electron dependency install', () => + runNpmInstall({ packageRoot: projectRoot }) + ); + if (installed) { + return installed; + } + + currentPackageRoot = resolveCurrentPackageRoot(); + } + + if (!currentPackageRoot && !overrideDistPath) { throw new Error( 'Electron could not be started because node_modules/electron is not installed.\n\n' + 'Run `npm install` from the repo root, then try `npm start` again.' ); } - const distDir = overrideDistPath || path.join(packageRoot, 'dist'); - const candidatePaths = electronBinaryCandidates({ packageRoot, distDir, platform }); - - const resolved = candidatePaths.find((candidate) => fs.existsSync(candidate)); - if (!resolved) { - if (packageRoot) { - if (runNpmRebuild({ packageRoot })) { - const rebuilt = electronBinaryCandidates({ packageRoot, distDir, platform }).find((candidate) => - fs.existsSync(candidate) - ); - if (rebuilt) { - return rebuilt; - } - } - - if (repairElectronInstall({ packageRoot })) { - const repaired = electronBinaryCandidates({ packageRoot, distDir, platform }).find((candidate) => - fs.existsSync(candidate) - ); - if (repaired) { - return repaired; - } - } - } - - throw new Error(buildMissingElectronError({ packageRoot, distDir, candidatePaths })); + const distDir = overrideDistPath || path.join(currentPackageRoot, 'dist'); + let candidatePaths = electronBinaryCandidates({ packageRoot: currentPackageRoot, distDir, platform }); + let resolved = candidatePaths.find((candidate) => fs.existsSync(candidate)); + if (resolved) { + return resolved; } - return resolved; + const repairAttempts = [ + ['Electron rebuild', () => runNpmRebuild({ packageRoot: currentPackageRoot })], + ['Electron install repair', () => repairElectronInstall({ packageRoot: currentPackageRoot })], + ['Electron dependency install', () => runNpmInstall({ packageRoot: projectRoot })], + ]; + + for (const [label, repairFn] of repairAttempts) { + const repaired = tryRepair(label, repairFn); + if (repaired) { + return repaired; + } + } + + throw new Error( + buildMissingElectronError({ + packageRoot: currentPackageRoot, + distDir, + candidatePaths, + }) + + (repairErrors.length + ? `\n\nAutomatic repair attempts failed:\n${repairErrors.map((error) => ` - ${error}`).join('\n')}` + : '') + ); } module.exports = { @@ -200,6 +290,7 @@ module.exports = { readElectronPathHint, repairElectronInstall, runNpmRebuild, + runNpmInstall, sanitizeElectronEnv, resolveElectronBinary, resolveElectronPackageRoot, diff --git a/tests/unit/electron-launcher.test.js b/tests/unit/electron-launcher.test.js index 42ff685..3cefaa1 100644 --- a/tests/unit/electron-launcher.test.js +++ b/tests/unit/electron-launcher.test.js @@ -112,6 +112,46 @@ test('rebuilds Electron through npm when the binary is missing', (t) => { ); }); +test('falls back to npm install when rebuild does not repair the runtime', (t) => { + const root = makeTmpDir('electron-install-fallback'); + t.after(() => rmrf(root)); + + fs.mkdirSync(path.join(root, 'dist'), { recursive: true }); + const fakeNpmCli = path.join(root, 'fake-npm-cli.js'); + fs.writeFileSync( + fakeNpmCli, + [ + "const fs = require('node:fs');", + "const path = require('node:path');", + "const command = process.argv[2];", + "if (command === 'rebuild') process.exit(1);", + "if (command === 'install') {", + " fs.mkdirSync(path.join(__dirname, 'dist'), { recursive: true });", + " fs.writeFileSync(path.join(__dirname, 'dist', 'electron.exe'), 'binary');", + " fs.writeFileSync(path.join(__dirname, 'path.txt'), 'electron.exe');", + " process.exit(0);", + "}", + "process.exit(1);", + ].join('\n') + ); + + const originalNpmExecPath = process.env.npm_execpath; + const originalNpmNodeExecPath = process.env.npm_node_execpath; + process.env.npm_execpath = fakeNpmCli; + process.env.npm_node_execpath = process.execPath; + t.after(() => { + if (originalNpmExecPath === undefined) delete process.env.npm_execpath; + else process.env.npm_execpath = originalNpmExecPath; + if (originalNpmNodeExecPath === undefined) delete process.env.npm_node_execpath; + else process.env.npm_node_execpath = originalNpmNodeExecPath; + }); + + assert.equal( + resolveElectronBinary({ packageRoot: root, projectRoot: root, platform: 'win32' }), + path.join(root, 'dist', 'electron.exe') + ); +}); + test('reports a helpful error when the runtime is missing', (t) => { const root = makeTmpDir('electron-missing'); t.after(() => rmrf(root)); From 1a1dbb846b3f4403d7818459979e7f4c3f07c6e9 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Tue, 23 Jun 2026 18:14:12 -0500 Subject: [PATCH 04/19] fix settings AI test connection wiring --- app/renderer/app.js | 1 + app/renderer/dialogs.js | 1 + app/renderer/editor.js | 1 + 3 files changed, 3 insertions(+) diff --git a/app/renderer/app.js b/app/renderer/app.js index 8baae92..619d041 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -861,6 +861,7 @@ class StepForgeApp { const settings = await api.settings.all(); const placeholders = await api.settings.globalPlaceholders(); await dialogs.showSettingsDialog({ + api, settings, placeholders, onSave: async (next) => { diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index 8ce8a84..16cf2c9 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -285,6 +285,7 @@ function showQuickActions({ query = '', commands = [], searchFn, onOpenItem, onC } function showSettingsDialog({ + api, settings, placeholders = {}, onSave, diff --git a/app/renderer/editor.js b/app/renderer/editor.js index dd51028..4825afd 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -1755,6 +1755,7 @@ class GuideEditor { const settings = await api.settings.all(); const placeholders = await api.settings.globalPlaceholders(); await dialogs.showSettingsDialog({ + api, settings, placeholders, onSave: async (next) => { From 375c14b0289adbd361759dacc6a0e2545ef6ba08 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 08:55:19 -0500 Subject: [PATCH 05/19] prefer OCR for capture titles --- app/text-intel.js | 6 + core/text-intel.js | 200 ++++++++++++++++++++++++++++------ tests/unit/text-intel.test.js | 42 ++++++- 3 files changed, 211 insertions(+), 37 deletions(-) diff --git a/app/text-intel.js b/app/text-intel.js index 2e1e08d..4fd6d74 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -404,9 +404,15 @@ public static class Win32 { this.collectForegroundWindowContext(), this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint), ]); + const titleCandidate = buildCaptureTitle({ + mode: step.kind === 'image' ? 'fullscreen' : 'window', + metadata, + ocrText: ocr.text, + }); captureContext = { ...metadata, ocrText: ocr.text, + titleCandidate, mode: step.kind === 'image' ? 'fullscreen' : 'content', }; } diff --git a/core/text-intel.js b/core/text-intel.js index 78c7e6a..c539815 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -44,6 +44,41 @@ const GENERIC_OCR_PHRASES = new Set([ 'type', ]); +const BROWSER_NAME_PHRASES = new Set([ + 'google chrome', + 'chrome', + 'chromium', + 'microsoft edge', + 'edge', + 'brave', + 'firefox', + 'safari', + 'opera', + 'vivaldi', +]); + +const ACTION_PREFIXES = [ + 'click', + 'select', + 'open', + 'choose', + 'enter', + 'type', + 'search', + 'switch to', + 'go to', + 'navigate to', + 'toggle', + 'turn on', + 'turn off', + 'enable', + 'disable', + 'pick', + 'focus', + 'launch', + 'activate', +]; + function normalizeWhitespace(text) { return String(text == null ? '' : text) .replace(/\s+/g, ' ') @@ -68,6 +103,55 @@ function displayText(text) { return clean.replace(/\s+/g, ' '); } +function sentenceCase(text) { + const clean = displayText(text); + if (!clean) return ''; + return clean.charAt(0).toUpperCase() + clean.slice(1); +} + +function isPathOrUrlLike(text) { + return /^(?:https?:\/\/|file:\/\/|about:blank|chrome:\/\/|edge:\/\/|moz-extension:\/\/|view-source:|localhost(?:[:/]|$)|www\.)/i.test(text) || + /[A-Za-z]:\\/.test(text) || + /\/(?:[^/\s]+\/){2,}/.test(text) || + /\\/.test(text); +} + +function isBrowserNoise(text) { + const clean = normalizeWhitespace(text).toLowerCase(); + if (!clean) return true; + if (BROWSER_NAME_PHRASES.has(clean)) return true; + if (isPathOrUrlLike(clean)) return true; + let foundBrowserName = false; + for (const name of BROWSER_NAME_PHRASES) { + if (clean.includes(name)) { + foundBrowserName = true; + break; + } + } + return foundBrowserName && /[\s|•·*]{2,}|[-–—]|\/|\\/.test(clean); +} + +function isUsefulTitleCandidate(text, { source = 'ocr' } = {}) { + const clean = displayText(text); + if (!clean) return false; + const lower = clean.toLowerCase(); + if (GENERIC_OCR_PHRASES.has(lower)) return false; + if (BROWSER_NAME_PHRASES.has(lower)) return false; + if (isPathOrUrlLike(clean)) return false; + if ((source === 'window' || source === 'app') && isBrowserNoise(clean)) return false; + if (/^[\p{P}\p{S}0-9]+$/u.test(clean)) return false; + return true; +} + +function splitTitleFragments(text) { + const clean = normalizeWhitespace(text); + if (!clean) return []; + return clean + .split(/\s*(?:\*\*+|[|•·]+|::|\/+|\\+|\s[-–—]\s|\s{2,})\s*/g) + .map((part) => displayText(part)) + .filter(Boolean); +} + function candidateWords(text) { const clean = normalizeWhitespace(text); if (!clean) return []; @@ -80,13 +164,15 @@ function scoreCandidate(text, { source = 'ocr' } = {}) { const words = candidateWords(clean); if (!words.length) return -Infinity; let score = 0; - score += source === 'element' ? 110 : source === 'window' ? 95 : source === 'app' ? 80 : 100; - score += Math.min(words.length, 6) * 12; - score -= Math.max(0, words.length - 6) * 10; - score -= Math.max(0, clean.length - 56) * 0.6; + score += source === 'ocr' ? 140 : source === 'element' ? 95 : source === 'window' ? 35 : source === 'app' ? 25 : 90; + score += Math.min(words.length, 5) * 10; + score -= Math.max(0, words.length - 5) * 11; + score -= Math.max(0, clean.length - 42) * 0.8; if (GENERIC_OCR_PHRASES.has(clean.toLowerCase())) score -= 50; + if (BROWSER_NAME_PHRASES.has(clean.toLowerCase())) score -= 80; + if (isBrowserNoise(clean)) score -= 60; if (clean.length <= 24) score += 10; - if (/^(click|open|enter|type|choose|select|save|cancel|confirm)\b/i.test(clean)) score += 8; + if (/^(click|select|open|choose|enter|type|search|switch to|go to|navigate to|toggle|turn on|turn off|enable|disable|pick|focus|launch|activate)\b/i.test(clean)) score += 12; if (/^[\p{P}\p{S}0-9]+$/u.test(clean)) score -= 100; return score; } @@ -96,9 +182,8 @@ function pickBestOcrPhrase(ocrText) { if (!text) return ''; const parts = text .split(/\n+/) - .map((line) => displayText(line)) - .filter(Boolean) - .filter((line) => !GENERIC_OCR_PHRASES.has(line.toLowerCase())); + .flatMap((line) => splitTitleFragments(line)) + .filter((line) => isUsefulTitleCandidate(line, { source: 'ocr' })); if (!parts.length) return ''; let best = ''; let bestScore = -Infinity; @@ -117,39 +202,81 @@ function isShortUiLabel(text) { return words.length > 0 && words.length <= 2 && text.length <= 24; } -function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) { - const candidates = []; - if (metadata.elementLabel) candidates.push({ text: metadata.elementLabel, source: 'element' }); - if (metadata.windowTitle) candidates.push({ text: metadata.windowTitle, source: 'window' }); - if (metadata.appName && metadata.appName !== metadata.windowTitle) { - candidates.push({ text: metadata.appName, source: 'app' }); - } - const ocrPhrase = pickBestOcrPhrase(ocrText); - if (ocrPhrase) candidates.push({ text: ocrPhrase, source: 'ocr' }); +function isDirectiveTitle(text) { + const clean = displayText(text); + if (!clean) return false; + const lower = clean.toLowerCase(); + return ACTION_PREFIXES.some((prefix) => lower.startsWith(prefix)); +} - let winner = null; - let winnerScore = -Infinity; - for (const candidate of candidates) { - const score = scoreCandidate(candidate.text, { source: candidate.source }); - if (score > winnerScore) { - winner = candidate; - winnerScore = score; +function verbForElementRole(role) { + const clean = normalizeWhitespace(role).toLowerCase(); + if (!clean) return null; + if (/(tab|menu item|menuitem|option|list item|tree item|radio button|dropdown list|combo box option)/.test(clean)) { + return 'Select'; + } + if (/(button|check box|checkbox|toggle button|switch|link|item|command)/.test(clean)) { + return 'Click'; + } + if (/(text field|edit|search box|combo box|textbox|text box|input|field)/.test(clean)) { + return 'Click'; + } + return null; +} + +function formatCaptureTitle(text, { source = 'ocr', metadata = {} } = {}) { + const clean = displayText(text); + if (!clean) return ''; + + if (isDirectiveTitle(clean)) { + return sentenceCase(clean); + } + + const roleVerb = (source === 'ocr' || source === 'element') ? verbForElementRole(metadata.elementRole) : null; + if (roleVerb) { + return `${roleVerb} ${sentenceCase(clean)}`; + } + + if (source === 'window' || source === 'app') { + return `Open ${sentenceCase(clean)}`; + } + + if (source === 'ocr' || source === 'element') { + return isShortUiLabel(clean) ? `Click ${sentenceCase(clean)}` : sentenceCase(clean); + } + + return sentenceCase(clean); +} + +function pickBestTitleFragment(text, { source = 'window', metadata = {} } = {}) { + const fragments = splitTitleFragments(text).filter((line) => isUsefulTitleCandidate(line, { source })); + if (!fragments.length) return ''; + let best = ''; + let bestScore = -Infinity; + for (const part of fragments) { + const score = scoreCandidate(part, { source }); + if (score > bestScore) { + best = part; + bestScore = score; } } + return best ? formatCaptureTitle(best, { source, metadata }) : ''; +} - const cleanedWinner = winner ? displayText(winner.text) : ''; - if (!cleanedWinner) return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; +function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) { + const ocrPhrase = pickBestOcrPhrase(ocrText); + if (ocrPhrase) return formatCaptureTitle(ocrPhrase, { source: 'ocr', metadata }); - if (winner.source === 'window' || winner.source === 'app') { - return `Open ${cleanedWinner}`; - } - if (winner.source === 'element') { - return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; - } - if (winner.source === 'ocr') { - return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; - } - return cleanedWinner; + const elementPhrase = pickBestTitleFragment(metadata.elementLabel, { source: 'element', metadata }); + if (elementPhrase) return elementPhrase; + + const windowPhrase = pickBestTitleFragment(metadata.windowTitle, { source: 'window', metadata }); + if (windowPhrase) return windowPhrase; + + const appPhrase = pickBestTitleFragment(metadata.appName, { source: 'app', metadata }); + if (appPhrase && appPhrase !== windowPhrase) return appPhrase; + + return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; } function plainTextToHtml(text) { @@ -326,6 +453,7 @@ function buildAiPrompt({ `Active window title: ${captureContext.windowTitle || '(unknown)'}`, `Active app: ${captureContext.appName || '(unknown)'}`, `OCR text near click: ${captureContext.ocrText || '(none)'}`, + `Deterministic title candidate: ${captureContext.titleCandidate || '(none)'}`, `Capture mode: ${captureContext.mode || '(unknown)'}`, ].join('\n') : 'Capture context: (not provided)', '', diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 63f254e..5e75262 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -7,6 +7,7 @@ const { makeTmpDir, rmrf } = require('./helpers'); const { createStep } = require('../../core/schema'); const { buildCaptureTitle, + buildAiPrompt, normalizeAiPatch, applyAiPatchToStep, } = require('../../core/text-intel'); @@ -36,7 +37,7 @@ test('capture titles prefer semantic metadata before OCR fallback', () => { metadata: { windowTitle: 'Reset user password in admin portal' }, ocrText: 'Save', }); - assert.equal(title, 'Open Reset user password in admin portal'); + assert.equal(title, 'Click Save'); }); test('capture titles prefer element metadata before window chrome and OCR', () => { @@ -51,6 +52,31 @@ test('capture titles prefer element metadata before window chrome and OCR', () = assert.equal(title, 'Open advanced settings'); }); +test('capture titles ignore browser chrome noise in favor of OCR', () => { + const title = buildCaptureTitle({ + mode: 'window', + metadata: { + windowTitle: 'Google Chrome ** PR reviews ** /chrome/tyler/autodoc', + appName: 'Google Chrome', + }, + ocrText: 'New tab', + }); + assert.equal(title, 'Click New tab'); +}); + +test('tab-like roles use select when OCR identifies a tab label', () => { + const title = buildCaptureTitle({ + mode: 'window', + metadata: { + elementLabel: 'New tab', + elementRole: 'tab item', + windowTitle: 'Google Chrome - PR reviews', + }, + ocrText: 'New tab', + }); + assert.equal(title, 'Select New tab'); +}); + test('capture titles fall back to OCR when metadata is absent', () => { const title = buildCaptureTitle({ mode: 'window', @@ -60,6 +86,20 @@ test('capture titles fall back to OCR when metadata is absent', () => { assert.equal(title, 'Click Save changes'); }); +test('ai prompts include the deterministic OCR-backed title candidate', () => { + const { prompt } = buildAiPrompt({ + captureContext: { + windowTitle: 'Google Chrome ** PR reviews ** /chrome/tyler/autodoc', + appName: 'Google Chrome', + ocrText: 'New tab', + titleCandidate: 'Click New tab', + mode: 'content', + }, + }); + + assert.match(prompt, /Deterministic title candidate: Click New tab/); +}); + test('ocr crop rectangles clamp to the image bounds', (t) => { const root = makeTmpDir('text-intel-crop'); t.after(() => rmrf(root)); From d2446265836fdd039020e20023de5501abaeb5a7 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 09:17:31 -0500 Subject: [PATCH 06/19] ai smart auto-documentation and text rewrite - Store captureMetadata (OCR text, window/app/element info) with each step at capture time so AI always has the original rich context. - Add buildCaptureContext() to TextIntelService; capture.js uses it instead of buildCaptureTitle() so both title and metadata come from one pass. - generateStepPatch() prefers stored captureMetadata over re-running OCR, giving the AI the best possible context when the user clicks an AI button later. - Add autoDoc setting: when enabled every capture (shoot, region, and session hotkey/click) is automatically documented by AI. Manual captures await AI before returning; session captures fire-and-forget and push a step:updated event so the renderer reloads seamlessly. - Add ai:rewriteText IPC and rewriteText() method for plain-text polishing via a separate callOllamaText() that skips JSON mode. - Add "AI Rewrite" section in the editor right panel: textarea + AI button that rewrites whatever the user types in place. - Improve buildAiPrompt() rules: action-focused title instructions, explicit anti-junk rules (no "Capture the screen / OCR" blocks), and a context-quality gate that suppresses blocks when context is thin. - Add autoDoc checkbox to AI settings dialog. - Renderer handles step:updated to reload the selected step after background auto-doc finishes. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 14 ++--- app/main.js | 52 ++++++++++++++++++- app/preload.js | 2 + app/renderer/app.js | 12 +++++ app/renderer/dialogs.js | 7 ++- app/renderer/editor.js | 48 +++++++++++++++++ app/text-intel.js | 98 ++++++++++++++++++++++++++++++++--- core/schema.js | 3 ++ core/text-intel.js | 49 ++++++++++++------ tests/unit/text-intel.test.js | 2 +- 10 files changed, 255 insertions(+), 32 deletions(-) diff --git a/app/capture.js b/app/capture.js index aefbe36..db918a8 100644 --- a/app/capture.js +++ b/app/capture.js @@ -1316,15 +1316,15 @@ public static class SFMouseHook { }; } - async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) { + async buildStepMeta(mode, frame, clickPos = null, clickMeta = null) { try { - if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') { - return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta }); + if (this.textIntel && typeof this.textIntel.buildCaptureContext === 'function') { + return await this.textIntel.buildCaptureContext({ mode, frame, clickPos, clickMeta }); } } catch { - // fall back to the local semantic title below + // fall back } - return this.autoTitle(mode); + return { title: this.autoTitle(mode), captureMetadata: null }; } async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) { @@ -1350,8 +1350,10 @@ public static class SFMouseHook { } } + const { title, captureMetadata } = await this.buildStepMeta(mode, frame, clickPos, clickMeta); const step = this.store.addStep(guideId, { - title: await this.buildStepTitle(mode, frame, clickPos, clickMeta), + title, + captureMetadata, annotations, focusedView: { enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')), diff --git a/app/main.js b/app/main.js index 6a87840..2e52984 100644 --- a/app/main.js +++ b/app/main.js @@ -521,18 +521,49 @@ function setupIpc() { if (result.ok) reindex(guideId); return result; }); + h('ai:rewriteText', async ({ text, guideTitle = '', stepTitle = '' } = {}) => { + return textIntel.rewriteText({ text, guideTitle, stepTitle }); + }); h('placeholders:globals:get', () => settings.getGlobalPlaceholders()); h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values)); // capture h('capture:shoot', async ({ guideId, mode, delayMs }) => { const result = await capture.shoot({ guideId, mode, delayMs }); - if (result.ok) reindex(guideId); + if (result.ok) { + reindex(guideId); + const aiConf = settings.get('ai') || {}; + if (aiConf.enabled && aiConf.autoDoc && result.step) { + const aiResult = await textIntel.generateStepPatch({ + guideId, + stepId: result.step.stepId, + target: 'all', + }).catch(() => null); + if (aiResult?.ok) { + reindex(guideId); + result.step = aiResult.step; + } + } + } return result; }); h('capture:region', async ({ guideId }) => { const result = await capture.regionCapture(guideId); - if (result.ok) reindex(guideId); + if (result.ok) { + reindex(guideId); + const aiConf = settings.get('ai') || {}; + if (aiConf.enabled && aiConf.autoDoc && result.step) { + const aiResult = await textIntel.generateStepPatch({ + guideId, + stepId: result.step.stepId, + target: 'all', + }).catch(() => null); + if (aiResult?.ok) { + reindex(guideId); + result.step = aiResult.step; + } + } + } return result; }); let capturePowerBlocker = -1; @@ -777,6 +808,23 @@ if (!gotLock) { try { keepProcessesResponsive(app.getAppMetrics().map((m) => m.pid)); } catch { /* best effort */ } } } + // Auto-document session captures in the background when autoDoc is enabled. + // Single-shot captures (capture:shoot) are handled synchronously in the IPC handler. + if (channel === 'capture:added' && payload?.step && payload?.guideId) { + const aiConf = settings.get('ai') || {}; + if (aiConf.enabled && aiConf.autoDoc) { + textIntel.generateStepPatch({ + guideId: payload.guideId, + stepId: payload.step.stepId, + target: 'all', + }).then((result) => { + if (result.ok) { + reindex(payload.guideId); + sendToRenderer('step:updated', { guideId: payload.guideId, step: result.step }); + } + }).catch(() => {}); + } + } }; capture = new CaptureService({ diff --git a/app/preload.js b/app/preload.js index 71f60b8..2f48a7e 100644 --- a/app/preload.js +++ b/app/preload.js @@ -55,6 +55,7 @@ const api = { ai: { test: invoke('ai:test'), fillStep: invoke('ai:fillStep'), + rewriteText: invoke('ai:rewriteText'), }, capture: { shoot: invoke('capture:shoot'), @@ -63,6 +64,7 @@ const api = { state: invoke('capture:state'), onAdded: (fn) => ipcRenderer.on('capture:added', (e, payload) => fn(payload)), onState: (fn) => ipcRenderer.on('capture:state', (e, payload) => fn(payload)), + onStepUpdated: (fn) => ipcRenderer.on('step:updated', (e, payload) => fn(payload)), }, archive: { export: invoke('archive:export'), diff --git a/app/renderer/app.js b/app/renderer/app.js index 619d041..bf46b59 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -80,6 +80,18 @@ class StepForgeApp { api.capture.onAdded((payload) => this.onCaptureAdded(payload)); api.capture.onState((payload) => this.updateCaptureState(payload)); + api.capture.onStepUpdated((payload) => this.onStepUpdated(payload)); + } + + async onStepUpdated(payload) { + if (!payload || !payload.guideId || !payload.step) return; + if (this.state.view === 'editor' && this.editor.guideId === payload.guideId) { + const currentStep = this.editor.currentStep; + if (currentStep && currentStep.stepId === payload.step.stepId) { + await this.editor.reload(payload.step.stepId); + toast('Documentation generated.'); + } + } } async onCaptureAdded(payload) { diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index 16cf2c9..4f42f06 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -315,6 +315,7 @@ function showSettingsDialog({ const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) }); const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 }); const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) }); + const aiAutoDoc = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.autoDoc) }); const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434'); const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b'); const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.'); @@ -408,7 +409,8 @@ function showSettingsDialog({ ), el('fieldset', {}, el('legend', {}, 'AI'), - labeledRow('Enable AI text filling', aiEnabled), + labeledRow('Enable AI', aiEnabled), + labeledRow('Auto-document captures', aiAutoDoc), labeledRow('Ollama host', ollamaHost), labeledRow('Ollama model', ollamaModel), el('div.row', { style: { justifyContent: 'space-between' } }, @@ -416,7 +418,7 @@ function showSettingsDialog({ testAiBtn, ), el('div.muted', {}, - 'AI generation is manual only. Captures stay deterministic until you click a generate button.', + 'When auto-document is on, each capture is automatically documented by AI. Turn it off to use AI manually only.', ), ), el('fieldset', {}, @@ -465,6 +467,7 @@ function showSettingsDialog({ ai: { ...settings.ai, enabled: aiEnabled.checked, + autoDoc: aiAutoDoc.checked, ollama: { ...(settings.ai?.ollama || {}), host: ollamaHost.value.trim(), diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 4825afd..6db5c4d 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -167,6 +167,7 @@ class GuideEditor { const buttons = [ this.dom?.titleAiBtn, this.dom?.descAiBtn, + this.dom?.rewriteAiBtn, ...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []), ].filter(Boolean); for (const button of buttons) { @@ -222,6 +223,36 @@ class GuideEditor { return this.runAiGeneration('all', { button }); } + async runTextRewrite(button = null) { + if (!this.isAiEnabled()) { + this.onToast('Enable AI in Settings first.', { error: true }); + return; + } + const text = this.dom?.rewriteInput?.value?.trim(); + if (!text) { + this.onToast('Type some text to rewrite first.', { error: true }); + return; + } + if (button) setButtonLoading(button, true, 'Rewriting…'); + try { + const result = await api.ai.rewriteText({ + text, + guideTitle: this.guide?.title || '', + stepTitle: this.currentStep?.title || '', + }); + if (!result || !result.ok) { + this.onToast(result?.reason || 'Rewrite failed.', { error: true }); + return; + } + if (this.dom?.rewriteInput) this.dom.rewriteInput.value = result.text; + this.onToast('Text rewritten.'); + } catch (err) { + this.onToast(err.message || 'Rewrite failed.', { error: true }); + } finally { + if (button) setButtonLoading(button, false); + } + } + async generateBlockWithAi(kind, block, button = null) { if (!block) return null; return this.runAiGeneration('block', { blockId: block.id, button }); @@ -411,6 +442,21 @@ class GuideEditor { this.dom.addTableBlockBtn = el('button', { type: 'button' }, '+ Table'), ), ), + el('section', {}, + el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } }, + el('h3', { style: { margin: 0 } }, 'AI Rewrite'), + this.dom.rewriteAiBtn = el('button.ai', { + type: 'button', + title: 'Rewrite the text below with AI', + dataset: { aiAction: 'rewrite', aiTitle: 'Rewrite with AI' }, + }, 'Rewrite'), + ), + this.dom.rewriteInput = el('textarea', { + rows: 3, + placeholder: 'Type something to improve…', + style: { width: '100%', resize: 'vertical' }, + }), + ), el('section', {}, el('h3', {}, 'Guide'), this.dom.guideSummary = el('div.muted', {}), @@ -583,6 +629,8 @@ class GuideEditor { }); this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn)); + this.dom.rewriteAiBtn.addEventListener('click', () => this.runTextRewrite(this.dom.rewriteAiBtn)); + this.dom.descEditor.addEventListener('paste', (e) => { // Keep pasted text simple; backend sanitization will handle the rest. e.preventDefault(); diff --git a/app/text-intel.js b/app/text-intel.js index 4fd6d74..4ef136f 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -276,15 +276,27 @@ public static class Win32 { } async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) { + const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta }); + return ctx.title; + } + + async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) { const [metadata, ocr] = await Promise.all([ this.collectForegroundWindowContext(clickMeta?.osPoint || null), this.ocrAroundClick(frame, clickPos), ]); - return buildCaptureTitle({ - mode, - metadata, - ocrText: ocr.text, - }); + const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text }); + return { + title, + captureMetadata: { + ocrText: ocr.text || '', + windowTitle: metadata.windowTitle || '', + appName: metadata.appName || '', + elementLabel: metadata.elementLabel || '', + elementRole: metadata.elementRole || '', + mode, + }, + }; } aiEnabled() { @@ -335,6 +347,28 @@ public static class Win32 { }; } + async callOllamaText({ host, model, prompt, systemPrompt }) { + const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); + const response = await this.fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + stream: false, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: prompt }, + ], + options: { temperature: 0.4 }, + }), + }); + if (!response.ok) throw new Error(`Ollama request failed (${response.status})`); + const payload = await response.json(); + const content = payload?.message?.content; + if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response'); + return content.trim(); + } + async callOllama({ host, model, prompt, systemPrompt }) { const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); const response = await this.fetch(url, { @@ -393,7 +427,23 @@ public static class Win32 { } let captureContext = null; - if (step.image) { + // Use stored capture metadata when available (best context, from capture time). + // Fall back to re-running OCR on the stored image only when metadata is absent. + if (step.captureMetadata) { + captureContext = { + ...step.captureMetadata, + titleCandidate: buildCaptureTitle({ + mode: step.captureMetadata.mode || 'fullscreen', + metadata: { + windowTitle: step.captureMetadata.windowTitle, + appName: step.captureMetadata.appName, + elementLabel: step.captureMetadata.elementLabel, + elementRole: step.captureMetadata.elementRole, + }, + ocrText: step.captureMetadata.ocrText, + }), + }; + } else if (step.image) { const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original'); if (imagePath && fs.existsSync(imagePath)) { const { nativeImage } = require('electron'); @@ -442,6 +492,42 @@ public static class Win32 { } } + async rewriteText({ text, guideTitle = '', stepTitle = '' }) { + try { + const config = this.aiConfig(); + if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' }; + if (!config.ollama.host || !config.ollama.model) { + return { ok: false, reason: 'Configure Ollama host and model in Settings.' }; + } + const trimmed = normalizeWhitespace(text); + if (!trimmed) return { ok: false, reason: 'No text to rewrite.' }; + + const contextHint = [ + guideTitle ? `Guide: ${guideTitle}` : '', + stepTitle ? `Step: ${stepTitle}` : '', + ].filter(Boolean).join('\n'); + + const prompt = [ + contextHint, + contextHint ? '' : null, + 'Rewrite the following text to sound professional and clear as step-by-step documentation.', + 'Keep it concise. Do not add extra information. Return only the rewritten text.', + '', + trimmed, + ].filter((l) => l !== null).join('\n'); + + const result = await this.callOllamaText({ + host: config.ollama.host, + model: config.ollama.model, + prompt, + systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.', + }); + return { ok: true, text: result }; + } catch (err) { + return { ok: false, reason: err?.message || 'Rewrite failed.' }; + } + } + clickPointFromStep(step, image = null) { const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h)); if (!marker) return null; diff --git a/core/schema.js b/core/schema.js index 88e2b51..7bbd122 100644 --- a/core/schema.js +++ b/core/schema.js @@ -86,6 +86,9 @@ function createStep(fields = {}) { codeBlocks: (fields.codeBlocks || []).map((cb) => normalizeCodeBlock(cb, takeOrder(cb))), tableBlocks: (fields.tableBlocks || []).map((tb) => normalizeTableBlock(tb, takeOrder(tb))), links: fields.links || [], // { id, label, targetStepId } + captureMetadata: (fields.captureMetadata && typeof fields.captureMetadata === 'object' && !Array.isArray(fields.captureMetadata)) + ? { ...fields.captureMetadata } + : null, }; } diff --git a/core/text-intel.js b/core/text-intel.js index c539815..823cacf 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -402,6 +402,14 @@ function summarizeGuideForAi(guide = {}) { ].join('\n'); } +function hasRichCaptureContext(captureContext) { + if (!captureContext) return false; + const ocr = normalizeWhitespace(captureContext.ocrText || ''); + const win = normalizeWhitespace(captureContext.windowTitle || ''); + const app = normalizeWhitespace(captureContext.appName || ''); + return ocr.length > 3 || (win.length > 2 && !isBrowserNoise(win)) || app.length > 1; +} + function buildAiPrompt({ target = 'all', guide = null, @@ -416,6 +424,8 @@ function buildAiPrompt({ all: 'rewrite the step title, description, and any useful blocks', }[target] || 'rewrite the step'; + const richContext = hasRichCaptureContext(captureContext); + const allowedBlockNote = [ 'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.', 'Use block.kind = "code" for code snippets.', @@ -423,8 +433,16 @@ function buildAiPrompt({ 'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].', ].join(' '); + const contextLines = captureContext ? [ + captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null, + captureContext.appName ? `App: ${captureContext.appName}` : null, + captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null, + captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null, + captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null, + ].filter(Boolean) : []; + const prompt = [ - 'You write concise, human-sounding step-by-step documentation for an offline desktop app.', + 'You write concise, action-focused step-by-step documentation for a desktop application guide.', 'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.', 'Schema:', '{', @@ -449,26 +467,27 @@ function buildAiPrompt({ '', step ? summarizeStepForAi(step) : 'Step: (not provided)', '', - captureContext ? [ - `Active window title: ${captureContext.windowTitle || '(unknown)'}`, - `Active app: ${captureContext.appName || '(unknown)'}`, - `OCR text near click: ${captureContext.ocrText || '(none)'}`, - `Deterministic title candidate: ${captureContext.titleCandidate || '(none)'}`, - `Capture mode: ${captureContext.mode || '(unknown)'}`, - ].join('\n') : 'Capture context: (not provided)', + contextLines.length + ? `Capture context:\n${contextLines.join('\n')}` + : 'Capture context: (not available)', '', - block ? `Target block:\n${JSON.stringify(block, null, 2)}` : 'Target block: (not provided)', + block ? `Target block:\n${JSON.stringify(block, null, 2)}` : null, '', 'Rules:', - '- Keep the wording natural and specific.', - '- Prefer short titles.', - '- Only return blocks that add value.', - '- Do not invent details that are not supported by the capture context.', + '- Write titles as short imperative actions: "Click Save", "Select New document", "Open Settings".', + '- If the suggested title is already a good imperative action, use it as-is or refine it slightly.', + '- Write descriptions in 1–2 sentences describing exactly what the user does in this step.', + '- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).', + richContext + ? '- You have rich context: use the OCR text, window title, and element info to write specific documentation.' + : '- Context is limited: write a minimal title. Leave description empty and return no blocks unless the step already has meaningful content.', + '- Do NOT generate blocks that describe the technical capture process or mention OCR.', + '- Do NOT invent details not supported by the capture context.', '- If the target is one block, only rewrite that block.', - ].join('\n'); + ].filter((l) => l !== null).join('\n'); return { - systemPrompt: 'You are a documentation assistant that only emits JSON.', + systemPrompt: 'You are a technical documentation writer. Emit only valid JSON matching the schema. Never add commentary or markdown.', prompt, }; } diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 5e75262..16bf024 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -97,7 +97,7 @@ test('ai prompts include the deterministic OCR-backed title candidate', () => { }, }); - assert.match(prompt, /Deterministic title candidate: Click New tab/); + assert.match(prompt, /Suggested title: Click New tab/); }); test('ocr crop rectangles clamp to the image bounds', (t) => { From 68ceee83ef004cef639c9c5aaa67d6066735796c Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 09:27:23 -0500 Subject: [PATCH 07/19] fix ai title generation and remove separate rewrite section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove the standalone "AI Rewrite" section from the editor panel. The existing title and description AI buttons already flush the step first, so they rewrite whatever the user has typed in those fields. - Never pass a generic fallback title ("Screen capture", "Window capture", "Region capture", "Capture") as the AI title candidate or as step content. It is now treated as "(not set — generate a specific action title)" so the AI always produces something real. - hasRichCaptureContext now counts any non-trivial app name or window title as sufficient context, instead of requiring non-browser noise. - Prompt rules updated: "NEVER output Screen/Window/Region capture", separate paths for improving a user draft vs generating from context, and clearer guidance when context is limited (use app/window name). - isPlaceholderTitle helper guards summarizeStepForAi so a default-titled step presents itself as empty to the AI. Co-Authored-By: Claude Sonnet 4.6 --- app/renderer/editor.js | 48 ---------------------------------------- app/text-intel.js | 29 ++++++++++++++---------- core/text-intel.js | 50 ++++++++++++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 72 deletions(-) diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 6db5c4d..4825afd 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -167,7 +167,6 @@ class GuideEditor { const buttons = [ this.dom?.titleAiBtn, this.dom?.descAiBtn, - this.dom?.rewriteAiBtn, ...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []), ].filter(Boolean); for (const button of buttons) { @@ -223,36 +222,6 @@ class GuideEditor { return this.runAiGeneration('all', { button }); } - async runTextRewrite(button = null) { - if (!this.isAiEnabled()) { - this.onToast('Enable AI in Settings first.', { error: true }); - return; - } - const text = this.dom?.rewriteInput?.value?.trim(); - if (!text) { - this.onToast('Type some text to rewrite first.', { error: true }); - return; - } - if (button) setButtonLoading(button, true, 'Rewriting…'); - try { - const result = await api.ai.rewriteText({ - text, - guideTitle: this.guide?.title || '', - stepTitle: this.currentStep?.title || '', - }); - if (!result || !result.ok) { - this.onToast(result?.reason || 'Rewrite failed.', { error: true }); - return; - } - if (this.dom?.rewriteInput) this.dom.rewriteInput.value = result.text; - this.onToast('Text rewritten.'); - } catch (err) { - this.onToast(err.message || 'Rewrite failed.', { error: true }); - } finally { - if (button) setButtonLoading(button, false); - } - } - async generateBlockWithAi(kind, block, button = null) { if (!block) return null; return this.runAiGeneration('block', { blockId: block.id, button }); @@ -442,21 +411,6 @@ class GuideEditor { this.dom.addTableBlockBtn = el('button', { type: 'button' }, '+ Table'), ), ), - el('section', {}, - el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } }, - el('h3', { style: { margin: 0 } }, 'AI Rewrite'), - this.dom.rewriteAiBtn = el('button.ai', { - type: 'button', - title: 'Rewrite the text below with AI', - dataset: { aiAction: 'rewrite', aiTitle: 'Rewrite with AI' }, - }, 'Rewrite'), - ), - this.dom.rewriteInput = el('textarea', { - rows: 3, - placeholder: 'Type something to improve…', - style: { width: '100%', resize: 'vertical' }, - }), - ), el('section', {}, el('h3', {}, 'Guide'), this.dom.guideSummary = el('div.muted', {}), @@ -629,8 +583,6 @@ class GuideEditor { }); this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn)); - this.dom.rewriteAiBtn.addEventListener('click', () => this.runTextRewrite(this.dom.rewriteAiBtn)); - this.dom.descEditor.addEventListener('paste', (e) => { // Keep pasted text simple; backend sanitization will handle the rest. e.preventDefault(); diff --git a/app/text-intel.js b/app/text-intel.js index 4ef136f..3206670 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -5,6 +5,7 @@ const path = require('node:path'); const { execFileSync } = require('node:child_process'); const { + DEFAULT_CAPTURE_TITLES, buildCaptureTitle, normalizeOllamaHost, normalizeAiPatch, @@ -14,6 +15,8 @@ const { normalizeWhitespace, } = require('../core/text-intel'); +const DEFAULT_TITLE_VALUES = new Set(Object.values(DEFAULT_CAPTURE_TITLES).concat(['Capture'])); + const OCR_CROP = { width: 420, height: 220, @@ -430,18 +433,20 @@ public static class Win32 { // Use stored capture metadata when available (best context, from capture time). // Fall back to re-running OCR on the stored image only when metadata is absent. if (step.captureMetadata) { + const rawCandidate = buildCaptureTitle({ + mode: step.captureMetadata.mode || 'fullscreen', + metadata: { + windowTitle: step.captureMetadata.windowTitle, + appName: step.captureMetadata.appName, + elementLabel: step.captureMetadata.elementLabel, + elementRole: step.captureMetadata.elementRole, + }, + ocrText: step.captureMetadata.ocrText, + }); captureContext = { ...step.captureMetadata, - titleCandidate: buildCaptureTitle({ - mode: step.captureMetadata.mode || 'fullscreen', - metadata: { - windowTitle: step.captureMetadata.windowTitle, - appName: step.captureMetadata.appName, - elementLabel: step.captureMetadata.elementLabel, - elementRole: step.captureMetadata.elementRole, - }, - ocrText: step.captureMetadata.ocrText, - }), + // Don't suggest a generic fallback title — leave it blank so AI generates from context. + titleCandidate: DEFAULT_TITLE_VALUES.has(rawCandidate) ? '' : rawCandidate, }; } else if (step.image) { const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original'); @@ -454,7 +459,7 @@ public static class Win32 { this.collectForegroundWindowContext(), this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint), ]); - const titleCandidate = buildCaptureTitle({ + const rawCandidate2 = buildCaptureTitle({ mode: step.kind === 'image' ? 'fullscreen' : 'window', metadata, ocrText: ocr.text, @@ -462,7 +467,7 @@ public static class Win32 { captureContext = { ...metadata, ocrText: ocr.text, - titleCandidate, + titleCandidate: DEFAULT_TITLE_VALUES.has(rawCandidate2) ? '' : rawCandidate2, mode: step.kind === 'image' ? 'fullscreen' : 'content', }; } diff --git a/core/text-intel.js b/core/text-intel.js index 823cacf..0f0f698 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -386,10 +386,22 @@ function summarizeBlocks(step = {}) { return parts.length ? parts.join('\n') : '(none)'; } +const DEFAULT_PLACEHOLDER_TITLES = new Set( + Object.values(DEFAULT_CAPTURE_TITLES).concat(['Capture', 'Untitled step']), +); + +function isPlaceholderTitle(title) { + return !title || DEFAULT_PLACEHOLDER_TITLES.has(title); +} + function summarizeStepForAi(step = {}) { + const titleLine = isPlaceholderTitle(step.title) + ? 'Step title: (not set — generate a specific action title from the capture context)' + : `Step title: ${step.title}`; + const descText = htmlToText(step.descriptionHtml || ''); return [ - `Step title: ${step.title || '(empty)'}`, - `Step description: ${htmlToText(step.descriptionHtml || '') || '(empty)'}`, + titleLine, + `Step description: ${descText || '(empty)'}`, `Step status: ${step.status || 'todo'}`, `Blocks:\n${summarizeBlocks(step)}`, ].join('\n'); @@ -407,7 +419,9 @@ function hasRichCaptureContext(captureContext) { const ocr = normalizeWhitespace(captureContext.ocrText || ''); const win = normalizeWhitespace(captureContext.windowTitle || ''); const app = normalizeWhitespace(captureContext.appName || ''); - return ocr.length > 3 || (win.length > 2 && !isBrowserNoise(win)) || app.length > 1; + const element = normalizeWhitespace(captureContext.elementLabel || ''); + // Any non-trivial context signal is enough — even just an app name. + return ocr.length > 3 || win.length > 2 || app.length > 1 || element.length > 1; } function buildAiPrompt({ @@ -417,11 +431,18 @@ function buildAiPrompt({ captureContext = null, block = null, } = {}) { + const hasDraftTitle = step && !isPlaceholderTitle(step.title); + const hasDraftDesc = step && Boolean(htmlToText(step.descriptionHtml || '')); + const targetText = { - title: 'rewrite only the step title', - description: 'rewrite only the step description', - block: 'rewrite only one block', - all: 'rewrite the step title, description, and any useful blocks', + title: hasDraftTitle + ? 'improve the user\'s draft step title — keep their intent, make it read like professional documentation' + : 'write a specific action title for this step using the capture context', + description: hasDraftDesc + ? 'improve the user\'s draft description — keep their intent, make it read like professional documentation' + : 'write a 1–2 sentence description of what the user does in this step, using the capture context', + block: 'rewrite only the target block', + all: 'write the step title, description, and any useful blocks from the capture context', }[target] || 'rewrite the step'; const richContext = hasRichCaptureContext(captureContext); @@ -474,13 +495,18 @@ function buildAiPrompt({ block ? `Target block:\n${JSON.stringify(block, null, 2)}` : null, '', 'Rules:', - '- Write titles as short imperative actions: "Click Save", "Select New document", "Open Settings".', - '- If the suggested title is already a good imperative action, use it as-is or refine it slightly.', - '- Write descriptions in 1–2 sentences describing exactly what the user does in this step.', + '- Titles must be short imperative actions: "Click Save", "Select New document", "Open Settings".', + '- NEVER output "Screen capture", "Window capture", "Region capture", or "Capture" as a title — always produce something specific.', + hasDraftTitle + ? '- The user has a draft title. Improve its wording without changing their intent.' + : '- No title yet. Use the capture context (OCR text, window, app) to write a specific action title.', + hasDraftDesc + ? '- The user has a draft description. Polish it to read like professional documentation.' + : '- No description yet. Write 1–2 sentences describing exactly what the user does.', '- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).', richContext - ? '- You have rich context: use the OCR text, window title, and element info to write specific documentation.' - : '- Context is limited: write a minimal title. Leave description empty and return no blocks unless the step already has meaningful content.', + ? '- Use the OCR text, window title, app name, and element info to make the documentation specific.' + : '- Context is limited. Use the app name or window title if available; generate a reasonable action title.', '- Do NOT generate blocks that describe the technical capture process or mention OCR.', '- Do NOT invent details not supported by the capture context.', '- If the target is one block, only rewrite that block.', From 8d645c0aca295dc7d4b34163744ae76f172e3bd3 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 09:44:36 -0500 Subject: [PATCH 08/19] fix smart title generation from browser window context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The title engine was falling back to "Screen capture" for all browser captures because it treated the entire browser window title as noise. Changes: - stripBrowserNameSuffix: removes "- Google Chrome" / "| Firefox" etc. from the end of window titles, leaving just the page title. "oracle - Google Search - Google Chrome" → "oracle - Google Search" "Oracle | Cloud Applications - Google Chrome" → page title only - extractSearchQuery: detects "[query] - Google Search" / Bing / etc. patterns after stripping the browser suffix, and formats the result as "Search for oracle". - buildCaptureTitle: uses stripped page title + search detection before falling through to the "Screen capture" fallback. - pickBestOcrPhrase: considers the full OCR line (≤80 chars) as a candidate with a +35 completeness bonus before splitting on | or ·. This preserves "Oracle | Cloud Applications and Cloud Platform" as a single phrase instead of breaking it into fragments. - candidateWords: filters out standalone punctuation tokens (|, ·, •) so they don't inflate word-count penalties for compound brand names. - verbForElementRole: hyperlinks and links now produce "Select" instead of "Click"; search box / search field produces "Search for". Five new unit tests cover: browser title stripping, search query extraction, full pipe-separated link text, link and search box verbs. Co-Authored-By: Claude Sonnet 4.6 --- core/text-intel.js | 92 ++++++++++++++++++++++++++++------- tests/unit/text-intel.test.js | 54 ++++++++++++++++++++ 2 files changed, 129 insertions(+), 17 deletions(-) diff --git a/core/text-intel.js b/core/text-intel.js index 0f0f698..c235c94 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -57,6 +57,19 @@ const BROWSER_NAME_PHRASES = new Set([ 'vivaldi', ]); +// Known search engine page title suffixes (what appears after the query in the window title). +const SEARCH_ENGINE_PAGE_NAMES = new Set([ + 'google search', + 'google', + 'bing', + 'duckduckgo', + 'yahoo search', + 'yahoo', + 'startpage', + 'ecosia', + 'brave search', +]); + const ACTION_PREFIXES = [ 'click', 'select', @@ -155,7 +168,32 @@ function splitTitleFragments(text) { function candidateWords(text) { const clean = normalizeWhitespace(text); if (!clean) return []; - return clean.split(/\s+/).filter(Boolean); + // Exclude standalone punctuation tokens (e.g. "|" in "Oracle | Cloud...") from word count. + return clean.split(/\s+/).filter((w) => /[a-zA-Z0-9]/.test(w)); +} + +// Remove trailing "- Google Chrome", "| Firefox", etc. from a browser window title, +// leaving just the page title portion: "oracle - Google Search - Google Chrome" → "oracle - Google Search". +function stripBrowserNameSuffix(text) { + let clean = normalizeWhitespace(text); + for (const name of BROWSER_NAME_PHRASES) { + const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + clean = clean.replace(new RegExp(`\\s*[-–—·|•]\\s*${escaped}\\s*$`, 'i'), '').trim(); + } + return clean; +} + +// Detect "[query] - Google Search" or "[query] - Bing" patterns in a (already-stripped) page title. +// Returns the query word(s) if found, otherwise ''. +function extractSearchQuery(pageTitle) { + const frags = splitTitleFragments(pageTitle); + if (frags.length < 2) return ''; + const last = frags[frags.length - 1].toLowerCase(); + if (SEARCH_ENGINE_PAGE_NAMES.has(last)) { + const query = frags[0]; + if (query && isUsefulTitleCandidate(query, { source: 'ocr' })) return query; + } + return ''; } function scoreCandidate(text, { source = 'ocr' } = {}) { @@ -180,18 +218,24 @@ function scoreCandidate(text, { source = 'ocr' } = {}) { function pickBestOcrPhrase(ocrText) { const text = normalizeWhitespace(ocrText); if (!text) return ''; - const parts = text - .split(/\n+/) - .flatMap((line) => splitTitleFragments(line)) - .filter((line) => isUsefulTitleCandidate(line, { source: 'ocr' })); - if (!parts.length) return ''; let best = ''; let bestScore = -Infinity; - for (const part of parts) { - const score = scoreCandidate(part, { source: 'ocr' }); - if (score > bestScore) { - best = part; - bestScore = score; + for (const rawLine of text.split(/\n+/)) { + const line = normalizeWhitespace(rawLine); + if (!line) continue; + // For short lines (link text, button labels) try the FULL line first before splitting. + // This preserves "Oracle | Cloud Applications and Cloud Platform" instead of splitting on |. + // Full-line bonus (+35) nudges it ahead of its own fragments. + const candidates = line.length <= 80 + ? [[line, 35], ...splitTitleFragments(line).map((f) => [f, 0])] + : splitTitleFragments(line).map((f) => [f, 0]); + for (const [part, bonus] of candidates) { + if (!isUsefulTitleCandidate(part, { source: 'ocr' })) continue; + const score = scoreCandidate(part, { source: 'ocr' }) + bonus; + if (score > bestScore) { + best = part; + bestScore = score; + } } } return best; @@ -212,13 +256,16 @@ function isDirectiveTitle(text) { function verbForElementRole(role) { const clean = normalizeWhitespace(role).toLowerCase(); if (!clean) return null; - if (/(tab|menu item|menuitem|option|list item|tree item|radio button|dropdown list|combo box option)/.test(clean)) { + if (/(tab|menu item|menuitem|option|list item|tree item|radio button|dropdown list|combo box option|hyperlink|link)/.test(clean)) { return 'Select'; } - if (/(button|check box|checkbox|toggle button|switch|link|item|command)/.test(clean)) { + if (/(search box|searchbox|search field|search bar|search input)/.test(clean)) { + return 'Search for'; + } + if (/(button|check box|checkbox|toggle button|switch|item|command)/.test(clean)) { return 'Click'; } - if (/(text field|edit|search box|combo box|textbox|text box|input|field)/.test(clean)) { + if (/(text field|edit|combo box|textbox|text box|input|field)/.test(clean)) { return 'Click'; } return null; @@ -270,11 +317,22 @@ function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } const elementPhrase = pickBestTitleFragment(metadata.elementLabel, { source: 'element', metadata }); if (elementPhrase) return elementPhrase; - const windowPhrase = pickBestTitleFragment(metadata.windowTitle, { source: 'window', metadata }); - if (windowPhrase) return windowPhrase; + // Strip the browser name suffix before processing window titles so that + // "oracle - Google Search - Google Chrome" becomes "oracle - Google Search" + // and "Oracle | Cloud Applications - Google Chrome" becomes just the page title. + const rawWindowTitle = metadata.windowTitle || ''; + const strippedWindowTitle = stripBrowserNameSuffix(rawWindowTitle); + if (strippedWindowTitle) { + // Detect "[query] - Google Search" → "Search for oracle" + const searchQuery = extractSearchQuery(strippedWindowTitle); + if (searchQuery) return `Search for ${sentenceCase(searchQuery)}`; + // Use the page title (now free of browser name noise). + const windowPhrase = pickBestTitleFragment(strippedWindowTitle, { source: 'window', metadata }); + if (windowPhrase) return windowPhrase; + } const appPhrase = pickBestTitleFragment(metadata.appName, { source: 'app', metadata }); - if (appPhrase && appPhrase !== windowPhrase) return appPhrase; + if (appPhrase) return appPhrase; return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; } diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 16bf024..9471f1e 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -86,6 +86,60 @@ test('capture titles fall back to OCR when metadata is absent', () => { assert.equal(title, 'Click Save changes'); }); +test('browser window title strips browser name and falls back to page title', () => { + // OCR fails; browser window title should give something useful, not "Screen capture". + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { + windowTitle: 'Oracle | Cloud Applications and Cloud Platform - Google Chrome', + appName: 'chrome', + }, + ocrText: '', + }); + // Stripped title "Oracle | Cloud Applications and Cloud Platform" → best fragment + assert.ok(title !== 'Screen capture', `Expected smart title, got: ${title}`); + assert.ok(title.toLowerCase().includes('oracle') || title.toLowerCase().includes('cloud'), `Expected oracle/cloud in title, got: ${title}`); +}); + +test('search query is extracted from browser window title pattern', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { + windowTitle: 'oracle - Google Search - Google Chrome', + appName: 'chrome', + }, + ocrText: '', + }); + assert.equal(title, 'Search for Oracle'); +}); + +test('full link text with pipe separator is preserved in OCR phrases', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { elementRole: 'hyperlink' }, + ocrText: 'Oracle | Cloud Applications and Cloud Platform', + }); + assert.equal(title, 'Select Oracle | Cloud Applications and Cloud Platform'); +}); + +test('link element role uses Select verb', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { elementLabel: 'Sign in', elementRole: 'hyperlink' }, + ocrText: '', + }); + assert.equal(title, 'Select Sign in'); +}); + +test('search box element role uses Search for verb', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { elementLabel: 'oracle', elementRole: 'search box' }, + ocrText: '', + }); + assert.equal(title, 'Search for Oracle'); +}); + test('ai prompts include the deterministic OCR-backed title candidate', () => { const { prompt } = buildAiPrompt({ captureContext: { From 13b5f67de83991c4f14e80f48c394c314783c4d4 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 10:04:34 -0500 Subject: [PATCH 09/19] add keyboard tracking and app-qualified smart titles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Keyboard hook (Windows)** - Extends the existing C# WH_MOUSE_LL process to also install WH_KEYBOARD_LL alongside it (keyboard hook is optional — failure does not break mouse capture). - Emits CHAR for printable keystrokes, KEY for modifier combos (Ctrl+T) and special keys (Backspace, Enter). **Text accumulation in capture session** - CaptureService tracks _keyBuffer (typed chars since last step) and _lastShortcut (last modifier combo) using the new onKeyboardEvent() method. - snapshotKeyContext() is called at enqueueClickCapture time so each step's clickMeta.keyContext carries { recentTyped, recentShortcut }. - Buffer resets after each snapshot; stale input (>8s gap) is dropped. **UIAutomation element value** - collectWindowsWindowContext now reads ValuePattern.Current.Value from the clicked element — giving us what's actually typed in a search box or text field without needing the keyboard buffer. **Smart title generation (core/text-intel.js)** - Priority chain: keyboard shortcut → element value → typed text → OCR → element label → page title → app name. - SHORTCUT_TITLES maps 50+ common shortcuts (Ctrl+T, Ctrl+S, F5 …) to natural language descriptions: "Open new tab", "Save", etc. - qualifyTitleWithApp() appends "in Chrome / VS Code / Terminal / …" to every title when the app is known: "Click Save in VS Code", "Search for oracle in Chrome", "Open new tab in Chrome". - APP_DISPLAY_NAMES covers browsers, editors, terminals, office apps. Six new unit tests cover shortcuts, typed-text search, element value, and app-qualified OCR titles. Capture test updated for keyContext. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 174 ++++++++++++++++++++++++++++++-- app/text-intel.js | 19 +++- core/text-intel.js | 183 ++++++++++++++++++++++++++++++++-- tests/unit/capture.test.js | 2 +- tests/unit/text-intel.test.js | 55 +++++++++- 5 files changed, 409 insertions(+), 24 deletions(-) diff --git a/app/capture.js b/app/capture.js index db918a8..d7f8243 100644 --- a/app/capture.js +++ b/app/capture.js @@ -136,6 +136,9 @@ class CaptureService { this.linuxEvent = null; // event block currently being parsed this.pendingRawClick = null; // raw press waiting for its coordinate twin this.pendingClickOsPoint = null; + this._keyBuffer = ''; // printable chars typed since last capture + this._lastShortcut = ''; // last Ctrl+X / Alt+X combination + this._keyLastAt = 0; // timestamp of last key event this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -799,12 +802,15 @@ using System.Collections.Concurrent; using System.Runtime.InteropServices; using System.Threading; -public static class SFMouseHook { +public static class SFHook { private const int WH_MOUSE_LL = 14; + private const int WH_KEYBOARD_LL = 13; private const int WM_LBUTTONDOWN = 0x0201; private const int WM_RBUTTONDOWN = 0x0204; private const int WM_MBUTTONDOWN = 0x0207; private const int WM_XBUTTONDOWN = 0x020B; + private const int WM_KEYDOWN = 0x0100; + private const int WM_SYSKEYDOWN = 0x0104; private const long UnixEpochMilliseconds = 62135596800000L; // Opting this process out of Windows Power Throttling (EcoQoS). In a // power-saving plan the OS CPU-starves background processes; a starved @@ -818,7 +824,9 @@ public static class SFMouseHook { private const uint HIGH_PRIORITY_CLASS = 0x00000080; private static IntPtr hook = IntPtr.Zero; - private static LowLevelMouseProc proc = HookCallback; + private static IntPtr keyHook = IntPtr.Zero; + private static LowLevelMouseProc proc = MouseHookCallback; + private static LowLevelKeyboardProc keyProc = KeyboardHookCallback; private static readonly ConcurrentQueue queue = new ConcurrentQueue(); private static readonly AutoResetEvent signal = new AutoResetEvent(false); @@ -854,11 +862,34 @@ public static class SFMouseHook { public uint StateMask; } + [StructLayout(LayoutKind.Sequential)] + private struct KBDLLHOOKSTRUCT { + public uint vkCode; + public uint scanCode; + public uint flags; + public uint time; + public UIntPtr dwExtraInfo; + } + private delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + private delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam); [DllImport("user32.dll", SetLastError = true)] private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelKeyboardProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll")] + private static extern short GetKeyState(int nVirtKey); + + [DllImport("user32.dll")] + private static extern bool GetKeyboardState([Out] byte[] lpKeyState); + + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern int ToUnicode(uint wVirtKey, uint wScanCode, byte[] lpKeyState, + System.Text.StringBuilder pwszBuff, int cchBuff, uint wFlags); + [DllImport("user32.dll", SetLastError = true)] private static extern bool UnhookWindowsHookEx(IntPtr hhk); @@ -917,6 +948,8 @@ public static class SFMouseHook { if (hook == IntPtr.Zero) { throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); } + // Keyboard hook is optional — if it fails we still get click titles. + try { keyHook = SetWindowsHookEx(WH_KEYBOARD_LL, keyProc, GetModuleHandle(null), 0); } catch { } Console.Out.WriteLine("READY"); Console.Out.Flush(); @@ -928,6 +961,7 @@ public static class SFMouseHook { } UnhookWindowsHookEx(hook); + if (keyHook != IntPtr.Zero) UnhookWindowsHookEx(keyHook); } private static void WriterLoop() { @@ -941,7 +975,7 @@ public static class SFMouseHook { } } - private static IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { if (nCode >= 0) { int message = wParam.ToInt32(); string button = ButtonName(message, lParam); @@ -955,6 +989,75 @@ public static class SFMouseHook { return CallNextHookEx(hook, nCode, wParam, lParam); } + private static IntPtr KeyboardHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0 && (wParam.ToInt32() == WM_KEYDOWN || wParam.ToInt32() == WM_SYSKEYDOWN)) { + try { + KBDLLHOOKSTRUCT kb = (KBDLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(KBDLLHOOKSTRUCT)); + int vk = (int)kb.vkCode; + bool ctrl = (GetKeyState(0x11) & 0x8000) != 0; + bool alt = (GetKeyState(0x12) & 0x8000) != 0; + bool shift = (GetKeyState(0x10) & 0x8000) != 0; + // Skip standalone modifier keys. + bool isMod = vk == 0x10 || vk == 0x11 || vk == 0x12 || vk == 0x5B || vk == 0x5C; + if (!isMod) { + long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds; + if (ctrl || alt) { + // Emit shortcut: "KEY Ctrl+T 123456" + string name = VkName(vk); + if (name != null) { + string mods = (ctrl ? "Ctrl" : "") + (ctrl && (alt || shift) ? "+" : "") + + (alt ? "Alt" : "") + ((alt || ctrl) && shift ? "+" : "") + + (shift ? "Shift" : ""); + queue.Enqueue("KEY " + mods + "+" + name + " " + unixMs); + signal.Set(); + } + } else if (vk == 0x08) { + queue.Enqueue("KEY Backspace " + unixMs); signal.Set(); + } else if (vk == 0x1B) { + queue.Enqueue("KEY Escape " + unixMs); signal.Set(); + } else if (vk == 0x0D) { + queue.Enqueue("KEY Enter " + unixMs); signal.Set(); + } else { + // Map to Unicode character using current keyboard layout + shift state. + byte[] ks = new byte[256]; + GetKeyboardState(ks); + var sb = new System.Text.StringBuilder(4); + int res = ToUnicode(kb.vkCode, kb.scanCode, ks, sb, 4, 0); + if (res > 0) { + char ch = sb[0]; + if (ch >= 0x20 && ch < 0x7F) { + queue.Enqueue("CHAR " + (int)ch + " " + unixMs); + signal.Set(); + } + } + } + } + } catch { } + } + return CallNextHookEx(keyHook, nCode, wParam, lParam); + } + + private static string VkName(int vk) { + if (vk >= 0x41 && vk <= 0x5A) return ((char)vk).ToString(); // A-Z + if (vk >= 0x30 && vk <= 0x39) return ((char)vk).ToString(); // 0-9 + if (vk >= 0x70 && vk <= 0x87) return "F" + (vk - 0x6F); // F1-F24 + switch (vk) { + case 0x09: return "Tab"; + case 0x0D: return "Enter"; + case 0x1B: return "Escape"; + case 0x20: return "Space"; + case 0x21: return "PageUp"; case 0x22: return "PageDown"; + case 0x23: return "End"; case 0x24: return "Home"; + case 0x25: return "Left"; case 0x26: return "Up"; + case 0x27: return "Right"; case 0x28: return "Down"; + case 0x2E: return "Delete"; + case 0x6B: case 0xBB: return "Plus"; + case 0x6D: case 0xBD: return "Minus"; + case 0xBE: return "Period"; case 0xBF: return "Slash"; + default: return null; + } + } + private static string ButtonName(int message, IntPtr lParam) { if (message == WM_LBUTTONDOWN) return "left"; if (message == WM_RBUTTONDOWN) return "right"; @@ -968,7 +1071,7 @@ public static class SFMouseHook { } } '@ -[SFMouseHook]::Run() +[SFHook]::Run() `; this.clickWatcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', ps], { stdio: ['ignore', 'pipe', 'pipe'], @@ -1103,11 +1206,22 @@ public static class SFMouseHook { } if (platform === 'win32') { for (const line of lines) { - const m = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(line.trim()); - if (m) { - const osPoint = m[1] === undefined ? null : { x: Number(m[1]), y: Number(m[2]) }; - const eventAt = m[4] === undefined ? Date.now() : Number(m[4]); - this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, m[3] || 'mouse'); + const trimmed = line.trim(); + const clickM = /^CLICK(?:\s+(-?\d+)\s+(-?\d+)(?:\s+([A-Za-z0-9_-]+))?(?:\s+(\d+))?)?\s*$/.exec(trimmed); + if (clickM) { + const osPoint = clickM[1] === undefined ? null : { x: Number(clickM[1]), y: Number(clickM[2]) }; + const eventAt = clickM[4] === undefined ? Date.now() : Number(clickM[4]); + this.onOsClick(Number.isFinite(eventAt) ? eventAt : Date.now(), osPoint, clickM[3] || 'mouse'); + continue; + } + const keyM = /^KEY\s+(\S+)\s+(\d+)\s*$/.exec(trimmed); + if (keyM) { + this.onKeyboardEvent('KEY', keyM[1], Number(keyM[2])); + continue; + } + const charM = /^CHAR\s+(\d+)\s+(\d+)\s*$/.exec(trimmed); + if (charM) { + this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2])); } } } @@ -1285,6 +1399,8 @@ public static class SFMouseHook { if (osPoint) { clickMeta.osPoint = { x: osPoint.x, y: osPoint.y }; } + // Snapshot keyboard context accumulated since the last capture. + clickMeta.keyContext = this.snapshotKeyContext(); if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be // stored if the session stops while this click waits in the queue. @@ -1298,6 +1414,46 @@ public static class SFMouseHook { return this.clickQueue; } + // --- Keyboard context tracking ------------------------------------------- + + onKeyboardEvent(type, data, eventAt) { + const now = Number.isFinite(eventAt) ? eventAt : Date.now(); + // Discard stale typing that happened more than 8 seconds ago (user moved on). + if (now - this._keyLastAt > 8000) { + this._keyBuffer = ''; + } + this._keyLastAt = now; + + if (type === 'CHAR') { + const ch = typeof data === 'number' ? String.fromCharCode(data) : String(data); + this._keyBuffer = (this._keyBuffer + ch).slice(-200); + } else if (type === 'KEY') { + const key = String(data); + if (key === 'Backspace') { + this._keyBuffer = this._keyBuffer.slice(0, -1); + } else if (key === 'Escape') { + this._keyBuffer = ''; + } else if (key === 'Enter') { + // Keep typed text — Enter often submits what was typed. + } else { + // It's a modifier+key shortcut (Ctrl+T etc.). + this._lastShortcut = key; + this._keyBuffer = ''; // a shortcut resets the typed-text buffer + } + } + } + + snapshotKeyContext() { + const ctx = { + recentTyped: this._keyBuffer.trim(), + recentShortcut: this._lastShortcut, + }; + // Reset after snapshot so each step gets its own context. + this._keyBuffer = ''; + this._lastShortcut = ''; + return ctx; + } + async captureCurrentFrame(mode, capturePoint = null, startedAt = Date.now()) { const grabbed = await this.grab(mode, capturePoint); return { diff --git a/app/text-intel.js b/app/text-intel.js index 3206670..1e5ff39 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -181,6 +181,7 @@ class TextIntelService { $elementRole = ''; $elementClass = ''; $elementProcessId = 0; + $elementValue = ''; if (${hasPoint ? '$true' : '$false'}) { try { Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null @@ -192,6 +193,12 @@ class TextIntelService { $elementRole = $current.LocalizedControlType; $elementClass = $current.ClassName; $elementProcessId = $current.ProcessId; + try { + $valPattern = [System.Windows.Automation.ValuePattern]::Pattern; + if ($element.GetSupportedPatterns() -contains $valPattern) { + $elementValue = $element.GetCurrentPattern($valPattern).Current.Value; + } + } catch { } } } catch { } } @@ -218,6 +225,7 @@ public static class Win32 { elementLabel = $elementLabel; elementRole = $elementRole; elementClass = $elementClass; + elementValue = $elementValue; elementProcessId = $elementProcessId; pid = $pid; }; @@ -284,11 +292,14 @@ public static class Win32 { } async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) { + const keyContext = clickMeta?.keyContext || {}; + const recentTyped = keyContext.recentTyped || ''; + const recentShortcut = keyContext.recentShortcut || ''; const [metadata, ocr] = await Promise.all([ this.collectForegroundWindowContext(clickMeta?.osPoint || null), this.ocrAroundClick(frame, clickPos), ]); - const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text }); + const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text, recentTyped, recentShortcut }); return { title, captureMetadata: { @@ -297,6 +308,9 @@ public static class Win32 { appName: metadata.appName || '', elementLabel: metadata.elementLabel || '', elementRole: metadata.elementRole || '', + elementValue: metadata.elementValue || '', + recentTyped, + recentShortcut, mode, }, }; @@ -440,8 +454,11 @@ public static class Win32 { appName: step.captureMetadata.appName, elementLabel: step.captureMetadata.elementLabel, elementRole: step.captureMetadata.elementRole, + elementValue: step.captureMetadata.elementValue, }, ocrText: step.captureMetadata.ocrText, + recentTyped: step.captureMetadata.recentTyped, + recentShortcut: step.captureMetadata.recentShortcut, }); captureContext = { ...step.captureMetadata, diff --git a/core/text-intel.js b/core/text-intel.js index c235c94..7bf1d08 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -70,6 +70,121 @@ const SEARCH_ENGINE_PAGE_NAMES = new Set([ 'brave search', ]); +// Common keyboard shortcuts → short action descriptions used as step titles. +const SHORTCUT_TITLES = { + 'Ctrl+T': 'Open new tab', + 'Ctrl+N': 'Open new window', + 'Ctrl+W': 'Close tab', + 'Ctrl+Shift+T': 'Reopen closed tab', + 'Ctrl+Shift+N': 'Open incognito window', + 'Ctrl+S': 'Save', + 'Ctrl+Shift+S': 'Save as', + 'Ctrl+Z': 'Undo', + 'Ctrl+Y': 'Redo', + 'Ctrl+Shift+Z': 'Redo', + 'Ctrl+C': 'Copy selection', + 'Ctrl+V': 'Paste', + 'Ctrl+X': 'Cut selection', + 'Ctrl+A': 'Select all', + 'Ctrl+F': 'Open Find', + 'Ctrl+H': 'Open Find and Replace', + 'Ctrl+R': 'Reload page', + 'Ctrl+Shift+R': 'Hard reload page', + 'Ctrl+L': 'Focus address bar', + 'Ctrl+D': 'Bookmark page', + 'Ctrl+Tab': 'Switch to next tab', + 'Ctrl+Shift+Tab': 'Switch to previous tab', + 'Ctrl+Plus': 'Zoom in', + 'Ctrl+Minus': 'Zoom out', + 'Ctrl+0': 'Reset zoom', + 'Ctrl+P': 'Print', + 'Ctrl+O': 'Open file', + 'Ctrl+E': 'Focus search bar', + 'Ctrl+K': 'Focus search bar', + 'Ctrl+G': 'Go to line', + 'Ctrl+B': 'Toggle sidebar', + 'Ctrl+Shift+P': 'Open command palette', + 'Ctrl+Shift+E': 'Show file explorer', + 'Ctrl+Shift+G': 'Show source control', + 'Ctrl+Shift+D': 'Show debug panel', + 'Ctrl+Shift+X': 'Show extensions', + 'Alt+F4': 'Close window', + 'Alt+Left': 'Go back', + 'Alt+Right': 'Go forward', + 'Alt+Tab': 'Switch application', + 'F2': 'Rename', + 'F3': 'Find next', + 'F4': 'Open address bar', + 'F5': 'Reload page', + 'F11': 'Toggle fullscreen', + 'F12': 'Open developer tools', +}; + +// Process name → human-readable display name (used to append "in Chrome" etc. to titles). +const APP_DISPLAY_NAMES = { + chrome: 'Chrome', + msedge: 'Edge', + firefox: 'Firefox', + safari: 'Safari', + opera: 'Opera', + brave: 'Brave', + vivaldi: 'Vivaldi', + code: 'VS Code', + cursor: 'Cursor', + 'sublime_text': 'Sublime Text', + atom: 'Atom', + notepad: 'Notepad', + 'notepad++': 'Notepad++', + winword: 'Word', + excel: 'Excel', + powerpnt: 'PowerPoint', + outlook: 'Outlook', + teams: 'Teams', + slack: 'Slack', + discord: 'Discord', + zoom: 'Zoom', + figma: 'Figma', + postman: 'Postman', + insomnia: 'Insomnia', + notion: 'Notion', + obsidian: 'Obsidian', + spotify: 'Spotify', + terminal: 'Terminal', + cmd: 'Command Prompt', + powershell: 'PowerShell', + windowsterminal: 'Windows Terminal', + wt: 'Windows Terminal', + iterm2: 'iTerm', + wezterm: 'WezTerm', + alacritty: 'Alacritty', + kitty: 'Kitty', + 'gnome-terminal': 'Terminal', + konsole: 'Konsole', + xterm: 'Terminal', + xfce4terminal: 'Terminal', + bash: 'Terminal', + zsh: 'Terminal', + fish: 'Terminal', + finder: 'Finder', + explorer: 'File Explorer', + 'files-uwp': 'File Explorer', + steam: 'Steam', + 'steamwebhelper': 'Steam', +}; + +function cleanAppName(rawName) { + if (!rawName) return ''; + const key = normalizeWhitespace(rawName).toLowerCase().replace(/\.exe$/i, ''); + return APP_DISPLAY_NAMES[key] || sentenceCase(rawName.replace(/\.exe$/i, '')); +} + +function qualifyTitleWithApp(title, appName) { + const app = cleanAppName(appName); + if (!app) return title; + if (new RegExp(`\\b${app.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i').test(title)) return title; + return `${title} in ${app}`; +} + const ACTION_PREFIXES = [ 'click', 'select', @@ -310,27 +425,70 @@ function pickBestTitleFragment(text, { source = 'window', metadata = {} } = {}) return best ? formatCaptureTitle(best, { source, metadata }) : ''; } -function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) { +function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '', recentTyped = '', recentShortcut = '' } = {}) { + const app = cleanAppName(metadata.appName); + + // 1. Keyboard shortcut → most reliable signal for "what action did the user take". + if (recentShortcut && SHORTCUT_TITLES[recentShortcut]) { + const base = SHORTCUT_TITLES[recentShortcut]; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + + // 2. UIAutomation element value — what's actually typed inside the clicked field. + const elementValue = normalizeWhitespace(metadata.elementValue || ''); + if (elementValue) { + const roleLower = normalizeWhitespace(metadata.elementRole || '').toLowerCase(); + const labelLower = normalizeWhitespace(metadata.elementLabel || '').toLowerCase(); + const looksLikeSearch = /(search|find|query|omnibox|address bar)/.test(roleLower + ' ' + labelLower); + const action = looksLikeSearch ? 'Search for' : 'Type'; + const base = `${action} "${elementValue}"`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + + // 3. Keyboard-buffer text (typed between captures) + input role context. + const typed = normalizeWhitespace(recentTyped || ''); + if (typed) { + const roleLower = normalizeWhitespace(metadata.elementRole || '').toLowerCase(); + const labelLower = normalizeWhitespace(metadata.elementLabel || '').toLowerCase(); + const isSearchRole = /(search box|searchbox|search field|search bar|search input)/.test(roleLower); + const looksLikeSearch = isSearchRole || /(search|find|query|omnibox|address bar)/.test(roleLower + ' ' + labelLower); + const isAnyInput = /(text field|edit|input|field|combo box|textbox|text box)/.test(roleLower); + if (looksLikeSearch) { + const base = `Search for "${typed}"`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + if (isAnyInput) { + const base = `Type "${typed}"`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + } + + // 4. OCR text around the click — link text, button labels, menu items. const ocrPhrase = pickBestOcrPhrase(ocrText); - if (ocrPhrase) return formatCaptureTitle(ocrPhrase, { source: 'ocr', metadata }); + if (ocrPhrase) { + const title = formatCaptureTitle(ocrPhrase, { source: 'ocr', metadata }); + return app ? qualifyTitleWithApp(title, metadata.appName) : title; + } + // 5. UIAutomation element label. const elementPhrase = pickBestTitleFragment(metadata.elementLabel, { source: 'element', metadata }); - if (elementPhrase) return elementPhrase; + if (elementPhrase) { + return app ? qualifyTitleWithApp(elementPhrase, metadata.appName) : elementPhrase; + } - // Strip the browser name suffix before processing window titles so that - // "oracle - Google Search - Google Chrome" becomes "oracle - Google Search" - // and "Oracle | Cloud Applications - Google Chrome" becomes just the page title. - const rawWindowTitle = metadata.windowTitle || ''; - const strippedWindowTitle = stripBrowserNameSuffix(rawWindowTitle); + // 6. Browser window title (suffix stripped) → page title or search query. + const strippedWindowTitle = stripBrowserNameSuffix(metadata.windowTitle || ''); if (strippedWindowTitle) { - // Detect "[query] - Google Search" → "Search for oracle" const searchQuery = extractSearchQuery(strippedWindowTitle); - if (searchQuery) return `Search for ${sentenceCase(searchQuery)}`; - // Use the page title (now free of browser name noise). + if (searchQuery) { + const base = `Search for ${sentenceCase(searchQuery)}`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } const windowPhrase = pickBestTitleFragment(strippedWindowTitle, { source: 'window', metadata }); if (windowPhrase) return windowPhrase; } + // 7. App name alone as last resort. const appPhrase = pickBestTitleFragment(metadata.appName, { source: 'app', metadata }); if (appPhrase) return appPhrase; @@ -516,6 +674,9 @@ function buildAiPrompt({ captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null, captureContext.appName ? `App: ${captureContext.appName}` : null, captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null, + captureContext.elementValue ? `Element content (what was typed): ${captureContext.elementValue}` : null, + captureContext.recentTyped ? `Keyboard input before this step: ${captureContext.recentTyped}` : null, + captureContext.recentShortcut ? `Keyboard shortcut used: ${captureContext.recentShortcut}` : null, captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null, captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null, ].filter(Boolean) : []; diff --git a/tests/unit/capture.test.js b/tests/unit/capture.test.js index 51323ca..5e0414f 100644 --- a/tests/unit/capture.test.js +++ b/tests/unit/capture.test.js @@ -375,7 +375,7 @@ test('queued click captures preserve the original event time and button', async assert.deepEqual(seen, [{ trigger: 'click', clickPos: { x: 7, y: 8 }, - clickMeta: { at: 1770000000456, button: 'left' }, + clickMeta: { at: 1770000000456, button: 'left', keyContext: { recentTyped: '', recentShortcut: '' } }, }]); }); diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 9471f1e..4c7a7ce 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -61,7 +61,8 @@ test('capture titles ignore browser chrome noise in favor of OCR', () => { }, ocrText: 'New tab', }); - assert.equal(title, 'Click New tab'); + // OCR wins over the noisy window title; app name is appended for context. + assert.equal(title, 'Click New tab in Google Chrome'); }); test('tab-like roles use select when OCR identifies a tab label', () => { @@ -110,7 +111,7 @@ test('search query is extracted from browser window title pattern', () => { }, ocrText: '', }); - assert.equal(title, 'Search for Oracle'); + assert.equal(title, 'Search for Oracle in Chrome'); }); test('full link text with pipe separator is preserved in OCR phrases', () => { @@ -140,6 +141,56 @@ test('search box element role uses Search for verb', () => { assert.equal(title, 'Search for Oracle'); }); +test('keyboard shortcut produces action title qualified with app', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { appName: 'chrome' }, + ocrText: '', + recentShortcut: 'Ctrl+T', + }); + assert.equal(title, 'Open new tab in Chrome'); +}); + +test('keyboard shortcut title without app name', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: {}, + ocrText: '', + recentShortcut: 'Ctrl+S', + }); + assert.equal(title, 'Save'); +}); + +test('typed text with search input role produces Search for title', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { elementRole: 'search box', appName: 'chrome' }, + ocrText: '', + recentTyped: 'oracle', + }); + assert.equal(title, 'Search for "oracle" in Chrome'); +}); + +test('UIAutomation element value takes priority over keyboard buffer', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { elementRole: 'edit', elementValue: 'oracle', appName: 'chrome' }, + ocrText: '', + recentTyped: 'ignored', + }); + // elementValue (from UIAutomation) wins over the keyboard buffer + assert.ok(title.includes('oracle'), `expected oracle in title, got: ${title}`); +}); + +test('app-qualified OCR title includes app name', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { appName: 'code' }, + ocrText: 'Save', + }); + assert.equal(title, 'Click Save in VS Code'); +}); + test('ai prompts include the deterministic OCR-backed title candidate', () => { const { prompt } = buildAiPrompt({ captureContext: { From 879434f14f8fa88ed3b01a2f8e5f1e9dbbf4dee4 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 10:16:11 -0500 Subject: [PATCH 10/19] fix: capture window title from click watcher instead of spawning PowerShell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of all-"Screen capture" titles: collectWindowsWindowContext called execFileSync('powershell.exe') with a 1200 ms timeout, but PowerShell cold-start on typical Windows systems takes 1-3 seconds. Every capture timed out silently, returned empty metadata, and fell back to "Screen capture". Fix: - The click watcher C# process (already running, already compiled) now emits a CTX event immediately before each CLICK event using GetForegroundWindow → GetWindowText + QueryFullProcessImageName. These are synchronous Win32 calls, sub-millisecond, no startup cost. - CTX payload is base64-encoded to survive the line protocol safely: CTX - processClickWatcherData parses CTX lines and stores the decoded strings in this._lastWindowContext. - enqueueClickCapture attaches it as clickMeta.windowContext. - buildCaptureContext uses it directly (Promise.resolve) when present, bypassing the PowerShell spawn entirely. Fallback (manual captures without a session click watcher running): - collectWindowsWindowContext is now async and uses execFile with a 4 s timeout instead of execFileSync at 1200 ms, so it no longer blocks the event loop and has room to succeed on slower machines. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 72 +++++++++++++++++++++++++++++++++++++++++++++++ app/text-intel.js | 28 ++++++++++++------ 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/app/capture.js b/app/capture.js index d7f8243..a3a2350 100644 --- a/app/capture.js +++ b/app/capture.js @@ -139,6 +139,7 @@ class CaptureService { this._keyBuffer = ''; // printable chars typed since last capture this._lastShortcut = ''; // last Ctrl+X / Alt+X combination this._keyLastAt = 0; // timestamp of last key event + this._lastWindowContext = null; // last CTX event from the click watcher this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -920,6 +921,58 @@ public static class SFHook { [DllImport("kernel32.dll")] private static extern bool SetPriorityClass(IntPtr hProcess, uint dwPriorityClass); + [DllImport("user32.dll")] + private static extern IntPtr GetForegroundWindow(); + + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count); + + [DllImport("user32.dll")] + private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr OpenProcess(uint processAccess, bool bInheritHandle, uint processId); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool CloseHandle(IntPtr hObject); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern bool QueryFullProcessImageName(IntPtr hProcess, uint dwFlags, System.Text.StringBuilder lpExeName, ref uint lpdwSize); + + private static string GetFwTitle() { + try { + IntPtr hwnd = GetForegroundWindow(); + if (hwnd == IntPtr.Zero) return ""; + var sb = new System.Text.StringBuilder(512); + GetWindowText(hwnd, sb, sb.Capacity); + return sb.ToString(); + } catch { return ""; } + } + + private static string GetFwApp() { + try { + IntPtr hwnd = GetForegroundWindow(); + if (hwnd == IntPtr.Zero) return ""; + uint pid = 0; + GetWindowThreadProcessId(hwnd, out pid); + if (pid == 0) return ""; + IntPtr hProc = OpenProcess(0x1000u, false, pid); + if (hProc == IntPtr.Zero) return ""; + var sb = new System.Text.StringBuilder(260); + uint sz = (uint)sb.Capacity; + QueryFullProcessImageName(hProc, 0u, sb, ref sz); + CloseHandle(hProc); + string path = sb.ToString(); + return string.IsNullOrEmpty(path) ? "" : System.IO.Path.GetFileNameWithoutExtension(path); + } catch { return ""; } + } + + // Base64-encodes a string for safe line-protocol transmission; "-" for empty. + private static string B64(string s) { + if (string.IsNullOrEmpty(s)) return "-"; + try { return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(s)); } catch { return "-"; } + } + // Force this process to run at full CPU speed regardless of the power plan, // so the mouse-hook callback never trips LowLevelHooksTimeout and clicks // keep being delivered while the laptop is in eco / power-saving mode. @@ -982,6 +1035,12 @@ public static class SFHook { if (button != null) { MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT)); long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds; + // Capture window context while the user's app is still in foreground. + // GetForegroundWindow is synchronous and fast (no IPC overhead). + try { + string t = GetFwTitle(), a = GetFwApp(); + queue.Enqueue("CTX " + B64(t) + " " + B64(a) + " " + unixMs); + } catch { } queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs); signal.Set(); } @@ -1222,6 +1281,15 @@ public static class SFHook { const charM = /^CHAR\s+(\d+)\s+(\d+)\s*$/.exec(trimmed); if (charM) { this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2])); + continue; + } + const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+\d+$/.exec(trimmed); + if (ctxM) { + const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8'); + this._lastWindowContext = { + windowTitle: decode(ctxM[1]), + appName: decode(ctxM[2]), + }; } } } @@ -1401,6 +1469,10 @@ public static class SFHook { } // Snapshot keyboard context accumulated since the last capture. clickMeta.keyContext = this.snapshotKeyContext(); + // Attach the window context emitted by the click watcher alongside the click. + if (this._lastWindowContext) { + clickMeta.windowContext = this._lastWindowContext; + } if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be // stored if the session stops while this click waits in the queue. diff --git a/app/text-intel.js b/app/text-intel.js index 1e5ff39..2695bda 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -2,7 +2,7 @@ const fs = require('node:fs'); const path = require('node:path'); -const { execFileSync } = require('node:child_process'); +const { execFileSync, execFile } = require('node:child_process'); const { DEFAULT_CAPTURE_TITLES, @@ -170,7 +170,7 @@ class TextIntelService { return { appName: '', windowTitle: '' }; } - collectWindowsWindowContext(osPoint = null) { + async collectWindowsWindowContext(osPoint = null) { const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y); const clickX = hasPoint ? Number(osPoint.x) : 0; const clickY = hasPoint ? Number(osPoint.y) : 0; @@ -231,12 +231,17 @@ public static class Win32 { }; $out | ConvertTo-Json -Compress; `; - const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 1200, - }).trim(); - return JSON.parse(result || '{}'); + return new Promise(resolve => { + execFile('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], { + encoding: 'utf8', + timeout: 4000, + windowsHide: true, + }, (err, stdout) => { + if (err) { resolve({}); return; } + try { resolve(JSON.parse(stdout.trim() || '{}')); } + catch { resolve({}); } + }); + }); } collectMacWindowContext() { @@ -295,8 +300,13 @@ public static class Win32 { const keyContext = clickMeta?.keyContext || {}; const recentTyped = keyContext.recentTyped || ''; const recentShortcut = keyContext.recentShortcut || ''; + // Use window context pre-captured by the click watcher when available. + // This avoids a costly PowerShell cold-start (1–3 s) on every capture. + const fastContext = clickMeta?.windowContext || null; const [metadata, ocr] = await Promise.all([ - this.collectForegroundWindowContext(clickMeta?.osPoint || null), + fastContext + ? Promise.resolve(fastContext) + : this.collectForegroundWindowContext(clickMeta?.osPoint || null), this.ocrAroundClick(frame, clickPos), ]); const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text, recentTyped, recentShortcut }); From 162ae0ae0582cae2b4d91ef96f040caafabe0690 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 10:42:56 -0500 Subject: [PATCH 11/19] improve AI rewrite UX and title quality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **AI button: dynamic tooltip hints** - titleAiBtn and descAiBtn now show "Rewrite with AI" when the field already has user content, "Generate with AI" when empty. - updateAiButtonHints() fires on every title/description input event and whenever syncStepFields() runs to keep hints current. **Stronger rewrite prompt** - When the user has typed a draft title or description, the prompt now shows it explicitly: "User's draft title (rewrite this): '...'" - Rules changed from "improve its wording" to "Your only job is to polish its grammar and phrasing. Do NOT replace it with something different." — prevents the model from ignoring the user's text and generating fresh content from capture context. - The suggested-title hint is suppressed when a draft title exists so the model doesn't silently swap the user's text for the auto-title. **Title quality: generic window title filter** - GENERIC_WINDOW_TITLES Set filters "New Tab", "Untitled", "Loading" etc. from the window-title path so they no longer produce titles like "Open New Tab in Chrome". **Title quality: app name stripping for non-browser apps** - stripBrowserNameSuffix now accepts an optional appName; it strips the app's display name and process name from the window title suffix using the same pattern as browser names. - "Document1.docx - Word" with appName "winword" → "Document1.docx". - buildCaptureTitle passes metadata.appName into the strip call. Co-Authored-By: Claude Sonnet 4.6 --- app/renderer/editor.js | 18 +++++++++++ core/text-intel.js | 70 ++++++++++++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 19 deletions(-) diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 4825afd..fd3e4a8 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -227,6 +227,21 @@ class GuideEditor { return this.runAiGeneration('block', { blockId: block.id, button }); } + updateAiButtonHints() { + const PLACEHOLDER_TITLES = new Set([ + '', 'screen capture', 'window capture', 'region capture', 'capture', + ]); + if (this.dom.titleAiBtn) { + const val = (this.dom.titleInput?.value || '').trim(); + const hasDraft = Boolean(val) && !PLACEHOLDER_TITLES.has(val.toLowerCase()); + this.dom.titleAiBtn.title = hasDraft ? 'Rewrite step title with AI' : 'Generate step title with AI'; + } + if (this.dom.descAiBtn) { + const hasDesc = Boolean((this.dom.descEditor?.innerText || '').trim()); + this.dom.descAiBtn.title = hasDesc ? 'Rewrite description with AI' : 'Generate description with AI'; + } + } + get currentStep() { return this.stepMap.get(this.selectedStepId) || null; } @@ -504,6 +519,7 @@ class GuideEditor { this.saveStepDebounced(); this.renderStepList(); this.emitMeta(); + this.updateAiButtonHints(); }); this.dom.titleAiBtn.addEventListener('click', () => this.generateTitleWithAi(this.dom.titleAiBtn)); @@ -575,6 +591,7 @@ class GuideEditor { this.saveStepDebounced(); this.emitMeta(); this.updateToolbarState(); + this.updateAiButtonHints(); }); this.dom.descEditor.addEventListener('keyup', () => this.updateToolbarState()); this.dom.descEditor.addEventListener('mouseup', () => this.updateToolbarState()); @@ -1019,6 +1036,7 @@ class GuideEditor { if (document.activeElement !== this.dom.titleInput) this.dom.titleInput.value = step.title || ''; if (document.activeElement !== this.dom.descEditor) this.dom.descEditor.innerHTML = step.descriptionHtml || ''; this.dom.statusSelect.value = step.status || 'todo'; + this.updateAiButtonHints(); this.dom.hiddenToggle.querySelector('input').checked = Boolean(step.hidden); this.dom.skippedToggle.querySelector('input').checked = Boolean(step.skipped); this.dom.forceNewPageToggle.querySelector('input').checked = Boolean(step.forceNewPage); diff --git a/core/text-intel.js b/core/text-intel.js index 7bf1d08..ada8a47 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -44,6 +44,13 @@ const GENERIC_OCR_PHRASES = new Set([ 'type', ]); +// Generic OS/browser chrome titles that tell us nothing about what the user did. +const GENERIC_WINDOW_TITLES = new Set([ + 'new tab', 'new window', 'new incognito window', 'new incognito tab', + 'new document', 'untitled', 'blank page', 'home page', 'homepage', + 'start page', 'speed dial', 'loading', 'loading…', 'loading...', +]); + const BROWSER_NAME_PHRASES = new Set([ 'google chrome', 'chrome', @@ -267,6 +274,7 @@ function isUsefulTitleCandidate(text, { source = 'ocr' } = {}) { if (BROWSER_NAME_PHRASES.has(lower)) return false; if (isPathOrUrlLike(clean)) return false; if ((source === 'window' || source === 'app') && isBrowserNoise(clean)) return false; + if (source === 'window' && GENERIC_WINDOW_TITLES.has(lower)) return false; if (/^[\p{P}\p{S}0-9]+$/u.test(clean)) return false; return true; } @@ -287,14 +295,25 @@ function candidateWords(text) { return clean.split(/\s+/).filter((w) => /[a-zA-Z0-9]/.test(w)); } -// Remove trailing "- Google Chrome", "| Firefox", etc. from a browser window title, -// leaving just the page title portion: "oracle - Google Search - Google Chrome" → "oracle - Google Search". -function stripBrowserNameSuffix(text) { +// Remove trailing "- Google Chrome", "| Firefox", etc. from a window title. +// When appName is supplied, also strips the specific app's display name suffix: +// "Document1 - Word" → "Document1" when appName is "winword". +function stripBrowserNameSuffix(text, appName) { let clean = normalizeWhitespace(text); + // Always strip known browser names first. for (const name of BROWSER_NAME_PHRASES) { const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); clean = clean.replace(new RegExp(`\\s*[-–—·|•]\\s*${escaped}\\s*$`, 'i'), '').trim(); } + // Also strip the specific app's display name when provided. + if (appName) { + const display = cleanAppName(appName); + const raw = normalizeWhitespace(appName).replace(/\.exe$/i, ''); + for (const name of [display, raw].filter(Boolean)) { + const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + clean = clean.replace(new RegExp(`\\s*[-–—·|•]\\s*${escaped}\\s*$`, 'i'), '').trim(); + } + } return clean; } @@ -476,8 +495,8 @@ function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '', r return app ? qualifyTitleWithApp(elementPhrase, metadata.appName) : elementPhrase; } - // 6. Browser window title (suffix stripped) → page title or search query. - const strippedWindowTitle = stripBrowserNameSuffix(metadata.windowTitle || ''); + // 6. Window title (browser suffix + app name stripped) → page title or search query. + const strippedWindowTitle = stripBrowserNameSuffix(metadata.windowTitle || '', metadata.appName); if (strippedWindowTitle) { const searchQuery = extractSearchQuery(strippedWindowTitle); if (searchQuery) { @@ -670,16 +689,29 @@ function buildAiPrompt({ 'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].', ].join(' '); - const contextLines = captureContext ? [ - captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null, - captureContext.appName ? `App: ${captureContext.appName}` : null, - captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null, - captureContext.elementValue ? `Element content (what was typed): ${captureContext.elementValue}` : null, - captureContext.recentTyped ? `Keyboard input before this step: ${captureContext.recentTyped}` : null, - captureContext.recentShortcut ? `Keyboard shortcut used: ${captureContext.recentShortcut}` : null, - captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null, - captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null, - ].filter(Boolean) : []; + // When the user already has a draft, surface it prominently so the model + // knows exactly what text to polish rather than generating from scratch. + const descText = htmlToText(step?.descriptionHtml || ''); + const draftTitleLine = hasDraftTitle && (target === 'title' || target === 'all') + ? `User's draft title (rewrite this): "${step.title}"` : null; + const draftDescLine = hasDraftDesc && (target === 'description' || target === 'all') + ? `User's draft description (rewrite this): "${descText}"` : null; + + const contextLines = [ + ...(captureContext ? [ + captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null, + captureContext.appName ? `App: ${captureContext.appName}` : null, + captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null, + captureContext.elementValue ? `Element content (what was typed): ${captureContext.elementValue}` : null, + captureContext.recentTyped ? `Keyboard input before this step: ${captureContext.recentTyped}` : null, + captureContext.recentShortcut ? `Keyboard shortcut used: ${captureContext.recentShortcut}` : null, + captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null, + (!hasDraftTitle || target === 'description') && captureContext.titleCandidate + ? `Suggested title: ${captureContext.titleCandidate}` : null, + ] : []), + draftTitleLine, + draftDescLine, + ].filter(Boolean); const prompt = [ 'You write concise, action-focused step-by-step documentation for a desktop application guide.', @@ -716,11 +748,11 @@ function buildAiPrompt({ 'Rules:', '- Titles must be short imperative actions: "Click Save", "Select New document", "Open Settings".', '- NEVER output "Screen capture", "Window capture", "Region capture", or "Capture" as a title — always produce something specific.', - hasDraftTitle - ? '- The user has a draft title. Improve its wording without changing their intent.' + hasDraftTitle && (target === 'title' || target === 'all') + ? '- The user wrote their own title (shown above). Your only job is to polish its grammar and phrasing. Do NOT replace it with something different. Do NOT change what action or subject it describes.' : '- No title yet. Use the capture context (OCR text, window, app) to write a specific action title.', - hasDraftDesc - ? '- The user has a draft description. Polish it to read like professional documentation.' + hasDraftDesc && (target === 'description' || target === 'all') + ? '- The user wrote their own description (shown above). Polish the wording to sound professional but preserve every fact and intent they stated.' : '- No description yet. Write 1–2 sentences describing exactly what the user does.', '- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).', richContext From dfa65c894b56b009865711fb626d590441433983 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 10:53:30 -0500 Subject: [PATCH 12/19] ci: only run tests on push to main, not on PR commits Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a2828d..51b415a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,8 +3,6 @@ name: CI on: push: branches: [main] - pull_request: - branches: [main] # Cancel an in-progress run when newer commits are pushed to the same ref. concurrency: From d0b181617588b5cb9dbeba6b4998a4bd64c09d91 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 11:05:34 -0500 Subject: [PATCH 13/19] fix step-behind titles: UIAutomation element label from click watcher + smarter search fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Root cause** When OCR fails, the title fell back to the browser window title. For a click on a search-results page, the window title reflects the *previous* search query ("oracle - Google Search"), producing "Search for Oracle" even though the user is clicking a link *on* that page. **Fix 1: UIAutomation element label from the click watcher** The C# click-watcher hook now enriches each click in a background thread (ClickProcessorLoop) rather than in the hook callback: - MouseHookCallback captures window title synchronously (fast Win32), then queues a PendingClick and returns immediately. - ClickProcessorLoop calls AutomationElement.FromPoint() via reflection (no compile-time assembly reference → no startup failure if UIA is absent). Wrapped in a 300ms timeout thread so slow UIA calls don't delay the click event past the frame buffer window. - Emits CTX + ELEM (label/role/value) + CLICK as an atomic batch. Node.js: - Parses ELEM events, merges element info into _pendingWindowContext. - clickMeta.windowContext now carries elementLabel/elementRole/elementValue in addition to windowTitle/appName. - buildCaptureTitle priority-5 (element label) now fires from click-watcher data, giving "Select Oracle | Cloud Applications…" without OCR. **Fix 2: Wider OCR crop** ocrAroundClick now uses a full-display-width × 100px horizontal strip at the click height. The previous 420 px crop cropped through long link text (e.g. "Oracle | Cloud Applications and Cloud Platform"), causing fragments to be scored lower than the complete text. **Fix 3: Search-results window title fallback** extractSearchQuery now only produces "Search for Oracle" when recentTyped is non-empty (the user was actually typing a query). For a pure click on the search-results page (no recent typing), the fallback is "Select a Oracle result in Chrome" — honest about what we know without implying the user performed the search in this step. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 138 ++++++++++++++++++++++++++++++---- app/text-intel.js | 9 ++- core/text-intel.js | 10 ++- tests/unit/text-intel.test.js | 18 +++-- 4 files changed, 154 insertions(+), 21 deletions(-) diff --git a/app/capture.js b/app/capture.js index a3a2350..b1114fc 100644 --- a/app/capture.js +++ b/app/capture.js @@ -139,7 +139,7 @@ class CaptureService { this._keyBuffer = ''; // printable chars typed since last capture this._lastShortcut = ''; // last Ctrl+X / Alt+X combination this._keyLastAt = 0; // timestamp of last key event - this._lastWindowContext = null; // last CTX event from the click watcher + this._pendingWindowContext = null; // buffered CTX+ELEM from click watcher, consumed on CLICK this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -831,6 +831,99 @@ public static class SFHook { private static readonly ConcurrentQueue queue = new ConcurrentQueue(); private static readonly AutoResetEvent signal = new AutoResetEvent(false); + // Pending click queue for background UIAutomation enrichment. + private struct PendingClick { + public int x, y; + public string button; + public long ts; + public string wTitle; + public string wApp; + } + private static readonly ConcurrentQueue pendingClicks = new ConcurrentQueue(); + private static readonly AutoResetEvent clickSignal = new AutoResetEvent(false); + + // UIAutomation state — loaded once via reflection so the C# code has no + // compile-time assembly reference (avoids breaking the whole hook if UIA is absent). + private static bool uiaLoadAttempted = false; + private static System.Reflection.MethodInfo uiaFromPoint = null; + private static System.Reflection.PropertyInfo uiaCurrent = null; + private static System.Type uiaPointType = null; + private static System.Reflection.Assembly uiaClientAssembly = null; + + private static void TryLoadUia() { + if (uiaLoadAttempted) return; + uiaLoadAttempted = true; + try { + uiaClientAssembly = System.Reflection.Assembly.Load("UIAutomationClient"); + var wbase = System.Reflection.Assembly.Load("WindowsBase"); + var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement"); + uiaFromPoint = aeType.GetMethod("FromPoint"); + uiaCurrent = aeType.GetProperty("Current"); + uiaPointType = wbase.GetType("System.Windows.Point"); + } catch { } + } + + private static void GetUiaElement(int x, int y, out string label, out string role, out string value) { + label = ""; role = ""; value = ""; + if (uiaFromPoint == null || uiaPointType == null) return; + try { + var pt = Activator.CreateInstance(uiaPointType, (double)x, (double)y); + var element = uiaFromPoint.Invoke(null, new object[] { pt }); + if (element == null) return; + var current = uiaCurrent.GetValue(element); + var ct = current.GetType(); + label = ct.GetProperty("Name")?.GetValue(current) as string ?? ""; + role = ct.GetProperty("LocalizedControlType")?.GetValue(current) as string ?? ""; + // Try to read the element's current value (text box contents etc.). + try { + var vpType = uiaClientAssembly.GetType("System.Windows.Automation.ValuePattern"); + var vpPatternId = vpType?.GetField("Pattern")?.GetValue(null); + if (vpPatternId != null) { + var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement"); + var tryGet = aeType.GetMethod("TryGetCurrentPattern", + new System.Type[] { vpPatternId.GetType(), typeof(object).MakeByRefType() }); + if (tryGet != null) { + object[] args = new object[] { vpPatternId, null }; + if ((bool)tryGet.Invoke(element, args) && args[1] != null) { + var curr = args[1].GetType().GetProperty("Current")?.GetValue(args[1]); + value = curr?.GetType().GetProperty("Value")?.GetValue(curr) as string ?? ""; + } + } + } + } catch { } + } catch { } + } + + // Background thread: enriches each click with UIAutomation element info, then + // emits CTX + ELEM (optional) + CLICK as an atomic batch. + private static void ClickProcessorLoop() { + TryLoadUia(); + while (true) { + clickSignal.WaitOne(); + PendingClick click; + while (pendingClicks.TryDequeue(out click)) { + string label = "", role = "", val = ""; + // Run UIAutomation on a separate thread so we can impose a timeout. + // 300 ms is enough for most apps; if UIA is slow we skip element enrichment + // but still emit the click (no steps are lost). + string[] r = new string[] { "", "", "" }; + var t = new Thread(() => { + GetUiaElement(click.x, click.y, out r[0], out r[1], out r[2]); + }); + t.IsBackground = true; + t.Start(); + if (t.Join(300)) { label = r[0]; role = r[1]; val = r[2]; } + // Emit window context (always), element info (when available), then click. + queue.Enqueue("CTX " + B64(click.wTitle) + " " + B64(click.wApp) + " " + click.ts); + if (label.Length > 0 || role.Length > 0) { + queue.Enqueue("ELEM " + B64(label) + " " + B64(role) + " " + B64(val) + " " + click.ts); + } + queue.Enqueue("CLICK " + click.x + " " + click.y + " " + click.button + " " + click.ts); + signal.Set(); + } + } + } + [StructLayout(LayoutKind.Sequential)] private struct POINT { public int x; @@ -997,6 +1090,10 @@ public static class SFHook { writer.IsBackground = true; writer.Start(); + Thread clickProcessor = new Thread(ClickProcessorLoop); + clickProcessor.IsBackground = true; + clickProcessor.Start(); + hook = SetWindowsHookEx(WH_MOUSE_LL, proc, GetModuleHandle(null), 0); if (hook == IntPtr.Zero) { throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); @@ -1035,14 +1132,12 @@ public static class SFHook { if (button != null) { MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT)); long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds; - // Capture window context while the user's app is still in foreground. - // GetForegroundWindow is synchronous and fast (no IPC overhead). - try { - string t = GetFwTitle(), a = GetFwApp(); - queue.Enqueue("CTX " + B64(t) + " " + B64(a) + " " + unixMs); - } catch { } - queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs); - signal.Set(); + // Capture window title synchronously (fast Win32, safe in hook callback). + // UIAutomation element lookup runs on the background ClickProcessorLoop thread. + string wTitle = "", wApp = ""; + try { wTitle = GetFwTitle(); wApp = GetFwApp(); } catch { } + pendingClicks.Enqueue(new PendingClick { x = data.pt.x, y = data.pt.y, button = button, ts = unixMs, wTitle = wTitle, wApp = wApp }); + clickSignal.Set(); } } return CallNextHookEx(hook, nCode, wParam, lParam); @@ -1283,13 +1378,27 @@ public static class SFHook { this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2])); continue; } - const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+\d+$/.exec(trimmed); + // CTX always arrives just before its paired ELEM+CLICK from the same background thread batch. + const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed); if (ctxM) { const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8'); - this._lastWindowContext = { + this._pendingWindowContext = { windowTitle: decode(ctxM[1]), appName: decode(ctxM[2]), + ts: Number(ctxM[3]), }; + continue; + } + // ELEM carries UIAutomation element info for the same click as the preceding CTX. + const elemM = /^ELEM\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed); + if (elemM) { + const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8'); + if (this._pendingWindowContext && this._pendingWindowContext.ts === Number(elemM[4])) { + this._pendingWindowContext.elementLabel = decode(elemM[1]); + this._pendingWindowContext.elementRole = decode(elemM[2]); + this._pendingWindowContext.elementValue = decode(elemM[3]); + } + continue; } } } @@ -1469,9 +1578,10 @@ public static class SFHook { } // Snapshot keyboard context accumulated since the last capture. clickMeta.keyContext = this.snapshotKeyContext(); - // Attach the window context emitted by the click watcher alongside the click. - if (this._lastWindowContext) { - clickMeta.windowContext = this._lastWindowContext; + // Attach the window + element context emitted by the click watcher. + if (this._pendingWindowContext) { + clickMeta.windowContext = this._pendingWindowContext; + this._pendingWindowContext = null; } if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be diff --git a/app/text-intel.js b/app/text-intel.js index 2695bda..961db39 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -151,7 +151,14 @@ class TextIntelService { async ocrAroundClick(frame, clickPos) { if (!frame || !frame.image) return { text: '', confidence: null }; - const rect = this.cropRectForPoint(frame, clickPos); + // Use a full-width horizontal strip at the click height. This preserves complete + // link text (e.g. "Oracle | Cloud Applications and Cloud Platform") rather than + // cropping through it when the element spans more than the 420 px default width. + const bounds = frame.display?.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height }; + const rect = this.cropRectForPoint(frame, clickPos, { + width: bounds.width, // full display width → full image width after DPI scaling + height: 100, // ~2 lines tall, enough context without too much noise + }); try { return await this.recognizeCrop(frame.image, rect); } catch { diff --git a/core/text-intel.js b/core/text-intel.js index ada8a47..5fe0a42 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -500,7 +500,15 @@ function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '', r if (strippedWindowTitle) { const searchQuery = extractSearchQuery(strippedWindowTitle); if (searchQuery) { - const base = `Search for ${sentenceCase(searchQuery)}`; + // Only claim this step IS the search action when the user was actually typing + // (recentTyped). Without typing context, the search page title is from the + // PREVIOUS step — the current step is a click ON the search results page. + if (recentTyped) { + const base = `Search for ${sentenceCase(searchQuery)}`; + return app ? qualifyTitleWithApp(base, metadata.appName) : base; + } + // User is clicking something on the search results page — don't claim they searched. + const base = `Select a ${sentenceCase(searchQuery)} result`; return app ? qualifyTitleWithApp(base, metadata.appName) : base; } const windowPhrase = pickBestTitleFragment(strippedWindowTitle, { source: 'window', metadata }); diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 4c7a7ce..86cd3ff 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -102,18 +102,26 @@ test('browser window title strips browser name and falls back to page title', () assert.ok(title.toLowerCase().includes('oracle') || title.toLowerCase().includes('cloud'), `Expected oracle/cloud in title, got: ${title}`); }); -test('search query is extracted from browser window title pattern', () => { +test('search query is extracted when user was typing (search step)', () => { const title = buildCaptureTitle({ mode: 'fullscreen', - metadata: { - windowTitle: 'oracle - Google Search - Google Chrome', - appName: 'chrome', - }, + metadata: { windowTitle: 'oracle - Google Search - Google Chrome', appName: 'chrome' }, ocrText: '', + recentTyped: 'oracle', // user was typing → this IS the search step }); assert.equal(title, 'Search for Oracle in Chrome'); }); +test('search results window title produces select-result title when no typing (click on results page)', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { windowTitle: 'oracle - Google Search - Google Chrome', appName: 'chrome' }, + ocrText: '', + recentTyped: '', // no recent typing → user is clicking a result, not searching + }); + assert.equal(title, 'Select a Oracle result in Chrome'); +}); + test('full link text with pipe separator is preserved in OCR phrases', () => { const title = buildCaptureTitle({ mode: 'fullscreen', From 3fa366a0d0db1e9153cbd2afb9fbc9bcc94aaa18 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 11:14:49 -0500 Subject: [PATCH 14/19] =?UTF-8?q?revert=20UIAutomation=20background=20thre?= =?UTF-8?q?ad=20=E2=80=94=20it=20stalled=20click=20detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TryLoadUia() blocked for several seconds loading reflection assemblies at ClickProcessorLoop startup, during which no CLICK events were emitted. Restored MouseHookCallback to emit CTX + CLICK synchronously (fast Win32 only) as before — no startup cost, clicks are never delayed. The wider OCR crop and smarter search-results title fallback are kept. UIAutomation element lookup can be revisited with a pre-warmed approach. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 127 ++++--------------------------------------------- 1 file changed, 10 insertions(+), 117 deletions(-) diff --git a/app/capture.js b/app/capture.js index b1114fc..eebe825 100644 --- a/app/capture.js +++ b/app/capture.js @@ -831,99 +831,6 @@ public static class SFHook { private static readonly ConcurrentQueue queue = new ConcurrentQueue(); private static readonly AutoResetEvent signal = new AutoResetEvent(false); - // Pending click queue for background UIAutomation enrichment. - private struct PendingClick { - public int x, y; - public string button; - public long ts; - public string wTitle; - public string wApp; - } - private static readonly ConcurrentQueue pendingClicks = new ConcurrentQueue(); - private static readonly AutoResetEvent clickSignal = new AutoResetEvent(false); - - // UIAutomation state — loaded once via reflection so the C# code has no - // compile-time assembly reference (avoids breaking the whole hook if UIA is absent). - private static bool uiaLoadAttempted = false; - private static System.Reflection.MethodInfo uiaFromPoint = null; - private static System.Reflection.PropertyInfo uiaCurrent = null; - private static System.Type uiaPointType = null; - private static System.Reflection.Assembly uiaClientAssembly = null; - - private static void TryLoadUia() { - if (uiaLoadAttempted) return; - uiaLoadAttempted = true; - try { - uiaClientAssembly = System.Reflection.Assembly.Load("UIAutomationClient"); - var wbase = System.Reflection.Assembly.Load("WindowsBase"); - var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement"); - uiaFromPoint = aeType.GetMethod("FromPoint"); - uiaCurrent = aeType.GetProperty("Current"); - uiaPointType = wbase.GetType("System.Windows.Point"); - } catch { } - } - - private static void GetUiaElement(int x, int y, out string label, out string role, out string value) { - label = ""; role = ""; value = ""; - if (uiaFromPoint == null || uiaPointType == null) return; - try { - var pt = Activator.CreateInstance(uiaPointType, (double)x, (double)y); - var element = uiaFromPoint.Invoke(null, new object[] { pt }); - if (element == null) return; - var current = uiaCurrent.GetValue(element); - var ct = current.GetType(); - label = ct.GetProperty("Name")?.GetValue(current) as string ?? ""; - role = ct.GetProperty("LocalizedControlType")?.GetValue(current) as string ?? ""; - // Try to read the element's current value (text box contents etc.). - try { - var vpType = uiaClientAssembly.GetType("System.Windows.Automation.ValuePattern"); - var vpPatternId = vpType?.GetField("Pattern")?.GetValue(null); - if (vpPatternId != null) { - var aeType = uiaClientAssembly.GetType("System.Windows.Automation.AutomationElement"); - var tryGet = aeType.GetMethod("TryGetCurrentPattern", - new System.Type[] { vpPatternId.GetType(), typeof(object).MakeByRefType() }); - if (tryGet != null) { - object[] args = new object[] { vpPatternId, null }; - if ((bool)tryGet.Invoke(element, args) && args[1] != null) { - var curr = args[1].GetType().GetProperty("Current")?.GetValue(args[1]); - value = curr?.GetType().GetProperty("Value")?.GetValue(curr) as string ?? ""; - } - } - } - } catch { } - } catch { } - } - - // Background thread: enriches each click with UIAutomation element info, then - // emits CTX + ELEM (optional) + CLICK as an atomic batch. - private static void ClickProcessorLoop() { - TryLoadUia(); - while (true) { - clickSignal.WaitOne(); - PendingClick click; - while (pendingClicks.TryDequeue(out click)) { - string label = "", role = "", val = ""; - // Run UIAutomation on a separate thread so we can impose a timeout. - // 300 ms is enough for most apps; if UIA is slow we skip element enrichment - // but still emit the click (no steps are lost). - string[] r = new string[] { "", "", "" }; - var t = new Thread(() => { - GetUiaElement(click.x, click.y, out r[0], out r[1], out r[2]); - }); - t.IsBackground = true; - t.Start(); - if (t.Join(300)) { label = r[0]; role = r[1]; val = r[2]; } - // Emit window context (always), element info (when available), then click. - queue.Enqueue("CTX " + B64(click.wTitle) + " " + B64(click.wApp) + " " + click.ts); - if (label.Length > 0 || role.Length > 0) { - queue.Enqueue("ELEM " + B64(label) + " " + B64(role) + " " + B64(val) + " " + click.ts); - } - queue.Enqueue("CLICK " + click.x + " " + click.y + " " + click.button + " " + click.ts); - signal.Set(); - } - } - } - [StructLayout(LayoutKind.Sequential)] private struct POINT { public int x; @@ -1090,10 +997,6 @@ public static class SFHook { writer.IsBackground = true; writer.Start(); - Thread clickProcessor = new Thread(ClickProcessorLoop); - clickProcessor.IsBackground = true; - clickProcessor.Start(); - hook = SetWindowsHookEx(WH_MOUSE_LL, proc, GetModuleHandle(null), 0); if (hook == IntPtr.Zero) { throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); @@ -1132,12 +1035,14 @@ public static class SFHook { if (button != null) { MSLLHOOKSTRUCT data = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT)); long unixMs = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond - UnixEpochMilliseconds; - // Capture window title synchronously (fast Win32, safe in hook callback). - // UIAutomation element lookup runs on the background ClickProcessorLoop thread. - string wTitle = "", wApp = ""; - try { wTitle = GetFwTitle(); wApp = GetFwApp(); } catch { } - pendingClicks.Enqueue(new PendingClick { x = data.pt.x, y = data.pt.y, button = button, ts = unixMs, wTitle = wTitle, wApp = wApp }); - clickSignal.Set(); + // Capture window context while the user's app is still in foreground. + // GetForegroundWindow is synchronous and fast (no IPC overhead). + try { + string t = GetFwTitle(), a = GetFwApp(); + queue.Enqueue("CTX " + B64(t) + " " + B64(a) + " " + unixMs); + } catch { } + queue.Enqueue("CLICK " + data.pt.x + " " + data.pt.y + " " + button + " " + unixMs); + signal.Set(); } } return CallNextHookEx(hook, nCode, wParam, lParam); @@ -1378,28 +1283,16 @@ public static class SFHook { this.onKeyboardEvent('CHAR', Number(charM[1]), Number(charM[2])); continue; } - // CTX always arrives just before its paired ELEM+CLICK from the same background thread batch. - const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed); + // CTX is emitted just before its paired CLICK from MouseHookCallback. + const ctxM = /^CTX\s+(\S+)\s+(\S+)\s+\d+$/.exec(trimmed); if (ctxM) { const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8'); this._pendingWindowContext = { windowTitle: decode(ctxM[1]), appName: decode(ctxM[2]), - ts: Number(ctxM[3]), }; continue; } - // ELEM carries UIAutomation element info for the same click as the preceding CTX. - const elemM = /^ELEM\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)$/.exec(trimmed); - if (elemM) { - const decode = s => s === '-' ? '' : Buffer.from(s, 'base64').toString('utf8'); - if (this._pendingWindowContext && this._pendingWindowContext.ts === Number(elemM[4])) { - this._pendingWindowContext.elementLabel = decode(elemM[1]); - this._pendingWindowContext.elementRole = decode(elemM[2]); - this._pendingWindowContext.elementValue = decode(elemM[3]); - } - continue; - } } } } From cf056d2b972764d9bc5b1e597135d3cb9da57338 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 11:18:41 -0500 Subject: [PATCH 15/19] stop AI from generating text/code/table blocks on generate-all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The model was producing Text and Code blocks on every 'generate all fields' action — uninstructed, generic, and never useful for the user. Blocks are only appropriate when the user explicitly edits an existing block (target === 'block'). For title/description/all generation: - Schema collapses to { "title", "description" } — no blocks key - targetText no longer mentions blocks - allowedBlockNote is null (omitted from prompt) - Rule explicitly says "Do NOT add any blocks array" Co-Authored-By: Claude Sonnet 4.6 --- core/text-intel.js | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/core/text-intel.js b/core/text-intel.js index 5fe0a42..90ac831 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -685,17 +685,17 @@ function buildAiPrompt({ ? 'improve the user\'s draft description — keep their intent, make it read like professional documentation' : 'write a 1–2 sentence description of what the user does in this step, using the capture context', block: 'rewrite only the target block', - all: 'write the step title, description, and any useful blocks from the capture context', + all: 'write the step title and description from the capture context', }[target] || 'rewrite the step'; const richContext = hasRichCaptureContext(captureContext); - const allowedBlockNote = [ + const allowedBlockNote = target === 'block' ? [ 'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.', 'Use block.kind = "code" for code snippets.', 'Use block.kind = "table" for tables, with rows as arrays of strings.', 'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].', - ].join(' '); + ].join(' ') : null; // When the user already has a draft, surface it prominently so the model // knows exactly what text to polish rather than generating from scratch. @@ -725,20 +725,22 @@ function buildAiPrompt({ 'You write concise, action-focused step-by-step documentation for a desktop application guide.', 'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.', 'Schema:', - '{', - ' "title": string,', - ' "description": string,', - ' "blocks": [{', - ' "kind": "text" | "code" | "table",', - ' "position"?: "before-title" | "after-title" | "before-image" | "after-image" | "before-description" | "after-description",', - ' "level"?: "info" | "warn" | "error" | "success",', - ' "title"?: string,', - ' "body"?: string,', - ' "language"?: string,', - ' "code"?: string,', - ' "rows"?: string[][]', - ' }]', - '}', + target === 'block' ? [ + '{', + ' "title": string,', + ' "description": string,', + ' "blocks": [{', + ' "kind": "text" | "code" | "table",', + ' "position"?: "before-title" | "after-title" | "before-image" | "after-image" | "before-description" | "after-description",', + ' "level"?: "info" | "warn" | "error" | "success",', + ' "title"?: string,', + ' "body"?: string,', + ' "language"?: string,', + ' "code"?: string,', + ' "rows"?: string[][]', + ' }]', + '}', + ].join('\n') : '{ "title": string, "description": string }', '', `Target: ${targetText}.`, allowedBlockNote, @@ -762,7 +764,9 @@ function buildAiPrompt({ hasDraftDesc && (target === 'description' || target === 'all') ? '- The user wrote their own description (shown above). Polish the wording to sound professional but preserve every fact and intent they stated.' : '- No description yet. Write 1–2 sentences describing exactly what the user does.', - '- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).', + target === 'block' + ? '- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).' + : '- Do NOT add any blocks array. Only output "title" and "description".', richContext ? '- Use the OCR text, window title, app name, and element info to make the documentation specific.' : '- Context is limited. Use the app name or window title if available; generate a reasonable action title.', From c60b7247af706da90d25ef997e3e72f124907a6f Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 11:22:27 -0500 Subject: [PATCH 16/19] generate all text fields with AI now fills every step, not just current MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the menu action only ran AI generation on the selected step. Now it loops through all steps sequentially, showing progress ("AI: filling step N of M…"), then reloads the visible step when done. Partial failures are reported in the final toast. Co-Authored-By: Claude Sonnet 4.6 --- app/renderer/editor.js | 44 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/app/renderer/editor.js b/app/renderer/editor.js index fd3e4a8..0f0e28b 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -219,7 +219,49 @@ class GuideEditor { } async generateAllTextFieldsWithAi(button = null) { - return this.runAiGeneration('all', { button }); + if (!this.steps.length) { + this.onToast('No steps to generate.', { error: true }); + return; + } + if (!this.isAiEnabled()) { + this.onToast('Enable AI in Settings first.', { error: true }); + return; + } + if (this.pendingSave) await this.flushStep(); + if (button) setButtonLoading(button, true, 'Generating…'); + let done = 0; + let failed = 0; + const total = this.steps.length; + try { + for (const step of this.steps) { + this.onToast(`AI: filling step ${done + 1} of ${total}…`); + try { + const result = await api.ai.fillStep({ + guideId: this.guideId, + stepId: step.stepId, + target: 'all', + }); + if (result?.ok) { + done++; + // Keep the in-memory steps list fresh so subsequent steps see updated guide context. + const idx = this.steps.findIndex((s) => s.stepId === step.stepId); + if (idx >= 0) this.steps[idx] = result.step; + } else { + failed++; + } + } catch { + failed++; + } + } + // Reload the currently-visible step so the editor reflects its new text. + if (this.selectedStepId) await this.reload(this.selectedStepId); + const msg = failed + ? `AI filled ${done} step${done === 1 ? '' : 's'} (${failed} failed).` + : `AI filled all ${done} step${done === 1 ? '' : 's'}.`; + this.onToast(msg, failed ? { error: true } : undefined); + } finally { + if (button) setButtonLoading(button, false); + } } async generateBlockWithAi(kind, block, button = null) { From 2d1b474931a8070f1634b15df421cf9ea95cdcf6 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 11:27:24 -0500 Subject: [PATCH 17/19] fix generate-all-steps: never overwrite existing content, right target per step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous version called target:'all' on every step unconditionally, which caused the AI to rewrite steps that already had good titles and descriptions — often making them worse or inaccurate. New behaviour: - Build a queue of only the steps that are actually missing content (placeholder title and/or empty description). - Determine target per step: 'title', 'description', or 'all' — only fill what is blank; leave existing user-written text completely alone. - If all steps already have titles and descriptions, show a toast and stop. - Call reload() once at the end instead of patching this.steps mid-loop, so the editor and step list both update atomically from the store. Co-Authored-By: Claude Sonnet 4.6 --- app/renderer/editor.js | 50 ++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 0f0e28b..420425b 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -228,36 +228,48 @@ class GuideEditor { return; } if (this.pendingSave) await this.flushStep(); + + // Only fill fields that are actually empty — never overwrite user-written content. + const PLACEHOLDER_TITLES = new Set([ + '', 'screen capture', 'window capture', 'region capture', 'capture', 'untitled step', + ]); + const isEmptyDesc = (html) => !(html || '').replace(/<[^>]*>/g, '').trim(); + + const queue = this.steps + .map((step) => { + const titleEmpty = !step.title || PLACEHOLDER_TITLES.has(step.title.toLowerCase()); + const descEmpty = isEmptyDesc(step.descriptionHtml); + if (!titleEmpty && !descEmpty) return null; + const target = titleEmpty && descEmpty ? 'all' : titleEmpty ? 'title' : 'description'; + return { stepId: step.stepId, target }; + }) + .filter(Boolean); + + if (!queue.length) { + this.onToast('All steps already have titles and descriptions.'); + return; + } + if (button) setButtonLoading(button, true, 'Generating…'); let done = 0; let failed = 0; - const total = this.steps.length; + const total = queue.length; try { - for (const step of this.steps) { - this.onToast(`AI: filling step ${done + 1} of ${total}…`); + for (const { stepId, target } of queue) { + this.onToast(`AI: filling step ${done + failed + 1} of ${total}…`); try { - const result = await api.ai.fillStep({ - guideId: this.guideId, - stepId: step.stepId, - target: 'all', - }); - if (result?.ok) { - done++; - // Keep the in-memory steps list fresh so subsequent steps see updated guide context. - const idx = this.steps.findIndex((s) => s.stepId === step.stepId); - if (idx >= 0) this.steps[idx] = result.step; - } else { - failed++; - } + const result = await api.ai.fillStep({ guideId: this.guideId, stepId, target }); + if (result?.ok) done++; + else failed++; } catch { failed++; } } - // Reload the currently-visible step so the editor reflects its new text. - if (this.selectedStepId) await this.reload(this.selectedStepId); + // Reload re-fetches all steps from the store so the editor and list both reflect the new text. + await this.reload(this.selectedStepId); const msg = failed ? `AI filled ${done} step${done === 1 ? '' : 's'} (${failed} failed).` - : `AI filled all ${done} step${done === 1 ? '' : 's'}.`; + : `AI filled ${done} step${done === 1 ? '' : 's'}.`; this.onToast(msg, failed ? { error: true } : undefined); } finally { if (button) setButtonLoading(button, false); From f60cf1a688223cba0f10f27cd994aec1065d9c4c Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 12:02:59 -0500 Subject: [PATCH 18/19] label generate-all-steps menu item as experimental Co-Authored-By: Claude Sonnet 4.6 --- app/renderer/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/renderer/app.js b/app/renderer/app.js index bf46b59..cba9994 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -333,7 +333,7 @@ class StepForgeApp { { label: 'Guide information…', action: () => this.editor.openGuideInfo() }, { label: 'Guide placeholders…', action: () => this.editor.openGuidePlaceholders() }, { label: 'Backups & snapshots…', action: () => this.editor.openBackupsDialog() }, - { label: 'Generate all text fields with AI', action: () => this.editor.generateAllTextFieldsWithAi() }, + { label: 'Generate all text fields with AI (experimental)', action: () => this.editor.generateAllTextFieldsWithAi() }, { label: guide && guide.linkedSource ? 'Linked guide…' : 'Linked guide (not linked)', action: () => this.editor.openLinkedGuide() }, 'sep', { label: 'Keyboard shortcuts…', action: () => this.editor.openShortcutsHelp() }, From ebeb6ac57837e668674bf2b8145a7562a832de32 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 12:04:40 -0500 Subject: [PATCH 19/19] label Enable AI setting as experimental Co-Authored-By: Claude Sonnet 4.6 --- app/renderer/dialogs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index 4f42f06..e92181a 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -409,7 +409,7 @@ function showSettingsDialog({ ), el('fieldset', {}, el('legend', {}, 'AI'), - labeledRow('Enable AI', aiEnabled), + labeledRow('Enable AI (experimental)', aiEnabled), labeledRow('Auto-document captures', aiAutoDoc), labeledRow('Ollama host', ollamaHost), labeledRow('Ollama model', ollamaModel),