From d2446265836fdd039020e20023de5501abaeb5a7 Mon Sep 17 00:00:00 2001 From: Twest2 Date: Wed, 24 Jun 2026 09:17:31 -0500 Subject: [PATCH] ai smart auto-documentation and text rewrite - Store captureMetadata (OCR text, window/app/element info) with each step at capture time so AI always has the original rich context. - Add buildCaptureContext() to TextIntelService; capture.js uses it instead of buildCaptureTitle() so both title and metadata come from one pass. - generateStepPatch() prefers stored captureMetadata over re-running OCR, giving the AI the best possible context when the user clicks an AI button later. - Add autoDoc setting: when enabled every capture (shoot, region, and session hotkey/click) is automatically documented by AI. Manual captures await AI before returning; session captures fire-and-forget and push a step:updated event so the renderer reloads seamlessly. - Add ai:rewriteText IPC and rewriteText() method for plain-text polishing via a separate callOllamaText() that skips JSON mode. - Add "AI Rewrite" section in the editor right panel: textarea + AI button that rewrites whatever the user types in place. - Improve buildAiPrompt() rules: action-focused title instructions, explicit anti-junk rules (no "Capture the screen / OCR" blocks), and a context-quality gate that suppresses blocks when context is thin. - Add autoDoc checkbox to AI settings dialog. - Renderer handles step:updated to reload the selected step after background auto-doc finishes. Co-Authored-By: Claude Sonnet 4.6 --- app/capture.js | 14 ++--- app/main.js | 52 ++++++++++++++++++- app/preload.js | 2 + app/renderer/app.js | 12 +++++ app/renderer/dialogs.js | 7 ++- app/renderer/editor.js | 48 +++++++++++++++++ app/text-intel.js | 98 ++++++++++++++++++++++++++++++++--- core/schema.js | 3 ++ core/text-intel.js | 49 ++++++++++++------ tests/unit/text-intel.test.js | 2 +- 10 files changed, 255 insertions(+), 32 deletions(-) diff --git a/app/capture.js b/app/capture.js index aefbe36..db918a8 100644 --- a/app/capture.js +++ b/app/capture.js @@ -1316,15 +1316,15 @@ public static class SFMouseHook { }; } - async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) { + async buildStepMeta(mode, frame, clickPos = null, clickMeta = null) { try { - if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') { - return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta }); + if (this.textIntel && typeof this.textIntel.buildCaptureContext === 'function') { + return await this.textIntel.buildCaptureContext({ mode, frame, clickPos, clickMeta }); } } catch { - // fall back to the local semantic title below + // fall back } - return this.autoTitle(mode); + return { title: this.autoTitle(mode), captureMetadata: null }; } async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) { @@ -1350,8 +1350,10 @@ public static class SFMouseHook { } } + const { title, captureMetadata } = await this.buildStepMeta(mode, frame, clickPos, clickMeta); const step = this.store.addStep(guideId, { - title: await this.buildStepTitle(mode, frame, clickPos, clickMeta), + title, + captureMetadata, annotations, focusedView: { enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')), diff --git a/app/main.js b/app/main.js index 6a87840..2e52984 100644 --- a/app/main.js +++ b/app/main.js @@ -521,18 +521,49 @@ function setupIpc() { if (result.ok) reindex(guideId); return result; }); + h('ai:rewriteText', async ({ text, guideTitle = '', stepTitle = '' } = {}) => { + return textIntel.rewriteText({ text, guideTitle, stepTitle }); + }); h('placeholders:globals:get', () => settings.getGlobalPlaceholders()); h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values)); // capture h('capture:shoot', async ({ guideId, mode, delayMs }) => { const result = await capture.shoot({ guideId, mode, delayMs }); - if (result.ok) reindex(guideId); + if (result.ok) { + reindex(guideId); + const aiConf = settings.get('ai') || {}; + if (aiConf.enabled && aiConf.autoDoc && result.step) { + const aiResult = await textIntel.generateStepPatch({ + guideId, + stepId: result.step.stepId, + target: 'all', + }).catch(() => null); + if (aiResult?.ok) { + reindex(guideId); + result.step = aiResult.step; + } + } + } return result; }); h('capture:region', async ({ guideId }) => { const result = await capture.regionCapture(guideId); - if (result.ok) reindex(guideId); + if (result.ok) { + reindex(guideId); + const aiConf = settings.get('ai') || {}; + if (aiConf.enabled && aiConf.autoDoc && result.step) { + const aiResult = await textIntel.generateStepPatch({ + guideId, + stepId: result.step.stepId, + target: 'all', + }).catch(() => null); + if (aiResult?.ok) { + reindex(guideId); + result.step = aiResult.step; + } + } + } return result; }); let capturePowerBlocker = -1; @@ -777,6 +808,23 @@ if (!gotLock) { try { keepProcessesResponsive(app.getAppMetrics().map((m) => m.pid)); } catch { /* best effort */ } } } + // Auto-document session captures in the background when autoDoc is enabled. + // Single-shot captures (capture:shoot) are handled synchronously in the IPC handler. + if (channel === 'capture:added' && payload?.step && payload?.guideId) { + const aiConf = settings.get('ai') || {}; + if (aiConf.enabled && aiConf.autoDoc) { + textIntel.generateStepPatch({ + guideId: payload.guideId, + stepId: payload.step.stepId, + target: 'all', + }).then((result) => { + if (result.ok) { + reindex(payload.guideId); + sendToRenderer('step:updated', { guideId: payload.guideId, step: result.step }); + } + }).catch(() => {}); + } + } }; capture = new CaptureService({ diff --git a/app/preload.js b/app/preload.js index 71f60b8..2f48a7e 100644 --- a/app/preload.js +++ b/app/preload.js @@ -55,6 +55,7 @@ const api = { ai: { test: invoke('ai:test'), fillStep: invoke('ai:fillStep'), + rewriteText: invoke('ai:rewriteText'), }, capture: { shoot: invoke('capture:shoot'), @@ -63,6 +64,7 @@ const api = { state: invoke('capture:state'), onAdded: (fn) => ipcRenderer.on('capture:added', (e, payload) => fn(payload)), onState: (fn) => ipcRenderer.on('capture:state', (e, payload) => fn(payload)), + onStepUpdated: (fn) => ipcRenderer.on('step:updated', (e, payload) => fn(payload)), }, archive: { export: invoke('archive:export'), diff --git a/app/renderer/app.js b/app/renderer/app.js index 619d041..bf46b59 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -80,6 +80,18 @@ class StepForgeApp { api.capture.onAdded((payload) => this.onCaptureAdded(payload)); api.capture.onState((payload) => this.updateCaptureState(payload)); + api.capture.onStepUpdated((payload) => this.onStepUpdated(payload)); + } + + async onStepUpdated(payload) { + if (!payload || !payload.guideId || !payload.step) return; + if (this.state.view === 'editor' && this.editor.guideId === payload.guideId) { + const currentStep = this.editor.currentStep; + if (currentStep && currentStep.stepId === payload.step.stepId) { + await this.editor.reload(payload.step.stepId); + toast('Documentation generated.'); + } + } } async onCaptureAdded(payload) { diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index 16cf2c9..4f42f06 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -315,6 +315,7 @@ function showSettingsDialog({ const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) }); const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 }); const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) }); + const aiAutoDoc = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.autoDoc) }); const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434'); const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b'); const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.'); @@ -408,7 +409,8 @@ function showSettingsDialog({ ), el('fieldset', {}, el('legend', {}, 'AI'), - labeledRow('Enable AI text filling', aiEnabled), + labeledRow('Enable AI', aiEnabled), + labeledRow('Auto-document captures', aiAutoDoc), labeledRow('Ollama host', ollamaHost), labeledRow('Ollama model', ollamaModel), el('div.row', { style: { justifyContent: 'space-between' } }, @@ -416,7 +418,7 @@ function showSettingsDialog({ testAiBtn, ), el('div.muted', {}, - 'AI generation is manual only. Captures stay deterministic until you click a generate button.', + 'When auto-document is on, each capture is automatically documented by AI. Turn it off to use AI manually only.', ), ), el('fieldset', {}, @@ -465,6 +467,7 @@ function showSettingsDialog({ ai: { ...settings.ai, enabled: aiEnabled.checked, + autoDoc: aiAutoDoc.checked, ollama: { ...(settings.ai?.ollama || {}), host: ollamaHost.value.trim(), diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 4825afd..6db5c4d 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -167,6 +167,7 @@ class GuideEditor { const buttons = [ this.dom?.titleAiBtn, this.dom?.descAiBtn, + this.dom?.rewriteAiBtn, ...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []), ].filter(Boolean); for (const button of buttons) { @@ -222,6 +223,36 @@ class GuideEditor { return this.runAiGeneration('all', { button }); } + async runTextRewrite(button = null) { + if (!this.isAiEnabled()) { + this.onToast('Enable AI in Settings first.', { error: true }); + return; + } + const text = this.dom?.rewriteInput?.value?.trim(); + if (!text) { + this.onToast('Type some text to rewrite first.', { error: true }); + return; + } + if (button) setButtonLoading(button, true, 'Rewriting…'); + try { + const result = await api.ai.rewriteText({ + text, + guideTitle: this.guide?.title || '', + stepTitle: this.currentStep?.title || '', + }); + if (!result || !result.ok) { + this.onToast(result?.reason || 'Rewrite failed.', { error: true }); + return; + } + if (this.dom?.rewriteInput) this.dom.rewriteInput.value = result.text; + this.onToast('Text rewritten.'); + } catch (err) { + this.onToast(err.message || 'Rewrite failed.', { error: true }); + } finally { + if (button) setButtonLoading(button, false); + } + } + async generateBlockWithAi(kind, block, button = null) { if (!block) return null; return this.runAiGeneration('block', { blockId: block.id, button }); @@ -411,6 +442,21 @@ class GuideEditor { this.dom.addTableBlockBtn = el('button', { type: 'button' }, '+ Table'), ), ), + el('section', {}, + el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } }, + el('h3', { style: { margin: 0 } }, 'AI Rewrite'), + this.dom.rewriteAiBtn = el('button.ai', { + type: 'button', + title: 'Rewrite the text below with AI', + dataset: { aiAction: 'rewrite', aiTitle: 'Rewrite with AI' }, + }, 'Rewrite'), + ), + this.dom.rewriteInput = el('textarea', { + rows: 3, + placeholder: 'Type something to improve…', + style: { width: '100%', resize: 'vertical' }, + }), + ), el('section', {}, el('h3', {}, 'Guide'), this.dom.guideSummary = el('div.muted', {}), @@ -583,6 +629,8 @@ class GuideEditor { }); this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn)); + this.dom.rewriteAiBtn.addEventListener('click', () => this.runTextRewrite(this.dom.rewriteAiBtn)); + this.dom.descEditor.addEventListener('paste', (e) => { // Keep pasted text simple; backend sanitization will handle the rest. e.preventDefault(); diff --git a/app/text-intel.js b/app/text-intel.js index 4fd6d74..4ef136f 100644 --- a/app/text-intel.js +++ b/app/text-intel.js @@ -276,15 +276,27 @@ public static class Win32 { } async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) { + const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta }); + return ctx.title; + } + + async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) { const [metadata, ocr] = await Promise.all([ this.collectForegroundWindowContext(clickMeta?.osPoint || null), this.ocrAroundClick(frame, clickPos), ]); - return buildCaptureTitle({ - mode, - metadata, - ocrText: ocr.text, - }); + const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text }); + return { + title, + captureMetadata: { + ocrText: ocr.text || '', + windowTitle: metadata.windowTitle || '', + appName: metadata.appName || '', + elementLabel: metadata.elementLabel || '', + elementRole: metadata.elementRole || '', + mode, + }, + }; } aiEnabled() { @@ -335,6 +347,28 @@ public static class Win32 { }; } + async callOllamaText({ host, model, prompt, systemPrompt }) { + const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); + const response = await this.fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + stream: false, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: prompt }, + ], + options: { temperature: 0.4 }, + }), + }); + if (!response.ok) throw new Error(`Ollama request failed (${response.status})`); + const payload = await response.json(); + const content = payload?.message?.content; + if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response'); + return content.trim(); + } + async callOllama({ host, model, prompt, systemPrompt }) { const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); const response = await this.fetch(url, { @@ -393,7 +427,23 @@ public static class Win32 { } let captureContext = null; - if (step.image) { + // Use stored capture metadata when available (best context, from capture time). + // Fall back to re-running OCR on the stored image only when metadata is absent. + if (step.captureMetadata) { + captureContext = { + ...step.captureMetadata, + titleCandidate: buildCaptureTitle({ + mode: step.captureMetadata.mode || 'fullscreen', + metadata: { + windowTitle: step.captureMetadata.windowTitle, + appName: step.captureMetadata.appName, + elementLabel: step.captureMetadata.elementLabel, + elementRole: step.captureMetadata.elementRole, + }, + ocrText: step.captureMetadata.ocrText, + }), + }; + } else if (step.image) { const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original'); if (imagePath && fs.existsSync(imagePath)) { const { nativeImage } = require('electron'); @@ -442,6 +492,42 @@ public static class Win32 { } } + async rewriteText({ text, guideTitle = '', stepTitle = '' }) { + try { + const config = this.aiConfig(); + if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' }; + if (!config.ollama.host || !config.ollama.model) { + return { ok: false, reason: 'Configure Ollama host and model in Settings.' }; + } + const trimmed = normalizeWhitespace(text); + if (!trimmed) return { ok: false, reason: 'No text to rewrite.' }; + + const contextHint = [ + guideTitle ? `Guide: ${guideTitle}` : '', + stepTitle ? `Step: ${stepTitle}` : '', + ].filter(Boolean).join('\n'); + + const prompt = [ + contextHint, + contextHint ? '' : null, + 'Rewrite the following text to sound professional and clear as step-by-step documentation.', + 'Keep it concise. Do not add extra information. Return only the rewritten text.', + '', + trimmed, + ].filter((l) => l !== null).join('\n'); + + const result = await this.callOllamaText({ + host: config.ollama.host, + model: config.ollama.model, + prompt, + systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.', + }); + return { ok: true, text: result }; + } catch (err) { + return { ok: false, reason: err?.message || 'Rewrite failed.' }; + } + } + clickPointFromStep(step, image = null) { const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h)); if (!marker) return null; diff --git a/core/schema.js b/core/schema.js index 88e2b51..7bbd122 100644 --- a/core/schema.js +++ b/core/schema.js @@ -86,6 +86,9 @@ function createStep(fields = {}) { codeBlocks: (fields.codeBlocks || []).map((cb) => normalizeCodeBlock(cb, takeOrder(cb))), tableBlocks: (fields.tableBlocks || []).map((tb) => normalizeTableBlock(tb, takeOrder(tb))), links: fields.links || [], // { id, label, targetStepId } + captureMetadata: (fields.captureMetadata && typeof fields.captureMetadata === 'object' && !Array.isArray(fields.captureMetadata)) + ? { ...fields.captureMetadata } + : null, }; } diff --git a/core/text-intel.js b/core/text-intel.js index c539815..823cacf 100644 --- a/core/text-intel.js +++ b/core/text-intel.js @@ -402,6 +402,14 @@ function summarizeGuideForAi(guide = {}) { ].join('\n'); } +function hasRichCaptureContext(captureContext) { + if (!captureContext) return false; + const ocr = normalizeWhitespace(captureContext.ocrText || ''); + const win = normalizeWhitespace(captureContext.windowTitle || ''); + const app = normalizeWhitespace(captureContext.appName || ''); + return ocr.length > 3 || (win.length > 2 && !isBrowserNoise(win)) || app.length > 1; +} + function buildAiPrompt({ target = 'all', guide = null, @@ -416,6 +424,8 @@ function buildAiPrompt({ all: 'rewrite the step title, description, and any useful blocks', }[target] || 'rewrite the step'; + const richContext = hasRichCaptureContext(captureContext); + const allowedBlockNote = [ 'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.', 'Use block.kind = "code" for code snippets.', @@ -423,8 +433,16 @@ function buildAiPrompt({ 'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].', ].join(' '); + const contextLines = captureContext ? [ + captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null, + captureContext.appName ? `App: ${captureContext.appName}` : null, + captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null, + captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null, + captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null, + ].filter(Boolean) : []; + const prompt = [ - 'You write concise, human-sounding step-by-step documentation for an offline desktop app.', + 'You write concise, action-focused step-by-step documentation for a desktop application guide.', 'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.', 'Schema:', '{', @@ -449,26 +467,27 @@ function buildAiPrompt({ '', step ? summarizeStepForAi(step) : 'Step: (not provided)', '', - captureContext ? [ - `Active window title: ${captureContext.windowTitle || '(unknown)'}`, - `Active app: ${captureContext.appName || '(unknown)'}`, - `OCR text near click: ${captureContext.ocrText || '(none)'}`, - `Deterministic title candidate: ${captureContext.titleCandidate || '(none)'}`, - `Capture mode: ${captureContext.mode || '(unknown)'}`, - ].join('\n') : 'Capture context: (not provided)', + contextLines.length + ? `Capture context:\n${contextLines.join('\n')}` + : 'Capture context: (not available)', '', - block ? `Target block:\n${JSON.stringify(block, null, 2)}` : 'Target block: (not provided)', + block ? `Target block:\n${JSON.stringify(block, null, 2)}` : null, '', 'Rules:', - '- Keep the wording natural and specific.', - '- Prefer short titles.', - '- Only return blocks that add value.', - '- Do not invent details that are not supported by the capture context.', + '- Write titles as short imperative actions: "Click Save", "Select New document", "Open Settings".', + '- If the suggested title is already a good imperative action, use it as-is or refine it slightly.', + '- Write descriptions in 1–2 sentences describing exactly what the user does in this step.', + '- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).', + richContext + ? '- You have rich context: use the OCR text, window title, and element info to write specific documentation.' + : '- Context is limited: write a minimal title. Leave description empty and return no blocks unless the step already has meaningful content.', + '- Do NOT generate blocks that describe the technical capture process or mention OCR.', + '- Do NOT invent details not supported by the capture context.', '- If the target is one block, only rewrite that block.', - ].join('\n'); + ].filter((l) => l !== null).join('\n'); return { - systemPrompt: 'You are a documentation assistant that only emits JSON.', + systemPrompt: 'You are a technical documentation writer. Emit only valid JSON matching the schema. Never add commentary or markdown.', prompt, }; } diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js index 5e75262..16bf024 100644 --- a/tests/unit/text-intel.test.js +++ b/tests/unit/text-intel.test.js @@ -97,7 +97,7 @@ test('ai prompts include the deterministic OCR-backed title candidate', () => { }, }); - assert.match(prompt, /Deterministic title candidate: Click New tab/); + assert.match(prompt, /Suggested title: Click New tab/); }); test('ocr crop rectangles clamp to the image bounds', (t) => {