ai smart auto-documentation and text rewrite

- Store captureMetadata (OCR text, window/app/element info) with each
  step at capture time so AI always has the original rich context.
- Add buildCaptureContext() to TextIntelService; capture.js uses it
  instead of buildCaptureTitle() so both title and metadata come from
  one pass.
- generateStepPatch() prefers stored captureMetadata over re-running
  OCR, giving the AI the best possible context when the user clicks
  an AI button later.
- Add autoDoc setting: when enabled every capture (shoot, region, and
  session hotkey/click) is automatically documented by AI. Manual
  captures await AI before returning; session captures fire-and-forget
  and push a step:updated event so the renderer reloads seamlessly.
- Add ai:rewriteText IPC and rewriteText() method for plain-text
  polishing via a separate callOllamaText() that skips JSON mode.
- Add "AI Rewrite" section in the editor right panel: textarea + AI
  button that rewrites whatever the user types in place.
- Improve buildAiPrompt() rules: action-focused title instructions,
  explicit anti-junk rules (no "Capture the screen / OCR" blocks),
  and a context-quality gate that suppresses blocks when context is
  thin.
- Add autoDoc checkbox to AI settings dialog.
- Renderer handles step:updated to reload the selected step after
  background auto-doc finishes.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>

app/capture.js

@@ -1316,15 +1316,15 @@ public static class SFMouseHook {
     };
   }
 
-  async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) {
+  async buildStepMeta(mode, frame, clickPos = null, clickMeta = null) {
     try {
-      if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') {
+      if (this.textIntel && typeof this.textIntel.buildCaptureContext === 'function') {
-        return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta });
+        return await this.textIntel.buildCaptureContext({ mode, frame, clickPos, clickMeta });
       }
     } catch {
-      // fall back to the local semantic title below
+      // fall back
     }
-    return this.autoTitle(mode);
+    return { title: this.autoTitle(mode), captureMetadata: null };
   }
 
   async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) {
@@ -1350,8 +1350,10 @@ public static class SFMouseHook {
       }
     }
 
+    const { title, captureMetadata } = await this.buildStepMeta(mode, frame, clickPos, clickMeta);
     const step = this.store.addStep(guideId, {
-      title: await this.buildStepTitle(mode, frame, clickPos, clickMeta),
+      title,
+      captureMetadata,
       annotations,
       focusedView: {
         enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),

app/main.js

@@ -521,18 +521,49 @@ function setupIpc() {
     if (result.ok) reindex(guideId);
     return result;
   });
+  h('ai:rewriteText', async ({ text, guideTitle = '', stepTitle = '' } = {}) => {
+    return textIntel.rewriteText({ text, guideTitle, stepTitle });
+  });
   h('placeholders:globals:get', () => settings.getGlobalPlaceholders());
   h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values));
 
   // capture
   h('capture:shoot', async ({ guideId, mode, delayMs }) => {
     const result = await capture.shoot({ guideId, mode, delayMs });
-    if (result.ok) reindex(guideId);
+    if (result.ok) {
+      reindex(guideId);
+      const aiConf = settings.get('ai') || {};
+      if (aiConf.enabled && aiConf.autoDoc && result.step) {
+        const aiResult = await textIntel.generateStepPatch({
+          guideId,
+          stepId: result.step.stepId,
+          target: 'all',
+        }).catch(() => null);
+        if (aiResult?.ok) {
+          reindex(guideId);
+          result.step = aiResult.step;
+        }
+      }
+    }
     return result;
   });
   h('capture:region', async ({ guideId }) => {
     const result = await capture.regionCapture(guideId);
-    if (result.ok) reindex(guideId);
+    if (result.ok) {
+      reindex(guideId);
+      const aiConf = settings.get('ai') || {};
+      if (aiConf.enabled && aiConf.autoDoc && result.step) {
+        const aiResult = await textIntel.generateStepPatch({
+          guideId,
+          stepId: result.step.stepId,
+          target: 'all',
+        }).catch(() => null);
+        if (aiResult?.ok) {
+          reindex(guideId);
+          result.step = aiResult.step;
+        }
+      }
+    }
     return result;
   });
   let capturePowerBlocker = -1;
@@ -777,6 +808,23 @@ if (!gotLock) {
           try { keepProcessesResponsive(app.getAppMetrics().map((m) => m.pid)); } catch { /* best effort */ }
         }
       }
+      // Auto-document session captures in the background when autoDoc is enabled.
+      // Single-shot captures (capture:shoot) are handled synchronously in the IPC handler.
+      if (channel === 'capture:added' && payload?.step && payload?.guideId) {
+        const aiConf = settings.get('ai') || {};
+        if (aiConf.enabled && aiConf.autoDoc) {
+          textIntel.generateStepPatch({
+            guideId: payload.guideId,
+            stepId: payload.step.stepId,
+            target: 'all',
+          }).then((result) => {
+            if (result.ok) {
+              reindex(payload.guideId);
+              sendToRenderer('step:updated', { guideId: payload.guideId, step: result.step });
+            }
+          }).catch(() => {});
+        }
+      }
     };
 
     capture = new CaptureService({

app/preload.js

@@ -55,6 +55,7 @@ const api = {
   ai: {
     test: invoke('ai:test'),
     fillStep: invoke('ai:fillStep'),
+    rewriteText: invoke('ai:rewriteText'),
   },
   capture: {
     shoot: invoke('capture:shoot'),
@@ -63,6 +64,7 @@ const api = {
     state: invoke('capture:state'),
     onAdded: (fn) => ipcRenderer.on('capture:added', (e, payload) => fn(payload)),
     onState: (fn) => ipcRenderer.on('capture:state', (e, payload) => fn(payload)),
+    onStepUpdated: (fn) => ipcRenderer.on('step:updated', (e, payload) => fn(payload)),
   },
   archive: {
     export: invoke('archive:export'),

app/renderer/app.js

@@ -80,6 +80,18 @@ class StepForgeApp {
 
     api.capture.onAdded((payload) => this.onCaptureAdded(payload));
     api.capture.onState((payload) => this.updateCaptureState(payload));
+    api.capture.onStepUpdated((payload) => this.onStepUpdated(payload));
+  }
+
+  async onStepUpdated(payload) {
+    if (!payload || !payload.guideId || !payload.step) return;
+    if (this.state.view === 'editor' && this.editor.guideId === payload.guideId) {
+      const currentStep = this.editor.currentStep;
+      if (currentStep && currentStep.stepId === payload.step.stepId) {
+        await this.editor.reload(payload.step.stepId);
+        toast('Documentation generated.');
+      }
+    }
   }
 
   async onCaptureAdded(payload) {

app/renderer/dialogs.js

@@ -315,6 +315,7 @@ function showSettingsDialog({
     const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) });
     const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 });
     const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) });
+    const aiAutoDoc = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.autoDoc) });
     const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434');
     const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b');
     const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.');
@@ -408,7 +409,8 @@ function showSettingsDialog({
       ),
       el('fieldset', {},
         el('legend', {}, 'AI'),
-        labeledRow('Enable AI text filling', aiEnabled),
+        labeledRow('Enable AI', aiEnabled),
+        labeledRow('Auto-document captures', aiAutoDoc),
         labeledRow('Ollama host', ollamaHost),
         labeledRow('Ollama model', ollamaModel),
         el('div.row', { style: { justifyContent: 'space-between' } },
@@ -416,7 +418,7 @@ function showSettingsDialog({
           testAiBtn,
         ),
         el('div.muted', {},
-          'AI generation is manual only. Captures stay deterministic until you click a generate button.',
+          'When auto-document is on, each capture is automatically documented by AI. Turn it off to use AI manually only.',
         ),
       ),
       el('fieldset', {},
@@ -465,6 +467,7 @@ function showSettingsDialog({
               ai: {
                 ...settings.ai,
                 enabled: aiEnabled.checked,
+                autoDoc: aiAutoDoc.checked,
                 ollama: {
                   ...(settings.ai?.ollama || {}),
                   host: ollamaHost.value.trim(),

app/renderer/editor.js

@@ -167,6 +167,7 @@ class GuideEditor {
     const buttons = [
       this.dom?.titleAiBtn,
       this.dom?.descAiBtn,
+      this.dom?.rewriteAiBtn,
       ...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []),
     ].filter(Boolean);
     for (const button of buttons) {
@@ -222,6 +223,36 @@ class GuideEditor {
     return this.runAiGeneration('all', { button });
   }
 
+  async runTextRewrite(button = null) {
+    if (!this.isAiEnabled()) {
+      this.onToast('Enable AI in Settings first.', { error: true });
+      return;
+    }
+    const text = this.dom?.rewriteInput?.value?.trim();
+    if (!text) {
+      this.onToast('Type some text to rewrite first.', { error: true });
+      return;
+    }
+    if (button) setButtonLoading(button, true, 'Rewriting…');
+    try {
+      const result = await api.ai.rewriteText({
+        text,
+        guideTitle: this.guide?.title || '',
+        stepTitle: this.currentStep?.title || '',
+      });
+      if (!result || !result.ok) {
+        this.onToast(result?.reason || 'Rewrite failed.', { error: true });
+        return;
+      }
+      if (this.dom?.rewriteInput) this.dom.rewriteInput.value = result.text;
+      this.onToast('Text rewritten.');
+    } catch (err) {
+      this.onToast(err.message || 'Rewrite failed.', { error: true });
+    } finally {
+      if (button) setButtonLoading(button, false);
+    }
+  }
+
   async generateBlockWithAi(kind, block, button = null) {
     if (!block) return null;
     return this.runAiGeneration('block', { blockId: block.id, button });
@@ -411,6 +442,21 @@ class GuideEditor {
             this.dom.addTableBlockBtn = el('button', { type: 'button' }, '+ Table'),
           ),
         ),
+        el('section', {},
+          el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } },
+            el('h3', { style: { margin: 0 } }, 'AI Rewrite'),
+            this.dom.rewriteAiBtn = el('button.ai', {
+              type: 'button',
+              title: 'Rewrite the text below with AI',
+              dataset: { aiAction: 'rewrite', aiTitle: 'Rewrite with AI' },
+            }, 'Rewrite'),
+          ),
+          this.dom.rewriteInput = el('textarea', {
+            rows: 3,
+            placeholder: 'Type something to improve…',
+            style: { width: '100%', resize: 'vertical' },
+          }),
+        ),
         el('section', {},
           el('h3', {}, 'Guide'),
           this.dom.guideSummary = el('div.muted', {}),
@@ -583,6 +629,8 @@ class GuideEditor {
     });
     this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn));
 
+    this.dom.rewriteAiBtn.addEventListener('click', () => this.runTextRewrite(this.dom.rewriteAiBtn));
+
     this.dom.descEditor.addEventListener('paste', (e) => {
       // Keep pasted text simple; backend sanitization will handle the rest.
       e.preventDefault();

app/text-intel.js

@@ -276,15 +276,27 @@ public static class Win32 {
   }
 
   async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
+    const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta });
+    return ctx.title;
+  }
+
+  async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
     const [metadata, ocr] = await Promise.all([
       this.collectForegroundWindowContext(clickMeta?.osPoint || null),
       this.ocrAroundClick(frame, clickPos),
     ]);
-    return buildCaptureTitle({
+    const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text });
+    return {
+      title,
+      captureMetadata: {
+        ocrText: ocr.text || '',
+        windowTitle: metadata.windowTitle || '',
+        appName: metadata.appName || '',
+        elementLabel: metadata.elementLabel || '',
+        elementRole: metadata.elementRole || '',
         mode,
-      metadata,
+      },
-      ocrText: ocr.text,
+    };
-    });
   }
 
   aiEnabled() {
@@ -335,6 +347,28 @@ public static class Win32 {
     };
   }
 
+  async callOllamaText({ host, model, prompt, systemPrompt }) {
+    const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
+    const response = await this.fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model,
+        stream: false,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: prompt },
+        ],
+        options: { temperature: 0.4 },
+      }),
+    });
+    if (!response.ok) throw new Error(`Ollama request failed (${response.status})`);
+    const payload = await response.json();
+    const content = payload?.message?.content;
+    if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response');
+    return content.trim();
+  }
+
   async callOllama({ host, model, prompt, systemPrompt }) {
     const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
     const response = await this.fetch(url, {
@@ -393,7 +427,23 @@ public static class Win32 {
       }
 
       let captureContext = null;
-      if (step.image) {
+      // Use stored capture metadata when available (best context, from capture time).
+      // Fall back to re-running OCR on the stored image only when metadata is absent.
+      if (step.captureMetadata) {
+        captureContext = {
+          ...step.captureMetadata,
+          titleCandidate: buildCaptureTitle({
+            mode: step.captureMetadata.mode || 'fullscreen',
+            metadata: {
+              windowTitle: step.captureMetadata.windowTitle,
+              appName: step.captureMetadata.appName,
+              elementLabel: step.captureMetadata.elementLabel,
+              elementRole: step.captureMetadata.elementRole,
+            },
+            ocrText: step.captureMetadata.ocrText,
+          }),
+        };
+      } else if (step.image) {
         const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
         if (imagePath && fs.existsSync(imagePath)) {
           const { nativeImage } = require('electron');
@@ -442,6 +492,42 @@ public static class Win32 {
     }
   }
 
+  async rewriteText({ text, guideTitle = '', stepTitle = '' }) {
+    try {
+      const config = this.aiConfig();
+      if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' };
+      if (!config.ollama.host || !config.ollama.model) {
+        return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
+      }
+      const trimmed = normalizeWhitespace(text);
+      if (!trimmed) return { ok: false, reason: 'No text to rewrite.' };
+
+      const contextHint = [
+        guideTitle ? `Guide: ${guideTitle}` : '',
+        stepTitle ? `Step: ${stepTitle}` : '',
+      ].filter(Boolean).join('\n');
+
+      const prompt = [
+        contextHint,
+        contextHint ? '' : null,
+        'Rewrite the following text to sound professional and clear as step-by-step documentation.',
+        'Keep it concise. Do not add extra information. Return only the rewritten text.',
+        '',
+        trimmed,
+      ].filter((l) => l !== null).join('\n');
+
+      const result = await this.callOllamaText({
+        host: config.ollama.host,
+        model: config.ollama.model,
+        prompt,
+        systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.',
+      });
+      return { ok: true, text: result };
+    } catch (err) {
+      return { ok: false, reason: err?.message || 'Rewrite failed.' };
+    }
+  }
+
   clickPointFromStep(step, image = null) {
     const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
     if (!marker) return null;

core/schema.js

@@ -86,6 +86,9 @@ function createStep(fields = {}) {
     codeBlocks: (fields.codeBlocks || []).map((cb) => normalizeCodeBlock(cb, takeOrder(cb))),
     tableBlocks: (fields.tableBlocks || []).map((tb) => normalizeTableBlock(tb, takeOrder(tb))),
     links: fields.links || [], // { id, label, targetStepId }
+    captureMetadata: (fields.captureMetadata && typeof fields.captureMetadata === 'object' && !Array.isArray(fields.captureMetadata))
+      ? { ...fields.captureMetadata }
+      : null,
   };
 }
 

core/text-intel.js

@@ -402,6 +402,14 @@ function summarizeGuideForAi(guide = {}) {
   ].join('\n');
 }
 
+function hasRichCaptureContext(captureContext) {
+  if (!captureContext) return false;
+  const ocr = normalizeWhitespace(captureContext.ocrText || '');
+  const win = normalizeWhitespace(captureContext.windowTitle || '');
+  const app = normalizeWhitespace(captureContext.appName || '');
+  return ocr.length > 3 || (win.length > 2 && !isBrowserNoise(win)) || app.length > 1;
+}
+
 function buildAiPrompt({
   target = 'all',
   guide = null,
@@ -416,6 +424,8 @@ function buildAiPrompt({
     all: 'rewrite the step title, description, and any useful blocks',
   }[target] || 'rewrite the step';
 
+  const richContext = hasRichCaptureContext(captureContext);
+
   const allowedBlockNote = [
     'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.',
     'Use block.kind = "code" for code snippets.',
@@ -423,8 +433,16 @@ function buildAiPrompt({
     'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].',
   ].join(' ');
 
+  const contextLines = captureContext ? [
+    captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null,
+    captureContext.appName ? `App: ${captureContext.appName}` : null,
+    captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null,
+    captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null,
+    captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null,
+  ].filter(Boolean) : [];
+
   const prompt = [
-    'You write concise, human-sounding step-by-step documentation for an offline desktop app.',
+    'You write concise, action-focused step-by-step documentation for a desktop application guide.',
     'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.',
     'Schema:',
     '{',
@@ -449,26 +467,27 @@ function buildAiPrompt({
     '',
     step ? summarizeStepForAi(step) : 'Step: (not provided)',
     '',
-    captureContext ? [
+    contextLines.length
-      `Active window title: ${captureContext.windowTitle || '(unknown)'}`,
+      ? `Capture context:\n${contextLines.join('\n')}`
-      `Active app: ${captureContext.appName || '(unknown)'}`,
+      : 'Capture context: (not available)',
-      `OCR text near click: ${captureContext.ocrText || '(none)'}`,
-      `Deterministic title candidate: ${captureContext.titleCandidate || '(none)'}`,
-      `Capture mode: ${captureContext.mode || '(unknown)'}`,
-    ].join('\n') : 'Capture context: (not provided)',
     '',
-    block ? `Target block:\n${JSON.stringify(block, null, 2)}` : 'Target block: (not provided)',
+    block ? `Target block:\n${JSON.stringify(block, null, 2)}` : null,
     '',
     'Rules:',
-    '- Keep the wording natural and specific.',
+    '- Write titles as short imperative actions: "Click Save", "Select New document", "Open Settings".',
-    '- Prefer short titles.',
+    '- If the suggested title is already a good imperative action, use it as-is or refine it slightly.',
-    '- Only return blocks that add value.',
+    '- Write descriptions in 1–2 sentences describing exactly what the user does in this step.',
-    '- Do not invent details that are not supported by the capture context.',
+    '- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).',
+    richContext
+      ? '- You have rich context: use the OCR text, window title, and element info to write specific documentation.'
+      : '- Context is limited: write a minimal title. Leave description empty and return no blocks unless the step already has meaningful content.',
+    '- Do NOT generate blocks that describe the technical capture process or mention OCR.',
+    '- Do NOT invent details not supported by the capture context.',
     '- If the target is one block, only rewrite that block.',
-  ].join('\n');
+  ].filter((l) => l !== null).join('\n');
 
   return {
-    systemPrompt: 'You are a documentation assistant that only emits JSON.',
+    systemPrompt: 'You are a technical documentation writer. Emit only valid JSON matching the schema. Never add commentary or markdown.',
     prompt,
   };
 }

tests/unit/text-intel.test.js

@@ -97,7 +97,7 @@ test('ai prompts include the deterministic OCR-backed title candidate', () => {
     },
   });
 
-  assert.match(prompt, /Deterministic title candidate: Click New tab/);
+  assert.match(prompt, /Suggested title: Click New tab/);
 });
 
 test('ocr crop rectangles clamp to the image bounds', (t) => {