ai smart auto-documentation and text rewrite

- Store captureMetadata (OCR text, window/app/element info) with each
  step at capture time so AI always has the original rich context.
- Add buildCaptureContext() to TextIntelService; capture.js uses it
  instead of buildCaptureTitle() so both title and metadata come from
  one pass.
- generateStepPatch() prefers stored captureMetadata over re-running
  OCR, giving the AI the best possible context when the user clicks
  an AI button later.
- Add autoDoc setting: when enabled every capture (shoot, region, and
  session hotkey/click) is automatically documented by AI. Manual
  captures await AI before returning; session captures fire-and-forget
  and push a step:updated event so the renderer reloads seamlessly.
- Add ai:rewriteText IPC and rewriteText() method for plain-text
  polishing via a separate callOllamaText() that skips JSON mode.
- Add "AI Rewrite" section in the editor right panel: textarea + AI
  button that rewrites whatever the user types in place.
- Improve buildAiPrompt() rules: action-focused title instructions,
  explicit anti-junk rules (no "Capture the screen / OCR" blocks),
  and a context-quality gate that suppresses blocks when context is
  thin.
- Add autoDoc checkbox to AI settings dialog.
- Renderer handles step:updated to reload the selected step after
  background auto-doc finishes.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
2026-06-24 09:17:31 -05:00
co-authored by Claude Sonnet 4.6
parent 375c14b028
commit d244626583
10 changed files with 255 additions and 32 deletions
+92 -6
View File
@@ -276,15 +276,27 @@ public static class Win32 {
}
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta });
return ctx.title;
}
async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
this.ocrAroundClick(frame, clickPos),
]);
return buildCaptureTitle({
mode,
metadata,
ocrText: ocr.text,
});
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text });
return {
title,
captureMetadata: {
ocrText: ocr.text || '',
windowTitle: metadata.windowTitle || '',
appName: metadata.appName || '',
elementLabel: metadata.elementLabel || '',
elementRole: metadata.elementRole || '',
mode,
},
};
}
aiEnabled() {
@@ -335,6 +347,28 @@ public static class Win32 {
};
}
async callOllamaText({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
stream: false,
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: prompt },
],
options: { temperature: 0.4 },
}),
});
if (!response.ok) throw new Error(`Ollama request failed (${response.status})`);
const payload = await response.json();
const content = payload?.message?.content;
if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response');
return content.trim();
}
async callOllama({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
@@ -393,7 +427,23 @@ public static class Win32 {
}
let captureContext = null;
if (step.image) {
// Use stored capture metadata when available (best context, from capture time).
// Fall back to re-running OCR on the stored image only when metadata is absent.
if (step.captureMetadata) {
captureContext = {
...step.captureMetadata,
titleCandidate: buildCaptureTitle({
mode: step.captureMetadata.mode || 'fullscreen',
metadata: {
windowTitle: step.captureMetadata.windowTitle,
appName: step.captureMetadata.appName,
elementLabel: step.captureMetadata.elementLabel,
elementRole: step.captureMetadata.elementRole,
},
ocrText: step.captureMetadata.ocrText,
}),
};
} else if (step.image) {
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
if (imagePath && fs.existsSync(imagePath)) {
const { nativeImage } = require('electron');
@@ -442,6 +492,42 @@ public static class Win32 {
}
}
async rewriteText({ text, guideTitle = '', stepTitle = '' }) {
try {
const config = this.aiConfig();
if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' };
if (!config.ollama.host || !config.ollama.model) {
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
}
const trimmed = normalizeWhitespace(text);
if (!trimmed) return { ok: false, reason: 'No text to rewrite.' };
const contextHint = [
guideTitle ? `Guide: ${guideTitle}` : '',
stepTitle ? `Step: ${stepTitle}` : '',
].filter(Boolean).join('\n');
const prompt = [
contextHint,
contextHint ? '' : null,
'Rewrite the following text to sound professional and clear as step-by-step documentation.',
'Keep it concise. Do not add extra information. Return only the rewritten text.',
'',
trimmed,
].filter((l) => l !== null).join('\n');
const result = await this.callOllamaText({
host: config.ollama.host,
model: config.ollama.model,
prompt,
systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.',
});
return { ok: true, text: result };
} catch (err) {
return { ok: false, reason: err?.message || 'Rewrite failed.' };
}
}
clickPointFromStep(step, image = null) {
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
if (!marker) return null;