ai smart auto-documentation and text rewrite
- Store captureMetadata (OCR text, window/app/element info) with each step at capture time so AI always has the original rich context. - Add buildCaptureContext() to TextIntelService; capture.js uses it instead of buildCaptureTitle() so both title and metadata come from one pass. - generateStepPatch() prefers stored captureMetadata over re-running OCR, giving the AI the best possible context when the user clicks an AI button later. - Add autoDoc setting: when enabled every capture (shoot, region, and session hotkey/click) is automatically documented by AI. Manual captures await AI before returning; session captures fire-and-forget and push a step:updated event so the renderer reloads seamlessly. - Add ai:rewriteText IPC and rewriteText() method for plain-text polishing via a separate callOllamaText() that skips JSON mode. - Add "AI Rewrite" section in the editor right panel: textarea + AI button that rewrites whatever the user types in place. - Improve buildAiPrompt() rules: action-focused title instructions, explicit anti-junk rules (no "Capture the screen / OCR" blocks), and a context-quality gate that suppresses blocks when context is thin. - Add autoDoc checkbox to AI settings dialog. - Renderer handles step:updated to reload the selected step after background auto-doc finishes. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
+92
-6
@@ -276,15 +276,27 @@ public static class Win32 {
|
||||
}
|
||||
|
||||
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
|
||||
const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta });
|
||||
return ctx.title;
|
||||
}
|
||||
|
||||
async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
|
||||
const [metadata, ocr] = await Promise.all([
|
||||
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
|
||||
this.ocrAroundClick(frame, clickPos),
|
||||
]);
|
||||
return buildCaptureTitle({
|
||||
mode,
|
||||
metadata,
|
||||
ocrText: ocr.text,
|
||||
});
|
||||
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text });
|
||||
return {
|
||||
title,
|
||||
captureMetadata: {
|
||||
ocrText: ocr.text || '',
|
||||
windowTitle: metadata.windowTitle || '',
|
||||
appName: metadata.appName || '',
|
||||
elementLabel: metadata.elementLabel || '',
|
||||
elementRole: metadata.elementRole || '',
|
||||
mode,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
aiEnabled() {
|
||||
@@ -335,6 +347,28 @@ public static class Win32 {
|
||||
};
|
||||
}
|
||||
|
||||
async callOllamaText({ host, model, prompt, systemPrompt }) {
|
||||
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
|
||||
const response = await this.fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
stream: false,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
options: { temperature: 0.4 },
|
||||
}),
|
||||
});
|
||||
if (!response.ok) throw new Error(`Ollama request failed (${response.status})`);
|
||||
const payload = await response.json();
|
||||
const content = payload?.message?.content;
|
||||
if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response');
|
||||
return content.trim();
|
||||
}
|
||||
|
||||
async callOllama({ host, model, prompt, systemPrompt }) {
|
||||
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
|
||||
const response = await this.fetch(url, {
|
||||
@@ -393,7 +427,23 @@ public static class Win32 {
|
||||
}
|
||||
|
||||
let captureContext = null;
|
||||
if (step.image) {
|
||||
// Use stored capture metadata when available (best context, from capture time).
|
||||
// Fall back to re-running OCR on the stored image only when metadata is absent.
|
||||
if (step.captureMetadata) {
|
||||
captureContext = {
|
||||
...step.captureMetadata,
|
||||
titleCandidate: buildCaptureTitle({
|
||||
mode: step.captureMetadata.mode || 'fullscreen',
|
||||
metadata: {
|
||||
windowTitle: step.captureMetadata.windowTitle,
|
||||
appName: step.captureMetadata.appName,
|
||||
elementLabel: step.captureMetadata.elementLabel,
|
||||
elementRole: step.captureMetadata.elementRole,
|
||||
},
|
||||
ocrText: step.captureMetadata.ocrText,
|
||||
}),
|
||||
};
|
||||
} else if (step.image) {
|
||||
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
|
||||
if (imagePath && fs.existsSync(imagePath)) {
|
||||
const { nativeImage } = require('electron');
|
||||
@@ -442,6 +492,42 @@ public static class Win32 {
|
||||
}
|
||||
}
|
||||
|
||||
async rewriteText({ text, guideTitle = '', stepTitle = '' }) {
|
||||
try {
|
||||
const config = this.aiConfig();
|
||||
if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' };
|
||||
if (!config.ollama.host || !config.ollama.model) {
|
||||
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
|
||||
}
|
||||
const trimmed = normalizeWhitespace(text);
|
||||
if (!trimmed) return { ok: false, reason: 'No text to rewrite.' };
|
||||
|
||||
const contextHint = [
|
||||
guideTitle ? `Guide: ${guideTitle}` : '',
|
||||
stepTitle ? `Step: ${stepTitle}` : '',
|
||||
].filter(Boolean).join('\n');
|
||||
|
||||
const prompt = [
|
||||
contextHint,
|
||||
contextHint ? '' : null,
|
||||
'Rewrite the following text to sound professional and clear as step-by-step documentation.',
|
||||
'Keep it concise. Do not add extra information. Return only the rewritten text.',
|
||||
'',
|
||||
trimmed,
|
||||
].filter((l) => l !== null).join('\n');
|
||||
|
||||
const result = await this.callOllamaText({
|
||||
host: config.ollama.host,
|
||||
model: config.ollama.model,
|
||||
prompt,
|
||||
systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.',
|
||||
});
|
||||
return { ok: true, text: result };
|
||||
} catch (err) {
|
||||
return { ok: false, reason: err?.message || 'Rewrite failed.' };
|
||||
}
|
||||
}
|
||||
|
||||
clickPointFromStep(step, image = null) {
|
||||
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
|
||||
if (!marker) return null;
|
||||
|
||||
Reference in New Issue
Block a user