Tyler/StepForge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

ai smart auto-documentation and text rewrite

Browse Source 
- Store captureMetadata (OCR text, window/app/element info) with each
  step at capture time so AI always has the original rich context.
- Add buildCaptureContext() to TextIntelService; capture.js uses it
  instead of buildCaptureTitle() so both title and metadata come from
  one pass.
- generateStepPatch() prefers stored captureMetadata over re-running
  OCR, giving the AI the best possible context when the user clicks
  an AI button later.
- Add autoDoc setting: when enabled every capture (shoot, region, and
  session hotkey/click) is automatically documented by AI. Manual
  captures await AI before returning; session captures fire-and-forget
  and push a step:updated event so the renderer reloads seamlessly.
- Add ai:rewriteText IPC and rewriteText() method for plain-text
  polishing via a separate callOllamaText() that skips JSON mode.
- Add "AI Rewrite" section in the editor right panel: textarea + AI
  button that rewrites whatever the user types in place.
- Improve buildAiPrompt() rules: action-focused title instructions,
  explicit anti-junk rules (no "Capture the screen / OCR" blocks),
  and a context-quality gate that suppresses blocks when context is
  thin.
- Add autoDoc checkbox to AI settings dialog.
- Renderer handles step:updated to reload the selected step after
  background auto-doc finishes.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Tyler
2026-06-24 09:17:31 -05:00
co-authored by Claude Sonnet 4.6
parent 375c14b028
commit d244626583
10 changed files with 255 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/capture.js
+8 -6
View File
@@ -1316,15 +1316,15 @@ public static class SFMouseHook {
};
}
async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) {
async buildStepMeta(mode, frame, clickPos = null, clickMeta = null) {
try {
if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') {
return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta });
if (this.textIntel && typeof this.textIntel.buildCaptureContext === 'function') {
return await this.textIntel.buildCaptureContext({ mode, frame, clickPos, clickMeta });
}
} catch {
// fall back to the local semantic title below
// fall back
}
return this.autoTitle(mode);
return { title: this.autoTitle(mode), captureMetadata: null };
}
async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) {
@@ -1350,8 +1350,10 @@ public static class SFMouseHook {
}
}
const { title, captureMetadata } = await this.buildStepMeta(mode, frame, clickPos, clickMeta);
const step = this.store.addStep(guideId, {
title: await this.buildStepTitle(mode, frame, clickPos, clickMeta),
title,
captureMetadata,
annotations,
focusedView: {
enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),
app/main.js
+50 -2
View File
@@ -521,18 +521,49 @@ function setupIpc() {
if (result.ok) reindex(guideId);
return result;
});
h('ai:rewriteText', async ({ text, guideTitle = '', stepTitle = '' } = {}) => {
return textIntel.rewriteText({ text, guideTitle, stepTitle });
});
h('placeholders:globals:get', () => settings.getGlobalPlaceholders());
h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values));
// capture
h('capture:shoot', async ({ guideId, mode, delayMs }) => {
const result = await capture.shoot({ guideId, mode, delayMs });
if (result.ok) reindex(guideId);
if (result.ok) {
reindex(guideId);
const aiConf = settings.get('ai') || {};
if (aiConf.enabled && aiConf.autoDoc && result.step) {
const aiResult = await textIntel.generateStepPatch({
guideId,
stepId: result.step.stepId,
target: 'all',
}).catch(() => null);
if (aiResult?.ok) {
reindex(guideId);
result.step = aiResult.step;
}
}
}
return result;
});
h('capture:region', async ({ guideId }) => {
const result = await capture.regionCapture(guideId);
if (result.ok) reindex(guideId);
if (result.ok) {
reindex(guideId);
const aiConf = settings.get('ai') || {};
if (aiConf.enabled && aiConf.autoDoc && result.step) {
const aiResult = await textIntel.generateStepPatch({
guideId,
stepId: result.step.stepId,
target: 'all',
}).catch(() => null);
if (aiResult?.ok) {
reindex(guideId);
result.step = aiResult.step;
}
}
}
return result;
});
let capturePowerBlocker = -1;
@@ -777,6 +808,23 @@ if (!gotLock) {
try { keepProcessesResponsive(app.getAppMetrics().map((m) => m.pid)); } catch { /* best effort */ }
}
}
// Auto-document session captures in the background when autoDoc is enabled.
// Single-shot captures (capture:shoot) are handled synchronously in the IPC handler.
if (channel === 'capture:added' && payload?.step && payload?.guideId) {
const aiConf = settings.get('ai') || {};
if (aiConf.enabled && aiConf.autoDoc) {
textIntel.generateStepPatch({
guideId: payload.guideId,
stepId: payload.step.stepId,
target: 'all',
}).then((result) => {
if (result.ok) {
reindex(payload.guideId);
sendToRenderer('step:updated', { guideId: payload.guideId, step: result.step });
}
}).catch(() => {});
}
}
};
capture = new CaptureService({
app/preload.js
+2
View File
@@ -55,6 +55,7 @@ const api = {
ai: {
test: invoke('ai:test'),
fillStep: invoke('ai:fillStep'),
rewriteText: invoke('ai:rewriteText'),
},
capture: {
shoot: invoke('capture:shoot'),
@@ -63,6 +64,7 @@ const api = {
state: invoke('capture:state'),
onAdded: (fn) => ipcRenderer.on('capture:added', (e, payload) => fn(payload)),
onState: (fn) => ipcRenderer.on('capture:state', (e, payload) => fn(payload)),
onStepUpdated: (fn) => ipcRenderer.on('step:updated', (e, payload) => fn(payload)),
},
archive: {
export: invoke('archive:export'),
app/renderer/app.js
+12
View File
@@ -80,6 +80,18 @@ class StepForgeApp {
api.capture.onAdded((payload) => this.onCaptureAdded(payload));
api.capture.onState((payload) => this.updateCaptureState(payload));
api.capture.onStepUpdated((payload) => this.onStepUpdated(payload));
}
async onStepUpdated(payload) {
if (!payload || !payload.guideId || !payload.step) return;
if (this.state.view === 'editor' && this.editor.guideId === payload.guideId) {
const currentStep = this.editor.currentStep;
if (currentStep && currentStep.stepId === payload.step.stepId) {
await this.editor.reload(payload.step.stepId);
toast('Documentation generated.');
}
}
}
async onCaptureAdded(payload) {
app/renderer/dialogs.js
+5 -2
View File
@@ -315,6 +315,7 @@ function showSettingsDialog({
const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) });
const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 });
const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) });
const aiAutoDoc = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.autoDoc) });
const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434');
const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b');
const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.');
@@ -408,7 +409,8 @@ function showSettingsDialog({
),
el('fieldset', {},
el('legend', {}, 'AI'),
labeledRow('Enable AI text filling', aiEnabled),
labeledRow('Enable AI', aiEnabled),
labeledRow('Auto-document captures', aiAutoDoc),
labeledRow('Ollama host', ollamaHost),
labeledRow('Ollama model', ollamaModel),
el('div.row', { style: { justifyContent: 'space-between' } },
@@ -416,7 +418,7 @@ function showSettingsDialog({
testAiBtn,
),
el('div.muted', {},
'AI generation is manual only. Captures stay deterministic until you click a generate button.',
'When auto-document is on, each capture is automatically documented by AI. Turn it off to use AI manually only.',
),
),
el('fieldset', {},
@@ -465,6 +467,7 @@ function showSettingsDialog({
ai: {
...settings.ai,
enabled: aiEnabled.checked,
autoDoc: aiAutoDoc.checked,
ollama: {
...(settings.ai?.ollama || {}),
host: ollamaHost.value.trim(),
app/renderer/editor.js
+48
View File
@@ -167,6 +167,7 @@ class GuideEditor {
const buttons = [
this.dom?.titleAiBtn,
this.dom?.descAiBtn,
this.dom?.rewriteAiBtn,
...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []),
].filter(Boolean);
for (const button of buttons) {
@@ -222,6 +223,36 @@ class GuideEditor {
return this.runAiGeneration('all', { button });
}
async runTextRewrite(button = null) {
if (!this.isAiEnabled()) {
this.onToast('Enable AI in Settings first.', { error: true });
return;
}
const text = this.dom?.rewriteInput?.value?.trim();
if (!text) {
this.onToast('Type some text to rewrite first.', { error: true });
return;
}
if (button) setButtonLoading(button, true, 'Rewriting…');
try {
const result = await api.ai.rewriteText({
text,
guideTitle: this.guide?.title || '',
stepTitle: this.currentStep?.title || '',
});
if (!result || !result.ok) {
this.onToast(result?.reason || 'Rewrite failed.', { error: true });
return;
}
if (this.dom?.rewriteInput) this.dom.rewriteInput.value = result.text;
this.onToast('Text rewritten.');
} catch (err) {
this.onToast(err.message || 'Rewrite failed.', { error: true });
} finally {
if (button) setButtonLoading(button, false);
}
}
async generateBlockWithAi(kind, block, button = null) {
if (!block) return null;
return this.runAiGeneration('block', { blockId: block.id, button });
@@ -411,6 +442,21 @@ class GuideEditor {
this.dom.addTableBlockBtn = el('button', { type: 'button' }, '+ Table'),
),
),
el('section', {},
el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } },
el('h3', { style: { margin: 0 } }, 'AI Rewrite'),
this.dom.rewriteAiBtn = el('button.ai', {
type: 'button',
title: 'Rewrite the text below with AI',
dataset: { aiAction: 'rewrite', aiTitle: 'Rewrite with AI' },
}, 'Rewrite'),
),
this.dom.rewriteInput = el('textarea', {
rows: 3,
placeholder: 'Type something to improve…',
style: { width: '100%', resize: 'vertical' },
}),
),
el('section', {},
el('h3', {}, 'Guide'),
this.dom.guideSummary = el('div.muted', {}),
@@ -583,6 +629,8 @@ class GuideEditor {
});
this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn));
this.dom.rewriteAiBtn.addEventListener('click', () => this.runTextRewrite(this.dom.rewriteAiBtn));
this.dom.descEditor.addEventListener('paste', (e) => {
// Keep pasted text simple; backend sanitization will handle the rest.
e.preventDefault();
app/text-intel.js
+91 -5
View File
@@ -276,15 +276,27 @@ public static class Win32 {
}
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta });
return ctx.title;
}
async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
this.ocrAroundClick(frame, clickPos),
]);
return buildCaptureTitle({
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text });
return {
title,
captureMetadata: {
ocrText: ocr.text || '',
windowTitle: metadata.windowTitle || '',
appName: metadata.appName || '',
elementLabel: metadata.elementLabel || '',
elementRole: metadata.elementRole || '',
mode,
metadata,
ocrText: ocr.text,
});
},
};
}
aiEnabled() {
@@ -335,6 +347,28 @@ public static class Win32 {
};
}
async callOllamaText({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
stream: false,
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: prompt },
],
options: { temperature: 0.4 },
}),
});
if (!response.ok) throw new Error(`Ollama request failed (${response.status})`);
const payload = await response.json();
const content = payload?.message?.content;
if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response');
return content.trim();
}
async callOllama({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
@@ -393,7 +427,23 @@ public static class Win32 {
}
let captureContext = null;
if (step.image) {
// Use stored capture metadata when available (best context, from capture time).
// Fall back to re-running OCR on the stored image only when metadata is absent.
if (step.captureMetadata) {
captureContext = {
...step.captureMetadata,
titleCandidate: buildCaptureTitle({
mode: step.captureMetadata.mode || 'fullscreen',
metadata: {
windowTitle: step.captureMetadata.windowTitle,
appName: step.captureMetadata.appName,
elementLabel: step.captureMetadata.elementLabel,
elementRole: step.captureMetadata.elementRole,
},
ocrText: step.captureMetadata.ocrText,
}),
};
} else if (step.image) {
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
if (imagePath && fs.existsSync(imagePath)) {
const { nativeImage } = require('electron');
@@ -442,6 +492,42 @@ public static class Win32 {
}
}
async rewriteText({ text, guideTitle = '', stepTitle = '' }) {
try {
const config = this.aiConfig();
if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' };
if (!config.ollama.host || !config.ollama.model) {
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
}
const trimmed = normalizeWhitespace(text);
if (!trimmed) return { ok: false, reason: 'No text to rewrite.' };
const contextHint = [
guideTitle ? `Guide: ${guideTitle}` : '',
stepTitle ? `Step: ${stepTitle}` : '',
].filter(Boolean).join('\n');
const prompt = [
contextHint,
contextHint ? '' : null,
'Rewrite the following text to sound professional and clear as step-by-step documentation.',
'Keep it concise. Do not add extra information. Return only the rewritten text.',
'',
trimmed,
].filter((l) => l !== null).join('\n');
const result = await this.callOllamaText({
host: config.ollama.host,
model: config.ollama.model,
prompt,
systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.',
});
return { ok: true, text: result };
} catch (err) {
return { ok: false, reason: err?.message || 'Rewrite failed.' };
}
}
clickPointFromStep(step, image = null) {
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
if (!marker) return null;
core/schema.js
+3
View File
@@ -86,6 +86,9 @@ function createStep(fields = {}) {
codeBlocks: (fields.codeBlocks || []).map((cb) => normalizeCodeBlock(cb, takeOrder(cb))),
tableBlocks: (fields.tableBlocks || []).map((tb) => normalizeTableBlock(tb, takeOrder(tb))),
links: fields.links || [], // { id, label, targetStepId }
captureMetadata: (fields.captureMetadata && typeof fields.captureMetadata === 'object' && !Array.isArray(fields.captureMetadata))
? { ...fields.captureMetadata }
: null,
};
}
core/text-intel.js
+34 -15
View File
@@ -402,6 +402,14 @@ function summarizeGuideForAi(guide = {}) {
].join('\n');
}
function hasRichCaptureContext(captureContext) {
if (!captureContext) return false;
const ocr = normalizeWhitespace(captureContext.ocrText || '');
const win = normalizeWhitespace(captureContext.windowTitle || '');
const app = normalizeWhitespace(captureContext.appName || '');
return ocr.length > 3 || (win.length > 2 && !isBrowserNoise(win)) || app.length > 1;
}
function buildAiPrompt({
target = 'all',
guide = null,
@@ -416,6 +424,8 @@ function buildAiPrompt({
all: 'rewrite the step title, description, and any useful blocks',
}[target] || 'rewrite the step';
const richContext = hasRichCaptureContext(captureContext);
const allowedBlockNote = [
'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.',
'Use block.kind = "code" for code snippets.',
@@ -423,8 +433,16 @@ function buildAiPrompt({
'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].',
].join(' ');
const contextLines = captureContext ? [
captureContext.windowTitle ? `Active window: ${captureContext.windowTitle}` : null,
captureContext.appName ? `App: ${captureContext.appName}` : null,
captureContext.elementLabel ? `UI element: ${captureContext.elementLabel}${captureContext.elementRole ? ` (${captureContext.elementRole})` : ''}` : null,
captureContext.ocrText ? `OCR text near click:\n${captureContext.ocrText}` : null,
captureContext.titleCandidate ? `Suggested title: ${captureContext.titleCandidate}` : null,
].filter(Boolean) : [];
const prompt = [
'You write concise, human-sounding step-by-step documentation for an offline desktop app.',
'You write concise, action-focused step-by-step documentation for a desktop application guide.',
'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.',
'Schema:',
'{',
@@ -449,26 +467,27 @@ function buildAiPrompt({
'',
step ? summarizeStepForAi(step) : 'Step: (not provided)',
'',
captureContext ? [
`Active window title: ${captureContext.windowTitle || '(unknown)'}`,
`Active app: ${captureContext.appName || '(unknown)'}`,
`OCR text near click: ${captureContext.ocrText || '(none)'}`,
`Deterministic title candidate: ${captureContext.titleCandidate || '(none)'}`,
`Capture mode: ${captureContext.mode || '(unknown)'}`,
].join('\n') : 'Capture context: (not provided)',
contextLines.length
? `Capture context:\n${contextLines.join('\n')}`
: 'Capture context: (not available)',
'',
block ? `Target block:\n${JSON.stringify(block, null, 2)}` : 'Target block: (not provided)',
block ? `Target block:\n${JSON.stringify(block, null, 2)}` : null,
'',
'Rules:',
'- Keep the wording natural and specific.',
'- Prefer short titles.',
'- Only return blocks that add value.',
'- Do not invent details that are not supported by the capture context.',
'- Write titles as short imperative actions: "Click Save", "Select New document", "Open Settings".',
'- If the suggested title is already a good imperative action, use it as-is or refine it slightly.',
'- Write descriptions in 1–2 sentences describing exactly what the user does in this step.',
'- Only include blocks that provide genuinely useful supplemental information (warnings, tips, code).',
richContext
? '- You have rich context: use the OCR text, window title, and element info to write specific documentation.'
: '- Context is limited: write a minimal title. Leave description empty and return no blocks unless the step already has meaningful content.',
'- Do NOT generate blocks that describe the technical capture process or mention OCR.',
'- Do NOT invent details not supported by the capture context.',
'- If the target is one block, only rewrite that block.',
].join('\n');
].filter((l) => l !== null).join('\n');
return {
systemPrompt: 'You are a documentation assistant that only emits JSON.',
systemPrompt: 'You are a technical documentation writer. Emit only valid JSON matching the schema. Never add commentary or markdown.',
prompt,
};
}
tests/unit/text-intel.test.js
+1 -1
View File
@@ -97,7 +97,7 @@ test('ai prompts include the deterministic OCR-backed title candidate', () => {
},
});
assert.match(prompt, /Deterministic title candidate: Click New tab/);
assert.match(prompt, /Suggested title: Click New tab/);
});
test('ocr crop rectangles clamp to the image bounds', (t) => {