Tyler/StepForge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

ai smart auto-documentation and text rewrite

Browse Source 
- Store captureMetadata (OCR text, window/app/element info) with each
  step at capture time so AI always has the original rich context.
- Add buildCaptureContext() to TextIntelService; capture.js uses it
  instead of buildCaptureTitle() so both title and metadata come from
  one pass.
- generateStepPatch() prefers stored captureMetadata over re-running
  OCR, giving the AI the best possible context when the user clicks
  an AI button later.
- Add autoDoc setting: when enabled every capture (shoot, region, and
  session hotkey/click) is automatically documented by AI. Manual
  captures await AI before returning; session captures fire-and-forget
  and push a step:updated event so the renderer reloads seamlessly.
- Add ai:rewriteText IPC and rewriteText() method for plain-text
  polishing via a separate callOllamaText() that skips JSON mode.
- Add "AI Rewrite" section in the editor right panel: textarea + AI
  button that rewrites whatever the user types in place.
- Improve buildAiPrompt() rules: action-focused title instructions,
  explicit anti-junk rules (no "Capture the screen / OCR" blocks),
  and a context-quality gate that suppresses blocks when context is
  thin.
- Add autoDoc checkbox to AI settings dialog.
- Renderer handles step:updated to reload the selected step after
  background auto-doc finishes.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Tyler
2026-06-24 09:17:31 -05:00
co-authored by Claude Sonnet 4.6
parent 375c14b028
commit d244626583
10 changed files with 255 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/capture.js
+8 -6
View File
@@ -1316,15 +1316,15 @@ public static class SFMouseHook {
};
}
async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) {
async buildStepMeta(mode, frame, clickPos = null, clickMeta = null) {
try {
if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') {
return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta });
if (this.textIntel && typeof this.textIntel.buildCaptureContext === 'function') {
return await this.textIntel.buildCaptureContext({ mode, frame, clickPos, clickMeta });
}
} catch {
// fall back to the local semantic title below
// fall back
}
return this.autoTitle(mode);
return { title: this.autoTitle(mode), captureMetadata: null };
}
async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) {
@@ -1350,8 +1350,10 @@ public static class SFMouseHook {
}
}
const { title, captureMetadata } = await this.buildStepMeta(mode, frame, clickPos, clickMeta);
const step = this.store.addStep(guideId, {
title: await this.buildStepTitle(mode, frame, clickPos, clickMeta),
title,
captureMetadata,
annotations,
focusedView: {
enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),
app/main.js
+50 -2
View File
@@ -521,18 +521,49 @@ function setupIpc() {
if (result.ok) reindex(guideId);
return result;
});
h('ai:rewriteText', async ({ text, guideTitle = '', stepTitle = '' } = {}) => {
return textIntel.rewriteText({ text, guideTitle, stepTitle });
});
h('placeholders:globals:get', () => settings.getGlobalPlaceholders());
h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values));
// capture
h('capture:shoot', async ({ guideId, mode, delayMs }) => {
const result = await capture.shoot({ guideId, mode, delayMs });
if (result.ok) reindex(guideId);
if (result.ok) {
reindex(guideId);
const aiConf = settings.get('ai') || {};
if (aiConf.enabled && aiConf.autoDoc && result.step) {
const aiResult = await textIntel.generateStepPatch({
guideId,
stepId: result.step.stepId,
target: 'all',
}).catch(() => null);
if (aiResult?.ok) {
reindex(guideId);
result.step = aiResult.step;
}
}
}
return result;
});
h('capture:region', async ({ guideId }) => {
const result = await capture.regionCapture(guideId);
if (result.ok) reindex(guideId);
if (result.ok) {
reindex(guideId);
const aiConf = settings.get('ai') || {};
if (aiConf.enabled && aiConf.autoDoc && result.step) {
const aiResult = await textIntel.generateStepPatch({
guideId,
stepId: result.step.stepId,
target: 'all',
}).catch(() => null);
if (aiResult?.ok) {
reindex(guideId);
result.step = aiResult.step;
}
}
}
return result;
});
let capturePowerBlocker = -1;
@@ -777,6 +808,23 @@ if (!gotLock) {
try { keepProcessesResponsive(app.getAppMetrics().map((m) => m.pid)); } catch { /* best effort */ }
}
}
// Auto-document session captures in the background when autoDoc is enabled.
// Single-shot captures (capture:shoot) are handled synchronously in the IPC handler.
if (channel === 'capture:added' && payload?.step && payload?.guideId) {
const aiConf = settings.get('ai') || {};
if (aiConf.enabled && aiConf.autoDoc) {
textIntel.generateStepPatch({
guideId: payload.guideId,
stepId: payload.step.stepId,
target: 'all',
}).then((result) => {
if (result.ok) {
reindex(payload.guideId);
sendToRenderer('step:updated', { guideId: payload.guideId, step: result.step });
}
}).catch(() => {});
}
}
};
capture = new CaptureService({
app/preload.js
+2
View File
@@ -55,6 +55,7 @@ const api = {
ai: {
test: invoke('ai:test'),
fillStep: invoke('ai:fillStep'),
rewriteText: invoke('ai:rewriteText'),
},
capture: {
shoot: invoke('capture:shoot'),
@@ -63,6 +64,7 @@ const api = {
state: invoke('capture:state'),
onAdded: (fn) => ipcRenderer.on('capture:added', (e, payload) => fn(payload)),
onState: (fn) => ipcRenderer.on('capture:state', (e, payload) => fn(payload)),
onStepUpdated: (fn) => ipcRenderer.on('step:updated', (e, payload) => fn(payload)),
},
archive: {
export: invoke('archive:export'),
app/renderer/app.js
+12
View File
@@ -80,6 +80,18 @@ class StepForgeApp {
api.capture.onAdded((payload) => this.onCaptureAdded(payload));
api.capture.onState((payload) => this.updateCaptureState(payload));
api.capture.onStepUpdated((payload) => this.onStepUpdated(payload));
}
async onStepUpdated(payload) {
if (!payload || !payload.guideId || !payload.step) return;
if (this.state.view === 'editor' && this.editor.guideId === payload.guideId) {
const currentStep = this.editor.currentStep;
if (currentStep && currentStep.stepId === payload.step.stepId) {
await this.editor.reload(payload.step.stepId);
toast('Documentation generated.');
}
}
}
async onCaptureAdded(payload) {
app/renderer/dialogs.js
+5 -2
View File
@@ -315,6 +315,7 @@ function showSettingsDialog({
const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) });
const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 });
const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) });
const aiAutoDoc = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.autoDoc) });
const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434');
const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b');
const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.');
@@ -408,7 +409,8 @@ function showSettingsDialog({
),
el('fieldset', {},
el('legend', {}, 'AI'),
labeledRow('Enable AI text filling', aiEnabled),
labeledRow('Enable AI', aiEnabled),
labeledRow('Auto-document captures', aiAutoDoc),
labeledRow('Ollama host', ollamaHost),
labeledRow('Ollama model', ollamaModel),
el('div.row', { style: { justifyContent: 'space-between' } },
@@ -416,7 +418,7 @@ function showSettingsDialog({
testAiBtn,
),
el('div.muted', {},
'AI generation is manual only. Captures stay deterministic until you click a generate button.',
'When auto-document is on, each capture is automatically documented by AI. Turn it off to use AI manually only.',
),
),
el('fieldset', {},
@@ -465,6 +467,7 @@ function showSettingsDialog({
ai: {
...settings.ai,
enabled: aiEnabled.checked,
autoDoc: aiAutoDoc.checked,
ollama: {
...(settings.ai?.ollama || {}),
host: ollamaHost.value.trim(),
app/renderer/editor.js
+48
View File
@@ -167,6 +167,7 @@ class GuideEditor {
const buttons = [
this.dom?.titleAiBtn,
this.dom?.descAiBtn,
this.dom?.rewriteAiBtn,
...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []),
].filter(Boolean);
for (const button of buttons) {
@@ -222,6 +223,36 @@ class GuideEditor {
return this.runAiGeneration('all', { button });
}
async runTextRewrite(button = null) {
if (!this.isAiEnabled()) {
this.onToast('Enable AI in Settings first.', { error: true });
return;
}
const text = this.dom?.rewriteInput?.value?.trim();
if (!text) {
this.onToast('Type some text to rewrite first.', { error: true });
return;
}
if (button) setButtonLoading(button, true, 'Rewriting…');
try {
const result = await api.ai.rewriteText({
text,
guideTitle: this.guide?.title || '',
stepTitle: this.currentStep?.title || '',
});
if (!result || !result.ok) {
this.onToast(result?.reason || 'Rewrite failed.', { error: true });
return;
}
if (this.dom?.rewriteInput) this.dom.rewriteInput.value = result.text;
this.onToast('Text rewritten.');
} catch (err) {
this.onToast(err.message || 'Rewrite failed.', { error: true });
} finally {
if (button) setButtonLoading(button, false);
}
}
async generateBlockWithAi(kind, block, button = null) {
if (!block) return null;
return this.runAiGeneration('block', { blockId: block.id, button });
@@ -411,6 +442,21 @@ class GuideEditor {
this.dom.addTableBlockBtn = el('button', { type: 'button' }, '+ Table'),
),
),
el('section', {},
el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } },
el('h3', { style: { margin: 0 } }, 'AI Rewrite'),
this.dom.rewriteAiBtn = el('button.ai', {
type: 'button',
title: 'Rewrite the text below with AI',
dataset: { aiAction: 'rewrite', aiTitle: 'Rewrite with AI' },
}, 'Rewrite'),
),
this.dom.rewriteInput = el('textarea', {
rows: 3,
placeholder: 'Type something to improve…',
style: { width: '100%', resize: 'vertical' },
}),
),
el('section', {},
el('h3', {}, 'Guide'),
this.dom.guideSummary = el('div.muted', {}),
@@ -583,6 +629,8 @@ class GuideEditor {
});
this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn));
this.dom.rewriteAiBtn.addEventListener('click', () => this.runTextRewrite(this.dom.rewriteAiBtn));
this.dom.descEditor.addEventListener('paste', (e) => {
// Keep pasted text simple; backend sanitization will handle the rest.
e.preventDefault();
app/text-intel.js
+92 -6
View File
@@ -276,15 +276,27 @@ public static class Win32 {
}
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta });
return ctx.title;
}
async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
this.ocrAroundClick(frame, clickPos),
]);
return buildCaptureTitle({
mode,
metadata,
ocrText: ocr.text,
});
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text });
return {
title,
captureMetadata: {
ocrText: ocr.text || '',
windowTitle: metadata.windowTitle || '',
appName: metadata.appName || '',
elementLabel: metadata.elementLabel || '',
elementRole: metadata.elementRole || '',
mode,
},
};
}
aiEnabled() {
@@ -335,6 +347,28 @@ public static class Win32 {
};
}
async callOllamaText({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
stream: false,
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: prompt },
],
options: { temperature: 0.4 },
}),
});
if (!response.ok) throw new Error(`Ollama request failed (${response.status})`);
const payload = await response.json();
const content = payload?.message?.content;
if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response');
return content.trim();
}
async callOllama({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
@@ -393,7 +427,23 @@ public static class Win32 {
}
let captureContext = null;
if (step.image) {
// Use stored capture metadata when available (best context, from capture time).
// Fall back to re-running OCR on the stored image only when metadata is absent.
if (step.captureMetadata) {
captureContext = {
...step.captureMetadata,
titleCandidate: buildCaptureTitle({
mode: step.captureMetadata.mode || 'fullscreen',
metadata: {
windowTitle: step.captureMetadata.windowTitle,
appName: step.captureMetadata.appName,
elementLabel: step.captureMetadata.elementLabel,
elementRole: step.captureMetadata.elementRole,
},
ocrText: step.captureMetadata.ocrText,
}),
};
} else if (step.image) {
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
if (imagePath && fs.existsSync(imagePath)) {
const { nativeImage } = require('electron');
@@ -442,6 +492,42 @@ public static class Win32 {
}
}
async rewriteText({ text, guideTitle = '', stepTitle = '' }) {
try {
const config = this.aiConfig();
if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' };
if (!config.ollama.host || !config.ollama.model) {
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
}
const trimmed = normalizeWhitespace(text);
if (!trimmed) return { ok: false, reason: 'No text to rewrite.' };
const contextHint = [
guideTitle ? `Guide: ${guideTitle}` : '',
stepTitle ? `Step: ${stepTitle}` : '',
].filter(Boolean).join('\n');
const prompt = [
contextHint,
contextHint ? '' : null,
'Rewrite the following text to sound professional and clear as step-by-step documentation.',
'Keep it concise. Do not add extra information. Return only the rewritten text.',
'',
trimmed,
].filter((l) => l !== null).join('\n');
const result = await this.callOllamaText({
host: config.ollama.host,
model: config.ollama.model,
prompt,
systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.',
});
return { ok: true, text: result };
} catch (err) {
return { ok: false, reason: err?.message || 'Rewrite failed.' };
}
}
clickPointFromStep(step, image = null) {
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
if (!marker) return null;