Tyler/StepForge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

semantic titles ai assist

Browse Source
This commit is contained in:
Tyler
2026-06-23 17:46:33 -05:00
parent 2b27428849
commit 33eb0f745e
14 changed files with 1518 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/capture.js
+39 -15
View File
@@ -3,7 +3,6 @@
const path = require('node:path');
const { spawn, execFileSync } = require('node:child_process');
const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron');
const { expandPlaceholders } = require('../core/placeholders');
const raster = require('../core/raster');
const { encodePng } = require('../core/png');
const {
@@ -14,6 +13,7 @@ const {
DEFAULT_START_SLACK_MS,
} = require('./click-frames');
const { physicalToDip } = require('./coords');
const { DEFAULT_CAPTURE_TITLES } = require('../core/text-intel');
/**
* Capture service: full-screen, active-window, and region capture, plus a
@@ -114,6 +114,7 @@ class CaptureService {
getWindow,
notify,
screenApi = screen,
textIntel = null,
}) {
this.store = store;
this.settings = settings;
@@ -122,6 +123,7 @@ class CaptureService {
// Injectable for tests; the click/coordinate paths must never reach for
// the global `screen` directly so coordinate handling stays testable.
this.screen = screenApi;
this.textIntel = textIntel;
this.session = null; // { guideId, paused, count, intervalSec }
this.intervalTimer = null;
this.clickWatcher = null;
@@ -133,6 +135,7 @@ class CaptureService {
this.clickWatcherErrTail = '';
this.linuxEvent = null; // event block currently being parsed
this.pendingRawClick = null; // raw press waiting for its coordinate twin
this.pendingClickOsPoint = null;
this.clickQueue = Promise.resolve();
this.frameLoopInFlight = false;
this.frameLoopGrabStartedAt = null;
@@ -463,7 +466,7 @@ class CaptureService {
if (frame) {
clog('click@', clickAt, 'frame', frame.source || 'loop',
'started', frame.startedAt - clickAt, 'ms, captured', frame.capturedAt - clickAt, 'ms rel. click');
const result = this.storeFrameAsStep(guideId, frame.mode, frame, clickPos);
const result = await this.storeFrameAsStep(guideId, frame.mode, frame, clickPos, clickMeta);
if (result.ok) this.noteStepAdded(result.step, trigger, guideId);
return result;
}
@@ -1211,6 +1214,9 @@ public static class SFMouseHook {
let clickPos = osPoint ? this.osPointToDip(osPoint) : null;
if (!clickPos) clickPos = this.screen.getCursorScreenPoint();
clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos);
this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y)
? { x: osPoint.x, y: osPoint.y }
: null;
this.enqueueClickCapture(clickPos, clickAt, button || 'mouse');
}
@@ -1268,7 +1274,17 @@ public static class SFMouseHook {
* fast the user clicks or how slow the encoder is.
*/
enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') {
const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' };
const osPoint = this.pendingClickOsPoint && Number.isFinite(this.pendingClickOsPoint.x) && Number.isFinite(this.pendingClickOsPoint.y)
? this.pendingClickOsPoint
: null;
this.pendingClickOsPoint = null;
const clickMeta = {
at: Number.isFinite(clickAt) ? clickAt : Date.now(),
button: button || 'mouse',
};
if (osPoint) {
clickMeta.osPoint = { x: osPoint.x, y: osPoint.y };
}
if (this.session && !this.session.paused && !this.userIsInApp()) {
// The guide id pins the click to its recording so it can still be
// stored if the session stops while this click waits in the queue.
@@ -1300,7 +1316,18 @@ public static class SFMouseHook {
};
}
storeFrameAsStep(guideId, mode, frame, clickPos = null) {
async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) {
try {
if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') {
return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta });
}
} catch {
// fall back to the local semantic title below
}
return this.autoTitle(mode);
}
async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) {
if (!frame) return { ok: false, reason: 'no capture frame available' };
const annotations = [];
// The click position (DIP, read at event time) wins over the frame's
@@ -1324,7 +1351,7 @@ public static class SFMouseHook {
}
const step = this.store.addStep(guideId, {
title: this.autoTitle(mode),
title: await this.buildStepTitle(mode, frame, clickPos, clickMeta),
annotations,
focusedView: {
enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),
@@ -1335,14 +1362,7 @@ public static class SFMouseHook {
}
autoTitle(mode) {
const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]';
const now = new Date();
const pad = (n) => String(n).padStart(2, '0');
return expandPlaceholders(tplStr, {
Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen',
Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`,
Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`,
});
return DEFAULT_CAPTURE_TITLES[mode] || 'Capture';
}
/** Grab the screen/window image as { image, display } or throw. */
@@ -1451,8 +1471,12 @@ public static class SFMouseHook {
const cropped = image.crop(rect);
const size = cropped.getSize();
if (!size.width || !size.height) return { ok: false, reason: 'empty selection' };
const step = this.store.addStep(guideId, { title: this.autoTitle('region') },
cropped.toPNG(), size);
const step = await this.storeFrameAsStep(guideId, 'region', {
image: cropped,
size,
display,
cursor: null,
}, null, null);
return { ok: true, step };
}
app/main.js
+29
View File
@@ -19,6 +19,7 @@ const { exportGuideArchive, importGuideArchive, saveLinkedGuide, readArchive } =
const { createSnapshot, listSnapshots, restoreSnapshot } = require('../core/snapshots');
const { readLock } = require('../core/locks');
const CaptureService = require('./capture');
const { TextIntelService } = require('./text-intel');
const { keepProcessesResponsive } = require('./win-power');
const APP_ID = 'com.stepforge.app';
@@ -59,6 +60,7 @@ let settings;
let searchIndex;
let templates;
let capture;
let textIntel;
let mainWindow;
function reindex(guideId) {
@@ -503,6 +505,22 @@ function setupIpc() {
if (keyPath.startsWith('capture.hotkey')) registerHotkeys();
return settings.data;
});
h('ai:test', async ({ enabled = null, ollama = null } = {}) => {
return textIntel.testAiConnection({
enabled,
ollama,
});
});
h('ai:fillStep', async ({ guideId, stepId, target = 'all', blockId = null } = {}) => {
const result = await textIntel.generateStepPatch({
guideId,
stepId,
target,
blockId,
});
if (result.ok) reindex(guideId);
return result;
});
h('placeholders:globals:get', () => settings.getGlobalPlaceholders());
h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values));
@@ -739,6 +757,13 @@ if (!gotLock) {
settings = new Settings(store.settingsDir);
searchIndex = new SearchIndex(store.indexDir);
templates = new TemplateManager(store.templatesDir);
textIntel = new TextIntelService({
store,
settings,
getWindow: () => mainWindow,
dataDir,
screenApi: screen,
});
// Bringing up the desktop-capture stream spawns/upgrades Chromium's GPU
// and screen-capture utility processes — which can be born after a session
// already started, so the start-time EcoQoS opt-out misses them. Re-apply
@@ -759,6 +784,7 @@ if (!gotLock) {
settings,
getWindow: () => mainWindow,
notify: captureNotify,
textIntel,
});
applyTheme();
@@ -778,6 +804,9 @@ if (!gotLock) {
capture.stopClickWatcher();
capture.destroySessionTray();
}
if (textIntel) {
textIntel.shutdown().catch(() => {});
}
// clean preview temp files on close
try {
for (const entry of fs.readdirSync(store.tempDir)) {
app/preload.js
+4
View File
@@ -52,6 +52,10 @@ const api = {
globalPlaceholders: invoke('placeholders:globals:get'),
setGlobalPlaceholders: invoke('placeholders:globals:set'),
},
ai: {
test: invoke('ai:test'),
fillStep: invoke('ai:fillStep'),
},
capture: {
shoot: invoke('capture:shoot'),
region: invoke('capture:region'),
app/renderer/app.js
+3
View File
@@ -117,6 +117,7 @@ class StepForgeApp {
guideFolders: library.folders?.guideFolders || {},
};
this.state.trash = trash;
this.editor.setSettings(settings);
}
async refreshLibrary({ keepFilter = true } = {}) {
@@ -320,6 +321,7 @@ class StepForgeApp {
{ label: 'Guide information…', action: () => this.editor.openGuideInfo() },
{ label: 'Guide placeholders…', action: () => this.editor.openGuidePlaceholders() },
{ label: 'Backups & snapshots…', action: () => this.editor.openBackupsDialog() },
{ label: 'Generate all text fields with AI', action: () => this.editor.generateAllTextFieldsWithAi() },
{ label: guide && guide.linkedSource ? 'Linked guide…' : 'Linked guide (not linked)', action: () => this.editor.openLinkedGuide() },
'sep',
{ label: 'Keyboard shortcuts…', action: () => this.editor.openShortcutsHelp() },
@@ -866,6 +868,7 @@ class StepForgeApp {
await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck });
await api.settings.set({ keyPath: 'capture', value: next.capture });
await api.settings.set({ keyPath: 'editor', value: next.editor });
await api.settings.set({ keyPath: 'ai', value: next.ai });
await api.settings.set({ keyPath: 'exports', value: next.exports });
await api.settings.set({ keyPath: 'backups', value: next.backups });
await api.settings.setGlobalPlaceholders(next.placeholders || {});
app/renderer/dialogs.js
+66
View File
@@ -313,6 +313,42 @@ function showSettingsDialog({
const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) });
const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) });
const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 });
const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) });
const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434');
const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b');
const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.');
const testAiBtn = el('button', { type: 'button' }, 'Test connection');
const updateAiStatus = (message, { error = false } = {}) => {
aiStatus.textContent = message;
aiStatus.classList.toggle('error', Boolean(error));
};
const testAiConnection = async () => {
setButtonLoading(testAiBtn, true, 'Testing…');
updateAiStatus('Checking Ollama at the configured host…');
try {
const result = await api.ai.test({
ollama: {
host: ollamaHost.value.trim(),
model: ollamaModel.value.trim(),
},
});
if (!result.ok) {
updateAiStatus(result.reason || 'Could not connect to Ollama.', { error: true });
return;
}
if (result.installed) {
updateAiStatus(`Connected to ${result.host} with ${result.model}.`);
} else {
updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true });
}
} catch (err) {
updateAiStatus(err.message || 'Could not connect to Ollama.', { error: true });
} finally {
setButtonLoading(testAiBtn, false);
}
};
const placeholderRows = el('div', { className: 'placeholder-rows' });
const rows = [];
@@ -369,6 +405,19 @@ function showSettingsDialog({
el('legend', {}, 'Backups'),
labeledRow('Keep last snapshots', keepLast),
),
el('fieldset', {},
el('legend', {}, 'AI'),
labeledRow('Enable AI text filling', aiEnabled),
labeledRow('Ollama host', ollamaHost),
labeledRow('Ollama model', ollamaModel),
el('div.row', { style: { justifyContent: 'space-between' } },
aiStatus,
testAiBtn,
),
el('div.muted', {},
'AI generation is manual only. Captures stay deterministic until you click a generate button.',
),
),
el('fieldset', {},
el('legend', {}, 'Global placeholders'),
placeholderRows,
@@ -412,6 +461,15 @@ function showSettingsDialog({
...settings.backups,
keepLast: Number(keepLast.value || 0),
},
ai: {
...settings.ai,
enabled: aiEnabled.checked,
ollama: {
...(settings.ai?.ollama || {}),
host: ollamaHost.value.trim(),
model: ollamaModel.value.trim(),
},
},
placeholders: rows.reduce((acc, row) => {
const inputs = row.querySelectorAll('input');
const key = inputs[0].value.trim();
@@ -430,6 +488,14 @@ function showSettingsDialog({
});
form.addEventListener('submit', (e) => e.preventDefault());
testAiBtn.addEventListener('click', testAiConnection);
aiEnabled.addEventListener('change', () => {
updateAiStatus(
aiEnabled.checked
? 'AI generation will be available once Ollama is reachable.'
: 'AI generation is disabled. The settings are still saved for later.',
);
});
});
}
app/renderer/editor.js
+107 -2
View File
@@ -135,6 +135,7 @@ class GuideEditor {
this.descriptionDirty = false;
this.titleDirty = false;
this.active = true;
this.settings = {};
this.saveStepDebounced = debounce(() => this.flushStep(), 180);
this.saveGuideDebounced = debounce(() => this.flushGuide(), 180);
@@ -152,6 +153,80 @@ class GuideEditor {
this.active = Boolean(active);
}
setSettings(settings) {
this.settings = settings || {};
this.updateAiButtonState();
}
isAiEnabled() {
return Boolean(this.settings?.ai?.enabled);
}
updateAiButtonState() {
const enabled = this.isAiEnabled();
const buttons = [
this.dom?.titleAiBtn,
this.dom?.descAiBtn,
...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []),
].filter(Boolean);
for (const button of buttons) {
button.disabled = !enabled;
button.title = enabled
? button.dataset.aiTitle || 'Generate with AI'
: 'Enable AI in Settings first.';
}
}
async runAiGeneration(target, { blockId = null, button = null } = {}) {
if (!this.currentStep) {
this.onToast('Select a step first.', { error: true });
return null;
}
if (!this.isAiEnabled()) {
this.onToast('Enable AI in Settings first.', { error: true });
return null;
}
if (this.pendingSave) await this.flushStep();
if (button) setButtonLoading(button, true, 'Generating…');
try {
const result = await api.ai.fillStep({
guideId: this.guideId,
stepId: this.currentStep.stepId,
target,
blockId,
});
if (!result || !result.ok) {
this.onToast(result?.reason || 'AI generation failed.', { error: true });
return null;
}
await this.reload(result.step.stepId);
this.onToast('AI text filled.');
return result.step;
} catch (err) {
this.onToast(err.message || 'AI generation failed.', { error: true });
return null;
} finally {
if (button) setButtonLoading(button, false);
}
}
async generateTitleWithAi(button = null) {
return this.runAiGeneration('title', { button });
}
async generateDescriptionWithAi(button = null) {
return this.runAiGeneration('description', { button });
}
async generateAllTextFieldsWithAi(button = null) {
return this.runAiGeneration('all', { button });
}
async generateBlockWithAi(kind, block, button = null) {
if (!block) return null;
return this.runAiGeneration('block', { blockId: block.id, button });
}
get currentStep() {
return this.stepMap.get(this.selectedStepId) || null;
}
@@ -271,7 +346,14 @@ class GuideEditor {
el('aside.pane-props', {},
el('section', {},
el('h3', {}, 'Step'),
this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title' }),
el('div.row', {},
this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title', style: { flex: 1 } }),
this.dom.titleAiBtn = el('button.ai', {
type: 'button',
title: 'Generate the step title with AI',
dataset: { aiAction: 'title', aiTitle: 'Generate the step title with AI' },
}, 'AI'),
),
this.dom.statusSelect = makeSelect('todo', [
{ value: 'todo', label: 'Todo' },
{ value: 'in-progress', label: 'In progress' },
@@ -296,7 +378,14 @@ class GuideEditor {
),
),
el('section', {},
el('h3', {}, 'Description'),
el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } },
el('h3', { style: { margin: 0 } }, 'Description'),
this.dom.descAiBtn = el('button.ai', {
type: 'button',
title: 'Generate the description with AI',
dataset: { aiAction: 'description', aiTitle: 'Generate the description with AI' },
}, 'AI'),
),
this.dom.richToolbar = el('div.rich-toolbar', {},
this.toolbarBtn('bold', 'Bold'),
this.toolbarBtn('italic', 'Italic'),
@@ -416,6 +505,7 @@ class GuideEditor {
this.renderStepList();
this.emitMeta();
});
this.dom.titleAiBtn.addEventListener('click', () => this.generateTitleWithAi(this.dom.titleAiBtn));
this.dom.statusSelect.addEventListener('change', () => {
if (!this.currentStep) return;
@@ -491,6 +581,7 @@ class GuideEditor {
document.addEventListener('selectionchange', () => {
if (document.activeElement === this.dom.descEditor) this.updateToolbarState();
});
this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn));
this.dom.descEditor.addEventListener('paste', (e) => {
// Keep pasted text simple; backend sanitization will handle the rest.
@@ -504,6 +595,8 @@ class GuideEditor {
if (!item) return;
this.canvas.select(item.dataset.annId);
});
this.updateAiButtonState();
}
renderAll() {
@@ -513,6 +606,7 @@ class GuideEditor {
this.renderCanvas();
this.renderAnnotationPanel();
this.renderBlocksPanel();
this.updateAiButtonState();
this.emitMeta();
}
@@ -670,6 +764,15 @@ class GuideEditor {
el('span.spacer'),
el('button.icon', { type: 'button', title: 'Move block up', disabled: !canMoveUp, onClick: moveUp, dataset: { blockMove: 'up' } }, '↑'),
el('button.icon', { type: 'button', title: 'Move block down', disabled: !canMoveDown, onClick: moveDown, dataset: { blockMove: 'down' } }, '↓'),
(() => {
const aiBtn = el('button.ai', {
type: 'button',
title: 'Generate this block with AI',
dataset: { aiAction: 'block', aiTitle: 'Generate this block with AI' },
onClick: () => this.generateBlockWithAi(kind, block, aiBtn),
}, 'AI');
return aiBtn;
})(),
removeBtn(() => {
if (kind === 'text') step.textBlocks = (step.textBlocks || []).filter((b) => b !== block);
else if (kind === 'code') step.codeBlocks = (step.codeBlocks || []).filter((b) => b !== block);
@@ -761,6 +864,7 @@ class GuideEditor {
if (!blocks.length) {
this.dom.blocksList.append(el('div.muted', {}, 'Informational text, code, and table blocks can be reordered with drag handles or arrows.'));
}
this.updateAiButtonState();
}
renderStepList() {
@@ -1658,6 +1762,7 @@ class GuideEditor {
await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck });
await api.settings.set({ keyPath: 'capture', value: next.capture });
await api.settings.set({ keyPath: 'editor', value: next.editor });
await api.settings.set({ keyPath: 'ai', value: next.ai });
await api.settings.set({ keyPath: 'exports', value: next.exports });
await api.settings.set({ keyPath: 'backups', value: next.backups });
await api.settings.setGlobalPlaceholders(next.placeholders || {});
app/renderer/style.css
+11
View File
@@ -90,6 +90,16 @@ button.tool.active {
border-color: var(--accent);
color: var(--accent-fg);
}
button.ai {
padding: 5px 10px;
border-color: color-mix(in srgb, var(--accent) 38%, var(--border));
background: color-mix(in srgb, var(--accent) 10%, var(--panel-solid));
color: var(--accent-strong);
font-weight: 650;
letter-spacing: 0.02em;
}
button.ai:hover { background: color-mix(in srgb, var(--accent) 16%, var(--panel-2)); }
button.ai:disabled { opacity: 0.6; }
button:disabled { opacity: 0.6; cursor: default; }
button.loading {
display: inline-flex;
@@ -751,6 +761,7 @@ fieldset legend {
color: var(--muted);
line-height: 1.5;
}
.ai-status.error { color: var(--danger); }
#modal-root:not(:empty) {
position: fixed;
app/text-intel.js
+438
View File
@@ -0,0 +1,438 @@
'use strict';
const fs = require('node:fs');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const { createWorker } = require('tesseract.js');
const {
buildCaptureTitle,
normalizeOllamaHost,
normalizeAiPatch,
buildAiPrompt,
applyAiPatchToStep,
displayText,
normalizeWhitespace,
} = require('../core/text-intel');
const OCR_CROP = {
width: 420,
height: 220,
};
function hasBinary(name) {
try {
execFileSync('which', [name], { stdio: 'pipe' });
return true;
} catch {
return false;
}
}
function clamp(v, min, max) {
return Math.min(max, Math.max(min, v));
}
class TextIntelService {
constructor({
store,
settings,
getWindow = () => null,
dataDir,
fetchImpl = global.fetch,
screenApi = null,
}) {
this.store = store;
this.settings = settings;
this.getWindow = getWindow;
this.dataDir = dataDir;
this.fetch = fetchImpl;
this.screen = screenApi;
this.worker = null;
this.workerPromise = null;
this.workerQueue = Promise.resolve();
this.ocrDataDir = path.join(dataDir, 'ocr', 'eng');
}
async shutdown() {
if (this.worker) {
try {
await this.worker.terminate();
} catch {
// best effort
}
this.worker = null;
this.workerPromise = null;
}
}
ensureLangData() {
const packageDir = path.dirname(require.resolve('@tesseract.js-data/eng/package.json'));
const source = path.join(packageDir, '4.0.0_best_int', 'eng.traineddata.gz');
const targetDir = this.ocrDataDir;
const target = path.join(targetDir, 'eng.traineddata.gz');
if (!fs.existsSync(target)) {
fs.mkdirSync(targetDir, { recursive: true });
fs.copyFileSync(source, target);
}
return targetDir;
}
async getWorker() {
if (this.workerPromise) return this.workerPromise;
this.workerPromise = (async () => {
const langPath = this.ensureLangData();
const worker = await createWorker('eng', 1, {
langPath,
});
await worker.setParameters({
preserve_interword_spaces: '1',
});
this.worker = worker;
return worker;
})();
return this.workerPromise;
}
async recognizeCrop(image, rect = null) {
const worker = await this.getWorker();
const cropped = rect ? image.crop(rect) : image;
const buffer = cropped.toPNG();
const result = await worker.recognize(buffer);
const text = String(result?.data?.text || '').trim();
return {
text,
confidence: Number.isFinite(result?.data?.confidence) ? result.data.confidence : null,
raw: result,
};
}
cropRectForPoint(frame, clickPos, { width = OCR_CROP.width, height = OCR_CROP.height } = {}) {
if (!frame || !frame.size) return null;
const bounds = frame.display.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height };
const scaleX = frame.size.width / bounds.width;
const scaleY = frame.size.height / bounds.height;
const point = clickPos || {
x: bounds.x + bounds.width / 2,
y: bounds.y + bounds.height / 2,
};
const centerX = (point.x - bounds.x) * scaleX;
const centerY = (point.y - bounds.y) * scaleY;
const rectW = Math.max(1, Math.round(width * scaleX));
const rectH = Math.max(1, Math.round(height * scaleY));
const rect = {
x: Math.round(centerX - rectW / 2),
y: Math.round(centerY - rectH / 2),
width: rectW,
height: rectH,
};
rect.x = clamp(rect.x, 0, Math.max(0, frame.size.width - rect.width));
rect.y = clamp(rect.y, 0, Math.max(0, frame.size.height - rect.height));
rect.width = clamp(rect.width, 1, frame.size.width);
rect.height = clamp(rect.height, 1, frame.size.height);
return rect;
}
async ocrAroundClick(frame, clickPos) {
if (!frame || !frame.image) return { text: '', confidence: null };
const rect = this.cropRectForPoint(frame, clickPos);
try {
return await this.recognizeCrop(frame.image, rect);
} catch {
return { text: '', confidence: null };
}
}
async collectForegroundWindowContext(osPoint = null) {
try {
if (process.platform === 'win32') return this.collectWindowsWindowContext(osPoint);
if (process.platform === 'darwin') return this.collectMacWindowContext();
if (process.platform === 'linux') return this.collectLinuxWindowContext();
} catch {
// best effort only
}
return { appName: '', windowTitle: '' };
}
collectWindowsWindowContext(osPoint = null) {
const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y);
const clickX = hasPoint ? Number(osPoint.x) : 0;
const clickY = hasPoint ? Number(osPoint.y) : 0;
const script = `
$clickX = ${clickX};
$clickY = ${clickY};
$elementLabel = '';
$elementRole = '';
$elementClass = '';
$elementProcessId = 0;
if (${hasPoint ? '$true' : '$false'}) {
try {
Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null
$point = New-Object System.Windows.Point($clickX, $clickY);
$element = [System.Windows.Automation.AutomationElement]::FromPoint($point);
if ($element) {
$current = $element.Current;
$elementLabel = $current.Name;
$elementRole = $current.LocalizedControlType;
$elementClass = $current.ClassName;
$elementProcessId = $current.ProcessId;
}
} catch { }
}
Add-Type @"
using System;
using System.Runtime.InteropServices;
using System.Text;
public static class Win32 {
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
[DllImport("user32.dll", CharSet = CharSet.Unicode)]
public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId);
}
"@;
$hWnd = [Win32]::GetForegroundWindow();
$sb = New-Object System.Text.StringBuilder 512;
[void][Win32]::GetWindowText($hWnd, $sb, $sb.Capacity);
$pid = 0;
[void][Win32]::GetWindowThreadProcessId($hWnd, [ref]$pid);
$proc = Get-Process -Id $pid -ErrorAction SilentlyContinue | Select-Object -First 1;
$out = [ordered]@{
appName = if ($proc) { $proc.ProcessName } else { '' };
windowTitle = $sb.ToString();
elementLabel = $elementLabel;
elementRole = $elementRole;
elementClass = $elementClass;
elementProcessId = $elementProcessId;
pid = $pid;
};
$out | ConvertTo-Json -Compress;
`;
const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
}).trim();
return JSON.parse(result || '{}');
}
collectMacWindowContext() {
const script = `
set appName to ""
set windowTitle to ""
tell application "System Events"
try
set frontApp to first application process whose frontmost is true
set appName to name of frontApp
try
set windowTitle to name of front window of frontApp
end try
end try
end tell
return appName & linefeed & windowTitle
`;
const result = execFileSync('osascript', ['-e', script], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
}).trimEnd();
const [appName = '', windowTitle = ''] = result.split(/\r?\n/);
return { appName, windowTitle };
}
collectLinuxWindowContext() {
if (!hasBinary('xprop')) return { appName: '', windowTitle: '' };
const active = execFileSync('xprop', ['-root', '_NET_ACTIVE_WINDOW'], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
});
const activeMatch = active.match(/window id # (0x[0-9a-fA-F]+)/);
if (!activeMatch) return { appName: '', windowTitle: '' };
const winId = activeMatch[1];
const details = execFileSync('xprop', ['-id', winId, '_NET_WM_NAME', 'WM_NAME', 'WM_CLASS'], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
});
const titleMatch = details.match(/(?:_NET_WM_NAME\(UTF8_STRING\)|WM_NAME\(STRING\)|WM_NAME\(UTF8_STRING\)) = "([^"]*)"/);
const classMatch = details.match(/WM_CLASS\(STRING\) = "([^"]*)"(?:, "([^"]*)")?/);
return {
appName: classMatch ? (classMatch[2] || classMatch[1] || '') : '',
windowTitle: titleMatch ? titleMatch[1] : '',
};
}
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
this.ocrAroundClick(frame, clickPos),
]);
return buildCaptureTitle({
mode,
metadata,
ocrText: ocr.text,
});
}
aiEnabled() {
return Boolean(this.settings.get('ai.enabled'));
}
aiConfig(override = null) {
const stored = this.settings.get('ai') || {};
const merged = override ? {
...stored,
...override,
ollama: {
...(stored.ollama || {}),
...(override.ollama || {}),
},
} : stored;
return {
...merged,
enabled: override && Object.prototype.hasOwnProperty.call(override, 'enabled')
? Boolean(override.enabled)
: Boolean(stored.enabled),
ollama: {
host: normalizeOllamaHost(merged.ollama?.host || ''),
model: normalizeWhitespace(merged.ollama?.model || ''),
},
};
}
async testAiConnection(override = null) {
const config = this.aiConfig(override);
if (!config.ollama.host) {
return { ok: false, reason: 'Set an Ollama host first.' };
}
const tagsUrl = new URL('/api/tags', `${config.ollama.host.replace(/\/+$/, '')}/`);
const res = await this.fetch(tagsUrl, { method: 'GET' });
if (!res.ok) {
return { ok: false, reason: `Ollama check failed (${res.status})` };
}
const data = await res.json();
const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : [];
const installed = config.ollama.model ? models.includes(config.ollama.model) : false;
return {
ok: true,
installed,
models,
host: config.ollama.host,
model: config.ollama.model,
};
}
async callOllama({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
stream: false,
format: 'json',
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: prompt },
],
options: {
temperature: 0.2,
},
}),
});
if (!response.ok) {
throw new Error(`Ollama request failed (${response.status})`);
}
const payload = await response.json();
const content = payload?.message?.content;
if (typeof content !== 'string' || !content.trim()) {
throw new Error('Ollama returned an empty response');
}
return content;
}
async generateStepPatch({
guideId,
stepId,
target = 'all',
blockId = null,
}) {
try {
const config = this.aiConfig();
if (!config.enabled) {
return { ok: false, reason: 'Enable AI in settings first.' };
}
if (!config.ollama.host || !config.ollama.model) {
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
}
const guide = this.store.getGuide(guideId);
const step = this.store.getStep(guideId, stepId);
if (!guide || !step) {
return { ok: false, reason: 'Guide or step not found.' };
}
const currentBlock = blockId
? [...(step.textBlocks || []), ...(step.codeBlocks || []), ...(step.tableBlocks || [])].find((b) => b.id === blockId) || null
: null;
if (blockId && target === 'block' && !currentBlock) {
return { ok: false, reason: 'Block not found.' };
}
let captureContext = null;
if (step.image) {
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
if (imagePath && fs.existsSync(imagePath)) {
const { nativeImage } = require('electron');
const image = nativeImage.createFromPath(imagePath);
if (!image.isEmpty()) {
const clickPoint = this.clickPointFromStep(step, image);
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(),
this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint),
]);
captureContext = {
...metadata,
ocrText: ocr.text,
mode: step.kind === 'image' ? 'fullscreen' : 'content',
};
}
}
}
const { systemPrompt, prompt } = buildAiPrompt({
target,
guide,
step,
captureContext,
block: currentBlock,
});
const raw = await this.callOllama({
host: config.ollama.host,
model: config.ollama.model,
prompt,
systemPrompt,
});
const patch = normalizeAiPatch(raw);
const updated = applyAiPatchToStep(step, patch, { target, blockId });
const saved = this.store.saveStep(guideId, updated);
return { ok: true, step: saved, patch };
} catch (err) {
return { ok: false, reason: err && err.message ? err.message : 'AI generation failed.' };
}
}
clickPointFromStep(step, image = null) {
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
if (!marker) return null;
const size = image ? image.getSize() : step.image?.size || { width: 0, height: 0 };
if (!size.width || !size.height) return null;
return {
x: Math.round((marker.x + marker.w / 2) * size.width),
y: Math.round((marker.y + marker.h / 2) * size.height),
};
}
}
module.exports = { TextIntelService };
core/settings.js
+7
View File
@@ -47,6 +47,13 @@ const DEFAULT_SETTINGS = {
focusedViewDefaultForNewSteps: false,
autoTitleTemplate: '[[Mode]] capture [[Time]]',
},
ai: {
enabled: false,
ollama: {
host: 'http://127.0.0.1:11434',
model: 'llama3.2:1b',
},
},
exports: {
previewStepCount: 3,
openFolderAfterExport: true,
core/text-intel.js
+416
View File
@@ -0,0 +1,416 @@
'use strict';
const { deepClone, htmlToText, escapeHtml } = require('./util');
const { sanitizeHtml } = require('./sanitize');
const {
TEXTBLOCK_LEVELS,
TEXTBLOCK_POSITIONS,
normalizeTextBlock,
normalizeCodeBlock,
normalizeTableBlock,
} = require('./schema');
const DEFAULT_CAPTURE_TITLES = {
fullscreen: 'Screen capture',
window: 'Window capture',
region: 'Region capture',
};
const AI_LEVEL_ALIASES = new Map([
['note', 'info'],
['info', 'info'],
['tip', 'success'],
['success', 'success'],
['warning', 'warn'],
['warn', 'warn'],
['important', 'error'],
['error', 'error'],
]);
const GENERIC_OCR_PHRASES = new Set([
'button',
'click',
'double click',
'menu',
'item',
'field',
'text field',
'search',
'submit',
'cancel',
'ok',
'open',
'select',
'enter',
'type',
]);
function normalizeWhitespace(text) {
return String(text == null ? '' : text)
.replace(/\s+/g, ' ')
.trim();
}
function titleCaseWord(word) {
if (!word) return word;
if (/^[A-Z0-9]{2,}$/.test(word)) return word;
if (/^\d+$/.test(word)) return word;
return word[0].toUpperCase() + word.slice(1).toLowerCase();
}
function displayText(text) {
const clean = normalizeWhitespace(text)
.replace(/^[\s"'`([{<]+|[\s"'`)}\]>.,;:!?]+$/g, '')
.trim();
if (!clean) return '';
if (clean === clean.toUpperCase()) {
return clean.split(/\s+/).map(titleCaseWord).join(' ');
}
return clean.replace(/\s+/g, ' ');
}
function candidateWords(text) {
const clean = normalizeWhitespace(text);
if (!clean) return [];
return clean.split(/\s+/).filter(Boolean);
}
function scoreCandidate(text, { source = 'ocr' } = {}) {
const clean = displayText(text);
if (!clean) return -Infinity;
const words = candidateWords(clean);
if (!words.length) return -Infinity;
let score = 0;
score += source === 'element' ? 110 : source === 'window' ? 95 : source === 'app' ? 80 : 100;
score += Math.min(words.length, 6) * 12;
score -= Math.max(0, words.length - 6) * 10;
score -= Math.max(0, clean.length - 56) * 0.6;
if (GENERIC_OCR_PHRASES.has(clean.toLowerCase())) score -= 50;
if (clean.length <= 24) score += 10;
if (/^(click|open|enter|type|choose|select|save|cancel|confirm)\b/i.test(clean)) score += 8;
if (/^[\p{P}\p{S}0-9]+$/u.test(clean)) score -= 100;
return score;
}
function pickBestOcrPhrase(ocrText) {
const text = normalizeWhitespace(ocrText);
if (!text) return '';
const parts = text
.split(/\n+/)
.map((line) => displayText(line))
.filter(Boolean)
.filter((line) => !GENERIC_OCR_PHRASES.has(line.toLowerCase()));
if (!parts.length) return '';
let best = '';
let bestScore = -Infinity;
for (const part of parts) {
const score = scoreCandidate(part, { source: 'ocr' });
if (score > bestScore) {
best = part;
bestScore = score;
}
}
return best;
}
function isShortUiLabel(text) {
const words = candidateWords(text);
return words.length > 0 && words.length <= 2 && text.length <= 24;
}
function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) {
const candidates = [];
if (metadata.elementLabel) candidates.push({ text: metadata.elementLabel, source: 'element' });
if (metadata.windowTitle) candidates.push({ text: metadata.windowTitle, source: 'window' });
if (metadata.appName && metadata.appName !== metadata.windowTitle) {
candidates.push({ text: metadata.appName, source: 'app' });
}
const ocrPhrase = pickBestOcrPhrase(ocrText);
if (ocrPhrase) candidates.push({ text: ocrPhrase, source: 'ocr' });
let winner = null;
let winnerScore = -Infinity;
for (const candidate of candidates) {
const score = scoreCandidate(candidate.text, { source: candidate.source });
if (score > winnerScore) {
winner = candidate;
winnerScore = score;
}
}
const cleanedWinner = winner ? displayText(winner.text) : '';
if (!cleanedWinner) return DEFAULT_CAPTURE_TITLES[mode] || 'Capture';
if (winner.source === 'window' || winner.source === 'app') {
return `Open ${cleanedWinner}`;
}
if (winner.source === 'element') {
return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner;
}
if (winner.source === 'ocr') {
return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner;
}
return cleanedWinner;
}
function plainTextToHtml(text) {
const trimmed = normalizeWhitespace(text);
if (!trimmed) return '';
return trimmed
.split(/\n{2,}/)
.map((para) => `<p>${escapeHtml(para).replace(/\n/g, '<br>')}</p>`)
.join('');
}
function normalizeOllamaHost(host) {
const raw = normalizeWhitespace(host);
if (!raw) return '';
if (/^https?:\/\//i.test(raw)) return raw.replace(/\/+$/, '');
return `http://${raw.replace(/\/+$/, '')}`;
}
function normalizeAiLevel(level) {
const key = normalizeWhitespace(level).toLowerCase();
return AI_LEVEL_ALIASES.get(key) || (TEXTBLOCK_LEVELS.includes(key) ? key : 'info');
}
function normalizeAiPosition(position) {
const key = normalizeWhitespace(position).toLowerCase();
return TEXTBLOCK_POSITIONS.includes(key) ? key : 'after-description';
}
function normalizeAiBlock(block) {
if (!block || typeof block !== 'object') return null;
const kind = normalizeWhitespace(block.kind).toLowerCase();
if (kind === 'text') {
const normalized = normalizeTextBlock({
id: block.id,
order: Number.isFinite(block.order) ? block.order : null,
position: normalizeAiPosition(block.position),
level: normalizeAiLevel(block.level),
title: displayText(block.title),
descriptionHtml: plainTextToHtml(block.body ?? block.description ?? block.text ?? ''),
}, Number.isFinite(block.order) ? block.order : null);
return { ...normalized, kind: 'text' };
}
if (kind === 'code') {
return {
...normalizeCodeBlock({
id: block.id,
order: Number.isFinite(block.order) ? block.order : null,
language: displayText(block.language).toLowerCase(),
code: String(block.code ?? ''),
}, Number.isFinite(block.order) ? block.order : null),
kind: 'code',
};
}
if (kind === 'table') {
const rows = Array.isArray(block.rows)
? block.rows.map((row) => (Array.isArray(row) ? row.map((cell) => displayText(cell)) : []))
: [];
return {
...normalizeTableBlock({
id: block.id,
order: Number.isFinite(block.order) ? block.order : null,
rows,
}, Number.isFinite(block.order) ? block.order : null),
kind: 'table',
};
}
return null;
}
function normalizeAiPatch(raw) {
let data = raw;
if (typeof raw === 'string') {
const trimmed = raw.trim().replace(/^```(?:json)?\s*|\s*```$/g, '');
const start = trimmed.indexOf('{');
const end = trimmed.lastIndexOf('}');
const jsonText = start >= 0 && end > start ? trimmed.slice(start, end + 1) : trimmed;
data = JSON.parse(jsonText);
}
if (!data || typeof data !== 'object' || Array.isArray(data)) {
throw new Error('AI response must be a JSON object');
}
const out = {
title: displayText(data.title),
descriptionHtml: plainTextToHtml(data.description ?? data.descriptionText ?? ''),
blocks: Array.isArray(data.blocks)
? data.blocks.map((block) => normalizeAiBlock(block)).filter(Boolean)
: [],
};
return out;
}
function summarizeBlocks(step = {}) {
const parts = [];
for (const block of step.textBlocks || []) {
const body = htmlToText(block.descriptionHtml || '');
parts.push(`- Text (${block.level || 'info'}, ${block.position || 'after-description'}): ${block.title || ''}${body ? ` — ${body}` : ''}`.trim());
}
for (const block of step.codeBlocks || []) {
const code = String(block.code || '').trim();
parts.push(`- Code (${block.language || 'plain'}):\n${code || '(empty)'}`);
}
for (const block of step.tableBlocks || []) {
const rows = Array.isArray(block.rows) ? block.rows.length : 0;
const cols = rows > 0 && Array.isArray(block.rows[0]) ? block.rows[0].length : 0;
parts.push(`- Table (${rows}x${cols})`);
}
return parts.length ? parts.join('\n') : '(none)';
}
function summarizeStepForAi(step = {}) {
return [
`Step title: ${step.title || '(empty)'}`,
`Step description: ${htmlToText(step.descriptionHtml || '') || '(empty)'}`,
`Step status: ${step.status || 'todo'}`,
`Blocks:\n${summarizeBlocks(step)}`,
].join('\n');
}
function summarizeGuideForAi(guide = {}) {
return [
`Guide title: ${guide.title || '(untitled)'}`,
`Guide description: ${htmlToText(guide.descriptionHtml || '') || '(empty)'}`,
].join('\n');
}
function buildAiPrompt({
target = 'all',
guide = null,
step = null,
captureContext = null,
block = null,
} = {}) {
const targetText = {
title: 'rewrite only the step title',
description: 'rewrite only the step description',
block: 'rewrite only one block',
all: 'rewrite the step title, description, and any useful blocks',
}[target] || 'rewrite the step';
const allowedBlockNote = [
'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.',
'Use block.kind = "code" for code snippets.',
'Use block.kind = "table" for tables, with rows as arrays of strings.',
'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].',
].join(' ');
const prompt = [
'You write concise, human-sounding step-by-step documentation for an offline desktop app.',
'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.',
'Schema:',
'{',
' "title": string,',
' "description": string,',
' "blocks": [{',
' "kind": "text" | "code" | "table",',
' "position"?: "before-title" | "after-title" | "before-image" | "after-image" | "before-description" | "after-description",',
' "level"?: "info" | "warn" | "error" | "success",',
' "title"?: string,',
' "body"?: string,',
' "language"?: string,',
' "code"?: string,',
' "rows"?: string[][]',
' }]',
'}',
'',
`Target: ${targetText}.`,
allowedBlockNote,
'',
guide ? summarizeGuideForAi(guide) : 'Guide: (not provided)',
'',
step ? summarizeStepForAi(step) : 'Step: (not provided)',
'',
captureContext ? [
`Active window title: ${captureContext.windowTitle || '(unknown)'}`,
`Active app: ${captureContext.appName || '(unknown)'}`,
`OCR text near click: ${captureContext.ocrText || '(none)'}`,
`Capture mode: ${captureContext.mode || '(unknown)'}`,
].join('\n') : 'Capture context: (not provided)',
'',
block ? `Target block:\n${JSON.stringify(block, null, 2)}` : 'Target block: (not provided)',
'',
'Rules:',
'- Keep the wording natural and specific.',
'- Prefer short titles.',
'- Only return blocks that add value.',
'- Do not invent details that are not supported by the capture context.',
'- If the target is one block, only rewrite that block.',
].join('\n');
return {
systemPrompt: 'You are a documentation assistant that only emits JSON.',
prompt,
};
}
function applyAiPatchToStep(step, patch, { target = 'all', blockId = null } = {}) {
const next = deepClone(step);
if ((target === 'all' || target === 'title') && patch.title) {
next.title = displayText(patch.title);
}
if ((target === 'all' || target === 'description') && patch.descriptionHtml) {
next.descriptionHtml = sanitizeHtml(patch.descriptionHtml);
}
if (target === 'all' && Array.isArray(patch.blocks) && patch.blocks.length) {
const textBlocks = [];
const codeBlocks = [];
const tableBlocks = [];
let nextOrder = 1;
for (const block of patch.blocks) {
const clone = deepClone(block);
clone.order = nextOrder++;
if (clone.kind === 'text') textBlocks.push(clone);
else if (clone.kind === 'code') codeBlocks.push(clone);
else if (clone.kind === 'table') tableBlocks.push(clone);
}
next.textBlocks = textBlocks;
next.codeBlocks = codeBlocks;
next.tableBlocks = tableBlocks;
} else if (target === 'block' && blockId && Array.isArray(patch.blocks) && patch.blocks.length) {
const replacement = patch.blocks[0];
const textBlock = (next.textBlocks || []).find((block) => block.id === blockId);
const codeBlock = (next.codeBlocks || []).find((block) => block.id === blockId);
const tableBlock = (next.tableBlocks || []).find((block) => block.id === blockId);
if (textBlock && replacement.kind === 'text') {
if (replacement.position) textBlock.position = replacement.position;
if (replacement.level) textBlock.level = replacement.level;
if (replacement.title) textBlock.title = replacement.title;
if (replacement.descriptionHtml) textBlock.descriptionHtml = sanitizeHtml(replacement.descriptionHtml);
} else if (codeBlock && replacement.kind === 'code') {
if (replacement.language) codeBlock.language = replacement.language;
if (replacement.code) codeBlock.code = replacement.code;
} else if (tableBlock && replacement.kind === 'table') {
if (replacement.rows) tableBlock.rows = replacement.rows;
}
}
if (!next.image) {
const hasBody = Boolean(
next.title ||
htmlToText(next.descriptionHtml || '') ||
(next.textBlocks || []).length ||
(next.codeBlocks || []).length ||
(next.tableBlocks || []).length,
);
if (hasBody) next.kind = 'content';
}
return next;
}
module.exports = {
DEFAULT_CAPTURE_TITLES,
buildCaptureTitle,
plainTextToHtml,
normalizeOllamaHost,
normalizeAiPatch,
buildAiPrompt,
applyAiPatchToStep,
summarizeStepForAi,
summarizeGuideForAi,
displayText,
normalizeWhitespace,
scoreCandidate,
pickBestOcrPhrase,
};
docs/getting_started_with_ai.md
+75
View File
@@ -0,0 +1,75 @@
# Getting Started With AI
StepForge keeps AI local. It talks to your own Ollama server on your machine and does not send guide content to the cloud.
## 1. Install Ollama
Install Ollama from https://ollama.com and make sure the service is running.
On most systems you can verify it with:
```bash
ollama --version
```
## 2. Pull a lightweight model
The recommended default is:
```bash
ollama pull llama3.2:1b
```
That model is small enough to feel responsive on modest hardware, but still good enough for human-sounding titles and short text blocks.
If you need something even smaller, try:
```bash
ollama pull qwen3:0.6b
```
or:
```bash
ollama pull gemma3:270m
```
Those are lighter, but they are usually weaker at writing polished step text.
## 3. Open StepForge settings
In StepForge, open `Settings` and find the `AI` section.
Set:
* `Enable AI text filling` to on
* `Ollama host` to your local Ollama server
* `Ollama model` to `llama3.2:1b` or the smaller model you pulled
The default host is:
```text
http://127.0.0.1:11434
```
## 4. Test the connection
Use the `Test connection` button in the AI settings section.
If the model is installed, StepForge should confirm the host and model.
## 5. Use AI manually
AI is never automatic. After capture, use the `AI` button next to:
* the step title
* the step description
* each text, code, and table block
You can also use `More -> Generate all text fields with AI` to fill the whole step in one pass.
## Notes
* Capture titles are still generated automatically without AI.
* AI generation only works when `Enable AI text filling` is turned on.
* The app always uses local OCR around the click area first, then local AI only when you ask for it.
package-lock.json
Generated
+124
View File
@@ -8,6 +8,10 @@
"name": "stepforge",
"version": "0.1.0",
"license": "MPL-2.0",
"dependencies": {
"@tesseract.js-data/eng": "^1.0.0",
"tesseract.js": "^7.0.0"
},
"devDependencies": {
"electron": "^41.7.1",
"electron-builder": "^26.15.2"
@@ -624,6 +628,12 @@
"node": ">=10"
}
},
"node_modules/@tesseract.js-data/eng": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@tesseract.js-data/eng/-/eng-1.0.0.tgz",
"integrity": "sha512-mbTumm6KQPUHyzTPQaF3ObXYnx0SqqfV2nabqFVQBwD6Kl7PhGSLSzOlfFTWy0P3BjghaSKA2W9GB19Jk+ZcTg==",
"license": "MIT"
},
"node_modules/@types/cacheable-request": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz",
@@ -1057,6 +1067,12 @@
"dev": true,
"license": "MIT"
},
"node_modules/bmp-js": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
"integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==",
"license": "MIT"
},
"node_modules/boolean": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
@@ -2512,6 +2528,12 @@
"node": ">= 14"
}
},
"node_modules/idb-keyval": {
"version": "6.2.5",
"resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.5.tgz",
"integrity": "sha512-eKQkTnS0relYsSOYomx8ozIbmdsQCKUdhyuIaQ2DZgKuaxtyQQMkyD/wlnQN32pO3yutN1b1L8uqwcDKaJd7/Q==",
"license": "Apache-2.0"
},
"node_modules/inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -2541,6 +2563,12 @@
"node": ">=8"
}
},
"node_modules/is-url": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
"integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==",
"license": "MIT"
},
"node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
@@ -2903,6 +2931,26 @@
"node": ">=10"
}
},
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"license": "MIT",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/node-gyp": {
"version": "12.4.0",
"resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.4.0.tgz",
@@ -3024,6 +3072,15 @@
"wrappy": "1"
}
},
"node_modules/opencollective-postinstall": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz",
"integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==",
"license": "MIT",
"bin": {
"opencollective-postinstall": "index.js"
}
},
"node_modules/p-cancelable": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz",
@@ -3314,6 +3371,12 @@
"util-deprecate": "~1.0.1"
}
},
"node_modules/regenerator-runtime": {
"version": "0.13.11",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
"integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==",
"license": "MIT"
},
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -3728,6 +3791,30 @@
"node": ">= 10.0.0"
}
},
"node_modules/tesseract.js": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz",
"integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==",
"hasInstallScript": true,
"license": "Apache-2.0",
"dependencies": {
"bmp-js": "^0.1.0",
"idb-keyval": "^6.2.0",
"is-url": "^1.2.4",
"node-fetch": "^2.6.9",
"opencollective-postinstall": "^2.0.3",
"regenerator-runtime": "^0.13.3",
"tesseract.js-core": "^7.0.0",
"wasm-feature-detect": "^1.8.0",
"zlibjs": "^0.3.1"
}
},
"node_modules/tesseract.js-core": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz",
"integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==",
"license": "Apache-2.0"
},
"node_modules/tiny-async-pool": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.3.0.tgz",
@@ -3785,6 +3872,12 @@
"tmp": "^0.2.0"
}
},
"node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
"license": "MIT"
},
"node_modules/truncate-utf8-bytes": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz",
@@ -3909,6 +4002,12 @@
"dev": true,
"license": "MIT"
},
"node_modules/wasm-feature-detect": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz",
"integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==",
"license": "Apache-2.0"
},
"node_modules/webcrypto-core": {
"version": "1.9.2",
"resolved": "https://registry.npmjs.org/webcrypto-core/-/webcrypto-core-1.9.2.tgz",
@@ -3923,6 +4022,22 @@
"tslib": "^2.8.1"
}
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
"license": "BSD-2-Clause"
},
"node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"license": "MIT",
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
}
},
"node_modules/which": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/which/-/which-5.0.0.tgz",
@@ -4043,6 +4158,15 @@
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/zlibjs": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz",
"integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==",
"license": "MIT",
"engines": {
"node": "*"
}
}
}
}
package.json
+4
View File
@@ -15,6 +15,10 @@
"verify": "bash scripts/verify.sh",
"bootstrap": "bash scripts/bootstrap-offline.sh"
},
"dependencies": {
"@tesseract.js-data/eng": "^1.0.0",
"tesseract.js": "^7.0.0"
},
"devDependencies": {
"electron": "^41.7.1",
"electron-builder": "^26.15.2"
tests/unit/text-intel.test.js
+195
View File
@@ -0,0 +1,195 @@
'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const { makeTmpDir, rmrf } = require('./helpers');
const { createStep } = require('../../core/schema');
const {
buildCaptureTitle,
normalizeAiPatch,
applyAiPatchToStep,
} = require('../../core/text-intel');
const { TextIntelService } = require('../../app/text-intel');
function makeSettings(values = {}) {
const data = {
ai: {
enabled: true,
ollama: {
host: 'http://127.0.0.1:11434',
model: 'llama3.2:1b',
},
},
...values,
};
return {
get(key) {
return key.split('.').reduce((acc, part) => (acc == null ? undefined : acc[part]), data);
},
};
}
test('capture titles prefer semantic metadata before OCR fallback', () => {
const title = buildCaptureTitle({
mode: 'fullscreen',
metadata: { windowTitle: 'Reset user password in admin portal' },
ocrText: 'Save',
});
assert.equal(title, 'Open Reset user password in admin portal');
});
test('capture titles prefer element metadata before window chrome and OCR', () => {
const title = buildCaptureTitle({
mode: 'window',
metadata: {
elementLabel: 'Open advanced settings',
windowTitle: 'Preferences',
},
ocrText: 'Cancel',
});
assert.equal(title, 'Open advanced settings');
});
test('capture titles fall back to OCR when metadata is absent', () => {
const title = buildCaptureTitle({
mode: 'window',
metadata: {},
ocrText: 'Save changes',
});
assert.equal(title, 'Click Save changes');
});
test('ocr crop rectangles clamp to the image bounds', (t) => {
const root = makeTmpDir('text-intel-crop');
t.after(() => rmrf(root));
const service = new TextIntelService({
store: { settingsDir: root },
settings: makeSettings(),
getWindow: () => null,
dataDir: root,
fetchImpl: global.fetch,
});
const frame = {
size: { width: 1000, height: 500 },
display: { bounds: { x: 0, y: 0, width: 1000, height: 500 } },
};
const topLeft = service.cropRectForPoint(frame, { x: 5, y: 5 });
assert.deepEqual(topLeft, { x: 0, y: 0, width: 420, height: 220 });
const bottomRight = service.cropRectForPoint(frame, { x: 995, y: 495 });
assert.deepEqual(bottomRight, { x: 580, y: 280, width: 420, height: 220 });
});
test('ai response normalization and application keeps fields structured', () => {
const patch = normalizeAiPatch(JSON.stringify({
title: 'Open settings',
description: 'Pick the AI tab.',
blocks: [
{
kind: 'text',
position: 'after-description',
level: 'tip',
title: 'Tip',
body: 'Use the local Ollama model.',
},
{
kind: 'code',
language: 'bash',
code: 'ollama pull llama3.2:1b',
},
{
kind: 'table',
rows: [['Name', 'Value'], ['Host', '127.0.0.1']],
},
],
}));
const step = createStep({
title: 'Old title',
descriptionHtml: '<p>Old text</p>',
textBlocks: [{ id: 'tb1', order: 1, position: 'after-description', level: 'info', title: 'Old tip', descriptionHtml: '<p>Old body</p>' }],
codeBlocks: [{ id: 'cb1', order: 2, language: 'text', code: 'old' }],
tableBlocks: [{ id: 'tbl1', order: 3, rows: [['x']] }],
});
const updated = applyAiPatchToStep(step, patch, { target: 'all' });
assert.equal(updated.title, 'Open settings');
assert.equal(updated.descriptionHtml, '<p>Pick the AI tab.</p>');
assert.equal(updated.textBlocks.length, 1);
assert.equal(updated.textBlocks[0].level, 'success');
assert.equal(updated.textBlocks[0].descriptionHtml, '<p>Use the local Ollama model.</p>');
assert.equal(updated.codeBlocks[0].code, 'ollama pull llama3.2:1b');
assert.deepEqual(updated.tableBlocks[0].rows, [['Name', 'Value'], ['Host', '127.0.0.1']]);
});
test('ollama connection test reports installed models', async (t) => {
const root = makeTmpDir('text-intel-ai');
t.after(() => rmrf(root));
const service = new TextIntelService({
store: { settingsDir: root },
settings: makeSettings(),
getWindow: () => null,
dataDir: root,
fetchImpl: async () => ({
ok: true,
json: async () => ({
models: [
{ name: 'llama3.2:1b' },
{ name: 'qwen3:0.6b' },
],
}),
}),
});
const result = await service.testAiConnection();
assert.equal(result.ok, true);
assert.equal(result.installed, true);
assert.equal(result.model, 'llama3.2:1b');
});
test('invalid ollama output fails safely without saving the step', async (t) => {
const root = makeTmpDir('text-intel-ai-invalid');
t.after(() => rmrf(root));
let saveCalls = 0;
const step = createStep({
title: 'Old title',
descriptionHtml: '<p>Old text</p>',
});
const service = new TextIntelService({
store: {
settingsDir: root,
getGuide: () => ({ guideId: 'g1', title: 'Guide', descriptionHtml: '', stepsOrder: ['s1'] }),
getStep: () => step,
stepImagePath: () => null,
saveStep: () => {
saveCalls += 1;
throw new Error('save should not be called');
},
},
settings: makeSettings(),
getWindow: () => null,
dataDir: root,
fetchImpl: async () => ({
ok: true,
json: async () => ({
message: {
content: 'not json at all',
},
}),
}),
});
const result = await service.generateStepPatch({
guideId: 'g1',
stepId: 's1',
target: 'all',
});
assert.equal(result.ok, false);
assert.match(result.reason, /JSON/i);
assert.equal(saveCalls, 0);
assert.equal(step.title, 'Old title');
});