semantic titles ai assist

This commit is contained in:
2026-06-23 17:46:33 -05:00
parent 2b27428849
commit 33eb0f745e
14 changed files with 1518 additions and 17 deletions
+39 -15
View File
@@ -3,7 +3,6 @@
const path = require('node:path');
const { spawn, execFileSync } = require('node:child_process');
const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron');
const { expandPlaceholders } = require('../core/placeholders');
const raster = require('../core/raster');
const { encodePng } = require('../core/png');
const {
@@ -14,6 +13,7 @@ const {
DEFAULT_START_SLACK_MS,
} = require('./click-frames');
const { physicalToDip } = require('./coords');
const { DEFAULT_CAPTURE_TITLES } = require('../core/text-intel');
/**
* Capture service: full-screen, active-window, and region capture, plus a
@@ -114,6 +114,7 @@ class CaptureService {
getWindow,
notify,
screenApi = screen,
textIntel = null,
}) {
this.store = store;
this.settings = settings;
@@ -122,6 +123,7 @@ class CaptureService {
// Injectable for tests; the click/coordinate paths must never reach for
// the global `screen` directly so coordinate handling stays testable.
this.screen = screenApi;
this.textIntel = textIntel;
this.session = null; // { guideId, paused, count, intervalSec }
this.intervalTimer = null;
this.clickWatcher = null;
@@ -133,6 +135,7 @@ class CaptureService {
this.clickWatcherErrTail = '';
this.linuxEvent = null; // event block currently being parsed
this.pendingRawClick = null; // raw press waiting for its coordinate twin
this.pendingClickOsPoint = null;
this.clickQueue = Promise.resolve();
this.frameLoopInFlight = false;
this.frameLoopGrabStartedAt = null;
@@ -463,7 +466,7 @@ class CaptureService {
if (frame) {
clog('click@', clickAt, 'frame', frame.source || 'loop',
'started', frame.startedAt - clickAt, 'ms, captured', frame.capturedAt - clickAt, 'ms rel. click');
const result = this.storeFrameAsStep(guideId, frame.mode, frame, clickPos);
const result = await this.storeFrameAsStep(guideId, frame.mode, frame, clickPos, clickMeta);
if (result.ok) this.noteStepAdded(result.step, trigger, guideId);
return result;
}
@@ -1211,6 +1214,9 @@ public static class SFMouseHook {
let clickPos = osPoint ? this.osPointToDip(osPoint) : null;
if (!clickPos) clickPos = this.screen.getCursorScreenPoint();
clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos);
this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y)
? { x: osPoint.x, y: osPoint.y }
: null;
this.enqueueClickCapture(clickPos, clickAt, button || 'mouse');
}
@@ -1268,7 +1274,17 @@ public static class SFMouseHook {
* fast the user clicks or how slow the encoder is.
*/
enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') {
const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' };
const osPoint = this.pendingClickOsPoint && Number.isFinite(this.pendingClickOsPoint.x) && Number.isFinite(this.pendingClickOsPoint.y)
? this.pendingClickOsPoint
: null;
this.pendingClickOsPoint = null;
const clickMeta = {
at: Number.isFinite(clickAt) ? clickAt : Date.now(),
button: button || 'mouse',
};
if (osPoint) {
clickMeta.osPoint = { x: osPoint.x, y: osPoint.y };
}
if (this.session && !this.session.paused && !this.userIsInApp()) {
// The guide id pins the click to its recording so it can still be
// stored if the session stops while this click waits in the queue.
@@ -1300,7 +1316,18 @@ public static class SFMouseHook {
};
}
storeFrameAsStep(guideId, mode, frame, clickPos = null) {
async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) {
try {
if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') {
return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta });
}
} catch {
// fall back to the local semantic title below
}
return this.autoTitle(mode);
}
async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) {
if (!frame) return { ok: false, reason: 'no capture frame available' };
const annotations = [];
// The click position (DIP, read at event time) wins over the frame's
@@ -1324,7 +1351,7 @@ public static class SFMouseHook {
}
const step = this.store.addStep(guideId, {
title: this.autoTitle(mode),
title: await this.buildStepTitle(mode, frame, clickPos, clickMeta),
annotations,
focusedView: {
enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),
@@ -1335,14 +1362,7 @@ public static class SFMouseHook {
}
autoTitle(mode) {
const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]';
const now = new Date();
const pad = (n) => String(n).padStart(2, '0');
return expandPlaceholders(tplStr, {
Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen',
Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`,
Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`,
});
return DEFAULT_CAPTURE_TITLES[mode] || 'Capture';
}
/** Grab the screen/window image as { image, display } or throw. */
@@ -1451,8 +1471,12 @@ public static class SFMouseHook {
const cropped = image.crop(rect);
const size = cropped.getSize();
if (!size.width || !size.height) return { ok: false, reason: 'empty selection' };
const step = this.store.addStep(guideId, { title: this.autoTitle('region') },
cropped.toPNG(), size);
const step = await this.storeFrameAsStep(guideId, 'region', {
image: cropped,
size,
display,
cursor: null,
}, null, null);
return { ok: true, step };
}
+29
View File
@@ -19,6 +19,7 @@ const { exportGuideArchive, importGuideArchive, saveLinkedGuide, readArchive } =
const { createSnapshot, listSnapshots, restoreSnapshot } = require('../core/snapshots');
const { readLock } = require('../core/locks');
const CaptureService = require('./capture');
const { TextIntelService } = require('./text-intel');
const { keepProcessesResponsive } = require('./win-power');
const APP_ID = 'com.stepforge.app';
@@ -59,6 +60,7 @@ let settings;
let searchIndex;
let templates;
let capture;
let textIntel;
let mainWindow;
function reindex(guideId) {
@@ -503,6 +505,22 @@ function setupIpc() {
if (keyPath.startsWith('capture.hotkey')) registerHotkeys();
return settings.data;
});
h('ai:test', async ({ enabled = null, ollama = null } = {}) => {
return textIntel.testAiConnection({
enabled,
ollama,
});
});
h('ai:fillStep', async ({ guideId, stepId, target = 'all', blockId = null } = {}) => {
const result = await textIntel.generateStepPatch({
guideId,
stepId,
target,
blockId,
});
if (result.ok) reindex(guideId);
return result;
});
h('placeholders:globals:get', () => settings.getGlobalPlaceholders());
h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values));
@@ -739,6 +757,13 @@ if (!gotLock) {
settings = new Settings(store.settingsDir);
searchIndex = new SearchIndex(store.indexDir);
templates = new TemplateManager(store.templatesDir);
textIntel = new TextIntelService({
store,
settings,
getWindow: () => mainWindow,
dataDir,
screenApi: screen,
});
// Bringing up the desktop-capture stream spawns/upgrades Chromium's GPU
// and screen-capture utility processes — which can be born after a session
// already started, so the start-time EcoQoS opt-out misses them. Re-apply
@@ -759,6 +784,7 @@ if (!gotLock) {
settings,
getWindow: () => mainWindow,
notify: captureNotify,
textIntel,
});
applyTheme();
@@ -778,6 +804,9 @@ if (!gotLock) {
capture.stopClickWatcher();
capture.destroySessionTray();
}
if (textIntel) {
textIntel.shutdown().catch(() => {});
}
// clean preview temp files on close
try {
for (const entry of fs.readdirSync(store.tempDir)) {
+4
View File
@@ -52,6 +52,10 @@ const api = {
globalPlaceholders: invoke('placeholders:globals:get'),
setGlobalPlaceholders: invoke('placeholders:globals:set'),
},
ai: {
test: invoke('ai:test'),
fillStep: invoke('ai:fillStep'),
},
capture: {
shoot: invoke('capture:shoot'),
region: invoke('capture:region'),
+3
View File
@@ -117,6 +117,7 @@ class StepForgeApp {
guideFolders: library.folders?.guideFolders || {},
};
this.state.trash = trash;
this.editor.setSettings(settings);
}
async refreshLibrary({ keepFilter = true } = {}) {
@@ -320,6 +321,7 @@ class StepForgeApp {
{ label: 'Guide information…', action: () => this.editor.openGuideInfo() },
{ label: 'Guide placeholders…', action: () => this.editor.openGuidePlaceholders() },
{ label: 'Backups & snapshots…', action: () => this.editor.openBackupsDialog() },
{ label: 'Generate all text fields with AI', action: () => this.editor.generateAllTextFieldsWithAi() },
{ label: guide && guide.linkedSource ? 'Linked guide…' : 'Linked guide (not linked)', action: () => this.editor.openLinkedGuide() },
'sep',
{ label: 'Keyboard shortcuts…', action: () => this.editor.openShortcutsHelp() },
@@ -866,6 +868,7 @@ class StepForgeApp {
await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck });
await api.settings.set({ keyPath: 'capture', value: next.capture });
await api.settings.set({ keyPath: 'editor', value: next.editor });
await api.settings.set({ keyPath: 'ai', value: next.ai });
await api.settings.set({ keyPath: 'exports', value: next.exports });
await api.settings.set({ keyPath: 'backups', value: next.backups });
await api.settings.setGlobalPlaceholders(next.placeholders || {});
+66
View File
@@ -313,6 +313,42 @@ function showSettingsDialog({
const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) });
const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) });
const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 });
const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) });
const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434');
const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b');
const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.');
const testAiBtn = el('button', { type: 'button' }, 'Test connection');
const updateAiStatus = (message, { error = false } = {}) => {
aiStatus.textContent = message;
aiStatus.classList.toggle('error', Boolean(error));
};
const testAiConnection = async () => {
setButtonLoading(testAiBtn, true, 'Testing…');
updateAiStatus('Checking Ollama at the configured host…');
try {
const result = await api.ai.test({
ollama: {
host: ollamaHost.value.trim(),
model: ollamaModel.value.trim(),
},
});
if (!result.ok) {
updateAiStatus(result.reason || 'Could not connect to Ollama.', { error: true });
return;
}
if (result.installed) {
updateAiStatus(`Connected to ${result.host} with ${result.model}.`);
} else {
updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true });
}
} catch (err) {
updateAiStatus(err.message || 'Could not connect to Ollama.', { error: true });
} finally {
setButtonLoading(testAiBtn, false);
}
};
const placeholderRows = el('div', { className: 'placeholder-rows' });
const rows = [];
@@ -369,6 +405,19 @@ function showSettingsDialog({
el('legend', {}, 'Backups'),
labeledRow('Keep last snapshots', keepLast),
),
el('fieldset', {},
el('legend', {}, 'AI'),
labeledRow('Enable AI text filling', aiEnabled),
labeledRow('Ollama host', ollamaHost),
labeledRow('Ollama model', ollamaModel),
el('div.row', { style: { justifyContent: 'space-between' } },
aiStatus,
testAiBtn,
),
el('div.muted', {},
'AI generation is manual only. Captures stay deterministic until you click a generate button.',
),
),
el('fieldset', {},
el('legend', {}, 'Global placeholders'),
placeholderRows,
@@ -412,6 +461,15 @@ function showSettingsDialog({
...settings.backups,
keepLast: Number(keepLast.value || 0),
},
ai: {
...settings.ai,
enabled: aiEnabled.checked,
ollama: {
...(settings.ai?.ollama || {}),
host: ollamaHost.value.trim(),
model: ollamaModel.value.trim(),
},
},
placeholders: rows.reduce((acc, row) => {
const inputs = row.querySelectorAll('input');
const key = inputs[0].value.trim();
@@ -430,6 +488,14 @@ function showSettingsDialog({
});
form.addEventListener('submit', (e) => e.preventDefault());
testAiBtn.addEventListener('click', testAiConnection);
aiEnabled.addEventListener('change', () => {
updateAiStatus(
aiEnabled.checked
? 'AI generation will be available once Ollama is reachable.'
: 'AI generation is disabled. The settings are still saved for later.',
);
});
});
}
+107 -2
View File
@@ -135,6 +135,7 @@ class GuideEditor {
this.descriptionDirty = false;
this.titleDirty = false;
this.active = true;
this.settings = {};
this.saveStepDebounced = debounce(() => this.flushStep(), 180);
this.saveGuideDebounced = debounce(() => this.flushGuide(), 180);
@@ -152,6 +153,80 @@ class GuideEditor {
this.active = Boolean(active);
}
setSettings(settings) {
this.settings = settings || {};
this.updateAiButtonState();
}
isAiEnabled() {
return Boolean(this.settings?.ai?.enabled);
}
updateAiButtonState() {
const enabled = this.isAiEnabled();
const buttons = [
this.dom?.titleAiBtn,
this.dom?.descAiBtn,
...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []),
].filter(Boolean);
for (const button of buttons) {
button.disabled = !enabled;
button.title = enabled
? button.dataset.aiTitle || 'Generate with AI'
: 'Enable AI in Settings first.';
}
}
async runAiGeneration(target, { blockId = null, button = null } = {}) {
if (!this.currentStep) {
this.onToast('Select a step first.', { error: true });
return null;
}
if (!this.isAiEnabled()) {
this.onToast('Enable AI in Settings first.', { error: true });
return null;
}
if (this.pendingSave) await this.flushStep();
if (button) setButtonLoading(button, true, 'Generating…');
try {
const result = await api.ai.fillStep({
guideId: this.guideId,
stepId: this.currentStep.stepId,
target,
blockId,
});
if (!result || !result.ok) {
this.onToast(result?.reason || 'AI generation failed.', { error: true });
return null;
}
await this.reload(result.step.stepId);
this.onToast('AI text filled.');
return result.step;
} catch (err) {
this.onToast(err.message || 'AI generation failed.', { error: true });
return null;
} finally {
if (button) setButtonLoading(button, false);
}
}
async generateTitleWithAi(button = null) {
return this.runAiGeneration('title', { button });
}
async generateDescriptionWithAi(button = null) {
return this.runAiGeneration('description', { button });
}
async generateAllTextFieldsWithAi(button = null) {
return this.runAiGeneration('all', { button });
}
async generateBlockWithAi(kind, block, button = null) {
if (!block) return null;
return this.runAiGeneration('block', { blockId: block.id, button });
}
get currentStep() {
return this.stepMap.get(this.selectedStepId) || null;
}
@@ -271,7 +346,14 @@ class GuideEditor {
el('aside.pane-props', {},
el('section', {},
el('h3', {}, 'Step'),
this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title' }),
el('div.row', {},
this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title', style: { flex: 1 } }),
this.dom.titleAiBtn = el('button.ai', {
type: 'button',
title: 'Generate the step title with AI',
dataset: { aiAction: 'title', aiTitle: 'Generate the step title with AI' },
}, 'AI'),
),
this.dom.statusSelect = makeSelect('todo', [
{ value: 'todo', label: 'Todo' },
{ value: 'in-progress', label: 'In progress' },
@@ -296,7 +378,14 @@ class GuideEditor {
),
),
el('section', {},
el('h3', {}, 'Description'),
el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } },
el('h3', { style: { margin: 0 } }, 'Description'),
this.dom.descAiBtn = el('button.ai', {
type: 'button',
title: 'Generate the description with AI',
dataset: { aiAction: 'description', aiTitle: 'Generate the description with AI' },
}, 'AI'),
),
this.dom.richToolbar = el('div.rich-toolbar', {},
this.toolbarBtn('bold', 'Bold'),
this.toolbarBtn('italic', 'Italic'),
@@ -416,6 +505,7 @@ class GuideEditor {
this.renderStepList();
this.emitMeta();
});
this.dom.titleAiBtn.addEventListener('click', () => this.generateTitleWithAi(this.dom.titleAiBtn));
this.dom.statusSelect.addEventListener('change', () => {
if (!this.currentStep) return;
@@ -491,6 +581,7 @@ class GuideEditor {
document.addEventListener('selectionchange', () => {
if (document.activeElement === this.dom.descEditor) this.updateToolbarState();
});
this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn));
this.dom.descEditor.addEventListener('paste', (e) => {
// Keep pasted text simple; backend sanitization will handle the rest.
@@ -504,6 +595,8 @@ class GuideEditor {
if (!item) return;
this.canvas.select(item.dataset.annId);
});
this.updateAiButtonState();
}
renderAll() {
@@ -513,6 +606,7 @@ class GuideEditor {
this.renderCanvas();
this.renderAnnotationPanel();
this.renderBlocksPanel();
this.updateAiButtonState();
this.emitMeta();
}
@@ -670,6 +764,15 @@ class GuideEditor {
el('span.spacer'),
el('button.icon', { type: 'button', title: 'Move block up', disabled: !canMoveUp, onClick: moveUp, dataset: { blockMove: 'up' } }, '↑'),
el('button.icon', { type: 'button', title: 'Move block down', disabled: !canMoveDown, onClick: moveDown, dataset: { blockMove: 'down' } }, '↓'),
(() => {
const aiBtn = el('button.ai', {
type: 'button',
title: 'Generate this block with AI',
dataset: { aiAction: 'block', aiTitle: 'Generate this block with AI' },
onClick: () => this.generateBlockWithAi(kind, block, aiBtn),
}, 'AI');
return aiBtn;
})(),
removeBtn(() => {
if (kind === 'text') step.textBlocks = (step.textBlocks || []).filter((b) => b !== block);
else if (kind === 'code') step.codeBlocks = (step.codeBlocks || []).filter((b) => b !== block);
@@ -761,6 +864,7 @@ class GuideEditor {
if (!blocks.length) {
this.dom.blocksList.append(el('div.muted', {}, 'Informational text, code, and table blocks can be reordered with drag handles or arrows.'));
}
this.updateAiButtonState();
}
renderStepList() {
@@ -1658,6 +1762,7 @@ class GuideEditor {
await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck });
await api.settings.set({ keyPath: 'capture', value: next.capture });
await api.settings.set({ keyPath: 'editor', value: next.editor });
await api.settings.set({ keyPath: 'ai', value: next.ai });
await api.settings.set({ keyPath: 'exports', value: next.exports });
await api.settings.set({ keyPath: 'backups', value: next.backups });
await api.settings.setGlobalPlaceholders(next.placeholders || {});
+11
View File
@@ -90,6 +90,16 @@ button.tool.active {
border-color: var(--accent);
color: var(--accent-fg);
}
button.ai {
padding: 5px 10px;
border-color: color-mix(in srgb, var(--accent) 38%, var(--border));
background: color-mix(in srgb, var(--accent) 10%, var(--panel-solid));
color: var(--accent-strong);
font-weight: 650;
letter-spacing: 0.02em;
}
button.ai:hover { background: color-mix(in srgb, var(--accent) 16%, var(--panel-2)); }
button.ai:disabled { opacity: 0.6; }
button:disabled { opacity: 0.6; cursor: default; }
button.loading {
display: inline-flex;
@@ -751,6 +761,7 @@ fieldset legend {
color: var(--muted);
line-height: 1.5;
}
.ai-status.error { color: var(--danger); }
#modal-root:not(:empty) {
position: fixed;
+438
View File
@@ -0,0 +1,438 @@
'use strict';
const fs = require('node:fs');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const { createWorker } = require('tesseract.js');
const {
buildCaptureTitle,
normalizeOllamaHost,
normalizeAiPatch,
buildAiPrompt,
applyAiPatchToStep,
displayText,
normalizeWhitespace,
} = require('../core/text-intel');
const OCR_CROP = {
width: 420,
height: 220,
};
function hasBinary(name) {
try {
execFileSync('which', [name], { stdio: 'pipe' });
return true;
} catch {
return false;
}
}
function clamp(v, min, max) {
return Math.min(max, Math.max(min, v));
}
class TextIntelService {
constructor({
store,
settings,
getWindow = () => null,
dataDir,
fetchImpl = global.fetch,
screenApi = null,
}) {
this.store = store;
this.settings = settings;
this.getWindow = getWindow;
this.dataDir = dataDir;
this.fetch = fetchImpl;
this.screen = screenApi;
this.worker = null;
this.workerPromise = null;
this.workerQueue = Promise.resolve();
this.ocrDataDir = path.join(dataDir, 'ocr', 'eng');
}
async shutdown() {
if (this.worker) {
try {
await this.worker.terminate();
} catch {
// best effort
}
this.worker = null;
this.workerPromise = null;
}
}
ensureLangData() {
const packageDir = path.dirname(require.resolve('@tesseract.js-data/eng/package.json'));
const source = path.join(packageDir, '4.0.0_best_int', 'eng.traineddata.gz');
const targetDir = this.ocrDataDir;
const target = path.join(targetDir, 'eng.traineddata.gz');
if (!fs.existsSync(target)) {
fs.mkdirSync(targetDir, { recursive: true });
fs.copyFileSync(source, target);
}
return targetDir;
}
async getWorker() {
if (this.workerPromise) return this.workerPromise;
this.workerPromise = (async () => {
const langPath = this.ensureLangData();
const worker = await createWorker('eng', 1, {
langPath,
});
await worker.setParameters({
preserve_interword_spaces: '1',
});
this.worker = worker;
return worker;
})();
return this.workerPromise;
}
async recognizeCrop(image, rect = null) {
const worker = await this.getWorker();
const cropped = rect ? image.crop(rect) : image;
const buffer = cropped.toPNG();
const result = await worker.recognize(buffer);
const text = String(result?.data?.text || '').trim();
return {
text,
confidence: Number.isFinite(result?.data?.confidence) ? result.data.confidence : null,
raw: result,
};
}
cropRectForPoint(frame, clickPos, { width = OCR_CROP.width, height = OCR_CROP.height } = {}) {
if (!frame || !frame.size) return null;
const bounds = frame.display.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height };
const scaleX = frame.size.width / bounds.width;
const scaleY = frame.size.height / bounds.height;
const point = clickPos || {
x: bounds.x + bounds.width / 2,
y: bounds.y + bounds.height / 2,
};
const centerX = (point.x - bounds.x) * scaleX;
const centerY = (point.y - bounds.y) * scaleY;
const rectW = Math.max(1, Math.round(width * scaleX));
const rectH = Math.max(1, Math.round(height * scaleY));
const rect = {
x: Math.round(centerX - rectW / 2),
y: Math.round(centerY - rectH / 2),
width: rectW,
height: rectH,
};
rect.x = clamp(rect.x, 0, Math.max(0, frame.size.width - rect.width));
rect.y = clamp(rect.y, 0, Math.max(0, frame.size.height - rect.height));
rect.width = clamp(rect.width, 1, frame.size.width);
rect.height = clamp(rect.height, 1, frame.size.height);
return rect;
}
async ocrAroundClick(frame, clickPos) {
if (!frame || !frame.image) return { text: '', confidence: null };
const rect = this.cropRectForPoint(frame, clickPos);
try {
return await this.recognizeCrop(frame.image, rect);
} catch {
return { text: '', confidence: null };
}
}
async collectForegroundWindowContext(osPoint = null) {
try {
if (process.platform === 'win32') return this.collectWindowsWindowContext(osPoint);
if (process.platform === 'darwin') return this.collectMacWindowContext();
if (process.platform === 'linux') return this.collectLinuxWindowContext();
} catch {
// best effort only
}
return { appName: '', windowTitle: '' };
}
collectWindowsWindowContext(osPoint = null) {
const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y);
const clickX = hasPoint ? Number(osPoint.x) : 0;
const clickY = hasPoint ? Number(osPoint.y) : 0;
const script = `
$clickX = ${clickX};
$clickY = ${clickY};
$elementLabel = '';
$elementRole = '';
$elementClass = '';
$elementProcessId = 0;
if (${hasPoint ? '$true' : '$false'}) {
try {
Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null
$point = New-Object System.Windows.Point($clickX, $clickY);
$element = [System.Windows.Automation.AutomationElement]::FromPoint($point);
if ($element) {
$current = $element.Current;
$elementLabel = $current.Name;
$elementRole = $current.LocalizedControlType;
$elementClass = $current.ClassName;
$elementProcessId = $current.ProcessId;
}
} catch { }
}
Add-Type @"
using System;
using System.Runtime.InteropServices;
using System.Text;
public static class Win32 {
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
[DllImport("user32.dll", CharSet = CharSet.Unicode)]
public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId);
}
"@;
$hWnd = [Win32]::GetForegroundWindow();
$sb = New-Object System.Text.StringBuilder 512;
[void][Win32]::GetWindowText($hWnd, $sb, $sb.Capacity);
$pid = 0;
[void][Win32]::GetWindowThreadProcessId($hWnd, [ref]$pid);
$proc = Get-Process -Id $pid -ErrorAction SilentlyContinue | Select-Object -First 1;
$out = [ordered]@{
appName = if ($proc) { $proc.ProcessName } else { '' };
windowTitle = $sb.ToString();
elementLabel = $elementLabel;
elementRole = $elementRole;
elementClass = $elementClass;
elementProcessId = $elementProcessId;
pid = $pid;
};
$out | ConvertTo-Json -Compress;
`;
const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
}).trim();
return JSON.parse(result || '{}');
}
collectMacWindowContext() {
const script = `
set appName to ""
set windowTitle to ""
tell application "System Events"
try
set frontApp to first application process whose frontmost is true
set appName to name of frontApp
try
set windowTitle to name of front window of frontApp
end try
end try
end tell
return appName & linefeed & windowTitle
`;
const result = execFileSync('osascript', ['-e', script], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
}).trimEnd();
const [appName = '', windowTitle = ''] = result.split(/\r?\n/);
return { appName, windowTitle };
}
collectLinuxWindowContext() {
if (!hasBinary('xprop')) return { appName: '', windowTitle: '' };
const active = execFileSync('xprop', ['-root', '_NET_ACTIVE_WINDOW'], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
});
const activeMatch = active.match(/window id # (0x[0-9a-fA-F]+)/);
if (!activeMatch) return { appName: '', windowTitle: '' };
const winId = activeMatch[1];
const details = execFileSync('xprop', ['-id', winId, '_NET_WM_NAME', 'WM_NAME', 'WM_CLASS'], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 1200,
});
const titleMatch = details.match(/(?:_NET_WM_NAME\(UTF8_STRING\)|WM_NAME\(STRING\)|WM_NAME\(UTF8_STRING\)) = "([^"]*)"/);
const classMatch = details.match(/WM_CLASS\(STRING\) = "([^"]*)"(?:, "([^"]*)")?/);
return {
appName: classMatch ? (classMatch[2] || classMatch[1] || '') : '',
windowTitle: titleMatch ? titleMatch[1] : '',
};
}
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
this.ocrAroundClick(frame, clickPos),
]);
return buildCaptureTitle({
mode,
metadata,
ocrText: ocr.text,
});
}
aiEnabled() {
return Boolean(this.settings.get('ai.enabled'));
}
aiConfig(override = null) {
const stored = this.settings.get('ai') || {};
const merged = override ? {
...stored,
...override,
ollama: {
...(stored.ollama || {}),
...(override.ollama || {}),
},
} : stored;
return {
...merged,
enabled: override && Object.prototype.hasOwnProperty.call(override, 'enabled')
? Boolean(override.enabled)
: Boolean(stored.enabled),
ollama: {
host: normalizeOllamaHost(merged.ollama?.host || ''),
model: normalizeWhitespace(merged.ollama?.model || ''),
},
};
}
async testAiConnection(override = null) {
const config = this.aiConfig(override);
if (!config.ollama.host) {
return { ok: false, reason: 'Set an Ollama host first.' };
}
const tagsUrl = new URL('/api/tags', `${config.ollama.host.replace(/\/+$/, '')}/`);
const res = await this.fetch(tagsUrl, { method: 'GET' });
if (!res.ok) {
return { ok: false, reason: `Ollama check failed (${res.status})` };
}
const data = await res.json();
const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : [];
const installed = config.ollama.model ? models.includes(config.ollama.model) : false;
return {
ok: true,
installed,
models,
host: config.ollama.host,
model: config.ollama.model,
};
}
async callOllama({ host, model, prompt, systemPrompt }) {
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
const response = await this.fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
stream: false,
format: 'json',
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: prompt },
],
options: {
temperature: 0.2,
},
}),
});
if (!response.ok) {
throw new Error(`Ollama request failed (${response.status})`);
}
const payload = await response.json();
const content = payload?.message?.content;
if (typeof content !== 'string' || !content.trim()) {
throw new Error('Ollama returned an empty response');
}
return content;
}
async generateStepPatch({
guideId,
stepId,
target = 'all',
blockId = null,
}) {
try {
const config = this.aiConfig();
if (!config.enabled) {
return { ok: false, reason: 'Enable AI in settings first.' };
}
if (!config.ollama.host || !config.ollama.model) {
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
}
const guide = this.store.getGuide(guideId);
const step = this.store.getStep(guideId, stepId);
if (!guide || !step) {
return { ok: false, reason: 'Guide or step not found.' };
}
const currentBlock = blockId
? [...(step.textBlocks || []), ...(step.codeBlocks || []), ...(step.tableBlocks || [])].find((b) => b.id === blockId) || null
: null;
if (blockId && target === 'block' && !currentBlock) {
return { ok: false, reason: 'Block not found.' };
}
let captureContext = null;
if (step.image) {
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
if (imagePath && fs.existsSync(imagePath)) {
const { nativeImage } = require('electron');
const image = nativeImage.createFromPath(imagePath);
if (!image.isEmpty()) {
const clickPoint = this.clickPointFromStep(step, image);
const [metadata, ocr] = await Promise.all([
this.collectForegroundWindowContext(),
this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint),
]);
captureContext = {
...metadata,
ocrText: ocr.text,
mode: step.kind === 'image' ? 'fullscreen' : 'content',
};
}
}
}
const { systemPrompt, prompt } = buildAiPrompt({
target,
guide,
step,
captureContext,
block: currentBlock,
});
const raw = await this.callOllama({
host: config.ollama.host,
model: config.ollama.model,
prompt,
systemPrompt,
});
const patch = normalizeAiPatch(raw);
const updated = applyAiPatchToStep(step, patch, { target, blockId });
const saved = this.store.saveStep(guideId, updated);
return { ok: true, step: saved, patch };
} catch (err) {
return { ok: false, reason: err && err.message ? err.message : 'AI generation failed.' };
}
}
clickPointFromStep(step, image = null) {
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
if (!marker) return null;
const size = image ? image.getSize() : step.image?.size || { width: 0, height: 0 };
if (!size.width || !size.height) return null;
return {
x: Math.round((marker.x + marker.w / 2) * size.width),
y: Math.round((marker.y + marker.h / 2) * size.height),
};
}
}
module.exports = { TextIntelService };