semantic titles ai assist
This commit is contained in:
+39
-15
@@ -3,7 +3,6 @@
|
||||
const path = require('node:path');
|
||||
const { spawn, execFileSync } = require('node:child_process');
|
||||
const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron');
|
||||
const { expandPlaceholders } = require('../core/placeholders');
|
||||
const raster = require('../core/raster');
|
||||
const { encodePng } = require('../core/png');
|
||||
const {
|
||||
@@ -14,6 +13,7 @@ const {
|
||||
DEFAULT_START_SLACK_MS,
|
||||
} = require('./click-frames');
|
||||
const { physicalToDip } = require('./coords');
|
||||
const { DEFAULT_CAPTURE_TITLES } = require('../core/text-intel');
|
||||
|
||||
/**
|
||||
* Capture service: full-screen, active-window, and region capture, plus a
|
||||
@@ -114,6 +114,7 @@ class CaptureService {
|
||||
getWindow,
|
||||
notify,
|
||||
screenApi = screen,
|
||||
textIntel = null,
|
||||
}) {
|
||||
this.store = store;
|
||||
this.settings = settings;
|
||||
@@ -122,6 +123,7 @@ class CaptureService {
|
||||
// Injectable for tests; the click/coordinate paths must never reach for
|
||||
// the global `screen` directly so coordinate handling stays testable.
|
||||
this.screen = screenApi;
|
||||
this.textIntel = textIntel;
|
||||
this.session = null; // { guideId, paused, count, intervalSec }
|
||||
this.intervalTimer = null;
|
||||
this.clickWatcher = null;
|
||||
@@ -133,6 +135,7 @@ class CaptureService {
|
||||
this.clickWatcherErrTail = '';
|
||||
this.linuxEvent = null; // event block currently being parsed
|
||||
this.pendingRawClick = null; // raw press waiting for its coordinate twin
|
||||
this.pendingClickOsPoint = null;
|
||||
this.clickQueue = Promise.resolve();
|
||||
this.frameLoopInFlight = false;
|
||||
this.frameLoopGrabStartedAt = null;
|
||||
@@ -463,7 +466,7 @@ class CaptureService {
|
||||
if (frame) {
|
||||
clog('click@', clickAt, 'frame', frame.source || 'loop',
|
||||
'started', frame.startedAt - clickAt, 'ms, captured', frame.capturedAt - clickAt, 'ms rel. click');
|
||||
const result = this.storeFrameAsStep(guideId, frame.mode, frame, clickPos);
|
||||
const result = await this.storeFrameAsStep(guideId, frame.mode, frame, clickPos, clickMeta);
|
||||
if (result.ok) this.noteStepAdded(result.step, trigger, guideId);
|
||||
return result;
|
||||
}
|
||||
@@ -1211,6 +1214,9 @@ public static class SFMouseHook {
|
||||
let clickPos = osPoint ? this.osPointToDip(osPoint) : null;
|
||||
if (!clickPos) clickPos = this.screen.getCursorScreenPoint();
|
||||
clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos);
|
||||
this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y)
|
||||
? { x: osPoint.x, y: osPoint.y }
|
||||
: null;
|
||||
this.enqueueClickCapture(clickPos, clickAt, button || 'mouse');
|
||||
}
|
||||
|
||||
@@ -1268,7 +1274,17 @@ public static class SFMouseHook {
|
||||
* fast the user clicks or how slow the encoder is.
|
||||
*/
|
||||
enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') {
|
||||
const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' };
|
||||
const osPoint = this.pendingClickOsPoint && Number.isFinite(this.pendingClickOsPoint.x) && Number.isFinite(this.pendingClickOsPoint.y)
|
||||
? this.pendingClickOsPoint
|
||||
: null;
|
||||
this.pendingClickOsPoint = null;
|
||||
const clickMeta = {
|
||||
at: Number.isFinite(clickAt) ? clickAt : Date.now(),
|
||||
button: button || 'mouse',
|
||||
};
|
||||
if (osPoint) {
|
||||
clickMeta.osPoint = { x: osPoint.x, y: osPoint.y };
|
||||
}
|
||||
if (this.session && !this.session.paused && !this.userIsInApp()) {
|
||||
// The guide id pins the click to its recording so it can still be
|
||||
// stored if the session stops while this click waits in the queue.
|
||||
@@ -1300,7 +1316,18 @@ public static class SFMouseHook {
|
||||
};
|
||||
}
|
||||
|
||||
storeFrameAsStep(guideId, mode, frame, clickPos = null) {
|
||||
async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) {
|
||||
try {
|
||||
if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') {
|
||||
return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta });
|
||||
}
|
||||
} catch {
|
||||
// fall back to the local semantic title below
|
||||
}
|
||||
return this.autoTitle(mode);
|
||||
}
|
||||
|
||||
async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) {
|
||||
if (!frame) return { ok: false, reason: 'no capture frame available' };
|
||||
const annotations = [];
|
||||
// The click position (DIP, read at event time) wins over the frame's
|
||||
@@ -1324,7 +1351,7 @@ public static class SFMouseHook {
|
||||
}
|
||||
|
||||
const step = this.store.addStep(guideId, {
|
||||
title: this.autoTitle(mode),
|
||||
title: await this.buildStepTitle(mode, frame, clickPos, clickMeta),
|
||||
annotations,
|
||||
focusedView: {
|
||||
enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')),
|
||||
@@ -1335,14 +1362,7 @@ public static class SFMouseHook {
|
||||
}
|
||||
|
||||
autoTitle(mode) {
|
||||
const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]';
|
||||
const now = new Date();
|
||||
const pad = (n) => String(n).padStart(2, '0');
|
||||
return expandPlaceholders(tplStr, {
|
||||
Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen',
|
||||
Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`,
|
||||
Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`,
|
||||
});
|
||||
return DEFAULT_CAPTURE_TITLES[mode] || 'Capture';
|
||||
}
|
||||
|
||||
/** Grab the screen/window image as { image, display } or throw. */
|
||||
@@ -1451,8 +1471,12 @@ public static class SFMouseHook {
|
||||
const cropped = image.crop(rect);
|
||||
const size = cropped.getSize();
|
||||
if (!size.width || !size.height) return { ok: false, reason: 'empty selection' };
|
||||
const step = this.store.addStep(guideId, { title: this.autoTitle('region') },
|
||||
cropped.toPNG(), size);
|
||||
const step = await this.storeFrameAsStep(guideId, 'region', {
|
||||
image: cropped,
|
||||
size,
|
||||
display,
|
||||
cursor: null,
|
||||
}, null, null);
|
||||
return { ok: true, step };
|
||||
}
|
||||
|
||||
|
||||
+29
@@ -19,6 +19,7 @@ const { exportGuideArchive, importGuideArchive, saveLinkedGuide, readArchive } =
|
||||
const { createSnapshot, listSnapshots, restoreSnapshot } = require('../core/snapshots');
|
||||
const { readLock } = require('../core/locks');
|
||||
const CaptureService = require('./capture');
|
||||
const { TextIntelService } = require('./text-intel');
|
||||
const { keepProcessesResponsive } = require('./win-power');
|
||||
|
||||
const APP_ID = 'com.stepforge.app';
|
||||
@@ -59,6 +60,7 @@ let settings;
|
||||
let searchIndex;
|
||||
let templates;
|
||||
let capture;
|
||||
let textIntel;
|
||||
let mainWindow;
|
||||
|
||||
function reindex(guideId) {
|
||||
@@ -503,6 +505,22 @@ function setupIpc() {
|
||||
if (keyPath.startsWith('capture.hotkey')) registerHotkeys();
|
||||
return settings.data;
|
||||
});
|
||||
h('ai:test', async ({ enabled = null, ollama = null } = {}) => {
|
||||
return textIntel.testAiConnection({
|
||||
enabled,
|
||||
ollama,
|
||||
});
|
||||
});
|
||||
h('ai:fillStep', async ({ guideId, stepId, target = 'all', blockId = null } = {}) => {
|
||||
const result = await textIntel.generateStepPatch({
|
||||
guideId,
|
||||
stepId,
|
||||
target,
|
||||
blockId,
|
||||
});
|
||||
if (result.ok) reindex(guideId);
|
||||
return result;
|
||||
});
|
||||
h('placeholders:globals:get', () => settings.getGlobalPlaceholders());
|
||||
h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values));
|
||||
|
||||
@@ -739,6 +757,13 @@ if (!gotLock) {
|
||||
settings = new Settings(store.settingsDir);
|
||||
searchIndex = new SearchIndex(store.indexDir);
|
||||
templates = new TemplateManager(store.templatesDir);
|
||||
textIntel = new TextIntelService({
|
||||
store,
|
||||
settings,
|
||||
getWindow: () => mainWindow,
|
||||
dataDir,
|
||||
screenApi: screen,
|
||||
});
|
||||
// Bringing up the desktop-capture stream spawns/upgrades Chromium's GPU
|
||||
// and screen-capture utility processes — which can be born after a session
|
||||
// already started, so the start-time EcoQoS opt-out misses them. Re-apply
|
||||
@@ -759,6 +784,7 @@ if (!gotLock) {
|
||||
settings,
|
||||
getWindow: () => mainWindow,
|
||||
notify: captureNotify,
|
||||
textIntel,
|
||||
});
|
||||
|
||||
applyTheme();
|
||||
@@ -778,6 +804,9 @@ if (!gotLock) {
|
||||
capture.stopClickWatcher();
|
||||
capture.destroySessionTray();
|
||||
}
|
||||
if (textIntel) {
|
||||
textIntel.shutdown().catch(() => {});
|
||||
}
|
||||
// clean preview temp files on close
|
||||
try {
|
||||
for (const entry of fs.readdirSync(store.tempDir)) {
|
||||
|
||||
@@ -52,6 +52,10 @@ const api = {
|
||||
globalPlaceholders: invoke('placeholders:globals:get'),
|
||||
setGlobalPlaceholders: invoke('placeholders:globals:set'),
|
||||
},
|
||||
ai: {
|
||||
test: invoke('ai:test'),
|
||||
fillStep: invoke('ai:fillStep'),
|
||||
},
|
||||
capture: {
|
||||
shoot: invoke('capture:shoot'),
|
||||
region: invoke('capture:region'),
|
||||
|
||||
@@ -117,6 +117,7 @@ class StepForgeApp {
|
||||
guideFolders: library.folders?.guideFolders || {},
|
||||
};
|
||||
this.state.trash = trash;
|
||||
this.editor.setSettings(settings);
|
||||
}
|
||||
|
||||
async refreshLibrary({ keepFilter = true } = {}) {
|
||||
@@ -320,6 +321,7 @@ class StepForgeApp {
|
||||
{ label: 'Guide information…', action: () => this.editor.openGuideInfo() },
|
||||
{ label: 'Guide placeholders…', action: () => this.editor.openGuidePlaceholders() },
|
||||
{ label: 'Backups & snapshots…', action: () => this.editor.openBackupsDialog() },
|
||||
{ label: 'Generate all text fields with AI', action: () => this.editor.generateAllTextFieldsWithAi() },
|
||||
{ label: guide && guide.linkedSource ? 'Linked guide…' : 'Linked guide (not linked)', action: () => this.editor.openLinkedGuide() },
|
||||
'sep',
|
||||
{ label: 'Keyboard shortcuts…', action: () => this.editor.openShortcutsHelp() },
|
||||
@@ -866,6 +868,7 @@ class StepForgeApp {
|
||||
await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck });
|
||||
await api.settings.set({ keyPath: 'capture', value: next.capture });
|
||||
await api.settings.set({ keyPath: 'editor', value: next.editor });
|
||||
await api.settings.set({ keyPath: 'ai', value: next.ai });
|
||||
await api.settings.set({ keyPath: 'exports', value: next.exports });
|
||||
await api.settings.set({ keyPath: 'backups', value: next.backups });
|
||||
await api.settings.setGlobalPlaceholders(next.placeholders || {});
|
||||
|
||||
@@ -313,6 +313,42 @@ function showSettingsDialog({
|
||||
const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) });
|
||||
const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) });
|
||||
const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 });
|
||||
const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) });
|
||||
const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434');
|
||||
const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b');
|
||||
const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.');
|
||||
const testAiBtn = el('button', { type: 'button' }, 'Test connection');
|
||||
|
||||
const updateAiStatus = (message, { error = false } = {}) => {
|
||||
aiStatus.textContent = message;
|
||||
aiStatus.classList.toggle('error', Boolean(error));
|
||||
};
|
||||
|
||||
const testAiConnection = async () => {
|
||||
setButtonLoading(testAiBtn, true, 'Testing…');
|
||||
updateAiStatus('Checking Ollama at the configured host…');
|
||||
try {
|
||||
const result = await api.ai.test({
|
||||
ollama: {
|
||||
host: ollamaHost.value.trim(),
|
||||
model: ollamaModel.value.trim(),
|
||||
},
|
||||
});
|
||||
if (!result.ok) {
|
||||
updateAiStatus(result.reason || 'Could not connect to Ollama.', { error: true });
|
||||
return;
|
||||
}
|
||||
if (result.installed) {
|
||||
updateAiStatus(`Connected to ${result.host} with ${result.model}.`);
|
||||
} else {
|
||||
updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true });
|
||||
}
|
||||
} catch (err) {
|
||||
updateAiStatus(err.message || 'Could not connect to Ollama.', { error: true });
|
||||
} finally {
|
||||
setButtonLoading(testAiBtn, false);
|
||||
}
|
||||
};
|
||||
|
||||
const placeholderRows = el('div', { className: 'placeholder-rows' });
|
||||
const rows = [];
|
||||
@@ -369,6 +405,19 @@ function showSettingsDialog({
|
||||
el('legend', {}, 'Backups'),
|
||||
labeledRow('Keep last snapshots', keepLast),
|
||||
),
|
||||
el('fieldset', {},
|
||||
el('legend', {}, 'AI'),
|
||||
labeledRow('Enable AI text filling', aiEnabled),
|
||||
labeledRow('Ollama host', ollamaHost),
|
||||
labeledRow('Ollama model', ollamaModel),
|
||||
el('div.row', { style: { justifyContent: 'space-between' } },
|
||||
aiStatus,
|
||||
testAiBtn,
|
||||
),
|
||||
el('div.muted', {},
|
||||
'AI generation is manual only. Captures stay deterministic until you click a generate button.',
|
||||
),
|
||||
),
|
||||
el('fieldset', {},
|
||||
el('legend', {}, 'Global placeholders'),
|
||||
placeholderRows,
|
||||
@@ -412,6 +461,15 @@ function showSettingsDialog({
|
||||
...settings.backups,
|
||||
keepLast: Number(keepLast.value || 0),
|
||||
},
|
||||
ai: {
|
||||
...settings.ai,
|
||||
enabled: aiEnabled.checked,
|
||||
ollama: {
|
||||
...(settings.ai?.ollama || {}),
|
||||
host: ollamaHost.value.trim(),
|
||||
model: ollamaModel.value.trim(),
|
||||
},
|
||||
},
|
||||
placeholders: rows.reduce((acc, row) => {
|
||||
const inputs = row.querySelectorAll('input');
|
||||
const key = inputs[0].value.trim();
|
||||
@@ -430,6 +488,14 @@ function showSettingsDialog({
|
||||
});
|
||||
|
||||
form.addEventListener('submit', (e) => e.preventDefault());
|
||||
testAiBtn.addEventListener('click', testAiConnection);
|
||||
aiEnabled.addEventListener('change', () => {
|
||||
updateAiStatus(
|
||||
aiEnabled.checked
|
||||
? 'AI generation will be available once Ollama is reachable.'
|
||||
: 'AI generation is disabled. The settings are still saved for later.',
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+107
-2
@@ -135,6 +135,7 @@ class GuideEditor {
|
||||
this.descriptionDirty = false;
|
||||
this.titleDirty = false;
|
||||
this.active = true;
|
||||
this.settings = {};
|
||||
|
||||
this.saveStepDebounced = debounce(() => this.flushStep(), 180);
|
||||
this.saveGuideDebounced = debounce(() => this.flushGuide(), 180);
|
||||
@@ -152,6 +153,80 @@ class GuideEditor {
|
||||
this.active = Boolean(active);
|
||||
}
|
||||
|
||||
setSettings(settings) {
|
||||
this.settings = settings || {};
|
||||
this.updateAiButtonState();
|
||||
}
|
||||
|
||||
isAiEnabled() {
|
||||
return Boolean(this.settings?.ai?.enabled);
|
||||
}
|
||||
|
||||
updateAiButtonState() {
|
||||
const enabled = this.isAiEnabled();
|
||||
const buttons = [
|
||||
this.dom?.titleAiBtn,
|
||||
this.dom?.descAiBtn,
|
||||
...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []),
|
||||
].filter(Boolean);
|
||||
for (const button of buttons) {
|
||||
button.disabled = !enabled;
|
||||
button.title = enabled
|
||||
? button.dataset.aiTitle || 'Generate with AI'
|
||||
: 'Enable AI in Settings first.';
|
||||
}
|
||||
}
|
||||
|
||||
async runAiGeneration(target, { blockId = null, button = null } = {}) {
|
||||
if (!this.currentStep) {
|
||||
this.onToast('Select a step first.', { error: true });
|
||||
return null;
|
||||
}
|
||||
if (!this.isAiEnabled()) {
|
||||
this.onToast('Enable AI in Settings first.', { error: true });
|
||||
return null;
|
||||
}
|
||||
if (this.pendingSave) await this.flushStep();
|
||||
if (button) setButtonLoading(button, true, 'Generating…');
|
||||
try {
|
||||
const result = await api.ai.fillStep({
|
||||
guideId: this.guideId,
|
||||
stepId: this.currentStep.stepId,
|
||||
target,
|
||||
blockId,
|
||||
});
|
||||
if (!result || !result.ok) {
|
||||
this.onToast(result?.reason || 'AI generation failed.', { error: true });
|
||||
return null;
|
||||
}
|
||||
await this.reload(result.step.stepId);
|
||||
this.onToast('AI text filled.');
|
||||
return result.step;
|
||||
} catch (err) {
|
||||
this.onToast(err.message || 'AI generation failed.', { error: true });
|
||||
return null;
|
||||
} finally {
|
||||
if (button) setButtonLoading(button, false);
|
||||
}
|
||||
}
|
||||
|
||||
async generateTitleWithAi(button = null) {
|
||||
return this.runAiGeneration('title', { button });
|
||||
}
|
||||
|
||||
async generateDescriptionWithAi(button = null) {
|
||||
return this.runAiGeneration('description', { button });
|
||||
}
|
||||
|
||||
async generateAllTextFieldsWithAi(button = null) {
|
||||
return this.runAiGeneration('all', { button });
|
||||
}
|
||||
|
||||
async generateBlockWithAi(kind, block, button = null) {
|
||||
if (!block) return null;
|
||||
return this.runAiGeneration('block', { blockId: block.id, button });
|
||||
}
|
||||
|
||||
get currentStep() {
|
||||
return this.stepMap.get(this.selectedStepId) || null;
|
||||
}
|
||||
@@ -271,7 +346,14 @@ class GuideEditor {
|
||||
el('aside.pane-props', {},
|
||||
el('section', {},
|
||||
el('h3', {}, 'Step'),
|
||||
this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title' }),
|
||||
el('div.row', {},
|
||||
this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title', style: { flex: 1 } }),
|
||||
this.dom.titleAiBtn = el('button.ai', {
|
||||
type: 'button',
|
||||
title: 'Generate the step title with AI',
|
||||
dataset: { aiAction: 'title', aiTitle: 'Generate the step title with AI' },
|
||||
}, 'AI'),
|
||||
),
|
||||
this.dom.statusSelect = makeSelect('todo', [
|
||||
{ value: 'todo', label: 'Todo' },
|
||||
{ value: 'in-progress', label: 'In progress' },
|
||||
@@ -296,7 +378,14 @@ class GuideEditor {
|
||||
),
|
||||
),
|
||||
el('section', {},
|
||||
el('h3', {}, 'Description'),
|
||||
el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } },
|
||||
el('h3', { style: { margin: 0 } }, 'Description'),
|
||||
this.dom.descAiBtn = el('button.ai', {
|
||||
type: 'button',
|
||||
title: 'Generate the description with AI',
|
||||
dataset: { aiAction: 'description', aiTitle: 'Generate the description with AI' },
|
||||
}, 'AI'),
|
||||
),
|
||||
this.dom.richToolbar = el('div.rich-toolbar', {},
|
||||
this.toolbarBtn('bold', 'Bold'),
|
||||
this.toolbarBtn('italic', 'Italic'),
|
||||
@@ -416,6 +505,7 @@ class GuideEditor {
|
||||
this.renderStepList();
|
||||
this.emitMeta();
|
||||
});
|
||||
this.dom.titleAiBtn.addEventListener('click', () => this.generateTitleWithAi(this.dom.titleAiBtn));
|
||||
|
||||
this.dom.statusSelect.addEventListener('change', () => {
|
||||
if (!this.currentStep) return;
|
||||
@@ -491,6 +581,7 @@ class GuideEditor {
|
||||
document.addEventListener('selectionchange', () => {
|
||||
if (document.activeElement === this.dom.descEditor) this.updateToolbarState();
|
||||
});
|
||||
this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn));
|
||||
|
||||
this.dom.descEditor.addEventListener('paste', (e) => {
|
||||
// Keep pasted text simple; backend sanitization will handle the rest.
|
||||
@@ -504,6 +595,8 @@ class GuideEditor {
|
||||
if (!item) return;
|
||||
this.canvas.select(item.dataset.annId);
|
||||
});
|
||||
|
||||
this.updateAiButtonState();
|
||||
}
|
||||
|
||||
renderAll() {
|
||||
@@ -513,6 +606,7 @@ class GuideEditor {
|
||||
this.renderCanvas();
|
||||
this.renderAnnotationPanel();
|
||||
this.renderBlocksPanel();
|
||||
this.updateAiButtonState();
|
||||
this.emitMeta();
|
||||
}
|
||||
|
||||
@@ -670,6 +764,15 @@ class GuideEditor {
|
||||
el('span.spacer'),
|
||||
el('button.icon', { type: 'button', title: 'Move block up', disabled: !canMoveUp, onClick: moveUp, dataset: { blockMove: 'up' } }, '↑'),
|
||||
el('button.icon', { type: 'button', title: 'Move block down', disabled: !canMoveDown, onClick: moveDown, dataset: { blockMove: 'down' } }, '↓'),
|
||||
(() => {
|
||||
const aiBtn = el('button.ai', {
|
||||
type: 'button',
|
||||
title: 'Generate this block with AI',
|
||||
dataset: { aiAction: 'block', aiTitle: 'Generate this block with AI' },
|
||||
onClick: () => this.generateBlockWithAi(kind, block, aiBtn),
|
||||
}, 'AI');
|
||||
return aiBtn;
|
||||
})(),
|
||||
removeBtn(() => {
|
||||
if (kind === 'text') step.textBlocks = (step.textBlocks || []).filter((b) => b !== block);
|
||||
else if (kind === 'code') step.codeBlocks = (step.codeBlocks || []).filter((b) => b !== block);
|
||||
@@ -761,6 +864,7 @@ class GuideEditor {
|
||||
if (!blocks.length) {
|
||||
this.dom.blocksList.append(el('div.muted', {}, 'Informational text, code, and table blocks can be reordered with drag handles or arrows.'));
|
||||
}
|
||||
this.updateAiButtonState();
|
||||
}
|
||||
|
||||
renderStepList() {
|
||||
@@ -1658,6 +1762,7 @@ class GuideEditor {
|
||||
await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck });
|
||||
await api.settings.set({ keyPath: 'capture', value: next.capture });
|
||||
await api.settings.set({ keyPath: 'editor', value: next.editor });
|
||||
await api.settings.set({ keyPath: 'ai', value: next.ai });
|
||||
await api.settings.set({ keyPath: 'exports', value: next.exports });
|
||||
await api.settings.set({ keyPath: 'backups', value: next.backups });
|
||||
await api.settings.setGlobalPlaceholders(next.placeholders || {});
|
||||
|
||||
@@ -90,6 +90,16 @@ button.tool.active {
|
||||
border-color: var(--accent);
|
||||
color: var(--accent-fg);
|
||||
}
|
||||
button.ai {
|
||||
padding: 5px 10px;
|
||||
border-color: color-mix(in srgb, var(--accent) 38%, var(--border));
|
||||
background: color-mix(in srgb, var(--accent) 10%, var(--panel-solid));
|
||||
color: var(--accent-strong);
|
||||
font-weight: 650;
|
||||
letter-spacing: 0.02em;
|
||||
}
|
||||
button.ai:hover { background: color-mix(in srgb, var(--accent) 16%, var(--panel-2)); }
|
||||
button.ai:disabled { opacity: 0.6; }
|
||||
button:disabled { opacity: 0.6; cursor: default; }
|
||||
button.loading {
|
||||
display: inline-flex;
|
||||
@@ -751,6 +761,7 @@ fieldset legend {
|
||||
color: var(--muted);
|
||||
line-height: 1.5;
|
||||
}
|
||||
.ai-status.error { color: var(--danger); }
|
||||
|
||||
#modal-root:not(:empty) {
|
||||
position: fixed;
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
|
||||
const { createWorker } = require('tesseract.js');
|
||||
const {
|
||||
buildCaptureTitle,
|
||||
normalizeOllamaHost,
|
||||
normalizeAiPatch,
|
||||
buildAiPrompt,
|
||||
applyAiPatchToStep,
|
||||
displayText,
|
||||
normalizeWhitespace,
|
||||
} = require('../core/text-intel');
|
||||
|
||||
const OCR_CROP = {
|
||||
width: 420,
|
||||
height: 220,
|
||||
};
|
||||
|
||||
function hasBinary(name) {
|
||||
try {
|
||||
execFileSync('which', [name], { stdio: 'pipe' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function clamp(v, min, max) {
|
||||
return Math.min(max, Math.max(min, v));
|
||||
}
|
||||
|
||||
class TextIntelService {
|
||||
constructor({
|
||||
store,
|
||||
settings,
|
||||
getWindow = () => null,
|
||||
dataDir,
|
||||
fetchImpl = global.fetch,
|
||||
screenApi = null,
|
||||
}) {
|
||||
this.store = store;
|
||||
this.settings = settings;
|
||||
this.getWindow = getWindow;
|
||||
this.dataDir = dataDir;
|
||||
this.fetch = fetchImpl;
|
||||
this.screen = screenApi;
|
||||
this.worker = null;
|
||||
this.workerPromise = null;
|
||||
this.workerQueue = Promise.resolve();
|
||||
this.ocrDataDir = path.join(dataDir, 'ocr', 'eng');
|
||||
}
|
||||
|
||||
async shutdown() {
|
||||
if (this.worker) {
|
||||
try {
|
||||
await this.worker.terminate();
|
||||
} catch {
|
||||
// best effort
|
||||
}
|
||||
this.worker = null;
|
||||
this.workerPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
ensureLangData() {
|
||||
const packageDir = path.dirname(require.resolve('@tesseract.js-data/eng/package.json'));
|
||||
const source = path.join(packageDir, '4.0.0_best_int', 'eng.traineddata.gz');
|
||||
const targetDir = this.ocrDataDir;
|
||||
const target = path.join(targetDir, 'eng.traineddata.gz');
|
||||
if (!fs.existsSync(target)) {
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
fs.copyFileSync(source, target);
|
||||
}
|
||||
return targetDir;
|
||||
}
|
||||
|
||||
async getWorker() {
|
||||
if (this.workerPromise) return this.workerPromise;
|
||||
this.workerPromise = (async () => {
|
||||
const langPath = this.ensureLangData();
|
||||
const worker = await createWorker('eng', 1, {
|
||||
langPath,
|
||||
});
|
||||
await worker.setParameters({
|
||||
preserve_interword_spaces: '1',
|
||||
});
|
||||
this.worker = worker;
|
||||
return worker;
|
||||
})();
|
||||
return this.workerPromise;
|
||||
}
|
||||
|
||||
async recognizeCrop(image, rect = null) {
|
||||
const worker = await this.getWorker();
|
||||
const cropped = rect ? image.crop(rect) : image;
|
||||
const buffer = cropped.toPNG();
|
||||
const result = await worker.recognize(buffer);
|
||||
const text = String(result?.data?.text || '').trim();
|
||||
return {
|
||||
text,
|
||||
confidence: Number.isFinite(result?.data?.confidence) ? result.data.confidence : null,
|
||||
raw: result,
|
||||
};
|
||||
}
|
||||
|
||||
cropRectForPoint(frame, clickPos, { width = OCR_CROP.width, height = OCR_CROP.height } = {}) {
|
||||
if (!frame || !frame.size) return null;
|
||||
const bounds = frame.display.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height };
|
||||
const scaleX = frame.size.width / bounds.width;
|
||||
const scaleY = frame.size.height / bounds.height;
|
||||
const point = clickPos || {
|
||||
x: bounds.x + bounds.width / 2,
|
||||
y: bounds.y + bounds.height / 2,
|
||||
};
|
||||
const centerX = (point.x - bounds.x) * scaleX;
|
||||
const centerY = (point.y - bounds.y) * scaleY;
|
||||
const rectW = Math.max(1, Math.round(width * scaleX));
|
||||
const rectH = Math.max(1, Math.round(height * scaleY));
|
||||
const rect = {
|
||||
x: Math.round(centerX - rectW / 2),
|
||||
y: Math.round(centerY - rectH / 2),
|
||||
width: rectW,
|
||||
height: rectH,
|
||||
};
|
||||
rect.x = clamp(rect.x, 0, Math.max(0, frame.size.width - rect.width));
|
||||
rect.y = clamp(rect.y, 0, Math.max(0, frame.size.height - rect.height));
|
||||
rect.width = clamp(rect.width, 1, frame.size.width);
|
||||
rect.height = clamp(rect.height, 1, frame.size.height);
|
||||
return rect;
|
||||
}
|
||||
|
||||
async ocrAroundClick(frame, clickPos) {
|
||||
if (!frame || !frame.image) return { text: '', confidence: null };
|
||||
const rect = this.cropRectForPoint(frame, clickPos);
|
||||
try {
|
||||
return await this.recognizeCrop(frame.image, rect);
|
||||
} catch {
|
||||
return { text: '', confidence: null };
|
||||
}
|
||||
}
|
||||
|
||||
async collectForegroundWindowContext(osPoint = null) {
|
||||
try {
|
||||
if (process.platform === 'win32') return this.collectWindowsWindowContext(osPoint);
|
||||
if (process.platform === 'darwin') return this.collectMacWindowContext();
|
||||
if (process.platform === 'linux') return this.collectLinuxWindowContext();
|
||||
} catch {
|
||||
// best effort only
|
||||
}
|
||||
return { appName: '', windowTitle: '' };
|
||||
}
|
||||
|
||||
collectWindowsWindowContext(osPoint = null) {
|
||||
const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y);
|
||||
const clickX = hasPoint ? Number(osPoint.x) : 0;
|
||||
const clickY = hasPoint ? Number(osPoint.y) : 0;
|
||||
const script = `
|
||||
$clickX = ${clickX};
|
||||
$clickY = ${clickY};
|
||||
$elementLabel = '';
|
||||
$elementRole = '';
|
||||
$elementClass = '';
|
||||
$elementProcessId = 0;
|
||||
if (${hasPoint ? '$true' : '$false'}) {
|
||||
try {
|
||||
Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null
|
||||
$point = New-Object System.Windows.Point($clickX, $clickY);
|
||||
$element = [System.Windows.Automation.AutomationElement]::FromPoint($point);
|
||||
if ($element) {
|
||||
$current = $element.Current;
|
||||
$elementLabel = $current.Name;
|
||||
$elementRole = $current.LocalizedControlType;
|
||||
$elementClass = $current.ClassName;
|
||||
$elementProcessId = $current.ProcessId;
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
Add-Type @"
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
public static class Win32 {
|
||||
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
||||
[DllImport("user32.dll", CharSet = CharSet.Unicode)]
|
||||
public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
|
||||
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId);
|
||||
}
|
||||
"@;
|
||||
$hWnd = [Win32]::GetForegroundWindow();
|
||||
$sb = New-Object System.Text.StringBuilder 512;
|
||||
[void][Win32]::GetWindowText($hWnd, $sb, $sb.Capacity);
|
||||
$pid = 0;
|
||||
[void][Win32]::GetWindowThreadProcessId($hWnd, [ref]$pid);
|
||||
$proc = Get-Process -Id $pid -ErrorAction SilentlyContinue | Select-Object -First 1;
|
||||
$out = [ordered]@{
|
||||
appName = if ($proc) { $proc.ProcessName } else { '' };
|
||||
windowTitle = $sb.ToString();
|
||||
elementLabel = $elementLabel;
|
||||
elementRole = $elementRole;
|
||||
elementClass = $elementClass;
|
||||
elementProcessId = $elementProcessId;
|
||||
pid = $pid;
|
||||
};
|
||||
$out | ConvertTo-Json -Compress;
|
||||
`;
|
||||
const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 1200,
|
||||
}).trim();
|
||||
return JSON.parse(result || '{}');
|
||||
}
|
||||
|
||||
collectMacWindowContext() {
|
||||
const script = `
|
||||
set appName to ""
|
||||
set windowTitle to ""
|
||||
tell application "System Events"
|
||||
try
|
||||
set frontApp to first application process whose frontmost is true
|
||||
set appName to name of frontApp
|
||||
try
|
||||
set windowTitle to name of front window of frontApp
|
||||
end try
|
||||
end try
|
||||
end tell
|
||||
return appName & linefeed & windowTitle
|
||||
`;
|
||||
const result = execFileSync('osascript', ['-e', script], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 1200,
|
||||
}).trimEnd();
|
||||
const [appName = '', windowTitle = ''] = result.split(/\r?\n/);
|
||||
return { appName, windowTitle };
|
||||
}
|
||||
|
||||
collectLinuxWindowContext() {
|
||||
if (!hasBinary('xprop')) return { appName: '', windowTitle: '' };
|
||||
const active = execFileSync('xprop', ['-root', '_NET_ACTIVE_WINDOW'], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 1200,
|
||||
});
|
||||
const activeMatch = active.match(/window id # (0x[0-9a-fA-F]+)/);
|
||||
if (!activeMatch) return { appName: '', windowTitle: '' };
|
||||
const winId = activeMatch[1];
|
||||
const details = execFileSync('xprop', ['-id', winId, '_NET_WM_NAME', 'WM_NAME', 'WM_CLASS'], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 1200,
|
||||
});
|
||||
const titleMatch = details.match(/(?:_NET_WM_NAME\(UTF8_STRING\)|WM_NAME\(STRING\)|WM_NAME\(UTF8_STRING\)) = "([^"]*)"/);
|
||||
const classMatch = details.match(/WM_CLASS\(STRING\) = "([^"]*)"(?:, "([^"]*)")?/);
|
||||
return {
|
||||
appName: classMatch ? (classMatch[2] || classMatch[1] || '') : '',
|
||||
windowTitle: titleMatch ? titleMatch[1] : '',
|
||||
};
|
||||
}
|
||||
|
||||
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
|
||||
const [metadata, ocr] = await Promise.all([
|
||||
this.collectForegroundWindowContext(clickMeta?.osPoint || null),
|
||||
this.ocrAroundClick(frame, clickPos),
|
||||
]);
|
||||
return buildCaptureTitle({
|
||||
mode,
|
||||
metadata,
|
||||
ocrText: ocr.text,
|
||||
});
|
||||
}
|
||||
|
||||
aiEnabled() {
|
||||
return Boolean(this.settings.get('ai.enabled'));
|
||||
}
|
||||
|
||||
aiConfig(override = null) {
|
||||
const stored = this.settings.get('ai') || {};
|
||||
const merged = override ? {
|
||||
...stored,
|
||||
...override,
|
||||
ollama: {
|
||||
...(stored.ollama || {}),
|
||||
...(override.ollama || {}),
|
||||
},
|
||||
} : stored;
|
||||
return {
|
||||
...merged,
|
||||
enabled: override && Object.prototype.hasOwnProperty.call(override, 'enabled')
|
||||
? Boolean(override.enabled)
|
||||
: Boolean(stored.enabled),
|
||||
ollama: {
|
||||
host: normalizeOllamaHost(merged.ollama?.host || ''),
|
||||
model: normalizeWhitespace(merged.ollama?.model || ''),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async testAiConnection(override = null) {
|
||||
const config = this.aiConfig(override);
|
||||
if (!config.ollama.host) {
|
||||
return { ok: false, reason: 'Set an Ollama host first.' };
|
||||
}
|
||||
const tagsUrl = new URL('/api/tags', `${config.ollama.host.replace(/\/+$/, '')}/`);
|
||||
const res = await this.fetch(tagsUrl, { method: 'GET' });
|
||||
if (!res.ok) {
|
||||
return { ok: false, reason: `Ollama check failed (${res.status})` };
|
||||
}
|
||||
const data = await res.json();
|
||||
const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : [];
|
||||
const installed = config.ollama.model ? models.includes(config.ollama.model) : false;
|
||||
return {
|
||||
ok: true,
|
||||
installed,
|
||||
models,
|
||||
host: config.ollama.host,
|
||||
model: config.ollama.model,
|
||||
};
|
||||
}
|
||||
|
||||
async callOllama({ host, model, prompt, systemPrompt }) {
|
||||
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
|
||||
const response = await this.fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
stream: false,
|
||||
format: 'json',
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
options: {
|
||||
temperature: 0.2,
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Ollama request failed (${response.status})`);
|
||||
}
|
||||
const payload = await response.json();
|
||||
const content = payload?.message?.content;
|
||||
if (typeof content !== 'string' || !content.trim()) {
|
||||
throw new Error('Ollama returned an empty response');
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
async generateStepPatch({
|
||||
guideId,
|
||||
stepId,
|
||||
target = 'all',
|
||||
blockId = null,
|
||||
}) {
|
||||
try {
|
||||
const config = this.aiConfig();
|
||||
if (!config.enabled) {
|
||||
return { ok: false, reason: 'Enable AI in settings first.' };
|
||||
}
|
||||
if (!config.ollama.host || !config.ollama.model) {
|
||||
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
|
||||
}
|
||||
|
||||
const guide = this.store.getGuide(guideId);
|
||||
const step = this.store.getStep(guideId, stepId);
|
||||
if (!guide || !step) {
|
||||
return { ok: false, reason: 'Guide or step not found.' };
|
||||
}
|
||||
|
||||
const currentBlock = blockId
|
||||
? [...(step.textBlocks || []), ...(step.codeBlocks || []), ...(step.tableBlocks || [])].find((b) => b.id === blockId) || null
|
||||
: null;
|
||||
if (blockId && target === 'block' && !currentBlock) {
|
||||
return { ok: false, reason: 'Block not found.' };
|
||||
}
|
||||
|
||||
let captureContext = null;
|
||||
if (step.image) {
|
||||
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
|
||||
if (imagePath && fs.existsSync(imagePath)) {
|
||||
const { nativeImage } = require('electron');
|
||||
const image = nativeImage.createFromPath(imagePath);
|
||||
if (!image.isEmpty()) {
|
||||
const clickPoint = this.clickPointFromStep(step, image);
|
||||
const [metadata, ocr] = await Promise.all([
|
||||
this.collectForegroundWindowContext(),
|
||||
this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint),
|
||||
]);
|
||||
captureContext = {
|
||||
...metadata,
|
||||
ocrText: ocr.text,
|
||||
mode: step.kind === 'image' ? 'fullscreen' : 'content',
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { systemPrompt, prompt } = buildAiPrompt({
|
||||
target,
|
||||
guide,
|
||||
step,
|
||||
captureContext,
|
||||
block: currentBlock,
|
||||
});
|
||||
|
||||
const raw = await this.callOllama({
|
||||
host: config.ollama.host,
|
||||
model: config.ollama.model,
|
||||
prompt,
|
||||
systemPrompt,
|
||||
});
|
||||
const patch = normalizeAiPatch(raw);
|
||||
const updated = applyAiPatchToStep(step, patch, { target, blockId });
|
||||
const saved = this.store.saveStep(guideId, updated);
|
||||
return { ok: true, step: saved, patch };
|
||||
} catch (err) {
|
||||
return { ok: false, reason: err && err.message ? err.message : 'AI generation failed.' };
|
||||
}
|
||||
}
|
||||
|
||||
clickPointFromStep(step, image = null) {
|
||||
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
|
||||
if (!marker) return null;
|
||||
const size = image ? image.getSize() : step.image?.size || { width: 0, height: 0 };
|
||||
if (!size.width || !size.height) return null;
|
||||
return {
|
||||
x: Math.round((marker.x + marker.w / 2) * size.width),
|
||||
y: Math.round((marker.y + marker.h / 2) * size.height),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { TextIntelService };
|
||||
Reference in New Issue
Block a user