diff --git a/app/capture.js b/app/capture.js index 00b1ded..aefbe36 100644 --- a/app/capture.js +++ b/app/capture.js @@ -3,7 +3,6 @@ const path = require('node:path'); const { spawn, execFileSync } = require('node:child_process'); const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron'); -const { expandPlaceholders } = require('../core/placeholders'); const raster = require('../core/raster'); const { encodePng } = require('../core/png'); const { @@ -14,6 +13,7 @@ const { DEFAULT_START_SLACK_MS, } = require('./click-frames'); const { physicalToDip } = require('./coords'); +const { DEFAULT_CAPTURE_TITLES } = require('../core/text-intel'); /** * Capture service: full-screen, active-window, and region capture, plus a @@ -114,6 +114,7 @@ class CaptureService { getWindow, notify, screenApi = screen, + textIntel = null, }) { this.store = store; this.settings = settings; @@ -122,6 +123,7 @@ class CaptureService { // Injectable for tests; the click/coordinate paths must never reach for // the global `screen` directly so coordinate handling stays testable. this.screen = screenApi; + this.textIntel = textIntel; this.session = null; // { guideId, paused, count, intervalSec } this.intervalTimer = null; this.clickWatcher = null; @@ -133,6 +135,7 @@ class CaptureService { this.clickWatcherErrTail = ''; this.linuxEvent = null; // event block currently being parsed this.pendingRawClick = null; // raw press waiting for its coordinate twin + this.pendingClickOsPoint = null; this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -463,7 +466,7 @@ class CaptureService { if (frame) { clog('click@', clickAt, 'frame', frame.source || 'loop', 'started', frame.startedAt - clickAt, 'ms, captured', frame.capturedAt - clickAt, 'ms rel. click'); - const result = this.storeFrameAsStep(guideId, frame.mode, frame, clickPos); + const result = await this.storeFrameAsStep(guideId, frame.mode, frame, clickPos, clickMeta); if (result.ok) this.noteStepAdded(result.step, trigger, guideId); return result; } @@ -1211,6 +1214,9 @@ public static class SFMouseHook { let clickPos = osPoint ? this.osPointToDip(osPoint) : null; if (!clickPos) clickPos = this.screen.getCursorScreenPoint(); clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos); + this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y) + ? { x: osPoint.x, y: osPoint.y } + : null; this.enqueueClickCapture(clickPos, clickAt, button || 'mouse'); } @@ -1268,7 +1274,17 @@ public static class SFMouseHook { * fast the user clicks or how slow the encoder is. */ enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') { - const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' }; + const osPoint = this.pendingClickOsPoint && Number.isFinite(this.pendingClickOsPoint.x) && Number.isFinite(this.pendingClickOsPoint.y) + ? this.pendingClickOsPoint + : null; + this.pendingClickOsPoint = null; + const clickMeta = { + at: Number.isFinite(clickAt) ? clickAt : Date.now(), + button: button || 'mouse', + }; + if (osPoint) { + clickMeta.osPoint = { x: osPoint.x, y: osPoint.y }; + } if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be // stored if the session stops while this click waits in the queue. @@ -1300,7 +1316,18 @@ public static class SFMouseHook { }; } - storeFrameAsStep(guideId, mode, frame, clickPos = null) { + async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) { + try { + if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') { + return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta }); + } + } catch { + // fall back to the local semantic title below + } + return this.autoTitle(mode); + } + + async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) { if (!frame) return { ok: false, reason: 'no capture frame available' }; const annotations = []; // The click position (DIP, read at event time) wins over the frame's @@ -1324,7 +1351,7 @@ public static class SFMouseHook { } const step = this.store.addStep(guideId, { - title: this.autoTitle(mode), + title: await this.buildStepTitle(mode, frame, clickPos, clickMeta), annotations, focusedView: { enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')), @@ -1335,14 +1362,7 @@ public static class SFMouseHook { } autoTitle(mode) { - const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]'; - const now = new Date(); - const pad = (n) => String(n).padStart(2, '0'); - return expandPlaceholders(tplStr, { - Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen', - Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`, - Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`, - }); + return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; } /** Grab the screen/window image as { image, display } or throw. */ @@ -1451,8 +1471,12 @@ public static class SFMouseHook { const cropped = image.crop(rect); const size = cropped.getSize(); if (!size.width || !size.height) return { ok: false, reason: 'empty selection' }; - const step = this.store.addStep(guideId, { title: this.autoTitle('region') }, - cropped.toPNG(), size); + const step = await this.storeFrameAsStep(guideId, 'region', { + image: cropped, + size, + display, + cursor: null, + }, null, null); return { ok: true, step }; } diff --git a/app/main.js b/app/main.js index c89abc0..6a87840 100644 --- a/app/main.js +++ b/app/main.js @@ -19,6 +19,7 @@ const { exportGuideArchive, importGuideArchive, saveLinkedGuide, readArchive } = const { createSnapshot, listSnapshots, restoreSnapshot } = require('../core/snapshots'); const { readLock } = require('../core/locks'); const CaptureService = require('./capture'); +const { TextIntelService } = require('./text-intel'); const { keepProcessesResponsive } = require('./win-power'); const APP_ID = 'com.stepforge.app'; @@ -59,6 +60,7 @@ let settings; let searchIndex; let templates; let capture; +let textIntel; let mainWindow; function reindex(guideId) { @@ -503,6 +505,22 @@ function setupIpc() { if (keyPath.startsWith('capture.hotkey')) registerHotkeys(); return settings.data; }); + h('ai:test', async ({ enabled = null, ollama = null } = {}) => { + return textIntel.testAiConnection({ + enabled, + ollama, + }); + }); + h('ai:fillStep', async ({ guideId, stepId, target = 'all', blockId = null } = {}) => { + const result = await textIntel.generateStepPatch({ + guideId, + stepId, + target, + blockId, + }); + if (result.ok) reindex(guideId); + return result; + }); h('placeholders:globals:get', () => settings.getGlobalPlaceholders()); h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values)); @@ -739,6 +757,13 @@ if (!gotLock) { settings = new Settings(store.settingsDir); searchIndex = new SearchIndex(store.indexDir); templates = new TemplateManager(store.templatesDir); + textIntel = new TextIntelService({ + store, + settings, + getWindow: () => mainWindow, + dataDir, + screenApi: screen, + }); // Bringing up the desktop-capture stream spawns/upgrades Chromium's GPU // and screen-capture utility processes — which can be born after a session // already started, so the start-time EcoQoS opt-out misses them. Re-apply @@ -759,6 +784,7 @@ if (!gotLock) { settings, getWindow: () => mainWindow, notify: captureNotify, + textIntel, }); applyTheme(); @@ -778,6 +804,9 @@ if (!gotLock) { capture.stopClickWatcher(); capture.destroySessionTray(); } + if (textIntel) { + textIntel.shutdown().catch(() => {}); + } // clean preview temp files on close try { for (const entry of fs.readdirSync(store.tempDir)) { diff --git a/app/preload.js b/app/preload.js index 432e31b..71f60b8 100644 --- a/app/preload.js +++ b/app/preload.js @@ -52,6 +52,10 @@ const api = { globalPlaceholders: invoke('placeholders:globals:get'), setGlobalPlaceholders: invoke('placeholders:globals:set'), }, + ai: { + test: invoke('ai:test'), + fillStep: invoke('ai:fillStep'), + }, capture: { shoot: invoke('capture:shoot'), region: invoke('capture:region'), diff --git a/app/renderer/app.js b/app/renderer/app.js index bbd99c4..8baae92 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -117,6 +117,7 @@ class StepForgeApp { guideFolders: library.folders?.guideFolders || {}, }; this.state.trash = trash; + this.editor.setSettings(settings); } async refreshLibrary({ keepFilter = true } = {}) { @@ -320,6 +321,7 @@ class StepForgeApp { { label: 'Guide information…', action: () => this.editor.openGuideInfo() }, { label: 'Guide placeholders…', action: () => this.editor.openGuidePlaceholders() }, { label: 'Backups & snapshots…', action: () => this.editor.openBackupsDialog() }, + { label: 'Generate all text fields with AI', action: () => this.editor.generateAllTextFieldsWithAi() }, { label: guide && guide.linkedSource ? 'Linked guide…' : 'Linked guide (not linked)', action: () => this.editor.openLinkedGuide() }, 'sep', { label: 'Keyboard shortcuts…', action: () => this.editor.openShortcutsHelp() }, @@ -866,6 +868,7 @@ class StepForgeApp { await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck }); await api.settings.set({ keyPath: 'capture', value: next.capture }); await api.settings.set({ keyPath: 'editor', value: next.editor }); + await api.settings.set({ keyPath: 'ai', value: next.ai }); await api.settings.set({ keyPath: 'exports', value: next.exports }); await api.settings.set({ keyPath: 'backups', value: next.backups }); await api.settings.setGlobalPlaceholders(next.placeholders || {}); diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index 272c811..8ce8a84 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -313,6 +313,42 @@ function showSettingsDialog({ const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) }); const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) }); const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 }); + const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) }); + const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434'); + const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b'); + const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.'); + const testAiBtn = el('button', { type: 'button' }, 'Test connection'); + + const updateAiStatus = (message, { error = false } = {}) => { + aiStatus.textContent = message; + aiStatus.classList.toggle('error', Boolean(error)); + }; + + const testAiConnection = async () => { + setButtonLoading(testAiBtn, true, 'Testing…'); + updateAiStatus('Checking Ollama at the configured host…'); + try { + const result = await api.ai.test({ + ollama: { + host: ollamaHost.value.trim(), + model: ollamaModel.value.trim(), + }, + }); + if (!result.ok) { + updateAiStatus(result.reason || 'Could not connect to Ollama.', { error: true }); + return; + } + if (result.installed) { + updateAiStatus(`Connected to ${result.host} with ${result.model}.`); + } else { + updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true }); + } + } catch (err) { + updateAiStatus(err.message || 'Could not connect to Ollama.', { error: true }); + } finally { + setButtonLoading(testAiBtn, false); + } + }; const placeholderRows = el('div', { className: 'placeholder-rows' }); const rows = []; @@ -369,6 +405,19 @@ function showSettingsDialog({ el('legend', {}, 'Backups'), labeledRow('Keep last snapshots', keepLast), ), + el('fieldset', {}, + el('legend', {}, 'AI'), + labeledRow('Enable AI text filling', aiEnabled), + labeledRow('Ollama host', ollamaHost), + labeledRow('Ollama model', ollamaModel), + el('div.row', { style: { justifyContent: 'space-between' } }, + aiStatus, + testAiBtn, + ), + el('div.muted', {}, + 'AI generation is manual only. Captures stay deterministic until you click a generate button.', + ), + ), el('fieldset', {}, el('legend', {}, 'Global placeholders'), placeholderRows, @@ -412,6 +461,15 @@ function showSettingsDialog({ ...settings.backups, keepLast: Number(keepLast.value || 0), }, + ai: { + ...settings.ai, + enabled: aiEnabled.checked, + ollama: { + ...(settings.ai?.ollama || {}), + host: ollamaHost.value.trim(), + model: ollamaModel.value.trim(), + }, + }, placeholders: rows.reduce((acc, row) => { const inputs = row.querySelectorAll('input'); const key = inputs[0].value.trim(); @@ -430,6 +488,14 @@ function showSettingsDialog({ }); form.addEventListener('submit', (e) => e.preventDefault()); + testAiBtn.addEventListener('click', testAiConnection); + aiEnabled.addEventListener('change', () => { + updateAiStatus( + aiEnabled.checked + ? 'AI generation will be available once Ollama is reachable.' + : 'AI generation is disabled. The settings are still saved for later.', + ); + }); }); } diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 840f252..dd51028 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -135,6 +135,7 @@ class GuideEditor { this.descriptionDirty = false; this.titleDirty = false; this.active = true; + this.settings = {}; this.saveStepDebounced = debounce(() => this.flushStep(), 180); this.saveGuideDebounced = debounce(() => this.flushGuide(), 180); @@ -152,6 +153,80 @@ class GuideEditor { this.active = Boolean(active); } + setSettings(settings) { + this.settings = settings || {}; + this.updateAiButtonState(); + } + + isAiEnabled() { + return Boolean(this.settings?.ai?.enabled); + } + + updateAiButtonState() { + const enabled = this.isAiEnabled(); + const buttons = [ + this.dom?.titleAiBtn, + this.dom?.descAiBtn, + ...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []), + ].filter(Boolean); + for (const button of buttons) { + button.disabled = !enabled; + button.title = enabled + ? button.dataset.aiTitle || 'Generate with AI' + : 'Enable AI in Settings first.'; + } + } + + async runAiGeneration(target, { blockId = null, button = null } = {}) { + if (!this.currentStep) { + this.onToast('Select a step first.', { error: true }); + return null; + } + if (!this.isAiEnabled()) { + this.onToast('Enable AI in Settings first.', { error: true }); + return null; + } + if (this.pendingSave) await this.flushStep(); + if (button) setButtonLoading(button, true, 'Generating…'); + try { + const result = await api.ai.fillStep({ + guideId: this.guideId, + stepId: this.currentStep.stepId, + target, + blockId, + }); + if (!result || !result.ok) { + this.onToast(result?.reason || 'AI generation failed.', { error: true }); + return null; + } + await this.reload(result.step.stepId); + this.onToast('AI text filled.'); + return result.step; + } catch (err) { + this.onToast(err.message || 'AI generation failed.', { error: true }); + return null; + } finally { + if (button) setButtonLoading(button, false); + } + } + + async generateTitleWithAi(button = null) { + return this.runAiGeneration('title', { button }); + } + + async generateDescriptionWithAi(button = null) { + return this.runAiGeneration('description', { button }); + } + + async generateAllTextFieldsWithAi(button = null) { + return this.runAiGeneration('all', { button }); + } + + async generateBlockWithAi(kind, block, button = null) { + if (!block) return null; + return this.runAiGeneration('block', { blockId: block.id, button }); + } + get currentStep() { return this.stepMap.get(this.selectedStepId) || null; } @@ -271,7 +346,14 @@ class GuideEditor { el('aside.pane-props', {}, el('section', {}, el('h3', {}, 'Step'), - this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title' }), + el('div.row', {}, + this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title', style: { flex: 1 } }), + this.dom.titleAiBtn = el('button.ai', { + type: 'button', + title: 'Generate the step title with AI', + dataset: { aiAction: 'title', aiTitle: 'Generate the step title with AI' }, + }, 'AI'), + ), this.dom.statusSelect = makeSelect('todo', [ { value: 'todo', label: 'Todo' }, { value: 'in-progress', label: 'In progress' }, @@ -296,7 +378,14 @@ class GuideEditor { ), ), el('section', {}, - el('h3', {}, 'Description'), + el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } }, + el('h3', { style: { margin: 0 } }, 'Description'), + this.dom.descAiBtn = el('button.ai', { + type: 'button', + title: 'Generate the description with AI', + dataset: { aiAction: 'description', aiTitle: 'Generate the description with AI' }, + }, 'AI'), + ), this.dom.richToolbar = el('div.rich-toolbar', {}, this.toolbarBtn('bold', 'Bold'), this.toolbarBtn('italic', 'Italic'), @@ -416,6 +505,7 @@ class GuideEditor { this.renderStepList(); this.emitMeta(); }); + this.dom.titleAiBtn.addEventListener('click', () => this.generateTitleWithAi(this.dom.titleAiBtn)); this.dom.statusSelect.addEventListener('change', () => { if (!this.currentStep) return; @@ -491,6 +581,7 @@ class GuideEditor { document.addEventListener('selectionchange', () => { if (document.activeElement === this.dom.descEditor) this.updateToolbarState(); }); + this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn)); this.dom.descEditor.addEventListener('paste', (e) => { // Keep pasted text simple; backend sanitization will handle the rest. @@ -504,6 +595,8 @@ class GuideEditor { if (!item) return; this.canvas.select(item.dataset.annId); }); + + this.updateAiButtonState(); } renderAll() { @@ -513,6 +606,7 @@ class GuideEditor { this.renderCanvas(); this.renderAnnotationPanel(); this.renderBlocksPanel(); + this.updateAiButtonState(); this.emitMeta(); } @@ -670,6 +764,15 @@ class GuideEditor { el('span.spacer'), el('button.icon', { type: 'button', title: 'Move block up', disabled: !canMoveUp, onClick: moveUp, dataset: { blockMove: 'up' } }, '↑'), el('button.icon', { type: 'button', title: 'Move block down', disabled: !canMoveDown, onClick: moveDown, dataset: { blockMove: 'down' } }, '↓'), + (() => { + const aiBtn = el('button.ai', { + type: 'button', + title: 'Generate this block with AI', + dataset: { aiAction: 'block', aiTitle: 'Generate this block with AI' }, + onClick: () => this.generateBlockWithAi(kind, block, aiBtn), + }, 'AI'); + return aiBtn; + })(), removeBtn(() => { if (kind === 'text') step.textBlocks = (step.textBlocks || []).filter((b) => b !== block); else if (kind === 'code') step.codeBlocks = (step.codeBlocks || []).filter((b) => b !== block); @@ -761,6 +864,7 @@ class GuideEditor { if (!blocks.length) { this.dom.blocksList.append(el('div.muted', {}, 'Informational text, code, and table blocks can be reordered with drag handles or arrows.')); } + this.updateAiButtonState(); } renderStepList() { @@ -1658,6 +1762,7 @@ class GuideEditor { await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck }); await api.settings.set({ keyPath: 'capture', value: next.capture }); await api.settings.set({ keyPath: 'editor', value: next.editor }); + await api.settings.set({ keyPath: 'ai', value: next.ai }); await api.settings.set({ keyPath: 'exports', value: next.exports }); await api.settings.set({ keyPath: 'backups', value: next.backups }); await api.settings.setGlobalPlaceholders(next.placeholders || {}); diff --git a/app/renderer/style.css b/app/renderer/style.css index 6842b29..5ea2b5a 100644 --- a/app/renderer/style.css +++ b/app/renderer/style.css @@ -90,6 +90,16 @@ button.tool.active { border-color: var(--accent); color: var(--accent-fg); } +button.ai { + padding: 5px 10px; + border-color: color-mix(in srgb, var(--accent) 38%, var(--border)); + background: color-mix(in srgb, var(--accent) 10%, var(--panel-solid)); + color: var(--accent-strong); + font-weight: 650; + letter-spacing: 0.02em; +} +button.ai:hover { background: color-mix(in srgb, var(--accent) 16%, var(--panel-2)); } +button.ai:disabled { opacity: 0.6; } button:disabled { opacity: 0.6; cursor: default; } button.loading { display: inline-flex; @@ -751,6 +761,7 @@ fieldset legend { color: var(--muted); line-height: 1.5; } +.ai-status.error { color: var(--danger); } #modal-root:not(:empty) { position: fixed; diff --git a/app/text-intel.js b/app/text-intel.js new file mode 100644 index 0000000..0d305a4 --- /dev/null +++ b/app/text-intel.js @@ -0,0 +1,438 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +const { createWorker } = require('tesseract.js'); +const { + buildCaptureTitle, + normalizeOllamaHost, + normalizeAiPatch, + buildAiPrompt, + applyAiPatchToStep, + displayText, + normalizeWhitespace, +} = require('../core/text-intel'); + +const OCR_CROP = { + width: 420, + height: 220, +}; + +function hasBinary(name) { + try { + execFileSync('which', [name], { stdio: 'pipe' }); + return true; + } catch { + return false; + } +} + +function clamp(v, min, max) { + return Math.min(max, Math.max(min, v)); +} + +class TextIntelService { + constructor({ + store, + settings, + getWindow = () => null, + dataDir, + fetchImpl = global.fetch, + screenApi = null, + }) { + this.store = store; + this.settings = settings; + this.getWindow = getWindow; + this.dataDir = dataDir; + this.fetch = fetchImpl; + this.screen = screenApi; + this.worker = null; + this.workerPromise = null; + this.workerQueue = Promise.resolve(); + this.ocrDataDir = path.join(dataDir, 'ocr', 'eng'); + } + + async shutdown() { + if (this.worker) { + try { + await this.worker.terminate(); + } catch { + // best effort + } + this.worker = null; + this.workerPromise = null; + } + } + + ensureLangData() { + const packageDir = path.dirname(require.resolve('@tesseract.js-data/eng/package.json')); + const source = path.join(packageDir, '4.0.0_best_int', 'eng.traineddata.gz'); + const targetDir = this.ocrDataDir; + const target = path.join(targetDir, 'eng.traineddata.gz'); + if (!fs.existsSync(target)) { + fs.mkdirSync(targetDir, { recursive: true }); + fs.copyFileSync(source, target); + } + return targetDir; + } + + async getWorker() { + if (this.workerPromise) return this.workerPromise; + this.workerPromise = (async () => { + const langPath = this.ensureLangData(); + const worker = await createWorker('eng', 1, { + langPath, + }); + await worker.setParameters({ + preserve_interword_spaces: '1', + }); + this.worker = worker; + return worker; + })(); + return this.workerPromise; + } + + async recognizeCrop(image, rect = null) { + const worker = await this.getWorker(); + const cropped = rect ? image.crop(rect) : image; + const buffer = cropped.toPNG(); + const result = await worker.recognize(buffer); + const text = String(result?.data?.text || '').trim(); + return { + text, + confidence: Number.isFinite(result?.data?.confidence) ? result.data.confidence : null, + raw: result, + }; + } + + cropRectForPoint(frame, clickPos, { width = OCR_CROP.width, height = OCR_CROP.height } = {}) { + if (!frame || !frame.size) return null; + const bounds = frame.display.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height }; + const scaleX = frame.size.width / bounds.width; + const scaleY = frame.size.height / bounds.height; + const point = clickPos || { + x: bounds.x + bounds.width / 2, + y: bounds.y + bounds.height / 2, + }; + const centerX = (point.x - bounds.x) * scaleX; + const centerY = (point.y - bounds.y) * scaleY; + const rectW = Math.max(1, Math.round(width * scaleX)); + const rectH = Math.max(1, Math.round(height * scaleY)); + const rect = { + x: Math.round(centerX - rectW / 2), + y: Math.round(centerY - rectH / 2), + width: rectW, + height: rectH, + }; + rect.x = clamp(rect.x, 0, Math.max(0, frame.size.width - rect.width)); + rect.y = clamp(rect.y, 0, Math.max(0, frame.size.height - rect.height)); + rect.width = clamp(rect.width, 1, frame.size.width); + rect.height = clamp(rect.height, 1, frame.size.height); + return rect; + } + + async ocrAroundClick(frame, clickPos) { + if (!frame || !frame.image) return { text: '', confidence: null }; + const rect = this.cropRectForPoint(frame, clickPos); + try { + return await this.recognizeCrop(frame.image, rect); + } catch { + return { text: '', confidence: null }; + } + } + + async collectForegroundWindowContext(osPoint = null) { + try { + if (process.platform === 'win32') return this.collectWindowsWindowContext(osPoint); + if (process.platform === 'darwin') return this.collectMacWindowContext(); + if (process.platform === 'linux') return this.collectLinuxWindowContext(); + } catch { + // best effort only + } + return { appName: '', windowTitle: '' }; + } + + collectWindowsWindowContext(osPoint = null) { + const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y); + const clickX = hasPoint ? Number(osPoint.x) : 0; + const clickY = hasPoint ? Number(osPoint.y) : 0; + const script = ` + $clickX = ${clickX}; + $clickY = ${clickY}; + $elementLabel = ''; + $elementRole = ''; + $elementClass = ''; + $elementProcessId = 0; + if (${hasPoint ? '$true' : '$false'}) { + try { + Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null + $point = New-Object System.Windows.Point($clickX, $clickY); + $element = [System.Windows.Automation.AutomationElement]::FromPoint($point); + if ($element) { + $current = $element.Current; + $elementLabel = $current.Name; + $elementRole = $current.LocalizedControlType; + $elementClass = $current.ClassName; + $elementProcessId = $current.ProcessId; + } + } catch { } + } + Add-Type @" +using System; +using System.Runtime.InteropServices; +using System.Text; +public static class Win32 { + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count); + [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); +} +"@; + $hWnd = [Win32]::GetForegroundWindow(); + $sb = New-Object System.Text.StringBuilder 512; + [void][Win32]::GetWindowText($hWnd, $sb, $sb.Capacity); + $pid = 0; + [void][Win32]::GetWindowThreadProcessId($hWnd, [ref]$pid); + $proc = Get-Process -Id $pid -ErrorAction SilentlyContinue | Select-Object -First 1; + $out = [ordered]@{ + appName = if ($proc) { $proc.ProcessName } else { '' }; + windowTitle = $sb.ToString(); + elementLabel = $elementLabel; + elementRole = $elementRole; + elementClass = $elementClass; + elementProcessId = $elementProcessId; + pid = $pid; + }; + $out | ConvertTo-Json -Compress; + `; + const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }).trim(); + return JSON.parse(result || '{}'); + } + + collectMacWindowContext() { + const script = ` + set appName to "" + set windowTitle to "" + tell application "System Events" + try + set frontApp to first application process whose frontmost is true + set appName to name of frontApp + try + set windowTitle to name of front window of frontApp + end try + end try + end tell + return appName & linefeed & windowTitle + `; + const result = execFileSync('osascript', ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }).trimEnd(); + const [appName = '', windowTitle = ''] = result.split(/\r?\n/); + return { appName, windowTitle }; + } + + collectLinuxWindowContext() { + if (!hasBinary('xprop')) return { appName: '', windowTitle: '' }; + const active = execFileSync('xprop', ['-root', '_NET_ACTIVE_WINDOW'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }); + const activeMatch = active.match(/window id # (0x[0-9a-fA-F]+)/); + if (!activeMatch) return { appName: '', windowTitle: '' }; + const winId = activeMatch[1]; + const details = execFileSync('xprop', ['-id', winId, '_NET_WM_NAME', 'WM_NAME', 'WM_CLASS'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }); + const titleMatch = details.match(/(?:_NET_WM_NAME\(UTF8_STRING\)|WM_NAME\(STRING\)|WM_NAME\(UTF8_STRING\)) = "([^"]*)"/); + const classMatch = details.match(/WM_CLASS\(STRING\) = "([^"]*)"(?:, "([^"]*)")?/); + return { + appName: classMatch ? (classMatch[2] || classMatch[1] || '') : '', + windowTitle: titleMatch ? titleMatch[1] : '', + }; + } + + async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) { + const [metadata, ocr] = await Promise.all([ + this.collectForegroundWindowContext(clickMeta?.osPoint || null), + this.ocrAroundClick(frame, clickPos), + ]); + return buildCaptureTitle({ + mode, + metadata, + ocrText: ocr.text, + }); + } + + aiEnabled() { + return Boolean(this.settings.get('ai.enabled')); + } + + aiConfig(override = null) { + const stored = this.settings.get('ai') || {}; + const merged = override ? { + ...stored, + ...override, + ollama: { + ...(stored.ollama || {}), + ...(override.ollama || {}), + }, + } : stored; + return { + ...merged, + enabled: override && Object.prototype.hasOwnProperty.call(override, 'enabled') + ? Boolean(override.enabled) + : Boolean(stored.enabled), + ollama: { + host: normalizeOllamaHost(merged.ollama?.host || ''), + model: normalizeWhitespace(merged.ollama?.model || ''), + }, + }; + } + + async testAiConnection(override = null) { + const config = this.aiConfig(override); + if (!config.ollama.host) { + return { ok: false, reason: 'Set an Ollama host first.' }; + } + const tagsUrl = new URL('/api/tags', `${config.ollama.host.replace(/\/+$/, '')}/`); + const res = await this.fetch(tagsUrl, { method: 'GET' }); + if (!res.ok) { + return { ok: false, reason: `Ollama check failed (${res.status})` }; + } + const data = await res.json(); + const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : []; + const installed = config.ollama.model ? models.includes(config.ollama.model) : false; + return { + ok: true, + installed, + models, + host: config.ollama.host, + model: config.ollama.model, + }; + } + + async callOllama({ host, model, prompt, systemPrompt }) { + const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); + const response = await this.fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + stream: false, + format: 'json', + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: prompt }, + ], + options: { + temperature: 0.2, + }, + }), + }); + if (!response.ok) { + throw new Error(`Ollama request failed (${response.status})`); + } + const payload = await response.json(); + const content = payload?.message?.content; + if (typeof content !== 'string' || !content.trim()) { + throw new Error('Ollama returned an empty response'); + } + return content; + } + + async generateStepPatch({ + guideId, + stepId, + target = 'all', + blockId = null, + }) { + try { + const config = this.aiConfig(); + if (!config.enabled) { + return { ok: false, reason: 'Enable AI in settings first.' }; + } + if (!config.ollama.host || !config.ollama.model) { + return { ok: false, reason: 'Configure Ollama host and model in Settings.' }; + } + + const guide = this.store.getGuide(guideId); + const step = this.store.getStep(guideId, stepId); + if (!guide || !step) { + return { ok: false, reason: 'Guide or step not found.' }; + } + + const currentBlock = blockId + ? [...(step.textBlocks || []), ...(step.codeBlocks || []), ...(step.tableBlocks || [])].find((b) => b.id === blockId) || null + : null; + if (blockId && target === 'block' && !currentBlock) { + return { ok: false, reason: 'Block not found.' }; + } + + let captureContext = null; + if (step.image) { + const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original'); + if (imagePath && fs.existsSync(imagePath)) { + const { nativeImage } = require('electron'); + const image = nativeImage.createFromPath(imagePath); + if (!image.isEmpty()) { + const clickPoint = this.clickPointFromStep(step, image); + const [metadata, ocr] = await Promise.all([ + this.collectForegroundWindowContext(), + this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint), + ]); + captureContext = { + ...metadata, + ocrText: ocr.text, + mode: step.kind === 'image' ? 'fullscreen' : 'content', + }; + } + } + } + + const { systemPrompt, prompt } = buildAiPrompt({ + target, + guide, + step, + captureContext, + block: currentBlock, + }); + + const raw = await this.callOllama({ + host: config.ollama.host, + model: config.ollama.model, + prompt, + systemPrompt, + }); + const patch = normalizeAiPatch(raw); + const updated = applyAiPatchToStep(step, patch, { target, blockId }); + const saved = this.store.saveStep(guideId, updated); + return { ok: true, step: saved, patch }; + } catch (err) { + return { ok: false, reason: err && err.message ? err.message : 'AI generation failed.' }; + } + } + + clickPointFromStep(step, image = null) { + const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h)); + if (!marker) return null; + const size = image ? image.getSize() : step.image?.size || { width: 0, height: 0 }; + if (!size.width || !size.height) return null; + return { + x: Math.round((marker.x + marker.w / 2) * size.width), + y: Math.round((marker.y + marker.h / 2) * size.height), + }; + } +} + +module.exports = { TextIntelService }; diff --git a/core/settings.js b/core/settings.js index 62aa672..02b1552 100644 --- a/core/settings.js +++ b/core/settings.js @@ -47,6 +47,13 @@ const DEFAULT_SETTINGS = { focusedViewDefaultForNewSteps: false, autoTitleTemplate: '[[Mode]] capture [[Time]]', }, + ai: { + enabled: false, + ollama: { + host: 'http://127.0.0.1:11434', + model: 'llama3.2:1b', + }, + }, exports: { previewStepCount: 3, openFolderAfterExport: true, diff --git a/core/text-intel.js b/core/text-intel.js new file mode 100644 index 0000000..78c7e6a --- /dev/null +++ b/core/text-intel.js @@ -0,0 +1,416 @@ +'use strict'; +const { deepClone, htmlToText, escapeHtml } = require('./util'); +const { sanitizeHtml } = require('./sanitize'); +const { + TEXTBLOCK_LEVELS, + TEXTBLOCK_POSITIONS, + normalizeTextBlock, + normalizeCodeBlock, + normalizeTableBlock, +} = require('./schema'); + +const DEFAULT_CAPTURE_TITLES = { + fullscreen: 'Screen capture', + window: 'Window capture', + region: 'Region capture', +}; + +const AI_LEVEL_ALIASES = new Map([ + ['note', 'info'], + ['info', 'info'], + ['tip', 'success'], + ['success', 'success'], + ['warning', 'warn'], + ['warn', 'warn'], + ['important', 'error'], + ['error', 'error'], +]); + +const GENERIC_OCR_PHRASES = new Set([ + 'button', + 'click', + 'double click', + 'menu', + 'item', + 'field', + 'text field', + 'search', + 'submit', + 'cancel', + 'ok', + 'open', + 'select', + 'enter', + 'type', +]); + +function normalizeWhitespace(text) { + return String(text == null ? '' : text) + .replace(/\s+/g, ' ') + .trim(); +} + +function titleCaseWord(word) { + if (!word) return word; + if (/^[A-Z0-9]{2,}$/.test(word)) return word; + if (/^\d+$/.test(word)) return word; + return word[0].toUpperCase() + word.slice(1).toLowerCase(); +} + +function displayText(text) { + const clean = normalizeWhitespace(text) + .replace(/^[\s"'`([{<]+|[\s"'`)}\]>.,;:!?]+$/g, '') + .trim(); + if (!clean) return ''; + if (clean === clean.toUpperCase()) { + return clean.split(/\s+/).map(titleCaseWord).join(' '); + } + return clean.replace(/\s+/g, ' '); +} + +function candidateWords(text) { + const clean = normalizeWhitespace(text); + if (!clean) return []; + return clean.split(/\s+/).filter(Boolean); +} + +function scoreCandidate(text, { source = 'ocr' } = {}) { + const clean = displayText(text); + if (!clean) return -Infinity; + const words = candidateWords(clean); + if (!words.length) return -Infinity; + let score = 0; + score += source === 'element' ? 110 : source === 'window' ? 95 : source === 'app' ? 80 : 100; + score += Math.min(words.length, 6) * 12; + score -= Math.max(0, words.length - 6) * 10; + score -= Math.max(0, clean.length - 56) * 0.6; + if (GENERIC_OCR_PHRASES.has(clean.toLowerCase())) score -= 50; + if (clean.length <= 24) score += 10; + if (/^(click|open|enter|type|choose|select|save|cancel|confirm)\b/i.test(clean)) score += 8; + if (/^[\p{P}\p{S}0-9]+$/u.test(clean)) score -= 100; + return score; +} + +function pickBestOcrPhrase(ocrText) { + const text = normalizeWhitespace(ocrText); + if (!text) return ''; + const parts = text + .split(/\n+/) + .map((line) => displayText(line)) + .filter(Boolean) + .filter((line) => !GENERIC_OCR_PHRASES.has(line.toLowerCase())); + if (!parts.length) return ''; + let best = ''; + let bestScore = -Infinity; + for (const part of parts) { + const score = scoreCandidate(part, { source: 'ocr' }); + if (score > bestScore) { + best = part; + bestScore = score; + } + } + return best; +} + +function isShortUiLabel(text) { + const words = candidateWords(text); + return words.length > 0 && words.length <= 2 && text.length <= 24; +} + +function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) { + const candidates = []; + if (metadata.elementLabel) candidates.push({ text: metadata.elementLabel, source: 'element' }); + if (metadata.windowTitle) candidates.push({ text: metadata.windowTitle, source: 'window' }); + if (metadata.appName && metadata.appName !== metadata.windowTitle) { + candidates.push({ text: metadata.appName, source: 'app' }); + } + const ocrPhrase = pickBestOcrPhrase(ocrText); + if (ocrPhrase) candidates.push({ text: ocrPhrase, source: 'ocr' }); + + let winner = null; + let winnerScore = -Infinity; + for (const candidate of candidates) { + const score = scoreCandidate(candidate.text, { source: candidate.source }); + if (score > winnerScore) { + winner = candidate; + winnerScore = score; + } + } + + const cleanedWinner = winner ? displayText(winner.text) : ''; + if (!cleanedWinner) return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; + + if (winner.source === 'window' || winner.source === 'app') { + return `Open ${cleanedWinner}`; + } + if (winner.source === 'element') { + return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; + } + if (winner.source === 'ocr') { + return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; + } + return cleanedWinner; +} + +function plainTextToHtml(text) { + const trimmed = normalizeWhitespace(text); + if (!trimmed) return ''; + return trimmed + .split(/\n{2,}/) + .map((para) => `
${escapeHtml(para).replace(/\n/g, '
')}
Old text
', + textBlocks: [{ id: 'tb1', order: 1, position: 'after-description', level: 'info', title: 'Old tip', descriptionHtml: 'Old body
' }], + codeBlocks: [{ id: 'cb1', order: 2, language: 'text', code: 'old' }], + tableBlocks: [{ id: 'tbl1', order: 3, rows: [['x']] }], + }); + + const updated = applyAiPatchToStep(step, patch, { target: 'all' }); + assert.equal(updated.title, 'Open settings'); + assert.equal(updated.descriptionHtml, 'Pick the AI tab.
'); + assert.equal(updated.textBlocks.length, 1); + assert.equal(updated.textBlocks[0].level, 'success'); + assert.equal(updated.textBlocks[0].descriptionHtml, 'Use the local Ollama model.
'); + assert.equal(updated.codeBlocks[0].code, 'ollama pull llama3.2:1b'); + assert.deepEqual(updated.tableBlocks[0].rows, [['Name', 'Value'], ['Host', '127.0.0.1']]); +}); + +test('ollama connection test reports installed models', async (t) => { + const root = makeTmpDir('text-intel-ai'); + t.after(() => rmrf(root)); + const service = new TextIntelService({ + store: { settingsDir: root }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: async () => ({ + ok: true, + json: async () => ({ + models: [ + { name: 'llama3.2:1b' }, + { name: 'qwen3:0.6b' }, + ], + }), + }), + }); + + const result = await service.testAiConnection(); + assert.equal(result.ok, true); + assert.equal(result.installed, true); + assert.equal(result.model, 'llama3.2:1b'); +}); + +test('invalid ollama output fails safely without saving the step', async (t) => { + const root = makeTmpDir('text-intel-ai-invalid'); + t.after(() => rmrf(root)); + let saveCalls = 0; + const step = createStep({ + title: 'Old title', + descriptionHtml: 'Old text
', + }); + const service = new TextIntelService({ + store: { + settingsDir: root, + getGuide: () => ({ guideId: 'g1', title: 'Guide', descriptionHtml: '', stepsOrder: ['s1'] }), + getStep: () => step, + stepImagePath: () => null, + saveStep: () => { + saveCalls += 1; + throw new Error('save should not be called'); + }, + }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: async () => ({ + ok: true, + json: async () => ({ + message: { + content: 'not json at all', + }, + }), + }), + }); + + const result = await service.generateStepPatch({ + guideId: 'g1', + stepId: 's1', + target: 'all', + }); + + assert.equal(result.ok, false); + assert.match(result.reason, /JSON/i); + assert.equal(saveCalls, 0); + assert.equal(step.title, 'Old title'); +});