diff --git a/app/capture.js b/app/capture.js index 00b1ded..aefbe36 100644 --- a/app/capture.js +++ b/app/capture.js @@ -3,7 +3,6 @@ const path = require('node:path'); const { spawn, execFileSync } = require('node:child_process'); const { desktopCapturer, screen, BrowserWindow, nativeImage, Tray, Menu } = require('electron'); -const { expandPlaceholders } = require('../core/placeholders'); const raster = require('../core/raster'); const { encodePng } = require('../core/png'); const { @@ -14,6 +13,7 @@ const { DEFAULT_START_SLACK_MS, } = require('./click-frames'); const { physicalToDip } = require('./coords'); +const { DEFAULT_CAPTURE_TITLES } = require('../core/text-intel'); /** * Capture service: full-screen, active-window, and region capture, plus a @@ -114,6 +114,7 @@ class CaptureService { getWindow, notify, screenApi = screen, + textIntel = null, }) { this.store = store; this.settings = settings; @@ -122,6 +123,7 @@ class CaptureService { // Injectable for tests; the click/coordinate paths must never reach for // the global `screen` directly so coordinate handling stays testable. this.screen = screenApi; + this.textIntel = textIntel; this.session = null; // { guideId, paused, count, intervalSec } this.intervalTimer = null; this.clickWatcher = null; @@ -133,6 +135,7 @@ class CaptureService { this.clickWatcherErrTail = ''; this.linuxEvent = null; // event block currently being parsed this.pendingRawClick = null; // raw press waiting for its coordinate twin + this.pendingClickOsPoint = null; this.clickQueue = Promise.resolve(); this.frameLoopInFlight = false; this.frameLoopGrabStartedAt = null; @@ -463,7 +466,7 @@ class CaptureService { if (frame) { clog('click@', clickAt, 'frame', frame.source || 'loop', 'started', frame.startedAt - clickAt, 'ms, captured', frame.capturedAt - clickAt, 'ms rel. click'); - const result = this.storeFrameAsStep(guideId, frame.mode, frame, clickPos); + const result = await this.storeFrameAsStep(guideId, frame.mode, frame, clickPos, clickMeta); if (result.ok) this.noteStepAdded(result.step, trigger, guideId); return result; } @@ -1211,6 +1214,9 @@ public static class SFMouseHook { let clickPos = osPoint ? this.osPointToDip(osPoint) : null; if (!clickPos) clickPos = this.screen.getCursorScreenPoint(); clog('click@', clickAt, button, 'os', osPoint, '-> dip', clickPos); + this.pendingClickOsPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y) + ? { x: osPoint.x, y: osPoint.y } + : null; this.enqueueClickCapture(clickPos, clickAt, button || 'mouse'); } @@ -1268,7 +1274,17 @@ public static class SFMouseHook { * fast the user clicks or how slow the encoder is. */ enqueueClickCapture(clickPos, clickAt = Date.now(), button = 'mouse') { - const clickMeta = { at: Number.isFinite(clickAt) ? clickAt : Date.now(), button: button || 'mouse' }; + const osPoint = this.pendingClickOsPoint && Number.isFinite(this.pendingClickOsPoint.x) && Number.isFinite(this.pendingClickOsPoint.y) + ? this.pendingClickOsPoint + : null; + this.pendingClickOsPoint = null; + const clickMeta = { + at: Number.isFinite(clickAt) ? clickAt : Date.now(), + button: button || 'mouse', + }; + if (osPoint) { + clickMeta.osPoint = { x: osPoint.x, y: osPoint.y }; + } if (this.session && !this.session.paused && !this.userIsInApp()) { // The guide id pins the click to its recording so it can still be // stored if the session stops while this click waits in the queue. @@ -1300,7 +1316,18 @@ public static class SFMouseHook { }; } - storeFrameAsStep(guideId, mode, frame, clickPos = null) { + async buildStepTitle(mode, frame, clickPos = null, clickMeta = null) { + try { + if (this.textIntel && typeof this.textIntel.buildCaptureTitle === 'function') { + return await this.textIntel.buildCaptureTitle({ mode, frame, clickPos, clickMeta }); + } + } catch { + // fall back to the local semantic title below + } + return this.autoTitle(mode); + } + + async storeFrameAsStep(guideId, mode, frame, clickPos = null, clickMeta = null) { if (!frame) return { ok: false, reason: 'no capture frame available' }; const annotations = []; // The click position (DIP, read at event time) wins over the frame's @@ -1324,7 +1351,7 @@ public static class SFMouseHook { } const step = this.store.addStep(guideId, { - title: this.autoTitle(mode), + title: await this.buildStepTitle(mode, frame, clickPos, clickMeta), annotations, focusedView: { enabled: Boolean(this.settings.get('editor.focusedViewDefaultForNewSteps')), @@ -1335,14 +1362,7 @@ public static class SFMouseHook { } autoTitle(mode) { - const tplStr = this.settings.get('editor.autoTitleTemplate') || '[[Mode]] capture [[Time]]'; - const now = new Date(); - const pad = (n) => String(n).padStart(2, '0'); - return expandPlaceholders(tplStr, { - Mode: { fullscreen: 'Screen', window: 'Window', region: 'Region' }[mode] || 'Screen', - Time: `${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`, - Date: `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`, - }); + return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; } /** Grab the screen/window image as { image, display } or throw. */ @@ -1451,8 +1471,12 @@ public static class SFMouseHook { const cropped = image.crop(rect); const size = cropped.getSize(); if (!size.width || !size.height) return { ok: false, reason: 'empty selection' }; - const step = this.store.addStep(guideId, { title: this.autoTitle('region') }, - cropped.toPNG(), size); + const step = await this.storeFrameAsStep(guideId, 'region', { + image: cropped, + size, + display, + cursor: null, + }, null, null); return { ok: true, step }; } diff --git a/app/main.js b/app/main.js index c89abc0..6a87840 100644 --- a/app/main.js +++ b/app/main.js @@ -19,6 +19,7 @@ const { exportGuideArchive, importGuideArchive, saveLinkedGuide, readArchive } = const { createSnapshot, listSnapshots, restoreSnapshot } = require('../core/snapshots'); const { readLock } = require('../core/locks'); const CaptureService = require('./capture'); +const { TextIntelService } = require('./text-intel'); const { keepProcessesResponsive } = require('./win-power'); const APP_ID = 'com.stepforge.app'; @@ -59,6 +60,7 @@ let settings; let searchIndex; let templates; let capture; +let textIntel; let mainWindow; function reindex(guideId) { @@ -503,6 +505,22 @@ function setupIpc() { if (keyPath.startsWith('capture.hotkey')) registerHotkeys(); return settings.data; }); + h('ai:test', async ({ enabled = null, ollama = null } = {}) => { + return textIntel.testAiConnection({ + enabled, + ollama, + }); + }); + h('ai:fillStep', async ({ guideId, stepId, target = 'all', blockId = null } = {}) => { + const result = await textIntel.generateStepPatch({ + guideId, + stepId, + target, + blockId, + }); + if (result.ok) reindex(guideId); + return result; + }); h('placeholders:globals:get', () => settings.getGlobalPlaceholders()); h('placeholders:globals:set', (values) => settings.setGlobalPlaceholders(values)); @@ -739,6 +757,13 @@ if (!gotLock) { settings = new Settings(store.settingsDir); searchIndex = new SearchIndex(store.indexDir); templates = new TemplateManager(store.templatesDir); + textIntel = new TextIntelService({ + store, + settings, + getWindow: () => mainWindow, + dataDir, + screenApi: screen, + }); // Bringing up the desktop-capture stream spawns/upgrades Chromium's GPU // and screen-capture utility processes — which can be born after a session // already started, so the start-time EcoQoS opt-out misses them. Re-apply @@ -759,6 +784,7 @@ if (!gotLock) { settings, getWindow: () => mainWindow, notify: captureNotify, + textIntel, }); applyTheme(); @@ -778,6 +804,9 @@ if (!gotLock) { capture.stopClickWatcher(); capture.destroySessionTray(); } + if (textIntel) { + textIntel.shutdown().catch(() => {}); + } // clean preview temp files on close try { for (const entry of fs.readdirSync(store.tempDir)) { diff --git a/app/preload.js b/app/preload.js index 432e31b..71f60b8 100644 --- a/app/preload.js +++ b/app/preload.js @@ -52,6 +52,10 @@ const api = { globalPlaceholders: invoke('placeholders:globals:get'), setGlobalPlaceholders: invoke('placeholders:globals:set'), }, + ai: { + test: invoke('ai:test'), + fillStep: invoke('ai:fillStep'), + }, capture: { shoot: invoke('capture:shoot'), region: invoke('capture:region'), diff --git a/app/renderer/app.js b/app/renderer/app.js index bbd99c4..8baae92 100644 --- a/app/renderer/app.js +++ b/app/renderer/app.js @@ -117,6 +117,7 @@ class StepForgeApp { guideFolders: library.folders?.guideFolders || {}, }; this.state.trash = trash; + this.editor.setSettings(settings); } async refreshLibrary({ keepFilter = true } = {}) { @@ -320,6 +321,7 @@ class StepForgeApp { { label: 'Guide information…', action: () => this.editor.openGuideInfo() }, { label: 'Guide placeholders…', action: () => this.editor.openGuidePlaceholders() }, { label: 'Backups & snapshots…', action: () => this.editor.openBackupsDialog() }, + { label: 'Generate all text fields with AI', action: () => this.editor.generateAllTextFieldsWithAi() }, { label: guide && guide.linkedSource ? 'Linked guide…' : 'Linked guide (not linked)', action: () => this.editor.openLinkedGuide() }, 'sep', { label: 'Keyboard shortcuts…', action: () => this.editor.openShortcutsHelp() }, @@ -866,6 +868,7 @@ class StepForgeApp { await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck }); await api.settings.set({ keyPath: 'capture', value: next.capture }); await api.settings.set({ keyPath: 'editor', value: next.editor }); + await api.settings.set({ keyPath: 'ai', value: next.ai }); await api.settings.set({ keyPath: 'exports', value: next.exports }); await api.settings.set({ keyPath: 'backups', value: next.backups }); await api.settings.setGlobalPlaceholders(next.placeholders || {}); diff --git a/app/renderer/dialogs.js b/app/renderer/dialogs.js index 272c811..8ce8a84 100644 --- a/app/renderer/dialogs.js +++ b/app/renderer/dialogs.js @@ -313,6 +313,42 @@ function showSettingsDialog({ const captureOutside = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.captureOutsideClicks) }); const confirmSimple = el('input', { type: 'checkbox', checked: Boolean(settings.capture?.confirmSimpleCapture) }); const keepLast = makeInput(settings.backups?.keepLast ?? 10, 'number', { min: 0, step: 1 }); + const aiEnabled = el('input', { type: 'checkbox', checked: Boolean(settings.ai?.enabled) }); + const ollamaHost = makeInput(settings.ai?.ollama?.host || 'http://127.0.0.1:11434'); + const ollamaModel = makeInput(settings.ai?.ollama?.model || 'llama3.2:1b'); + const aiStatus = el('div', { className: 'muted ai-status' }, 'AI stays local through Ollama. Nothing is sent to the cloud.'); + const testAiBtn = el('button', { type: 'button' }, 'Test connection'); + + const updateAiStatus = (message, { error = false } = {}) => { + aiStatus.textContent = message; + aiStatus.classList.toggle('error', Boolean(error)); + }; + + const testAiConnection = async () => { + setButtonLoading(testAiBtn, true, 'Testing…'); + updateAiStatus('Checking Ollama at the configured host…'); + try { + const result = await api.ai.test({ + ollama: { + host: ollamaHost.value.trim(), + model: ollamaModel.value.trim(), + }, + }); + if (!result.ok) { + updateAiStatus(result.reason || 'Could not connect to Ollama.', { error: true }); + return; + } + if (result.installed) { + updateAiStatus(`Connected to ${result.host} with ${result.model}.`); + } else { + updateAiStatus(`Connected to ${result.host}. Model ${result.model} is not installed yet.`, { error: true }); + } + } catch (err) { + updateAiStatus(err.message || 'Could not connect to Ollama.', { error: true }); + } finally { + setButtonLoading(testAiBtn, false); + } + }; const placeholderRows = el('div', { className: 'placeholder-rows' }); const rows = []; @@ -369,6 +405,19 @@ function showSettingsDialog({ el('legend', {}, 'Backups'), labeledRow('Keep last snapshots', keepLast), ), + el('fieldset', {}, + el('legend', {}, 'AI'), + labeledRow('Enable AI text filling', aiEnabled), + labeledRow('Ollama host', ollamaHost), + labeledRow('Ollama model', ollamaModel), + el('div.row', { style: { justifyContent: 'space-between' } }, + aiStatus, + testAiBtn, + ), + el('div.muted', {}, + 'AI generation is manual only. Captures stay deterministic until you click a generate button.', + ), + ), el('fieldset', {}, el('legend', {}, 'Global placeholders'), placeholderRows, @@ -412,6 +461,15 @@ function showSettingsDialog({ ...settings.backups, keepLast: Number(keepLast.value || 0), }, + ai: { + ...settings.ai, + enabled: aiEnabled.checked, + ollama: { + ...(settings.ai?.ollama || {}), + host: ollamaHost.value.trim(), + model: ollamaModel.value.trim(), + }, + }, placeholders: rows.reduce((acc, row) => { const inputs = row.querySelectorAll('input'); const key = inputs[0].value.trim(); @@ -430,6 +488,14 @@ function showSettingsDialog({ }); form.addEventListener('submit', (e) => e.preventDefault()); + testAiBtn.addEventListener('click', testAiConnection); + aiEnabled.addEventListener('change', () => { + updateAiStatus( + aiEnabled.checked + ? 'AI generation will be available once Ollama is reachable.' + : 'AI generation is disabled. The settings are still saved for later.', + ); + }); }); } diff --git a/app/renderer/editor.js b/app/renderer/editor.js index 840f252..dd51028 100644 --- a/app/renderer/editor.js +++ b/app/renderer/editor.js @@ -135,6 +135,7 @@ class GuideEditor { this.descriptionDirty = false; this.titleDirty = false; this.active = true; + this.settings = {}; this.saveStepDebounced = debounce(() => this.flushStep(), 180); this.saveGuideDebounced = debounce(() => this.flushGuide(), 180); @@ -152,6 +153,80 @@ class GuideEditor { this.active = Boolean(active); } + setSettings(settings) { + this.settings = settings || {}; + this.updateAiButtonState(); + } + + isAiEnabled() { + return Boolean(this.settings?.ai?.enabled); + } + + updateAiButtonState() { + const enabled = this.isAiEnabled(); + const buttons = [ + this.dom?.titleAiBtn, + this.dom?.descAiBtn, + ...(this.dom?.blocksList ? [...this.dom.blocksList.querySelectorAll('button[data-ai-action]')] : []), + ].filter(Boolean); + for (const button of buttons) { + button.disabled = !enabled; + button.title = enabled + ? button.dataset.aiTitle || 'Generate with AI' + : 'Enable AI in Settings first.'; + } + } + + async runAiGeneration(target, { blockId = null, button = null } = {}) { + if (!this.currentStep) { + this.onToast('Select a step first.', { error: true }); + return null; + } + if (!this.isAiEnabled()) { + this.onToast('Enable AI in Settings first.', { error: true }); + return null; + } + if (this.pendingSave) await this.flushStep(); + if (button) setButtonLoading(button, true, 'Generating…'); + try { + const result = await api.ai.fillStep({ + guideId: this.guideId, + stepId: this.currentStep.stepId, + target, + blockId, + }); + if (!result || !result.ok) { + this.onToast(result?.reason || 'AI generation failed.', { error: true }); + return null; + } + await this.reload(result.step.stepId); + this.onToast('AI text filled.'); + return result.step; + } catch (err) { + this.onToast(err.message || 'AI generation failed.', { error: true }); + return null; + } finally { + if (button) setButtonLoading(button, false); + } + } + + async generateTitleWithAi(button = null) { + return this.runAiGeneration('title', { button }); + } + + async generateDescriptionWithAi(button = null) { + return this.runAiGeneration('description', { button }); + } + + async generateAllTextFieldsWithAi(button = null) { + return this.runAiGeneration('all', { button }); + } + + async generateBlockWithAi(kind, block, button = null) { + if (!block) return null; + return this.runAiGeneration('block', { blockId: block.id, button }); + } + get currentStep() { return this.stepMap.get(this.selectedStepId) || null; } @@ -271,7 +346,14 @@ class GuideEditor { el('aside.pane-props', {}, el('section', {}, el('h3', {}, 'Step'), - this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title' }), + el('div.row', {}, + this.dom.titleInput = el('input', { type: 'text', placeholder: 'Step title', style: { flex: 1 } }), + this.dom.titleAiBtn = el('button.ai', { + type: 'button', + title: 'Generate the step title with AI', + dataset: { aiAction: 'title', aiTitle: 'Generate the step title with AI' }, + }, 'AI'), + ), this.dom.statusSelect = makeSelect('todo', [ { value: 'todo', label: 'Todo' }, { value: 'in-progress', label: 'In progress' }, @@ -296,7 +378,14 @@ class GuideEditor { ), ), el('section', {}, - el('h3', {}, 'Description'), + el('div.row', { style: { justifyContent: 'space-between', alignItems: 'center' } }, + el('h3', { style: { margin: 0 } }, 'Description'), + this.dom.descAiBtn = el('button.ai', { + type: 'button', + title: 'Generate the description with AI', + dataset: { aiAction: 'description', aiTitle: 'Generate the description with AI' }, + }, 'AI'), + ), this.dom.richToolbar = el('div.rich-toolbar', {}, this.toolbarBtn('bold', 'Bold'), this.toolbarBtn('italic', 'Italic'), @@ -416,6 +505,7 @@ class GuideEditor { this.renderStepList(); this.emitMeta(); }); + this.dom.titleAiBtn.addEventListener('click', () => this.generateTitleWithAi(this.dom.titleAiBtn)); this.dom.statusSelect.addEventListener('change', () => { if (!this.currentStep) return; @@ -491,6 +581,7 @@ class GuideEditor { document.addEventListener('selectionchange', () => { if (document.activeElement === this.dom.descEditor) this.updateToolbarState(); }); + this.dom.descAiBtn.addEventListener('click', () => this.generateDescriptionWithAi(this.dom.descAiBtn)); this.dom.descEditor.addEventListener('paste', (e) => { // Keep pasted text simple; backend sanitization will handle the rest. @@ -504,6 +595,8 @@ class GuideEditor { if (!item) return; this.canvas.select(item.dataset.annId); }); + + this.updateAiButtonState(); } renderAll() { @@ -513,6 +606,7 @@ class GuideEditor { this.renderCanvas(); this.renderAnnotationPanel(); this.renderBlocksPanel(); + this.updateAiButtonState(); this.emitMeta(); } @@ -670,6 +764,15 @@ class GuideEditor { el('span.spacer'), el('button.icon', { type: 'button', title: 'Move block up', disabled: !canMoveUp, onClick: moveUp, dataset: { blockMove: 'up' } }, '↑'), el('button.icon', { type: 'button', title: 'Move block down', disabled: !canMoveDown, onClick: moveDown, dataset: { blockMove: 'down' } }, '↓'), + (() => { + const aiBtn = el('button.ai', { + type: 'button', + title: 'Generate this block with AI', + dataset: { aiAction: 'block', aiTitle: 'Generate this block with AI' }, + onClick: () => this.generateBlockWithAi(kind, block, aiBtn), + }, 'AI'); + return aiBtn; + })(), removeBtn(() => { if (kind === 'text') step.textBlocks = (step.textBlocks || []).filter((b) => b !== block); else if (kind === 'code') step.codeBlocks = (step.codeBlocks || []).filter((b) => b !== block); @@ -761,6 +864,7 @@ class GuideEditor { if (!blocks.length) { this.dom.blocksList.append(el('div.muted', {}, 'Informational text, code, and table blocks can be reordered with drag handles or arrows.')); } + this.updateAiButtonState(); } renderStepList() { @@ -1658,6 +1762,7 @@ class GuideEditor { await api.settings.set({ keyPath: 'spellcheck', value: next.spellcheck }); await api.settings.set({ keyPath: 'capture', value: next.capture }); await api.settings.set({ keyPath: 'editor', value: next.editor }); + await api.settings.set({ keyPath: 'ai', value: next.ai }); await api.settings.set({ keyPath: 'exports', value: next.exports }); await api.settings.set({ keyPath: 'backups', value: next.backups }); await api.settings.setGlobalPlaceholders(next.placeholders || {}); diff --git a/app/renderer/style.css b/app/renderer/style.css index 6842b29..5ea2b5a 100644 --- a/app/renderer/style.css +++ b/app/renderer/style.css @@ -90,6 +90,16 @@ button.tool.active { border-color: var(--accent); color: var(--accent-fg); } +button.ai { + padding: 5px 10px; + border-color: color-mix(in srgb, var(--accent) 38%, var(--border)); + background: color-mix(in srgb, var(--accent) 10%, var(--panel-solid)); + color: var(--accent-strong); + font-weight: 650; + letter-spacing: 0.02em; +} +button.ai:hover { background: color-mix(in srgb, var(--accent) 16%, var(--panel-2)); } +button.ai:disabled { opacity: 0.6; } button:disabled { opacity: 0.6; cursor: default; } button.loading { display: inline-flex; @@ -751,6 +761,7 @@ fieldset legend { color: var(--muted); line-height: 1.5; } +.ai-status.error { color: var(--danger); } #modal-root:not(:empty) { position: fixed; diff --git a/app/text-intel.js b/app/text-intel.js new file mode 100644 index 0000000..0d305a4 --- /dev/null +++ b/app/text-intel.js @@ -0,0 +1,438 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +const { createWorker } = require('tesseract.js'); +const { + buildCaptureTitle, + normalizeOllamaHost, + normalizeAiPatch, + buildAiPrompt, + applyAiPatchToStep, + displayText, + normalizeWhitespace, +} = require('../core/text-intel'); + +const OCR_CROP = { + width: 420, + height: 220, +}; + +function hasBinary(name) { + try { + execFileSync('which', [name], { stdio: 'pipe' }); + return true; + } catch { + return false; + } +} + +function clamp(v, min, max) { + return Math.min(max, Math.max(min, v)); +} + +class TextIntelService { + constructor({ + store, + settings, + getWindow = () => null, + dataDir, + fetchImpl = global.fetch, + screenApi = null, + }) { + this.store = store; + this.settings = settings; + this.getWindow = getWindow; + this.dataDir = dataDir; + this.fetch = fetchImpl; + this.screen = screenApi; + this.worker = null; + this.workerPromise = null; + this.workerQueue = Promise.resolve(); + this.ocrDataDir = path.join(dataDir, 'ocr', 'eng'); + } + + async shutdown() { + if (this.worker) { + try { + await this.worker.terminate(); + } catch { + // best effort + } + this.worker = null; + this.workerPromise = null; + } + } + + ensureLangData() { + const packageDir = path.dirname(require.resolve('@tesseract.js-data/eng/package.json')); + const source = path.join(packageDir, '4.0.0_best_int', 'eng.traineddata.gz'); + const targetDir = this.ocrDataDir; + const target = path.join(targetDir, 'eng.traineddata.gz'); + if (!fs.existsSync(target)) { + fs.mkdirSync(targetDir, { recursive: true }); + fs.copyFileSync(source, target); + } + return targetDir; + } + + async getWorker() { + if (this.workerPromise) return this.workerPromise; + this.workerPromise = (async () => { + const langPath = this.ensureLangData(); + const worker = await createWorker('eng', 1, { + langPath, + }); + await worker.setParameters({ + preserve_interword_spaces: '1', + }); + this.worker = worker; + return worker; + })(); + return this.workerPromise; + } + + async recognizeCrop(image, rect = null) { + const worker = await this.getWorker(); + const cropped = rect ? image.crop(rect) : image; + const buffer = cropped.toPNG(); + const result = await worker.recognize(buffer); + const text = String(result?.data?.text || '').trim(); + return { + text, + confidence: Number.isFinite(result?.data?.confidence) ? result.data.confidence : null, + raw: result, + }; + } + + cropRectForPoint(frame, clickPos, { width = OCR_CROP.width, height = OCR_CROP.height } = {}) { + if (!frame || !frame.size) return null; + const bounds = frame.display.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height }; + const scaleX = frame.size.width / bounds.width; + const scaleY = frame.size.height / bounds.height; + const point = clickPos || { + x: bounds.x + bounds.width / 2, + y: bounds.y + bounds.height / 2, + }; + const centerX = (point.x - bounds.x) * scaleX; + const centerY = (point.y - bounds.y) * scaleY; + const rectW = Math.max(1, Math.round(width * scaleX)); + const rectH = Math.max(1, Math.round(height * scaleY)); + const rect = { + x: Math.round(centerX - rectW / 2), + y: Math.round(centerY - rectH / 2), + width: rectW, + height: rectH, + }; + rect.x = clamp(rect.x, 0, Math.max(0, frame.size.width - rect.width)); + rect.y = clamp(rect.y, 0, Math.max(0, frame.size.height - rect.height)); + rect.width = clamp(rect.width, 1, frame.size.width); + rect.height = clamp(rect.height, 1, frame.size.height); + return rect; + } + + async ocrAroundClick(frame, clickPos) { + if (!frame || !frame.image) return { text: '', confidence: null }; + const rect = this.cropRectForPoint(frame, clickPos); + try { + return await this.recognizeCrop(frame.image, rect); + } catch { + return { text: '', confidence: null }; + } + } + + async collectForegroundWindowContext(osPoint = null) { + try { + if (process.platform === 'win32') return this.collectWindowsWindowContext(osPoint); + if (process.platform === 'darwin') return this.collectMacWindowContext(); + if (process.platform === 'linux') return this.collectLinuxWindowContext(); + } catch { + // best effort only + } + return { appName: '', windowTitle: '' }; + } + + collectWindowsWindowContext(osPoint = null) { + const hasPoint = osPoint && Number.isFinite(osPoint.x) && Number.isFinite(osPoint.y); + const clickX = hasPoint ? Number(osPoint.x) : 0; + const clickY = hasPoint ? Number(osPoint.y) : 0; + const script = ` + $clickX = ${clickX}; + $clickY = ${clickY}; + $elementLabel = ''; + $elementRole = ''; + $elementClass = ''; + $elementProcessId = 0; + if (${hasPoint ? '$true' : '$false'}) { + try { + Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes,WindowsBase | Out-Null + $point = New-Object System.Windows.Point($clickX, $clickY); + $element = [System.Windows.Automation.AutomationElement]::FromPoint($point); + if ($element) { + $current = $element.Current; + $elementLabel = $current.Name; + $elementRole = $current.LocalizedControlType; + $elementClass = $current.ClassName; + $elementProcessId = $current.ProcessId; + } + } catch { } + } + Add-Type @" +using System; +using System.Runtime.InteropServices; +using System.Text; +public static class Win32 { + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count); + [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); +} +"@; + $hWnd = [Win32]::GetForegroundWindow(); + $sb = New-Object System.Text.StringBuilder 512; + [void][Win32]::GetWindowText($hWnd, $sb, $sb.Capacity); + $pid = 0; + [void][Win32]::GetWindowThreadProcessId($hWnd, [ref]$pid); + $proc = Get-Process -Id $pid -ErrorAction SilentlyContinue | Select-Object -First 1; + $out = [ordered]@{ + appName = if ($proc) { $proc.ProcessName } else { '' }; + windowTitle = $sb.ToString(); + elementLabel = $elementLabel; + elementRole = $elementRole; + elementClass = $elementClass; + elementProcessId = $elementProcessId; + pid = $pid; + }; + $out | ConvertTo-Json -Compress; + `; + const result = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }).trim(); + return JSON.parse(result || '{}'); + } + + collectMacWindowContext() { + const script = ` + set appName to "" + set windowTitle to "" + tell application "System Events" + try + set frontApp to first application process whose frontmost is true + set appName to name of frontApp + try + set windowTitle to name of front window of frontApp + end try + end try + end tell + return appName & linefeed & windowTitle + `; + const result = execFileSync('osascript', ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }).trimEnd(); + const [appName = '', windowTitle = ''] = result.split(/\r?\n/); + return { appName, windowTitle }; + } + + collectLinuxWindowContext() { + if (!hasBinary('xprop')) return { appName: '', windowTitle: '' }; + const active = execFileSync('xprop', ['-root', '_NET_ACTIVE_WINDOW'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }); + const activeMatch = active.match(/window id # (0x[0-9a-fA-F]+)/); + if (!activeMatch) return { appName: '', windowTitle: '' }; + const winId = activeMatch[1]; + const details = execFileSync('xprop', ['-id', winId, '_NET_WM_NAME', 'WM_NAME', 'WM_CLASS'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 1200, + }); + const titleMatch = details.match(/(?:_NET_WM_NAME\(UTF8_STRING\)|WM_NAME\(STRING\)|WM_NAME\(UTF8_STRING\)) = "([^"]*)"/); + const classMatch = details.match(/WM_CLASS\(STRING\) = "([^"]*)"(?:, "([^"]*)")?/); + return { + appName: classMatch ? (classMatch[2] || classMatch[1] || '') : '', + windowTitle: titleMatch ? titleMatch[1] : '', + }; + } + + async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) { + const [metadata, ocr] = await Promise.all([ + this.collectForegroundWindowContext(clickMeta?.osPoint || null), + this.ocrAroundClick(frame, clickPos), + ]); + return buildCaptureTitle({ + mode, + metadata, + ocrText: ocr.text, + }); + } + + aiEnabled() { + return Boolean(this.settings.get('ai.enabled')); + } + + aiConfig(override = null) { + const stored = this.settings.get('ai') || {}; + const merged = override ? { + ...stored, + ...override, + ollama: { + ...(stored.ollama || {}), + ...(override.ollama || {}), + }, + } : stored; + return { + ...merged, + enabled: override && Object.prototype.hasOwnProperty.call(override, 'enabled') + ? Boolean(override.enabled) + : Boolean(stored.enabled), + ollama: { + host: normalizeOllamaHost(merged.ollama?.host || ''), + model: normalizeWhitespace(merged.ollama?.model || ''), + }, + }; + } + + async testAiConnection(override = null) { + const config = this.aiConfig(override); + if (!config.ollama.host) { + return { ok: false, reason: 'Set an Ollama host first.' }; + } + const tagsUrl = new URL('/api/tags', `${config.ollama.host.replace(/\/+$/, '')}/`); + const res = await this.fetch(tagsUrl, { method: 'GET' }); + if (!res.ok) { + return { ok: false, reason: `Ollama check failed (${res.status})` }; + } + const data = await res.json(); + const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : []; + const installed = config.ollama.model ? models.includes(config.ollama.model) : false; + return { + ok: true, + installed, + models, + host: config.ollama.host, + model: config.ollama.model, + }; + } + + async callOllama({ host, model, prompt, systemPrompt }) { + const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`); + const response = await this.fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + stream: false, + format: 'json', + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: prompt }, + ], + options: { + temperature: 0.2, + }, + }), + }); + if (!response.ok) { + throw new Error(`Ollama request failed (${response.status})`); + } + const payload = await response.json(); + const content = payload?.message?.content; + if (typeof content !== 'string' || !content.trim()) { + throw new Error('Ollama returned an empty response'); + } + return content; + } + + async generateStepPatch({ + guideId, + stepId, + target = 'all', + blockId = null, + }) { + try { + const config = this.aiConfig(); + if (!config.enabled) { + return { ok: false, reason: 'Enable AI in settings first.' }; + } + if (!config.ollama.host || !config.ollama.model) { + return { ok: false, reason: 'Configure Ollama host and model in Settings.' }; + } + + const guide = this.store.getGuide(guideId); + const step = this.store.getStep(guideId, stepId); + if (!guide || !step) { + return { ok: false, reason: 'Guide or step not found.' }; + } + + const currentBlock = blockId + ? [...(step.textBlocks || []), ...(step.codeBlocks || []), ...(step.tableBlocks || [])].find((b) => b.id === blockId) || null + : null; + if (blockId && target === 'block' && !currentBlock) { + return { ok: false, reason: 'Block not found.' }; + } + + let captureContext = null; + if (step.image) { + const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original'); + if (imagePath && fs.existsSync(imagePath)) { + const { nativeImage } = require('electron'); + const image = nativeImage.createFromPath(imagePath); + if (!image.isEmpty()) { + const clickPoint = this.clickPointFromStep(step, image); + const [metadata, ocr] = await Promise.all([ + this.collectForegroundWindowContext(), + this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint), + ]); + captureContext = { + ...metadata, + ocrText: ocr.text, + mode: step.kind === 'image' ? 'fullscreen' : 'content', + }; + } + } + } + + const { systemPrompt, prompt } = buildAiPrompt({ + target, + guide, + step, + captureContext, + block: currentBlock, + }); + + const raw = await this.callOllama({ + host: config.ollama.host, + model: config.ollama.model, + prompt, + systemPrompt, + }); + const patch = normalizeAiPatch(raw); + const updated = applyAiPatchToStep(step, patch, { target, blockId }); + const saved = this.store.saveStep(guideId, updated); + return { ok: true, step: saved, patch }; + } catch (err) { + return { ok: false, reason: err && err.message ? err.message : 'AI generation failed.' }; + } + } + + clickPointFromStep(step, image = null) { + const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h)); + if (!marker) return null; + const size = image ? image.getSize() : step.image?.size || { width: 0, height: 0 }; + if (!size.width || !size.height) return null; + return { + x: Math.round((marker.x + marker.w / 2) * size.width), + y: Math.round((marker.y + marker.h / 2) * size.height), + }; + } +} + +module.exports = { TextIntelService }; diff --git a/core/settings.js b/core/settings.js index 62aa672..02b1552 100644 --- a/core/settings.js +++ b/core/settings.js @@ -47,6 +47,13 @@ const DEFAULT_SETTINGS = { focusedViewDefaultForNewSteps: false, autoTitleTemplate: '[[Mode]] capture [[Time]]', }, + ai: { + enabled: false, + ollama: { + host: 'http://127.0.0.1:11434', + model: 'llama3.2:1b', + }, + }, exports: { previewStepCount: 3, openFolderAfterExport: true, diff --git a/core/text-intel.js b/core/text-intel.js new file mode 100644 index 0000000..78c7e6a --- /dev/null +++ b/core/text-intel.js @@ -0,0 +1,416 @@ +'use strict'; +const { deepClone, htmlToText, escapeHtml } = require('./util'); +const { sanitizeHtml } = require('./sanitize'); +const { + TEXTBLOCK_LEVELS, + TEXTBLOCK_POSITIONS, + normalizeTextBlock, + normalizeCodeBlock, + normalizeTableBlock, +} = require('./schema'); + +const DEFAULT_CAPTURE_TITLES = { + fullscreen: 'Screen capture', + window: 'Window capture', + region: 'Region capture', +}; + +const AI_LEVEL_ALIASES = new Map([ + ['note', 'info'], + ['info', 'info'], + ['tip', 'success'], + ['success', 'success'], + ['warning', 'warn'], + ['warn', 'warn'], + ['important', 'error'], + ['error', 'error'], +]); + +const GENERIC_OCR_PHRASES = new Set([ + 'button', + 'click', + 'double click', + 'menu', + 'item', + 'field', + 'text field', + 'search', + 'submit', + 'cancel', + 'ok', + 'open', + 'select', + 'enter', + 'type', +]); + +function normalizeWhitespace(text) { + return String(text == null ? '' : text) + .replace(/\s+/g, ' ') + .trim(); +} + +function titleCaseWord(word) { + if (!word) return word; + if (/^[A-Z0-9]{2,}$/.test(word)) return word; + if (/^\d+$/.test(word)) return word; + return word[0].toUpperCase() + word.slice(1).toLowerCase(); +} + +function displayText(text) { + const clean = normalizeWhitespace(text) + .replace(/^[\s"'`([{<]+|[\s"'`)}\]>.,;:!?]+$/g, '') + .trim(); + if (!clean) return ''; + if (clean === clean.toUpperCase()) { + return clean.split(/\s+/).map(titleCaseWord).join(' '); + } + return clean.replace(/\s+/g, ' '); +} + +function candidateWords(text) { + const clean = normalizeWhitespace(text); + if (!clean) return []; + return clean.split(/\s+/).filter(Boolean); +} + +function scoreCandidate(text, { source = 'ocr' } = {}) { + const clean = displayText(text); + if (!clean) return -Infinity; + const words = candidateWords(clean); + if (!words.length) return -Infinity; + let score = 0; + score += source === 'element' ? 110 : source === 'window' ? 95 : source === 'app' ? 80 : 100; + score += Math.min(words.length, 6) * 12; + score -= Math.max(0, words.length - 6) * 10; + score -= Math.max(0, clean.length - 56) * 0.6; + if (GENERIC_OCR_PHRASES.has(clean.toLowerCase())) score -= 50; + if (clean.length <= 24) score += 10; + if (/^(click|open|enter|type|choose|select|save|cancel|confirm)\b/i.test(clean)) score += 8; + if (/^[\p{P}\p{S}0-9]+$/u.test(clean)) score -= 100; + return score; +} + +function pickBestOcrPhrase(ocrText) { + const text = normalizeWhitespace(ocrText); + if (!text) return ''; + const parts = text + .split(/\n+/) + .map((line) => displayText(line)) + .filter(Boolean) + .filter((line) => !GENERIC_OCR_PHRASES.has(line.toLowerCase())); + if (!parts.length) return ''; + let best = ''; + let bestScore = -Infinity; + for (const part of parts) { + const score = scoreCandidate(part, { source: 'ocr' }); + if (score > bestScore) { + best = part; + bestScore = score; + } + } + return best; +} + +function isShortUiLabel(text) { + const words = candidateWords(text); + return words.length > 0 && words.length <= 2 && text.length <= 24; +} + +function buildCaptureTitle({ mode = 'fullscreen', metadata = {}, ocrText = '' } = {}) { + const candidates = []; + if (metadata.elementLabel) candidates.push({ text: metadata.elementLabel, source: 'element' }); + if (metadata.windowTitle) candidates.push({ text: metadata.windowTitle, source: 'window' }); + if (metadata.appName && metadata.appName !== metadata.windowTitle) { + candidates.push({ text: metadata.appName, source: 'app' }); + } + const ocrPhrase = pickBestOcrPhrase(ocrText); + if (ocrPhrase) candidates.push({ text: ocrPhrase, source: 'ocr' }); + + let winner = null; + let winnerScore = -Infinity; + for (const candidate of candidates) { + const score = scoreCandidate(candidate.text, { source: candidate.source }); + if (score > winnerScore) { + winner = candidate; + winnerScore = score; + } + } + + const cleanedWinner = winner ? displayText(winner.text) : ''; + if (!cleanedWinner) return DEFAULT_CAPTURE_TITLES[mode] || 'Capture'; + + if (winner.source === 'window' || winner.source === 'app') { + return `Open ${cleanedWinner}`; + } + if (winner.source === 'element') { + return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; + } + if (winner.source === 'ocr') { + return isShortUiLabel(cleanedWinner) ? `Click ${cleanedWinner}` : cleanedWinner; + } + return cleanedWinner; +} + +function plainTextToHtml(text) { + const trimmed = normalizeWhitespace(text); + if (!trimmed) return ''; + return trimmed + .split(/\n{2,}/) + .map((para) => `

${escapeHtml(para).replace(/\n/g, '
')}

`) + .join(''); +} + +function normalizeOllamaHost(host) { + const raw = normalizeWhitespace(host); + if (!raw) return ''; + if (/^https?:\/\//i.test(raw)) return raw.replace(/\/+$/, ''); + return `http://${raw.replace(/\/+$/, '')}`; +} + +function normalizeAiLevel(level) { + const key = normalizeWhitespace(level).toLowerCase(); + return AI_LEVEL_ALIASES.get(key) || (TEXTBLOCK_LEVELS.includes(key) ? key : 'info'); +} + +function normalizeAiPosition(position) { + const key = normalizeWhitespace(position).toLowerCase(); + return TEXTBLOCK_POSITIONS.includes(key) ? key : 'after-description'; +} + +function normalizeAiBlock(block) { + if (!block || typeof block !== 'object') return null; + const kind = normalizeWhitespace(block.kind).toLowerCase(); + if (kind === 'text') { + const normalized = normalizeTextBlock({ + id: block.id, + order: Number.isFinite(block.order) ? block.order : null, + position: normalizeAiPosition(block.position), + level: normalizeAiLevel(block.level), + title: displayText(block.title), + descriptionHtml: plainTextToHtml(block.body ?? block.description ?? block.text ?? ''), + }, Number.isFinite(block.order) ? block.order : null); + return { ...normalized, kind: 'text' }; + } + if (kind === 'code') { + return { + ...normalizeCodeBlock({ + id: block.id, + order: Number.isFinite(block.order) ? block.order : null, + language: displayText(block.language).toLowerCase(), + code: String(block.code ?? ''), + }, Number.isFinite(block.order) ? block.order : null), + kind: 'code', + }; + } + if (kind === 'table') { + const rows = Array.isArray(block.rows) + ? block.rows.map((row) => (Array.isArray(row) ? row.map((cell) => displayText(cell)) : [])) + : []; + return { + ...normalizeTableBlock({ + id: block.id, + order: Number.isFinite(block.order) ? block.order : null, + rows, + }, Number.isFinite(block.order) ? block.order : null), + kind: 'table', + }; + } + return null; +} + +function normalizeAiPatch(raw) { + let data = raw; + if (typeof raw === 'string') { + const trimmed = raw.trim().replace(/^```(?:json)?\s*|\s*```$/g, ''); + const start = trimmed.indexOf('{'); + const end = trimmed.lastIndexOf('}'); + const jsonText = start >= 0 && end > start ? trimmed.slice(start, end + 1) : trimmed; + data = JSON.parse(jsonText); + } + if (!data || typeof data !== 'object' || Array.isArray(data)) { + throw new Error('AI response must be a JSON object'); + } + const out = { + title: displayText(data.title), + descriptionHtml: plainTextToHtml(data.description ?? data.descriptionText ?? ''), + blocks: Array.isArray(data.blocks) + ? data.blocks.map((block) => normalizeAiBlock(block)).filter(Boolean) + : [], + }; + return out; +} + +function summarizeBlocks(step = {}) { + const parts = []; + for (const block of step.textBlocks || []) { + const body = htmlToText(block.descriptionHtml || ''); + parts.push(`- Text (${block.level || 'info'}, ${block.position || 'after-description'}): ${block.title || ''}${body ? ` — ${body}` : ''}`.trim()); + } + for (const block of step.codeBlocks || []) { + const code = String(block.code || '').trim(); + parts.push(`- Code (${block.language || 'plain'}):\n${code || '(empty)'}`); + } + for (const block of step.tableBlocks || []) { + const rows = Array.isArray(block.rows) ? block.rows.length : 0; + const cols = rows > 0 && Array.isArray(block.rows[0]) ? block.rows[0].length : 0; + parts.push(`- Table (${rows}x${cols})`); + } + return parts.length ? parts.join('\n') : '(none)'; +} + +function summarizeStepForAi(step = {}) { + return [ + `Step title: ${step.title || '(empty)'}`, + `Step description: ${htmlToText(step.descriptionHtml || '') || '(empty)'}`, + `Step status: ${step.status || 'todo'}`, + `Blocks:\n${summarizeBlocks(step)}`, + ].join('\n'); +} + +function summarizeGuideForAi(guide = {}) { + return [ + `Guide title: ${guide.title || '(untitled)'}`, + `Guide description: ${htmlToText(guide.descriptionHtml || '') || '(empty)'}`, + ].join('\n'); +} + +function buildAiPrompt({ + target = 'all', + guide = null, + step = null, + captureContext = null, + block = null, +} = {}) { + const targetText = { + title: 'rewrite only the step title', + description: 'rewrite only the step description', + block: 'rewrite only one block', + all: 'rewrite the step title, description, and any useful blocks', + }[target] || 'rewrite the step'; + + const allowedBlockNote = [ + 'Use block.kind = "text" with level in [info, warn, error, success] for note / warning / important / tip blocks.', + 'Use block.kind = "code" for code snippets.', + 'Use block.kind = "table" for tables, with rows as arrays of strings.', + 'Use block.position values from [before-title, after-title, before-image, after-image, before-description, after-description].', + ].join(' '); + + const prompt = [ + 'You write concise, human-sounding step-by-step documentation for an offline desktop app.', + 'Return JSON only. No markdown fences, no commentary, no extra keys outside the schema below.', + 'Schema:', + '{', + ' "title": string,', + ' "description": string,', + ' "blocks": [{', + ' "kind": "text" | "code" | "table",', + ' "position"?: "before-title" | "after-title" | "before-image" | "after-image" | "before-description" | "after-description",', + ' "level"?: "info" | "warn" | "error" | "success",', + ' "title"?: string,', + ' "body"?: string,', + ' "language"?: string,', + ' "code"?: string,', + ' "rows"?: string[][]', + ' }]', + '}', + '', + `Target: ${targetText}.`, + allowedBlockNote, + '', + guide ? summarizeGuideForAi(guide) : 'Guide: (not provided)', + '', + step ? summarizeStepForAi(step) : 'Step: (not provided)', + '', + captureContext ? [ + `Active window title: ${captureContext.windowTitle || '(unknown)'}`, + `Active app: ${captureContext.appName || '(unknown)'}`, + `OCR text near click: ${captureContext.ocrText || '(none)'}`, + `Capture mode: ${captureContext.mode || '(unknown)'}`, + ].join('\n') : 'Capture context: (not provided)', + '', + block ? `Target block:\n${JSON.stringify(block, null, 2)}` : 'Target block: (not provided)', + '', + 'Rules:', + '- Keep the wording natural and specific.', + '- Prefer short titles.', + '- Only return blocks that add value.', + '- Do not invent details that are not supported by the capture context.', + '- If the target is one block, only rewrite that block.', + ].join('\n'); + + return { + systemPrompt: 'You are a documentation assistant that only emits JSON.', + prompt, + }; +} + +function applyAiPatchToStep(step, patch, { target = 'all', blockId = null } = {}) { + const next = deepClone(step); + if ((target === 'all' || target === 'title') && patch.title) { + next.title = displayText(patch.title); + } + if ((target === 'all' || target === 'description') && patch.descriptionHtml) { + next.descriptionHtml = sanitizeHtml(patch.descriptionHtml); + } + + if (target === 'all' && Array.isArray(patch.blocks) && patch.blocks.length) { + const textBlocks = []; + const codeBlocks = []; + const tableBlocks = []; + let nextOrder = 1; + for (const block of patch.blocks) { + const clone = deepClone(block); + clone.order = nextOrder++; + if (clone.kind === 'text') textBlocks.push(clone); + else if (clone.kind === 'code') codeBlocks.push(clone); + else if (clone.kind === 'table') tableBlocks.push(clone); + } + next.textBlocks = textBlocks; + next.codeBlocks = codeBlocks; + next.tableBlocks = tableBlocks; + } else if (target === 'block' && blockId && Array.isArray(patch.blocks) && patch.blocks.length) { + const replacement = patch.blocks[0]; + const textBlock = (next.textBlocks || []).find((block) => block.id === blockId); + const codeBlock = (next.codeBlocks || []).find((block) => block.id === blockId); + const tableBlock = (next.tableBlocks || []).find((block) => block.id === blockId); + if (textBlock && replacement.kind === 'text') { + if (replacement.position) textBlock.position = replacement.position; + if (replacement.level) textBlock.level = replacement.level; + if (replacement.title) textBlock.title = replacement.title; + if (replacement.descriptionHtml) textBlock.descriptionHtml = sanitizeHtml(replacement.descriptionHtml); + } else if (codeBlock && replacement.kind === 'code') { + if (replacement.language) codeBlock.language = replacement.language; + if (replacement.code) codeBlock.code = replacement.code; + } else if (tableBlock && replacement.kind === 'table') { + if (replacement.rows) tableBlock.rows = replacement.rows; + } + } + if (!next.image) { + const hasBody = Boolean( + next.title || + htmlToText(next.descriptionHtml || '') || + (next.textBlocks || []).length || + (next.codeBlocks || []).length || + (next.tableBlocks || []).length, + ); + if (hasBody) next.kind = 'content'; + } + return next; +} + +module.exports = { + DEFAULT_CAPTURE_TITLES, + buildCaptureTitle, + plainTextToHtml, + normalizeOllamaHost, + normalizeAiPatch, + buildAiPrompt, + applyAiPatchToStep, + summarizeStepForAi, + summarizeGuideForAi, + displayText, + normalizeWhitespace, + scoreCandidate, + pickBestOcrPhrase, +}; diff --git a/docs/getting_started_with_ai.md b/docs/getting_started_with_ai.md new file mode 100644 index 0000000..193218e --- /dev/null +++ b/docs/getting_started_with_ai.md @@ -0,0 +1,75 @@ +# Getting Started With AI + +StepForge keeps AI local. It talks to your own Ollama server on your machine and does not send guide content to the cloud. + +## 1. Install Ollama + +Install Ollama from https://ollama.com and make sure the service is running. + +On most systems you can verify it with: + +```bash +ollama --version +``` + +## 2. Pull a lightweight model + +The recommended default is: + +```bash +ollama pull llama3.2:1b +``` + +That model is small enough to feel responsive on modest hardware, but still good enough for human-sounding titles and short text blocks. + +If you need something even smaller, try: + +```bash +ollama pull qwen3:0.6b +``` + +or: + +```bash +ollama pull gemma3:270m +``` + +Those are lighter, but they are usually weaker at writing polished step text. + +## 3. Open StepForge settings + +In StepForge, open `Settings` and find the `AI` section. + +Set: + +* `Enable AI text filling` to on +* `Ollama host` to your local Ollama server +* `Ollama model` to `llama3.2:1b` or the smaller model you pulled + +The default host is: + +```text +http://127.0.0.1:11434 +``` + +## 4. Test the connection + +Use the `Test connection` button in the AI settings section. + +If the model is installed, StepForge should confirm the host and model. + +## 5. Use AI manually + +AI is never automatic. After capture, use the `AI` button next to: + +* the step title +* the step description +* each text, code, and table block + +You can also use `More -> Generate all text fields with AI` to fill the whole step in one pass. + +## Notes + +* Capture titles are still generated automatically without AI. +* AI generation only works when `Enable AI text filling` is turned on. +* The app always uses local OCR around the click area first, then local AI only when you ask for it. diff --git a/package-lock.json b/package-lock.json index 49e5bb5..9cc3468 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,10 @@ "name": "stepforge", "version": "0.1.0", "license": "MPL-2.0", + "dependencies": { + "@tesseract.js-data/eng": "^1.0.0", + "tesseract.js": "^7.0.0" + }, "devDependencies": { "electron": "^41.7.1", "electron-builder": "^26.15.2" @@ -624,6 +628,12 @@ "node": ">=10" } }, + "node_modules/@tesseract.js-data/eng": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@tesseract.js-data/eng/-/eng-1.0.0.tgz", + "integrity": "sha512-mbTumm6KQPUHyzTPQaF3ObXYnx0SqqfV2nabqFVQBwD6Kl7PhGSLSzOlfFTWy0P3BjghaSKA2W9GB19Jk+ZcTg==", + "license": "MIT" + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -1057,6 +1067,12 @@ "dev": true, "license": "MIT" }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/boolean": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", @@ -2512,6 +2528,12 @@ "node": ">= 14" } }, + "node_modules/idb-keyval": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.5.tgz", + "integrity": "sha512-eKQkTnS0relYsSOYomx8ozIbmdsQCKUdhyuIaQ2DZgKuaxtyQQMkyD/wlnQN32pO3yutN1b1L8uqwcDKaJd7/Q==", + "license": "Apache-2.0" + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -2541,6 +2563,12 @@ "node": ">=8" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -2903,6 +2931,26 @@ "node": ">=10" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.4.0.tgz", @@ -3024,6 +3072,15 @@ "wrappy": "1" } }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/p-cancelable": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", @@ -3314,6 +3371,12 @@ "util-deprecate": "~1.0.1" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3728,6 +3791,30 @@ "node": ">= 10.0.0" } }, + "node_modules/tesseract.js": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz", + "integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^7.0.0", + "wasm-feature-detect": "^1.8.0", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz", + "integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==", + "license": "Apache-2.0" + }, "node_modules/tiny-async-pool": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.3.0.tgz", @@ -3785,6 +3872,12 @@ "tmp": "^0.2.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/truncate-utf8-bytes": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", @@ -3909,6 +4002,12 @@ "dev": true, "license": "MIT" }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, "node_modules/webcrypto-core": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/webcrypto-core/-/webcrypto-core-1.9.2.tgz", @@ -3923,6 +4022,22 @@ "tslib": "^2.8.1" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/which/-/which-5.0.0.tgz", @@ -4043,6 +4158,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } } } } diff --git a/package.json b/package.json index 8e44b59..41fa808 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,10 @@ "verify": "bash scripts/verify.sh", "bootstrap": "bash scripts/bootstrap-offline.sh" }, + "dependencies": { + "@tesseract.js-data/eng": "^1.0.0", + "tesseract.js": "^7.0.0" + }, "devDependencies": { "electron": "^41.7.1", "electron-builder": "^26.15.2" diff --git a/tests/unit/text-intel.test.js b/tests/unit/text-intel.test.js new file mode 100644 index 0000000..861e322 --- /dev/null +++ b/tests/unit/text-intel.test.js @@ -0,0 +1,195 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { makeTmpDir, rmrf } = require('./helpers'); +const { createStep } = require('../../core/schema'); +const { + buildCaptureTitle, + normalizeAiPatch, + applyAiPatchToStep, +} = require('../../core/text-intel'); +const { TextIntelService } = require('../../app/text-intel'); + +function makeSettings(values = {}) { + const data = { + ai: { + enabled: true, + ollama: { + host: 'http://127.0.0.1:11434', + model: 'llama3.2:1b', + }, + }, + ...values, + }; + return { + get(key) { + return key.split('.').reduce((acc, part) => (acc == null ? undefined : acc[part]), data); + }, + }; +} + +test('capture titles prefer semantic metadata before OCR fallback', () => { + const title = buildCaptureTitle({ + mode: 'fullscreen', + metadata: { windowTitle: 'Reset user password in admin portal' }, + ocrText: 'Save', + }); + assert.equal(title, 'Open Reset user password in admin portal'); +}); + +test('capture titles prefer element metadata before window chrome and OCR', () => { + const title = buildCaptureTitle({ + mode: 'window', + metadata: { + elementLabel: 'Open advanced settings', + windowTitle: 'Preferences', + }, + ocrText: 'Cancel', + }); + assert.equal(title, 'Open advanced settings'); +}); + +test('capture titles fall back to OCR when metadata is absent', () => { + const title = buildCaptureTitle({ + mode: 'window', + metadata: {}, + ocrText: 'Save changes', + }); + assert.equal(title, 'Click Save changes'); +}); + +test('ocr crop rectangles clamp to the image bounds', (t) => { + const root = makeTmpDir('text-intel-crop'); + t.after(() => rmrf(root)); + const service = new TextIntelService({ + store: { settingsDir: root }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: global.fetch, + }); + + const frame = { + size: { width: 1000, height: 500 }, + display: { bounds: { x: 0, y: 0, width: 1000, height: 500 } }, + }; + + const topLeft = service.cropRectForPoint(frame, { x: 5, y: 5 }); + assert.deepEqual(topLeft, { x: 0, y: 0, width: 420, height: 220 }); + + const bottomRight = service.cropRectForPoint(frame, { x: 995, y: 495 }); + assert.deepEqual(bottomRight, { x: 580, y: 280, width: 420, height: 220 }); +}); + +test('ai response normalization and application keeps fields structured', () => { + const patch = normalizeAiPatch(JSON.stringify({ + title: 'Open settings', + description: 'Pick the AI tab.', + blocks: [ + { + kind: 'text', + position: 'after-description', + level: 'tip', + title: 'Tip', + body: 'Use the local Ollama model.', + }, + { + kind: 'code', + language: 'bash', + code: 'ollama pull llama3.2:1b', + }, + { + kind: 'table', + rows: [['Name', 'Value'], ['Host', '127.0.0.1']], + }, + ], + })); + + const step = createStep({ + title: 'Old title', + descriptionHtml: '

Old text

', + textBlocks: [{ id: 'tb1', order: 1, position: 'after-description', level: 'info', title: 'Old tip', descriptionHtml: '

Old body

' }], + codeBlocks: [{ id: 'cb1', order: 2, language: 'text', code: 'old' }], + tableBlocks: [{ id: 'tbl1', order: 3, rows: [['x']] }], + }); + + const updated = applyAiPatchToStep(step, patch, { target: 'all' }); + assert.equal(updated.title, 'Open settings'); + assert.equal(updated.descriptionHtml, '

Pick the AI tab.

'); + assert.equal(updated.textBlocks.length, 1); + assert.equal(updated.textBlocks[0].level, 'success'); + assert.equal(updated.textBlocks[0].descriptionHtml, '

Use the local Ollama model.

'); + assert.equal(updated.codeBlocks[0].code, 'ollama pull llama3.2:1b'); + assert.deepEqual(updated.tableBlocks[0].rows, [['Name', 'Value'], ['Host', '127.0.0.1']]); +}); + +test('ollama connection test reports installed models', async (t) => { + const root = makeTmpDir('text-intel-ai'); + t.after(() => rmrf(root)); + const service = new TextIntelService({ + store: { settingsDir: root }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: async () => ({ + ok: true, + json: async () => ({ + models: [ + { name: 'llama3.2:1b' }, + { name: 'qwen3:0.6b' }, + ], + }), + }), + }); + + const result = await service.testAiConnection(); + assert.equal(result.ok, true); + assert.equal(result.installed, true); + assert.equal(result.model, 'llama3.2:1b'); +}); + +test('invalid ollama output fails safely without saving the step', async (t) => { + const root = makeTmpDir('text-intel-ai-invalid'); + t.after(() => rmrf(root)); + let saveCalls = 0; + const step = createStep({ + title: 'Old title', + descriptionHtml: '

Old text

', + }); + const service = new TextIntelService({ + store: { + settingsDir: root, + getGuide: () => ({ guideId: 'g1', title: 'Guide', descriptionHtml: '', stepsOrder: ['s1'] }), + getStep: () => step, + stepImagePath: () => null, + saveStep: () => { + saveCalls += 1; + throw new Error('save should not be called'); + }, + }, + settings: makeSettings(), + getWindow: () => null, + dataDir: root, + fetchImpl: async () => ({ + ok: true, + json: async () => ({ + message: { + content: 'not json at all', + }, + }), + }), + }); + + const result = await service.generateStepPatch({ + guideId: 'g1', + stepId: 's1', + target: 'all', + }); + + assert.equal(result.ok, false); + assert.match(result.reason, /JSON/i); + assert.equal(saveCalls, 0); + assert.equal(step.title, 'Old title'); +});