TylerandClaude Fable 5
970d76a780
Template tests / tests (pull_request) Failing after 33s
Phase 3 groundwork of the improvement plan (PR 7 of the sequence). The Linux
work is a platform rewrite, so this first establishes the interface boundary
and moves an OS-specific piece behind it with Windows behavior preserved — no
new process.platform branches in shared code.
- app/platform/index.js is the single factory that selects a platform
implementation; shared code asks it for adapters and never inspects
process.platform itself.
- app/platform/interfaces.js documents the adapter contracts
(WindowContextProvider, ClickSource, PowerPolicy) and the explicit click-
source vocabulary.
- Extracted the foreground-window/element detection into per-OS adapters,
verbatim from text-intel.js:
app/platform/windows/window-context.js (PowerShell UIAutomation)
app/platform/linux/window-context.js (xprop)
app/platform/darwin/window-context.js (AppleScript)
text-intel.js now delegates to the injected provider and its three
platform-branching methods (and the now-dead child_process import) are gone.
- app/platform/linux/diagnostics.js detects session type, portal/PipeWire,
xinput, readable input devices, and the resulting click/screen-capture
profile, returning actionable messages for the UI. Exposed via a new
platform:capabilities IPC + preload method.
This is behavior-preserving: the Windows/macOS/Linux window-context code is
the same, just relocated behind the factory, and the capture pipeline is
untouched.
Tests: platform selection for every OS, provider validity + null-object for
unsupported OS, the shared service delegating to an injected provider, Linux
capability detection (x11/xinput, Wayland-without-PipeWire messaging, no-click
fallback, evdev), the capability facade, and a guard that text-intel no longer
branches on process.platform. 268 unit tests pass; startup smoke passes and
the click self-test is unchanged (stream source, markers 3/3, burst 8/8).
Co-Authored-By: Claude Fable 5 <[email protected]>
677 lines
24 KiB
JavaScript
677 lines
24 KiB
JavaScript
'use strict';
|
||
|
||
const fs = require('node:fs');
|
||
const path = require('node:path');
|
||
|
||
const {
|
||
DEFAULT_CAPTURE_TITLES,
|
||
buildCaptureTitle,
|
||
normalizeOllamaHost,
|
||
validateOllamaHost,
|
||
normalizeAiPatch,
|
||
buildAiPrompt,
|
||
applyAiPatchToStep,
|
||
displayText,
|
||
normalizeWhitespace,
|
||
} = require('../core/text-intel');
|
||
|
||
const DEFAULT_TITLE_VALUES = new Set(Object.values(DEFAULT_CAPTURE_TITLES).concat(['Capture']));
|
||
|
||
const OCR_CROP = {
|
||
width: 420,
|
||
height: 220,
|
||
};
|
||
|
||
function clamp(v, min, max) {
|
||
return Math.min(max, Math.max(min, v));
|
||
}
|
||
|
||
function modelLooksVisionCapable(model) {
|
||
const clean = normalizeWhitespace(model).toLowerCase();
|
||
if (!clean) return false;
|
||
return clean.includes('vision')
|
||
|| clean.includes('llava')
|
||
|| clean.includes('gemma4')
|
||
|| /(^|[^a-z0-9])qwen[23](?:\.[0-9]+)?vl([^a-z0-9]|$)/.test(clean);
|
||
}
|
||
|
||
let createWorkerImpl = null;
|
||
function loadCreateWorker() {
|
||
if (createWorkerImpl) return createWorkerImpl;
|
||
// OCR is optional at startup; lazy-load it so the app can still boot when
|
||
// the dependency has not been installed yet.
|
||
// eslint-disable-next-line global-require
|
||
({ createWorker: createWorkerImpl } = require('tesseract.js'));
|
||
return createWorkerImpl;
|
||
}
|
||
|
||
class TextIntelService {
|
||
constructor({
|
||
store,
|
||
settings,
|
||
getWindow = () => null,
|
||
dataDir,
|
||
fetchImpl = global.fetch,
|
||
screenApi = null,
|
||
windowContextProvider = null,
|
||
}) {
|
||
this.store = store;
|
||
this.settings = settings;
|
||
this.getWindow = getWindow;
|
||
this.dataDir = dataDir;
|
||
this.fetch = fetchImpl;
|
||
this.screen = screenApi;
|
||
// OS-specific foreground-window/element detection is a platform adapter.
|
||
// This code no longer branches on process.platform; the factory selects it.
|
||
this.windowContext = windowContextProvider
|
||
|| require('./platform').createWindowContextProvider();
|
||
this.worker = null;
|
||
this.workerPromise = null;
|
||
this.workerQueue = Promise.resolve();
|
||
this.ocrDataDir = path.join(dataDir, 'ocr', 'eng');
|
||
this.modelCapabilityCache = new Map();
|
||
// In-flight AI request controllers, grouped by guide, so closing a guide
|
||
// (or quitting) can cancel outstanding requests instead of leaving them
|
||
// to resolve against stale data.
|
||
this.inflight = new Set();
|
||
// Bounded concurrency for AI network calls.
|
||
this.maxConcurrent = 2;
|
||
this.activeCount = 0;
|
||
this.waitQueue = [];
|
||
}
|
||
|
||
aiNetworkOptions() {
|
||
const ai = this.settings.get('ai') || {};
|
||
const timeoutMs = Number.isFinite(ai.timeoutMs) && ai.timeoutMs > 0 ? ai.timeoutMs : 60000;
|
||
const maxImageBytes = Number.isFinite(ai.maxImageBytes) && ai.maxImageBytes > 0
|
||
? ai.maxImageBytes
|
||
: 12 * 1024 * 1024;
|
||
return {
|
||
allowRemote: Boolean(ai.allowRemoteHost),
|
||
attachScreenshots: ai.attachScreenshots !== false,
|
||
timeoutMs,
|
||
maxImageBytes,
|
||
};
|
||
}
|
||
|
||
// Resolve the endpoint against the local-first policy or throw a clear error.
|
||
resolveHost(host) {
|
||
const { allowRemote } = this.aiNetworkOptions();
|
||
const result = validateOllamaHost(host, { allowRemote });
|
||
if (!result.ok) {
|
||
const err = new Error(result.reason);
|
||
err.code = 'STEPFORGE_AI_HOST_BLOCKED';
|
||
throw err;
|
||
}
|
||
return result.host;
|
||
}
|
||
|
||
// fetch with a hard deadline and cooperative cancellation. Every AI network
|
||
// call goes through here so a dead endpoint can never hang the UI.
|
||
async fetchJson(url, { method = 'GET', body = null, guideId = null } = {}) {
|
||
const { timeoutMs } = this.aiNetworkOptions();
|
||
const controller = new AbortController();
|
||
if (guideId) controller._guideId = guideId;
|
||
this.inflight.add(controller);
|
||
const timer = setTimeout(() => controller.abort(new Error('timeout')), timeoutMs);
|
||
try {
|
||
const res = await this.fetch(url, {
|
||
method,
|
||
headers: body ? { 'Content-Type': 'application/json' } : undefined,
|
||
body: body ? JSON.stringify(body) : undefined,
|
||
signal: controller.signal,
|
||
});
|
||
return res;
|
||
} catch (err) {
|
||
if (controller.signal.aborted) {
|
||
// The abort reason distinguishes an explicit cancel from a timeout.
|
||
const reasonMsg = controller.signal.reason && controller.signal.reason.message;
|
||
const cancelled = reasonMsg === 'cancelled';
|
||
const wrapped = new Error(cancelled ? 'AI request cancelled.' : 'AI request timed out.');
|
||
wrapped.code = 'STEPFORGE_AI_ABORTED';
|
||
throw wrapped;
|
||
}
|
||
throw err;
|
||
} finally {
|
||
clearTimeout(timer);
|
||
this.inflight.delete(controller);
|
||
}
|
||
}
|
||
|
||
// Cancel in-flight AI requests, optionally scoped to one guide.
|
||
cancelInflight(guideId = null) {
|
||
for (const controller of [...this.inflight]) {
|
||
if (!guideId || controller._guideId === guideId) {
|
||
try { controller.abort(new Error('cancelled')); } catch { /* already settled */ }
|
||
this.inflight.delete(controller);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Bounded-concurrency gate for AI network work.
|
||
async withConcurrency(fn) {
|
||
if (this.activeCount >= this.maxConcurrent) {
|
||
await new Promise((resolve) => this.waitQueue.push(resolve));
|
||
}
|
||
this.activeCount += 1;
|
||
try {
|
||
return await fn();
|
||
} finally {
|
||
this.activeCount -= 1;
|
||
const next = this.waitQueue.shift();
|
||
if (next) next();
|
||
}
|
||
}
|
||
|
||
async shutdown() {
|
||
this.cancelInflight();
|
||
if (this.worker) {
|
||
try {
|
||
await this.worker.terminate();
|
||
} catch {
|
||
// best effort
|
||
}
|
||
this.worker = null;
|
||
this.workerPromise = null;
|
||
}
|
||
}
|
||
|
||
ensureLangData() {
|
||
const packageDir = path.dirname(require.resolve('@tesseract.js-data/eng/package.json'));
|
||
const source = path.join(packageDir, '4.0.0_best_int', 'eng.traineddata.gz');
|
||
const targetDir = this.ocrDataDir;
|
||
const target = path.join(targetDir, 'eng.traineddata.gz');
|
||
if (!fs.existsSync(target)) {
|
||
fs.mkdirSync(targetDir, { recursive: true });
|
||
fs.copyFileSync(source, target);
|
||
}
|
||
return targetDir;
|
||
}
|
||
|
||
async getWorker() {
|
||
if (this.workerPromise) return this.workerPromise;
|
||
this.workerPromise = (async () => {
|
||
const workerFactory = loadCreateWorker();
|
||
const langPath = this.ensureLangData();
|
||
const worker = await workerFactory('eng', 1, {
|
||
langPath,
|
||
});
|
||
await worker.setParameters({
|
||
preserve_interword_spaces: '1',
|
||
});
|
||
this.worker = worker;
|
||
return worker;
|
||
})();
|
||
this.workerPromise.catch(() => {
|
||
this.workerPromise = null;
|
||
});
|
||
return this.workerPromise;
|
||
}
|
||
|
||
async recognizeCrop(image, rect = null) {
|
||
const worker = await this.getWorker();
|
||
const cropped = rect ? image.crop(rect) : image;
|
||
const buffer = cropped.toPNG();
|
||
const result = await worker.recognize(buffer);
|
||
const text = String(result?.data?.text || '').trim();
|
||
return {
|
||
text,
|
||
confidence: Number.isFinite(result?.data?.confidence) ? result.data.confidence : null,
|
||
raw: result,
|
||
};
|
||
}
|
||
|
||
cropRectForPoint(frame, clickPos, { width = OCR_CROP.width, height = OCR_CROP.height } = {}) {
|
||
if (!frame || !frame.size) return null;
|
||
const bounds = frame.display.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height };
|
||
const scaleX = frame.size.width / bounds.width;
|
||
const scaleY = frame.size.height / bounds.height;
|
||
const point = clickPos || {
|
||
x: bounds.x + bounds.width / 2,
|
||
y: bounds.y + bounds.height / 2,
|
||
};
|
||
const centerX = (point.x - bounds.x) * scaleX;
|
||
const centerY = (point.y - bounds.y) * scaleY;
|
||
const rectW = Math.max(1, Math.round(width * scaleX));
|
||
const rectH = Math.max(1, Math.round(height * scaleY));
|
||
const rect = {
|
||
x: Math.round(centerX - rectW / 2),
|
||
y: Math.round(centerY - rectH / 2),
|
||
width: rectW,
|
||
height: rectH,
|
||
};
|
||
rect.x = clamp(rect.x, 0, Math.max(0, frame.size.width - rect.width));
|
||
rect.y = clamp(rect.y, 0, Math.max(0, frame.size.height - rect.height));
|
||
rect.width = clamp(rect.width, 1, frame.size.width);
|
||
rect.height = clamp(rect.height, 1, frame.size.height);
|
||
return rect;
|
||
}
|
||
|
||
async ocrAroundClick(frame, clickPos) {
|
||
if (!frame || !frame.image) return { text: '', confidence: null };
|
||
// Use a full-width horizontal strip at the click height. This preserves complete
|
||
// link text (e.g. "Oracle | Cloud Applications and Cloud Platform") rather than
|
||
// cropping through it when the element spans more than the 420 px default width.
|
||
const bounds = frame.display?.bounds || { x: 0, y: 0, width: frame.size.width, height: frame.size.height };
|
||
const rect = this.cropRectForPoint(frame, clickPos, {
|
||
width: bounds.width, // full display width → full image width after DPI scaling
|
||
height: 100, // ~2 lines tall, enough context without too much noise
|
||
});
|
||
try {
|
||
return await this.recognizeCrop(frame.image, rect);
|
||
} catch {
|
||
return { text: '', confidence: null };
|
||
}
|
||
}
|
||
|
||
async collectForegroundWindowContext(osPoint = null) {
|
||
try {
|
||
return await this.windowContext.collect(osPoint);
|
||
} catch {
|
||
// best effort only
|
||
return { appName: '', windowTitle: '' };
|
||
}
|
||
}
|
||
|
||
|
||
async buildCaptureTitle({ mode, frame, clickPos, clickMeta = null }) {
|
||
const ctx = await this.buildCaptureContext({ mode, frame, clickPos, clickMeta });
|
||
return ctx.title;
|
||
}
|
||
|
||
async buildCaptureContext({ mode, frame, clickPos, clickMeta = null }) {
|
||
const keyContext = clickMeta?.keyContext || {};
|
||
const recentTyped = keyContext.recentTyped || '';
|
||
const recentShortcut = keyContext.recentShortcut || '';
|
||
// Use window context pre-captured by the click watcher when available.
|
||
// This avoids a costly PowerShell cold-start (1–3 s) on every capture.
|
||
const fastContext = clickMeta?.windowContext || null;
|
||
const [metadata, ocr] = await Promise.all([
|
||
fastContext
|
||
? Promise.resolve(fastContext)
|
||
: this.collectForegroundWindowContext(clickMeta?.osPoint || null),
|
||
this.ocrAroundClick(frame, clickPos),
|
||
]);
|
||
const title = buildCaptureTitle({ mode, metadata, ocrText: ocr.text, recentTyped, recentShortcut });
|
||
return {
|
||
title,
|
||
captureMetadata: {
|
||
ocrText: ocr.text || '',
|
||
windowTitle: metadata.windowTitle || '',
|
||
appName: metadata.appName || '',
|
||
elementLabel: metadata.elementLabel || '',
|
||
elementRole: metadata.elementRole || '',
|
||
elementValue: metadata.elementValue || '',
|
||
recentTyped,
|
||
recentShortcut,
|
||
mode,
|
||
},
|
||
};
|
||
}
|
||
|
||
aiEnabled() {
|
||
return Boolean(this.settings.get('ai.enabled'));
|
||
}
|
||
|
||
aiConfig(override = null) {
|
||
const stored = this.settings.get('ai') || {};
|
||
const merged = override ? {
|
||
...stored,
|
||
...override,
|
||
ollama: {
|
||
...(stored.ollama || {}),
|
||
...(override.ollama || {}),
|
||
},
|
||
} : stored;
|
||
return {
|
||
...merged,
|
||
enabled: override && Object.prototype.hasOwnProperty.call(override, 'enabled')
|
||
? Boolean(override.enabled)
|
||
: Boolean(stored.enabled),
|
||
ollama: {
|
||
host: normalizeOllamaHost(merged.ollama?.host || ''),
|
||
model: normalizeWhitespace(merged.ollama?.model || ''),
|
||
},
|
||
};
|
||
}
|
||
|
||
async testAiConnection(override = null) {
|
||
const config = this.aiConfig(override);
|
||
if (!config.ollama.host) {
|
||
return { ok: false, reason: 'Set an Ollama host first.' };
|
||
}
|
||
let host;
|
||
try {
|
||
host = this.resolveHost(config.ollama.host);
|
||
} catch (err) {
|
||
return { ok: false, reason: err.message };
|
||
}
|
||
const tagsUrl = new URL('/api/tags', `${host.replace(/\/+$/, '')}/`);
|
||
let res;
|
||
try {
|
||
res = await this.fetchJson(tagsUrl, { method: 'GET' });
|
||
} catch (err) {
|
||
return { ok: false, reason: err.message };
|
||
}
|
||
if (!res.ok) {
|
||
return { ok: false, reason: `Ollama check failed (${res.status})` };
|
||
}
|
||
const data = await res.json();
|
||
const models = Array.isArray(data?.models) ? data.models.map((model) => model.name).filter(Boolean) : [];
|
||
const installed = config.ollama.model ? models.includes(config.ollama.model) : false;
|
||
const vision = installed ? await this.modelSupportsVision({
|
||
host,
|
||
model: config.ollama.model,
|
||
}) : false;
|
||
return {
|
||
ok: true,
|
||
installed,
|
||
vision,
|
||
models,
|
||
host,
|
||
model: config.ollama.model,
|
||
};
|
||
}
|
||
|
||
async modelCapabilities({ host, model }) {
|
||
const normalizedHost = normalizeOllamaHost(host);
|
||
const normalizedModel = normalizeWhitespace(model);
|
||
if (!normalizedHost || !normalizedModel) return [];
|
||
const cacheKey = `${normalizedHost}::${normalizedModel}`;
|
||
if (this.modelCapabilityCache.has(cacheKey)) {
|
||
return this.modelCapabilityCache.get(cacheKey);
|
||
}
|
||
const url = new URL('/api/show', `${normalizedHost.replace(/\/+$/, '')}/`);
|
||
let capabilities = [];
|
||
try {
|
||
const response = await this.fetchJson(url, {
|
||
method: 'POST',
|
||
body: { model: normalizedModel },
|
||
});
|
||
if (response.ok) {
|
||
const payload = await response.json();
|
||
capabilities = Array.isArray(payload?.capabilities)
|
||
? payload.capabilities.map((cap) => normalizeWhitespace(cap).toLowerCase()).filter(Boolean)
|
||
: [];
|
||
}
|
||
} catch {
|
||
capabilities = [];
|
||
}
|
||
if (!capabilities.includes('vision') && modelLooksVisionCapable(normalizedModel)) {
|
||
capabilities = [...capabilities, 'vision'];
|
||
}
|
||
this.modelCapabilityCache.set(cacheKey, capabilities);
|
||
return capabilities;
|
||
}
|
||
|
||
async modelSupportsVision({ host, model }) {
|
||
const capabilities = await this.modelCapabilities({ host, model });
|
||
return capabilities.includes('vision');
|
||
}
|
||
|
||
readStepImageBase64(guideId, stepId) {
|
||
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
|
||
if (!imagePath || !fs.existsSync(imagePath)) return '';
|
||
return fs.readFileSync(imagePath).toString('base64');
|
||
}
|
||
|
||
async callOllamaText({ host, model, prompt, systemPrompt, guideId = null }) {
|
||
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
|
||
const response = await this.withConcurrency(() => this.fetchJson(url, {
|
||
method: 'POST',
|
||
guideId,
|
||
body: {
|
||
model,
|
||
stream: false,
|
||
messages: [
|
||
{ role: 'system', content: systemPrompt },
|
||
{ role: 'user', content: prompt },
|
||
],
|
||
options: { temperature: 0.4 },
|
||
},
|
||
}));
|
||
if (!response.ok) throw new Error(`Ollama request failed (${response.status})`);
|
||
const payload = await response.json();
|
||
const content = payload?.message?.content;
|
||
if (typeof content !== 'string' || !content.trim()) throw new Error('Ollama returned an empty response');
|
||
return content.trim();
|
||
}
|
||
|
||
async callOllama({ host, model, prompt, systemPrompt, images = [], guideId = null }) {
|
||
const url = new URL('/api/chat', `${host.replace(/\/+$/, '')}/`);
|
||
const userMessage = { role: 'user', content: prompt };
|
||
if (Array.isArray(images) && images.length) {
|
||
userMessage.images = images;
|
||
}
|
||
const response = await this.withConcurrency(() => this.fetchJson(url, {
|
||
method: 'POST',
|
||
guideId,
|
||
body: {
|
||
model,
|
||
stream: false,
|
||
format: 'json',
|
||
messages: [
|
||
{ role: 'system', content: systemPrompt },
|
||
userMessage,
|
||
],
|
||
options: {
|
||
temperature: 0.2,
|
||
},
|
||
},
|
||
}));
|
||
if (!response.ok) {
|
||
throw new Error(`Ollama request failed (${response.status})`);
|
||
}
|
||
const payload = await response.json();
|
||
const content = payload?.message?.content;
|
||
if (typeof content !== 'string' || !content.trim()) {
|
||
throw new Error('Ollama returned an empty response');
|
||
}
|
||
return content;
|
||
}
|
||
|
||
async generateStepPatch({
|
||
guideId,
|
||
stepId,
|
||
target = 'all',
|
||
blockId = null,
|
||
}) {
|
||
try {
|
||
const config = this.aiConfig();
|
||
if (!config.enabled) {
|
||
return { ok: false, reason: 'Enable AI in settings first.' };
|
||
}
|
||
if (!config.ollama.host || !config.ollama.model) {
|
||
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
|
||
}
|
||
let host;
|
||
try {
|
||
host = this.resolveHost(config.ollama.host);
|
||
} catch (err) {
|
||
return { ok: false, reason: err.message };
|
||
}
|
||
const netOptions = this.aiNetworkOptions();
|
||
|
||
const guide = this.store.getGuide(guideId);
|
||
const step = this.store.getStep(guideId, stepId);
|
||
if (!guide || !step) {
|
||
return { ok: false, reason: 'Guide or step not found.' };
|
||
}
|
||
// Snapshot the revision now: AI generation is slow, and the user may
|
||
// edit the step meanwhile. We save with this expectedRevision so a
|
||
// response built from stale data cannot overwrite a newer user edit.
|
||
const baseRevision = Number.isInteger(step.revision) ? step.revision : 0;
|
||
|
||
const currentBlock = blockId
|
||
? [...(step.textBlocks || []), ...(step.codeBlocks || []), ...(step.tableBlocks || [])].find((b) => b.id === blockId) || null
|
||
: null;
|
||
if (blockId && target === 'block' && !currentBlock) {
|
||
return { ok: false, reason: 'Block not found.' };
|
||
}
|
||
|
||
// Only attach a screenshot when the user allows it, the model can use
|
||
// it, and it is within the size budget (a full 4K PNG base64-expands to
|
||
// tens of MB in the request body).
|
||
let screenshotBase64 = '';
|
||
if (netOptions.attachScreenshots && step.image) {
|
||
const candidate = this.readStepImageBase64(guideId, stepId);
|
||
const bytes = candidate ? Math.floor((candidate.length * 3) / 4) : 0;
|
||
if (candidate && bytes <= netOptions.maxImageBytes) {
|
||
screenshotBase64 = candidate;
|
||
}
|
||
}
|
||
const screenshotAttached = Boolean(screenshotBase64)
|
||
? await this.modelSupportsVision({
|
||
host,
|
||
model: config.ollama.model,
|
||
})
|
||
: false;
|
||
|
||
let captureContext = null;
|
||
// Use stored capture metadata when available (best context, from capture time).
|
||
// Fall back to re-running OCR on the stored image only when metadata is absent.
|
||
if (step.captureMetadata) {
|
||
const rawCandidate = buildCaptureTitle({
|
||
mode: step.captureMetadata.mode || 'fullscreen',
|
||
metadata: {
|
||
windowTitle: step.captureMetadata.windowTitle,
|
||
appName: step.captureMetadata.appName,
|
||
elementLabel: step.captureMetadata.elementLabel,
|
||
elementRole: step.captureMetadata.elementRole,
|
||
elementValue: step.captureMetadata.elementValue,
|
||
},
|
||
ocrText: step.captureMetadata.ocrText,
|
||
recentTyped: step.captureMetadata.recentTyped,
|
||
recentShortcut: step.captureMetadata.recentShortcut,
|
||
});
|
||
captureContext = {
|
||
...step.captureMetadata,
|
||
// Don't suggest a generic fallback title — leave it blank so AI generates from context.
|
||
titleCandidate: DEFAULT_TITLE_VALUES.has(rawCandidate) ? '' : rawCandidate,
|
||
};
|
||
} else if (step.image) {
|
||
const imagePath = this.store.stepImagePath(guideId, stepId, 'working') || this.store.stepImagePath(guideId, stepId, 'original');
|
||
if (imagePath && fs.existsSync(imagePath)) {
|
||
const { nativeImage } = require('electron');
|
||
const image = nativeImage.createFromPath(imagePath);
|
||
if (!image.isEmpty()) {
|
||
const clickPoint = this.clickPointFromStep(step, image);
|
||
const [metadata, ocr] = await Promise.all([
|
||
this.collectForegroundWindowContext(),
|
||
this.ocrAroundClick({ image, size: image.getSize(), display: { bounds: { x: 0, y: 0, width: image.getSize().width, height: image.getSize().height } } }, clickPoint),
|
||
]);
|
||
const rawCandidate2 = buildCaptureTitle({
|
||
mode: step.kind === 'image' ? 'fullscreen' : 'window',
|
||
metadata,
|
||
ocrText: ocr.text,
|
||
});
|
||
captureContext = {
|
||
...metadata,
|
||
ocrText: ocr.text,
|
||
titleCandidate: DEFAULT_TITLE_VALUES.has(rawCandidate2) ? '' : rawCandidate2,
|
||
mode: step.kind === 'image' ? 'fullscreen' : 'content',
|
||
};
|
||
}
|
||
}
|
||
}
|
||
|
||
const { systemPrompt, prompt } = buildAiPrompt({
|
||
target,
|
||
guide,
|
||
step,
|
||
captureContext,
|
||
block: currentBlock,
|
||
screenshotAttached,
|
||
});
|
||
|
||
const raw = await this.callOllama({
|
||
host,
|
||
model: config.ollama.model,
|
||
prompt,
|
||
systemPrompt,
|
||
images: screenshotAttached ? [screenshotBase64] : [],
|
||
guideId,
|
||
});
|
||
const patch = normalizeAiPatch(raw);
|
||
// Re-read the step: while generation ran, a capture auto-doc or another
|
||
// background write may have advanced it. Apply the patch to the current
|
||
// step and save with the original expected revision so a user edit made
|
||
// during generation causes a conflict instead of a silent overwrite.
|
||
let currentStep = step;
|
||
try {
|
||
currentStep = this.store.getStep(guideId, stepId) || step;
|
||
} catch {
|
||
currentStep = step;
|
||
}
|
||
const updated = applyAiPatchToStep(currentStep, patch, { target, blockId });
|
||
let saved;
|
||
try {
|
||
saved = this.store.saveStep(guideId, updated, { expectedRevision: baseRevision });
|
||
} catch (err) {
|
||
if (err && err.code === 'STEPFORGE_REVISION_CONFLICT') {
|
||
return { ok: false, reason: 'The step changed while AI was generating; nothing was overwritten.' };
|
||
}
|
||
throw err;
|
||
}
|
||
return { ok: true, step: saved, patch };
|
||
} catch (err) {
|
||
return { ok: false, reason: err && err.message ? err.message : 'AI generation failed.' };
|
||
}
|
||
}
|
||
|
||
async rewriteText({ text, guideTitle = '', stepTitle = '' }) {
|
||
try {
|
||
const config = this.aiConfig();
|
||
if (!config.enabled) return { ok: false, reason: 'Enable AI in settings first.' };
|
||
if (!config.ollama.host || !config.ollama.model) {
|
||
return { ok: false, reason: 'Configure Ollama host and model in Settings.' };
|
||
}
|
||
let host;
|
||
try {
|
||
host = this.resolveHost(config.ollama.host);
|
||
} catch (err) {
|
||
return { ok: false, reason: err.message };
|
||
}
|
||
const trimmed = normalizeWhitespace(text);
|
||
if (!trimmed) return { ok: false, reason: 'No text to rewrite.' };
|
||
|
||
const contextHint = [
|
||
guideTitle ? `Guide: ${guideTitle}` : '',
|
||
stepTitle ? `Step: ${stepTitle}` : '',
|
||
].filter(Boolean).join('\n');
|
||
|
||
const prompt = [
|
||
contextHint,
|
||
contextHint ? '' : null,
|
||
'Rewrite the following text to sound professional and clear as step-by-step documentation.',
|
||
'Keep it concise. Do not add extra information. Return only the rewritten text.',
|
||
'',
|
||
trimmed,
|
||
].filter((l) => l !== null).join('\n');
|
||
|
||
const result = await this.callOllamaText({
|
||
host,
|
||
model: config.ollama.model,
|
||
prompt,
|
||
systemPrompt: 'You are a documentation editor. Return only the improved text, nothing else.',
|
||
});
|
||
return { ok: true, text: result };
|
||
} catch (err) {
|
||
return { ok: false, reason: err?.message || 'Rewrite failed.' };
|
||
}
|
||
}
|
||
|
||
clickPointFromStep(step, image = null) {
|
||
const marker = (step.annotations || []).find((ann) => ann.type === 'oval' && Number.isFinite(ann.x) && Number.isFinite(ann.y) && Number.isFinite(ann.w) && Number.isFinite(ann.h));
|
||
if (!marker) return null;
|
||
const size = image ? image.getSize() : step.image?.size || { width: 0, height: 0 };
|
||
if (!size.width || !size.height) return null;
|
||
return {
|
||
x: Math.round((marker.x + marker.w / 2) * size.width),
|
||
y: Math.round((marker.y + marker.h / 2) * size.height),
|
||
};
|
||
}
|
||
}
|
||
|
||
module.exports = { TextIntelService };
|