Files
StepForge/core/archive.js
T
TylerandClaude Fable 5 8aa9756b8a
Template tests / tests (pull_request) Failing after 33s
Harden archives, snapshots, locks, and search against corruption and races
Phase 1/2 of the improvement plan (PR 6 of the sequence). Recovery and
resource-limit hardening for the storage-adjacent modules.

ZIP resource limits (core/zip.js):
- unzipSync now enforces entry-count, total compressed, total inflated, and
  per-entry inflated budgets, and caps inflation with inflateRawSync
  maxOutputLength so a deflate bomb can't exhaust memory. Exact inflated-size
  match (not "at least") and CRC verification are kept. Import uses the
  default limits.

Transactional archive import (core/archive.js):
- The import validates the guide AND every step before writing anything, then
  stages the whole guide in a temp directory and publishes it with a single
  atomic rename. A corrupt step no longer leaves a partial guide in the
  library; a failure cleans up the staging directory.

Atomic snapshot restore (core/snapshots.js):
- Restore extracts and validates into a temp directory first; only then does
  it swap content in, moving live content aside so a mid-swap failure rolls
  back. A corrupt/truncated snapshot can no longer destroy the live guide
  (the old restore deleted live content before extracting).
- Fixed snapshot filename collisions: names kept milliseconds so two backups
  in the same second no longer overwrite each other.

Automatic backups (core/snapshots.js):
- Implemented the previously-dead backups.automatic/everyNSaves/keepLast
  settings: autoSnapshotIfDue snapshots every N saves and prunes to keepLast,
  wired into the save choke point in main.js. Never throws — a backup failure
  cannot break the save that triggered it.

Exclusive locks (core/locks.js):
- acquireLock uses O_CREAT|O_EXCL (flag 'wx') so only one writer wins the
  race; the old read-then-write left a window where two writers both believed
  they held the lock. Added a per-acquisition token so release only removes
  the exact lock it took (never one a force-steal replaced). Same-process
  re-acquire still succeeds; cross-process fresh locks conflict.

Search reconciliation (core/search.js):
- New reconcile(store) rebuilds/repairs the index against the library at
  startup using per-guide fingerprints (updatedAt+revision): reindexes new/
  changed guides, drops entries for deleted ones, and exposes a recovery
  status ('ok'|'reset'|'reconciled'). A missing/corrupt/version-mismatched
  index recovers instead of silently returning nothing. Wired into startup.

Recovery surface:
- New recovery:status IPC + preload method returns quarantined files (this
  session) and the search index status so the UI can surface data issues.

Tests: ZIP bomb/limits, transactional import abort with no partial guide,
atomic snapshot restore preserving the live guide on corruption, exclusive
lock conflict/steal/release-by-token, same-process re-acquire, search
reconcile (rebuild/drop/reindex/corrupt-reset), and automatic backup
cadence/pruning. 255 unit tests pass; startup smoke and workflow E2E pass.

Co-Authored-By: Claude Fable 5 <[email protected]>
2026-07-03 23:10:21 -05:00

198 lines
7.3 KiB
JavaScript

'use strict';
const fs = require('node:fs');
const path = require('node:path');
const { zipSync, unzipSync } = require('./zip');
const { newId, nowIso, atomicWriteFileSync, writeJsonSync, deepClone } = require('./util');
const { normalizeGuide, normalizeStep, validateGuide, validateStep, SCHEMA_VERSION } = require('./schema');
const { acquireLock, releaseLock } = require('./locks');
const ARCHIVE_FORMAT = 'stepforge-guide-archive';
const APP_VERSION = require('../package.json').version;
/**
* Single-file share archive (.sfgz). Zip layout:
* manifest.json
* guide.json
* steps/<stepId>/step.json
* steps/<stepId>/<image files>
*/
function buildArchiveEntries(store, guideId) {
const guide = store.getGuide(guideId);
const steps = store.listSteps(guideId);
const entries = [];
const manifest = {
format: ARCHIVE_FORMAT,
formatVersion: 1,
schemaVersion: SCHEMA_VERSION,
appVersion: APP_VERSION,
guideId: guide.guideId,
title: guide.title,
exportedAt: nowIso(),
stepCount: guide.stepsOrder.length,
};
entries.push({ name: 'manifest.json', data: JSON.stringify(manifest, null, 2) });
const portableGuide = deepClone(guide);
portableGuide.linkedSource = null; // links are a property of the library, not the file
entries.push({ name: 'guide.json', data: JSON.stringify(portableGuide, null, 2) });
for (const stepId of guide.stepsOrder) {
const step = steps.get(stepId);
if (!step) continue;
entries.push({ name: `steps/${stepId}/step.json`, data: JSON.stringify(step, null, 2) });
const dir = store.stepDir(guideId, stepId);
for (const file of fs.readdirSync(dir)) {
if (file === 'step.json') continue;
entries.push({ name: `steps/${stepId}/${file}`, data: fs.readFileSync(path.join(dir, file)), store: /\.(png|jpg|jpeg|gif|webp)$/i.test(file) });
}
}
return entries;
}
/** Export a guide to a .sfgz file. Returns the manifest written. */
function exportGuideArchive(store, guideId, destFile) {
const entries = buildArchiveEntries(store, guideId);
atomicWriteFileSync(destFile, zipSync(entries));
return JSON.parse(entries[0].data);
}
function readArchive(file) {
const entries = unzipSync(fs.readFileSync(file));
const byName = new Map(entries.map((e) => [e.name, e.data]));
if (!byName.has('manifest.json') || !byName.has('guide.json')) {
throw new Error('not a StepForge guide archive (missing manifest)');
}
const manifest = JSON.parse(byName.get('manifest.json').toString('utf8'));
if (manifest.format !== ARCHIVE_FORMAT) throw new Error(`unsupported archive format: ${manifest.format}`);
if (manifest.schemaVersion > SCHEMA_VERSION) {
throw new Error(`archive uses newer schema (${manifest.schemaVersion}) than this app supports`);
}
const guide = normalizeGuide(JSON.parse(byName.get('guide.json').toString('utf8')));
validateGuide(guide);
return { manifest, guide, entries };
}
/**
* Import a .sfgz into the library.
* mode 'copy' — fresh ids, fully independent local guide.
* mode 'linked' — keeps archive identity; edits autosave locally and write
* back to the file only on explicit save (saveLinkedGuide).
*/
function importGuideArchive(store, file, { mode = 'copy' } = {}) {
const { guide, entries } = readArchive(file);
const idMap = new Map();
const stepFiles = new Map(); // newStepId -> [{name, data}]
const stepJsons = new Map();
for (const { name, data } of entries) {
const m = /^steps\/([^/]+)\/(.+)$/.exec(name);
if (!m) continue;
const [, oldStepId, rest] = m;
if (!idMap.has(oldStepId)) {
idMap.set(oldStepId, mode === 'copy' ? newId('step') : oldStepId);
}
const stepId = idMap.get(oldStepId);
if (rest === 'step.json') stepJsons.set(stepId, { oldStepId, raw: JSON.parse(data.toString('utf8')) });
else {
if (!stepFiles.has(stepId)) stepFiles.set(stepId, []);
if (!/^[A-Za-z0-9._-]+$/.test(rest)) continue; // only flat, safe file names
stepFiles.get(stepId).push({ name: rest, data });
}
}
const newGuide = deepClone(guide);
if (mode === 'copy') {
newGuide.guideId = newId('guide');
newGuide.linkedSource = null;
} else {
if (store.guideExists(newGuide.guideId)) {
throw new Error('this shared guide is already in the library');
}
newGuide.linkedSource = {
path: path.resolve(file),
openedAt: nowIso(),
lastSavedAt: null,
};
}
newGuide.stepsOrder = guide.stepsOrder.map((id) => idMap.get(id)).filter(Boolean);
return finalizeImport(store, newGuide, idMap, stepJsons, stepFiles);
}
function finalizeImport(store, newGuide, idMap, stepJsons, stepFiles) {
// Transactional import: validate the guide and EVERY step first, then write
// the whole guide into a temporary staging directory, and only publish it
// with a single atomic rename. Previously guide.json was written before the
// steps validated, so a bad step left a partial guide in the library.
validateGuide(newGuide);
const normalizedSteps = [];
for (const [stepId, { raw }] of stepJsons) {
const step = normalizeStep({ ...raw, stepId });
step.parentStepId = raw.parentStepId ? idMap.get(raw.parentStepId) || null : null;
validateStep(step); // throws before anything is written on a bad step
normalizedSteps.push([stepId, step]);
}
const finalDir = store.guideDir(newGuide.guideId);
if (fs.existsSync(finalDir)) throw new Error(`guide already exists: ${newGuide.guideId}`);
const stagingDir = `${finalDir}.importing-${Date.now()}`;
fs.rmSync(stagingDir, { recursive: true, force: true });
try {
fs.mkdirSync(stagingDir, { recursive: true });
writeJsonSync(path.join(stagingDir, 'guide.json'), newGuide);
for (const [stepId, step] of normalizedSteps) {
const dir = path.join(stagingDir, 'steps', stepId);
fs.mkdirSync(dir, { recursive: true });
writeJsonSync(path.join(dir, 'step.json'), step);
for (const { name, data } of stepFiles.get(stepId) || []) {
atomicWriteFileSync(path.join(dir, name), data);
}
}
// Publish atomically. If the final dir appeared meanwhile, fail cleanly.
if (fs.existsSync(finalDir)) throw new Error(`guide already exists: ${newGuide.guideId}`);
fs.renameSync(stagingDir, finalDir);
} catch (err) {
fs.rmSync(stagingDir, { recursive: true, force: true });
throw err;
}
return store.getGuide(newGuide.guideId);
}
/**
* Write a linked guide back to its shared archive (explicit Ctrl+S save).
* Takes the advisory lock for the duration of the write.
*/
function saveLinkedGuide(store, guideId, { force = false } = {}) {
const guide = store.getGuide(guideId);
if (!guide.linkedSource || !guide.linkedSource.path) {
throw new Error('guide is not linked to a shared archive');
}
const target = guide.linkedSource.path;
const result = acquireLock(target, { force });
if (!result.acquired) {
return { saved: false, conflict: result.conflict };
}
try {
exportGuideArchive(store, guideId, target);
guide.linkedSource.lastSavedAt = nowIso();
store.saveGuide(guide, { touch: false });
return { saved: true, path: target };
} finally {
// Release by our acquisition token so we never remove a lock a concurrent
// force-steal replaced with theirs.
releaseLock(target, { lock: result.lock });
}
}
module.exports = {
ARCHIVE_FORMAT,
exportGuideArchive,
readArchive,
importGuideArchive,
saveLinkedGuide,
};