Rearchitect click capture: strict click-time frames, off-main-process recorder, exact marker coordinates
Template tests / tests (push) Successful in 1m50s

Implements the architecture change from ai_prompts/prompt3.md:

- New app/click-frames.js: shared timestamped frame ring + strict
  click-to-frame pairing (never a frame whose grab started after the
  click); legacy slack behavior kept behind capture.strictClickFrames=false.
- New stream capture backend (app/stream-backend.js + hidden worker
  window): per-display desktop media streams sampled into ring buffers
  and PNG-encoded entirely off the main process, so click delivery is
  never starved by capture work. Auto-degrades to the legacy in-process
  frame loop when streams cannot start or the worker stops answering.
- Clicks are paired with their frame at event time (eager pairing in
  enqueueClickCapture); only the storing is serialized, so slow encodes
  cannot skew later clicks in a fast burst.
- Linux watcher: restored event-time root coordinates from
  xinput test-xi2 and merge raw/regular twin events structurally.
- Replaced the 40ms time debounce with source-aware duplicate
  suppression: fast legitimate clicks are never dropped.
- New app/coords.js: physical-to-DIP conversion with multi-monitor and
  scale-factor handling; Windows keeps screenToDipPoint.
- STEPFORGE_CLICK_SELFTEST end-to-end hook: 3/3 clicks become steps via
  the stream backend with 0.00% marker offset on this host.
- Tests rewritten/added: strict selection, coords, stream backend,
  Linux coordinate parsing, twin merge, burst clicking (126 passing).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Iisyourdad
2026-06-11 21:33:31 -05:00
parent c6d0e9e356
commit a0b69f8cc7
14 changed files with 2109 additions and 162 deletions
+56
View File
@@ -98,6 +98,62 @@ function createWindow() {
}
}, 1500);
}
// Dev-only self-test: exercise the full click-capture pipeline — resume
// session, wait for the frame recorder, inject OS-level clicks the way
// the watcher would, and verify one stored step per click.
if (process.env.STEPFORGE_CLICK_SELFTEST) {
setTimeout(async () => {
try {
const guide = store.createGuide({ title: 'click selftest' });
capture.startSession(guide.guideId, { intervalSec: 0 });
capture.togglePause(false);
mainWindow.hide();
// Arm the frame recorder directly: this host may lack the click
// watcher binary (xinput), which normally gates the recorder, but
// the recorder itself must still be testable end to end.
await capture.startClickFrameBackend();
// Let the stream backend (or the fallback loop) come up and buffer.
await new Promise((res) => setTimeout(res, 3000));
console.log('CLICK-SELFTEST source:', capture.state().clickFrameSource);
const clicks = [
{ x: 200, y: 150 },
{ x: 400, y: 300 },
{ x: 600, y: 450 },
];
for (const point of clicks) {
capture.onOsClick(Date.now(), point, 'button-1');
await new Promise((res) => setTimeout(res, 120)); // fast clicking
}
// Wait for the queue to drain (encodes can take seconds on WSLg).
await capture.clickQueue;
await new Promise((res) => setTimeout(res, 500));
const stepIds = store.getGuide(guide.guideId).stepsOrder;
const steps = store.listSteps(guide.guideId);
const markers = stepIds.map((id) => (steps.get(id).annotations || []).length);
console.log('CLICK-SELFTEST steps:', stepIds.length, 'of', clicks.length,
'markers:', JSON.stringify(markers));
// Marker accuracy: each oval's center (fractional) must match the
// injected click position relative to the display bounds.
const { bounds } = screen.getPrimaryDisplay();
stepIds.forEach((id, i) => {
const a = (steps.get(id).annotations || [])[0];
if (!a) return;
const center = { x: a.x + a.w / 2, y: a.y + a.h / 2 };
const expected = {
x: (clicks[i].x - bounds.x) / bounds.width,
y: (clicks[i].y - bounds.y) / bounds.height,
};
const offBy = Math.hypot(center.x - expected.x, center.y - expected.y);
console.log(`CLICK-SELFTEST marker ${i}: off by ${(offBy * 100).toFixed(2)}% of screen`);
});
capture.finishSession();
} catch (err) {
console.log('CLICK-SELFTEST ERROR', err.message);
} finally {
app.quit();
}
}, 1500);
}
// Dev-only self-test: exercise the exact hotkey-session capture path
// (hide window -> grab -> showInactive) several times, then exit.
if (process.env.STEPFORGE_CAPTURE_SELFTEST) {