Rearchitect click capture: strict click-time frames, off-main-process recorder, exact marker coordinates
Template tests / tests (push) Successful in 1m50s
Template tests / tests (push) Successful in 1m50s
Implements the architecture change from ai_prompts/prompt3.md: - New app/click-frames.js: shared timestamped frame ring + strict click-to-frame pairing (never a frame whose grab started after the click); legacy slack behavior kept behind capture.strictClickFrames=false. - New stream capture backend (app/stream-backend.js + hidden worker window): per-display desktop media streams sampled into ring buffers and PNG-encoded entirely off the main process, so click delivery is never starved by capture work. Auto-degrades to the legacy in-process frame loop when streams cannot start or the worker stops answering. - Clicks are paired with their frame at event time (eager pairing in enqueueClickCapture); only the storing is serialized, so slow encodes cannot skew later clicks in a fast burst. - Linux watcher: restored event-time root coordinates from xinput test-xi2 and merge raw/regular twin events structurally. - Replaced the 40ms time debounce with source-aware duplicate suppression: fast legitimate clicks are never dropped. - New app/coords.js: physical-to-DIP conversion with multi-monitor and scale-factor handling; Windows keeps screenToDipPoint. - STEPFORGE_CLICK_SELFTEST end-to-end hook: 3/3 clicks become steps via the stream backend with 0.00% marker offset on this host. - Tests rewritten/added: strict selection, coords, stream backend, Linux coordinate parsing, twin merge, burst clicking (126 passing). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+56
@@ -98,6 +98,62 @@ function createWindow() {
|
||||
}
|
||||
}, 1500);
|
||||
}
|
||||
// Dev-only self-test: exercise the full click-capture pipeline — resume
|
||||
// session, wait for the frame recorder, inject OS-level clicks the way
|
||||
// the watcher would, and verify one stored step per click.
|
||||
if (process.env.STEPFORGE_CLICK_SELFTEST) {
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
const guide = store.createGuide({ title: 'click selftest' });
|
||||
capture.startSession(guide.guideId, { intervalSec: 0 });
|
||||
capture.togglePause(false);
|
||||
mainWindow.hide();
|
||||
// Arm the frame recorder directly: this host may lack the click
|
||||
// watcher binary (xinput), which normally gates the recorder, but
|
||||
// the recorder itself must still be testable end to end.
|
||||
await capture.startClickFrameBackend();
|
||||
// Let the stream backend (or the fallback loop) come up and buffer.
|
||||
await new Promise((res) => setTimeout(res, 3000));
|
||||
console.log('CLICK-SELFTEST source:', capture.state().clickFrameSource);
|
||||
const clicks = [
|
||||
{ x: 200, y: 150 },
|
||||
{ x: 400, y: 300 },
|
||||
{ x: 600, y: 450 },
|
||||
];
|
||||
for (const point of clicks) {
|
||||
capture.onOsClick(Date.now(), point, 'button-1');
|
||||
await new Promise((res) => setTimeout(res, 120)); // fast clicking
|
||||
}
|
||||
// Wait for the queue to drain (encodes can take seconds on WSLg).
|
||||
await capture.clickQueue;
|
||||
await new Promise((res) => setTimeout(res, 500));
|
||||
const stepIds = store.getGuide(guide.guideId).stepsOrder;
|
||||
const steps = store.listSteps(guide.guideId);
|
||||
const markers = stepIds.map((id) => (steps.get(id).annotations || []).length);
|
||||
console.log('CLICK-SELFTEST steps:', stepIds.length, 'of', clicks.length,
|
||||
'markers:', JSON.stringify(markers));
|
||||
// Marker accuracy: each oval's center (fractional) must match the
|
||||
// injected click position relative to the display bounds.
|
||||
const { bounds } = screen.getPrimaryDisplay();
|
||||
stepIds.forEach((id, i) => {
|
||||
const a = (steps.get(id).annotations || [])[0];
|
||||
if (!a) return;
|
||||
const center = { x: a.x + a.w / 2, y: a.y + a.h / 2 };
|
||||
const expected = {
|
||||
x: (clicks[i].x - bounds.x) / bounds.width,
|
||||
y: (clicks[i].y - bounds.y) / bounds.height,
|
||||
};
|
||||
const offBy = Math.hypot(center.x - expected.x, center.y - expected.y);
|
||||
console.log(`CLICK-SELFTEST marker ${i}: off by ${(offBy * 100).toFixed(2)}% of screen`);
|
||||
});
|
||||
capture.finishSession();
|
||||
} catch (err) {
|
||||
console.log('CLICK-SELFTEST ERROR', err.message);
|
||||
} finally {
|
||||
app.quit();
|
||||
}
|
||||
}, 1500);
|
||||
}
|
||||
// Dev-only self-test: exercise the exact hotkey-session capture path
|
||||
// (hide window -> grab -> showInactive) several times, then exit.
|
||||
if (process.env.STEPFORGE_CAPTURE_SELFTEST) {
|
||||
|
||||
Reference in New Issue
Block a user