Fix dropped clicks and late screenshots in fast recording
Root cause of 'I clicked many times but only got two screenshots':
finishing/pausing a session called backend.stop(), which cancelled every
in-flight frame request to null. Clicks whose PNG had not finished
encoding yet were then dropped — only the first few survived.
Fixes:
- Stream backend now *drains* on stop: it stops accepting new requests but
keeps the worker alive until frames already selected for queued clicks
finish encoding. stop({ immediate: true }) keeps the old abandon-now
behavior for an unhealthy worker.
- Two-stage worker reply: a fast 'frame-selected' ack pins the pairing and
proves liveness; the slow PNG payload follows. A slow encode (seconds on
software-rendered hosts) is no longer mistaken for a dead worker, which
had been forcing the post-click fresh-shot fallback (late screenshots).
- Queued clicks carry their guide id and are stored even if the session
ends while they wait in the queue.
- The tray gesture that stops a session is discarded by matching its
recorded screen position, not a time window — a fast workflow click near
the stop is no longer collateral damage. (Replaces the earlier grace
window, which dropped whole bursts.)
- A click on a display with no ready stream resolves null so the caller
fresh-shots the correct monitor instead of returning another screen.
- STEPFORGE_CAPTURE_LOG=1 prints one line per click decision; the
second-instance handler now surfaces the running window instead of
exiting silently.
- Self-test gains a fast-burst-then-finish scenario (8/8 saved) and the
marker/coordinate checks remain at 0.00% offset.
Tests: 133 unit + all repo checks passing.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -276,6 +276,92 @@ test('fast clicks are paired with their frames at event time, not behind the sto
|
||||
assert.equal(service.session.count, 2);
|
||||
});
|
||||
|
||||
test('clicks still queued when the session finishes are stored, not dropped', async () => {
|
||||
// Reported as "I clicked N times but only got two screenshots": with slow
|
||||
// encodes the queue lags, and finishing the session used to discard every
|
||||
// click still waiting in it.
|
||||
const service = makeService();
|
||||
service.session = { guideId: 'guide-finish', paused: false, count: 0, intervalSec: 0 };
|
||||
service.userIsInApp = () => false;
|
||||
let releaseFrame;
|
||||
const frameGate = new Promise((r) => { releaseFrame = r; });
|
||||
service.frameForClick = () => frameGate.then(() => makeFrame('late-stored-frame'));
|
||||
const added = [];
|
||||
service.store.addStep = (guideId, fields, png) => {
|
||||
added.push({ guideId, png: png.toString() });
|
||||
return { stepId: 'step-late' };
|
||||
};
|
||||
const events = [];
|
||||
service.notify = (channel, payload) => events.push({ channel, payload });
|
||||
|
||||
// Click happened comfortably before the user reached for the stop button.
|
||||
const queue = service.enqueueClickCapture({ x: 5, y: 5 }, Date.now() - 2000, 'left');
|
||||
service.finishSession();
|
||||
releaseFrame();
|
||||
await queue;
|
||||
|
||||
assert.deepEqual(added, [{ guideId: 'guide-finish', png: 'late-stored-frame' }],
|
||||
'the click was recorded while the session was live — it must become a step');
|
||||
const addedEvent = events.find((e) => e.channel === 'capture:added');
|
||||
assert.equal(addedEvent.payload.guideId, 'guide-finish');
|
||||
});
|
||||
|
||||
test('the tray click that stops the session does not become a junk step', async () => {
|
||||
// The tray gesture that stops capture is also seen by the OS hook; storing
|
||||
// it would append a step of the tray/menu to every recording. It is
|
||||
// matched by position so only that exact click is dropped.
|
||||
const service = makeService({
|
||||
screenApi: {
|
||||
getCursorScreenPoint: () => ({ x: 1900, y: 12 }), // over the tray
|
||||
getAllDisplays: () => [],
|
||||
},
|
||||
});
|
||||
service.session = { guideId: 'guide-stop', paused: false, count: 0, intervalSec: 0 };
|
||||
service.userIsInApp = () => false;
|
||||
service.frameForClick = async () => makeFrame('stop-click-frame');
|
||||
const added = [];
|
||||
service.store.addStep = (guideId, fields, png) => {
|
||||
added.push(png.toString());
|
||||
return { stepId: 'step-stop' };
|
||||
};
|
||||
|
||||
// The hook reports the tray click at the tray position.
|
||||
const queue = service.enqueueClickCapture({ x: 1900, y: 12 }, Date.now(), 'left');
|
||||
service.noteUiStopGesture(); // tray handler records where it was clicked
|
||||
service.finishSession();
|
||||
await queue;
|
||||
|
||||
assert.deepEqual(added, [], 'the stop click must be discarded');
|
||||
});
|
||||
|
||||
test('a fast workflow click near the stop time but elsewhere is NOT dropped', async () => {
|
||||
// Position matching is what makes this safe: the user clicks their
|
||||
// workflow, then reaches up to the tray. The last workflow click lands
|
||||
// far from the tray and must survive even though it is close in time.
|
||||
const service = makeService({
|
||||
screenApi: {
|
||||
getCursorScreenPoint: () => ({ x: 1900, y: 12 }), // tray location
|
||||
getAllDisplays: () => [],
|
||||
},
|
||||
});
|
||||
service.session = { guideId: 'guide-near', paused: false, count: 0, intervalSec: 0 };
|
||||
service.userIsInApp = () => false;
|
||||
service.frameForClick = async () => makeFrame('workflow-frame');
|
||||
const added = [];
|
||||
service.store.addStep = (guideId, fields, png) => {
|
||||
added.push(png.toString());
|
||||
return { stepId: 'step-near' };
|
||||
};
|
||||
|
||||
// Workflow click in the middle of the screen, then the tray stop.
|
||||
const queue = service.enqueueClickCapture({ x: 600, y: 500 }, Date.now(), 'left');
|
||||
service.noteUiStopGesture();
|
||||
service.finishSession();
|
||||
await queue;
|
||||
|
||||
assert.deepEqual(added, ['workflow-frame'], 'a click away from the tray must be kept');
|
||||
});
|
||||
|
||||
test('queued click captures preserve the original event time and button', async () => {
|
||||
const service = makeService();
|
||||
const seen = [];
|
||||
|
||||
@@ -31,7 +31,8 @@ function makeBackend({ autoReady = true, ...opts } = {}) {
|
||||
destroy() { destroyed = true; },
|
||||
};
|
||||
},
|
||||
frameTimeoutMs: 40,
|
||||
ackTimeoutMs: 40,
|
||||
encodeTimeoutMs: 120,
|
||||
startTimeoutMs: 100,
|
||||
...opts,
|
||||
});
|
||||
@@ -133,12 +134,12 @@ test('clicks on a multi-monitor setup route to the stream of the clicked display
|
||||
backend.stop();
|
||||
});
|
||||
|
||||
test('repeated frame-request timeouts mark the backend unhealthy exactly once', async () => {
|
||||
test('repeated unanswered frame requests mark the backend unhealthy exactly once', async () => {
|
||||
let unhealthy = 0;
|
||||
const { backend, isDestroyed } = makeBackend({ onUnhealthy: () => { unhealthy += 1; } });
|
||||
await backend.start({ displays: oneDisplay, sources: oneSource });
|
||||
|
||||
// Two consecutive timeouts (the worker never answers).
|
||||
// Two consecutive ack timeouts (the worker never answers at all).
|
||||
assert.equal(await backend.frameForClick({ clickAt: 1 }), null);
|
||||
assert.equal(await backend.frameForClick({ clickAt: 2 }), null);
|
||||
|
||||
@@ -147,6 +148,67 @@ test('repeated frame-request timeouts mark the backend unhealthy exactly once',
|
||||
assert.equal(isDestroyed(), true);
|
||||
});
|
||||
|
||||
test('a slow PNG encode after a prompt selection ack is not mistaken for a dead worker', async () => {
|
||||
// The ack window is 40ms here; the payload arrives at ~80ms — well past
|
||||
// the ack deadline but inside the encode deadline. The frame must land
|
||||
// and the failure counter must stay clean.
|
||||
let unhealthy = 0;
|
||||
const { backend, sent, worker } = makeBackend({ onUnhealthy: () => { unhealthy += 1; } });
|
||||
await backend.start({ displays: oneDisplay, sources: oneSource });
|
||||
|
||||
const promise = backend.frameForClick({ clickPos: { x: 10, y: 10 }, clickAt: 5000 });
|
||||
const request = sent.find((m) => m.type === 'frame-request');
|
||||
worker({ type: 'frame-selected', requestId: request.requestId, startedAt: 4900, capturedAt: 4910 });
|
||||
setTimeout(() => {
|
||||
worker({
|
||||
type: 'frame-response',
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
png: Uint8Array.from([7]),
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
startedAt: 4900,
|
||||
capturedAt: 4910,
|
||||
});
|
||||
}, 80);
|
||||
|
||||
const frame = await promise;
|
||||
|
||||
assert.ok(frame, 'the slowly-encoded frame must still be delivered');
|
||||
assert.deepEqual([...frame.png], [7]);
|
||||
assert.equal(unhealthy, 0);
|
||||
assert.equal(backend.isActive(), true);
|
||||
backend.stop();
|
||||
});
|
||||
|
||||
test('an acked request whose payload never arrives resolves null after the encode deadline', async () => {
|
||||
const { backend, sent, worker } = makeBackend();
|
||||
await backend.start({ displays: oneDisplay, sources: oneSource });
|
||||
|
||||
const promise = backend.frameForClick({ clickAt: 5000 });
|
||||
const request = sent.find((m) => m.type === 'frame-request');
|
||||
worker({ type: 'frame-selected', requestId: request.requestId });
|
||||
|
||||
assert.equal(await promise, null, 'a stuck encode must not hang the click forever');
|
||||
backend.stop();
|
||||
});
|
||||
|
||||
test('a click on a display without a ready stream is not served from another display', async () => {
|
||||
// Only display 1 has a screen source; a click on display 2 must resolve
|
||||
// null (the caller falls back to a fresh shot of the correct monitor)
|
||||
// rather than returning display 1 pixels with meaningless marker math.
|
||||
const displays = [display(1, 0, 0, 1920, 1080), display(2, 1920, 0, 1920, 1080)];
|
||||
const { backend, sent } = makeBackend();
|
||||
await backend.start({ displays, sources: [{ id: 'screen:1:0', display_id: '1' }] });
|
||||
|
||||
const frame = await backend.frameForClick({ clickPos: { x: 2500, y: 400 }, clickAt: 1 });
|
||||
|
||||
assert.equal(frame, null);
|
||||
assert.equal(sent.some((m) => m.type === 'frame-request'), false,
|
||||
'no request should even be sent for the wrong display');
|
||||
backend.stop();
|
||||
});
|
||||
|
||||
test('a late worker reply after the timeout is ignored', async () => {
|
||||
const { backend, sent, worker } = makeBackend();
|
||||
await backend.start({ displays: oneDisplay, sources: oneSource });
|
||||
@@ -159,15 +221,46 @@ test('a late worker reply after the timeout is ignored', async () => {
|
||||
backend.stop();
|
||||
});
|
||||
|
||||
test('stop() resolves all in-flight requests with null', async () => {
|
||||
const { backend } = makeBackend();
|
||||
test('stop() drains: a frame already selected at finish time still resolves', async () => {
|
||||
// This is the "I clicked many times but only got two screenshots" fix.
|
||||
// The session finishes (stop) while a click's frame is still encoding;
|
||||
// the frame must still come back, not be cancelled to null.
|
||||
const { backend, sent, worker, isDestroyed } = makeBackend();
|
||||
await backend.start({ displays: oneDisplay, sources: oneSource });
|
||||
|
||||
const pending = backend.frameForClick({ clickAt: 1 });
|
||||
backend.stop();
|
||||
const pending = backend.frameForClick({ clickPos: { x: 10, y: 10 }, clickAt: 1 });
|
||||
const request = sent.find((m) => m.type === 'frame-request');
|
||||
worker({ type: 'frame-selected', requestId: request.requestId, startedAt: 0, capturedAt: 0 });
|
||||
|
||||
backend.stop(); // user finishes the session while the encode is in flight
|
||||
assert.equal(backend.isActive(), false);
|
||||
assert.equal(isDestroyed(), false, 'the worker stays alive to finish encoding');
|
||||
|
||||
worker({
|
||||
type: 'frame-response',
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
png: Uint8Array.from([5]),
|
||||
width: 1,
|
||||
height: 1,
|
||||
});
|
||||
|
||||
const frame = await pending;
|
||||
assert.ok(frame, 'the in-flight frame must survive the stop');
|
||||
assert.deepEqual([...frame.png], [5]);
|
||||
assert.equal(isDestroyed(), true, 'the worker tears down once draining completes');
|
||||
});
|
||||
|
||||
test('stop({ immediate: true }) abandons in-flight requests at once', async () => {
|
||||
const { backend, isDestroyed } = makeBackend();
|
||||
await backend.start({ displays: oneDisplay, sources: oneSource });
|
||||
|
||||
const pending = backend.frameForClick({ clickPos: { x: 10, y: 10 }, clickAt: 1 });
|
||||
backend.stop({ immediate: true });
|
||||
|
||||
assert.equal(await pending, null);
|
||||
assert.equal(backend.isActive(), false);
|
||||
assert.equal(isDestroyed(), true);
|
||||
});
|
||||
|
||||
test('displays pair to screen sources by display_id; single display pairs to a lone source', () => {
|
||||
|
||||
Reference in New Issue
Block a user