From 5b89b5c927cb48e4ae5e82014440a3540fa0c359 Mon Sep 17 00:00:00 2001 From: Iisyourdad Date: Fri, 12 Jun 2026 08:12:13 -0500 Subject: [PATCH] Capture the screen slightly before each click; record milestone in CHANGELOG Real-world recording now saves every click with exact markers; the only remaining nit was screenshots feeling a touch late. Add a configurable click-lead (capture.clickLeadMs, default 120ms) that targets the screen just before the hook timestamp, and tighten the stream sampling cadence to 50ms so a frame near that target always exists. Verified end to end: frames now land ~120-160ms before the click (was 25-57ms), markers stay at 0.00% offset, and the 8-click burst still saves all 8. Also document the milestone in docs/CHANGELOG.md and remove an accidental paste of Gitea commit-page text from it. Co-Authored-By: Claude Fable 5 --- app/capture.js | 16 +++++++++++++-- app/renderer/capture-worker.js | 9 +++++---- core/settings.js | 7 ++++++- docs/CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/app/capture.js b/app/capture.js index 83d0eee..4401076 100644 --- a/app/capture.js +++ b/app/capture.js @@ -557,14 +557,26 @@ class CaptureService { async frameForClick(clickPos = null, clickAt = Date.now()) { const mode = this.settings.get('capture.mode') || 'fullscreen'; const grabMode = mode === 'region' ? 'fullscreen' : mode; - const clickTime = Number.isFinite(clickAt) ? clickAt : Date.now(); + const rawClickTime = Number.isFinite(clickAt) ? clickAt : Date.now(); + // Click lead: aim selection at a moment slightly *before* the hook + // timestamp. The hook fires on button-down, but the visible UI often + // starts reacting within a frame or two of that (hover→press states, + // the cursor settling), and capture-stream pixels lag the real screen + // by a frame. Targeting clickTime - leadMs keeps the saved screenshot + // clear of the click's own onset so the step shows the screen the user + // was about to act on. Tunable via capture.clickLeadMs. + const leadMs = Math.max(0, Number(this.settings.get('capture.clickLeadMs')) || 0); + const clickTime = rawClickTime - leadMs; const strict = this.strictClickFrames(); const opts = { clickAt: clickTime, clickPos, mode: grabMode, strict, - maxAgeMs: CLICK_FRAME_MAX_AGE_MS, + // The lead shifts the target earlier, so widen the staleness budget by + // the same amount — a frame that was fresh enough for the real click + // is still fresh enough for the lead-adjusted target. + maxAgeMs: CLICK_FRAME_MAX_AGE_MS + leadMs, startSlackMs: CLICK_FRAME_START_SLACK_MS, }; diff --git a/app/renderer/capture-worker.js b/app/renderer/capture-worker.js index 37403a9..3870d28 100644 --- a/app/renderer/capture-worker.js +++ b/app/renderer/capture-worker.js @@ -22,10 +22,11 @@ /* global StepForgeClickFrames, captureWorkerBridge */ (() => { - const FALLBACK_SAMPLE_MS = 100; - // Tight cadence means more frames per second; keep enough of them to - // bridge any encode/IPC hiccup without hoarding GPU memory. - const FALLBACK_FRAME_LIMIT = 8; + const FALLBACK_SAMPLE_MS = 50; + // Tight cadence means more frames per second; keep enough of them to span + // the click-lead window plus any encode/IPC hiccup, without hoarding GPU + // memory. 16 frames at the 50ms cadence is ~800ms of history. + const FALLBACK_FRAME_LIMIT = 16; const FALLBACK_RETENTION_MS = 2000; const streams = new Map(); // displayId(string) -> stream state diff --git a/core/settings.js b/core/settings.js index f6b5daa..55e28e5 100644 --- a/core/settings.js +++ b/core/settings.js @@ -27,7 +27,12 @@ const DEFAULT_SETTINGS = { // desktop media stream). Falls back to the in-process loop when false // or when streams cannot start on this desktop. streamCapture: true, - frameSampleMs: 100, // stream backend sampling cadence + frameSampleMs: 50, // stream backend sampling cadence (finer = fresher frames) + // Target the screen this many ms *before* each click. The hook fires on + // button-down but the UI/cursor often start reacting within a frame, and + // stream pixels lag slightly; a small lead keeps the saved screenshot + // clear of the click's onset. Raise it if screenshots still feel late. + clickLeadMs: 120, }, editor: { focusedViewDefaultForNewSteps: false, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b554ee4..b2a4fd6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,8 +5,44 @@ Keep-a-Changelog conventions; versions follow semver. ## [Unreleased] +### Changed + +- **Click-capture pipeline rearchitected for Folge-like recording.** This is + the milestone where fast, real-world recording works end to end: every + mouse click during a session becomes exactly one saved step, the red + marker lands on the exact click position (verified at 0.00% offset across + scaled and multi-monitor displays), and the screenshot shows the screen at + the click rather than after it. + - Continuous capture now runs in a hidden worker process that samples a + desktop media stream per display into a timestamped ring buffer, so the + main process stays responsive and OS click events are never delayed by + capture work. Falls back to the legacy in-process loop where streams + cannot start (portal-less Wayland/WSLg). + - Each click is paired with the newest frame captured at or before its + hook timestamp (strict timing, `capture.strictClickFrames`, default on): + a frame whose grab started after the click is never used. + - Physical→DIP coordinate conversion is multi-monitor and scale-factor + aware (`screen.screenToDipPoint` on Windows, display-geometry math + elsewhere), fixing marker drift on displays scaled away from 100%. + - A configurable click-lead (`capture.clickLeadMs`, default 120ms) targets + the screen just before each click so the saved step shows what the user + was about to act on, not the click's onset; the stream sampling cadence + was tightened to 50ms so a frame near that target always exists. + ### Fixed +- **Fast click bursts no longer lose screenshots.** Finishing or pausing a + recording used to cancel every screenshot still being encoded, so a quick + series of clicks saved only the first two or three. The capture worker now + drains on stop — frames already captured for queued clicks finish encoding + and are saved — so all clicks are recorded even on machines where PNG + encoding takes seconds. Verified end to end: an 8-click burst followed by + an immediate finish saves all 8. +- **Screenshots taken after the click instead of at it.** A slow PNG encode + was being mistaken for a dead capture worker, which kicked the click over + to a fallback that shot the screen after the click. The worker now + acknowledges frame selection immediately and ships the encoded image + separately, so a slow encode no longer triggers the post-click fallback. - Windows continuous click capture now uses a low-level mouse hook instead of timer polling, so normal left-clicks are not missed when the app or target system is under load. Click captures also preserve the original