From 13fd9bbda7c52617b08ce05591ead1fc8f80da04 Mon Sep 17 00:00:00 2001 From: xarmian Date: Mon, 11 May 2026 19:30:07 -0400 Subject: [PATCH] feat(web): IndexedDB persistence for localIndex (TASK-1356) (#503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(web): IndexedDB persistence for localIndex (TASK-1356) Adds `web/src/lib/stores/localIndexPersistence.ts` and wires it into the existing localIndex store. Cold loads still hit /items-index; warm loads paint from IDB before any network IO. - New `idb` (8.0.3) dependency — small wrapper around IndexedDB. - Per-workspace database `pad-local-index-{wsSlug}` with two object stores (`items` keyed by id, `meta` keyed by 'key' for cursor + schemaVersion). - `LOCAL_INDEX_SCHEMA_VERSION = 1` — bump it on incompatible changes to `ItemIndexRow` or the IDB layout; mismatches drop the store and force a full /items-index resync. Same pattern as the Yjs schemaVersion in `web/src/lib/collab/schemaVersion.ts`. - `bootstrap` now hydrates from IDB FIRST (paints from cache, flips state to 'ready'), then reconciles via /items-changes?since=cursor in the background. Cold cache falls through to /items-index and persists the result. - Every mutation path (`upsert`, `applyDelta`, `remove`, `reset`) writes through to IDB. Persistence failures degrade silently to in-memory only — the read path is never blocked. - SSR-safe (every IDB call gated on typeof indexedDB !== 'undefined'). - Best-effort: Safari private mode / quota / eviction all surface as "empty cache, re-bootstrap from network". Phase 2 acceptance: warm paint of a populated workspace should now appear before any /api/v1 request completes. Parent: PLAN-1343. See DOC-1342 design decision #4. * fix(web): localIndex reconcile loop + atomic delta persist (Codex round 1) - [P1] 403 from the warm-load reconcile is no longer swallowed. When /items-changes returns `forbidden`, drop the cache and re-throw so the registered access-revoked handler (TASK-1360) sees it. Other network blips remain non-fatal — cache stands and the next reconnect retries. - [P2] /items-changes is paged at DefaultItemChangesLimit (5000) per response. The previous one-shot reconcile would only catch up by a single page on a long-offline cache, and bootstrapState would pin at 'ready' forever with no later trigger to fetch the rest. Loop until the cursor stops advancing; defensively cap at 50 iterations. - [P2] New persistDelta() writes rows + meta cursor in a SINGLE IDB transaction. The previous separate persistUpserts + persistCursor could persist the cursor without the rows that produced it (tx interrupt, eviction), leaving the next warm hydrate with a cursor that skipped rows. applyDelta + the cold-path bootstrap snapshot now both use persistDelta. The standalone persistCursor helper is no longer used by localIndex but remains for callers that explicitly only need a cursor write. Parent: PLAN-1343. * fix(web): cold-path snapshot persists post-merge rows (Codex round 2) [P1] When the cold-path `/items-index` request is in flight, an SSE or `applyDelta` write can overlap and stamp a newer row into the in-RAM index. `mergeRow` correctly skips the stale response row for that id, but the previous IDB write used `resp.items.map(toSkinny)` — the unfiltered server response — so the cache got the stale row under the newer cursor. On the next warm boot, /items-changes?since would skip that row forever. Persist the POST-merge in-memory state (state.items.values()) so the on-disk rows match the in-RAM rows that won the seq guard, and the cursor stays consistent with them. Uses the same atomic persistDelta path applyDelta does. Parent: PLAN-1343. * fix(web): generation guard on bootstrap + user-scoped IDB cache (round 3) - [P1] WorkspaceState now carries a `generation` counter. Each `reset(ws)` bumps it on the prior state object before dropping the workspace from the map. Any in-flight bootstrap captures the generation at start and re-checks after every await — if the generation has advanced, the bootstrap silently bails out before reapplying rows or writing the snapshot to IDB. Without this, a sign-out / 403 purge during a slow /items-index could let the completed snapshot resurrect just-purged rows. - [P1] IDB databases are now keyed by (userId, workspaceSlug) instead of workspaceSlug alone. The cache is "what THIS user could see last sync" — if a different user signs into the same browser, their bootstrap opens a fresh per-user namespace and the previous user's rows never surface. Anonymous callers (pre-auth) use the `anon` namespace. `localIndex.bootstrap` takes an `{ userId }` opt the caller passes in (the workspace state captures it on first bootstrap and threads it through every persistence call). Parent: PLAN-1343. * fix(web): durable applyDelta + required userId opt (Codex round 4) - applyDelta now ALWAYS includes the existing in-RAM row in the persistDelta batch when it wins the seq guard. The previous version advanced the IDB cursor past those rows on the assumption their upsert()-fired persistUpserts had already landed — but that's a fire-and-forget background write that can lose the race or get aborted. Result: a row in RAM with seq S, no copy in IDB, and a persisted cursor of N > S — warm boot would skip it forever. One extra IDB put per redundant row trades cheaply against a missing- row class of bug. - localIndex.bootstrap's `opts.userId` is now REQUIRED (not optional with `null` default). Authenticated callers that forget to pass it would have silently landed their cache in the shared `anon` namespace — a later account on the same browser could then read the previous account's rows. TypeScript now enforces an explicit choice; pre-auth callers pass null deliberately. Parent: PLAN-1343. * fix(web): user-mismatch reset + transient resync retry (Codex round 5) - [P1] bootstrap() now resets the workspace state BEFORE the early-return for 'ready' / pending-promise when the caller's opts.userId doesn't match the cached state.userId. Without this, a user switch in the same tab could inherit the previous user's in-memory map and in-flight promise. - [P2] WorkspaceState.pendingResync tracks transient delta-sync failures on the warm path. When /items-changes fails (non-403) after warm cache hydrate, bootstrapState stays 'ready' so the UI keeps working off the cache, but pendingResync stays true and the next bootstrap() call retries the reconcile instead of no-opping. Cold path always finishes with pendingResync=false. - Documented the permission-revocation-without-row-change limitation inline: the cache can't see grants removed without a mutation, per DOC-1342 design decision #3 — that's the 403-on-click purge flow (TASK-1360), not this layer's job. Parent: PLAN-1343. * fix(web): only clear pendingResync when reconcile catches up (round 6) [P2] The /items-changes reconcile loop has a 50-page safety cap to prevent pathological tight loops. The previous code unconditionally cleared `pendingResync` after the loop exited, even on cap-hit, so a cache that's 50+ pages behind (250k+ rows) would record itself as fresh and skip retries on future bootstraps. Now `pendingResync` is only cleared when the loop exited because the server returned no new rows AND no cursor advance — the genuine "caught up" signal. Cap-hit leaves `pendingResync = true` so the next bootstrap call resumes. The permission-revocation-without-row-change concern Codex re-raised is the explicit DOC-1342 design decision #3 (best-effort cache; 403 purge handles stale-by-permission). The server emits grant-revocation tombstones through /items-changes per internal/store/grants.go, so the cache reconciles to-server-truth at the next reconnect. Anything that slips past that is the 403-on-click purge path (TASK-1360). The limitation is now explicitly noted inline. Parent: PLAN-1343. * fix(web): reentry order + identity-checked inflight cleanup (round 7) - [P2] Capture `reentry` BEFORE flipping bootstrapState to 'loading'. The previous version set state='loading' first, then checked `state.bootstrapState === 'ready'` to decide if we're in a pendingResync retry — that check always read false, so retries re-read IDB instead of just rerunning the reconcile. With fire-and-forget IDB writes, re-reading rows whose RAM copy was just removed but whose IDB delete hadn't landed would resurrect them. Reentry now also skips the 'loading' flip so the UI never blanks during a retry. - [P2] Inflight cleanup is now identity-checked. A `reset()` during an in-flight bootstrap can let a fresh bootstrap call re-occupy the inflight slot before the stale promise's `finally` runs; deleting unconditionally would remove the new entry and let a duplicate bootstrap start. We hold the promise in `slot.p` (a shared object so the closure can see assignment without TDZ issues), and only clear `inflight.delete(ws)` if `slot.p` is still the registered promise. Parent: PLAN-1343. * fix(web): 401 + empty-cache warm-load (Codex round 8) - [P1] 401 (unauthorized) from /items-changes reconcile is now treated like 403 (forbidden): drop the cache, mark state=error, re-throw. The api.items.changes path throws PadApiError with code='unauthorized' on 401 (after the redirect-to-login is fired), so the cache shouldn't keep showing private rows while the redirect is in flight. Other network failures remain transient. - [P2] A populated IDB cache is now defined as "has rows OR cursor > 0", not "has rows". An empty workspace (or a guest with item-level grants but no items granted yet) legitimately has zero rows but a real meta cursor from the prior sync. The previous check forced those workspaces through the cold /items-index path on every page load, defeating the warm-load fast path. Parent: PLAN-1343. --- web/package-lock.json | 7 + web/package.json | 1 + web/src/lib/stores/localIndex.svelte.ts | 386 +++++++++++++++++--- web/src/lib/stores/localIndexPersistence.ts | 325 ++++++++++++++++ 4 files changed, 674 insertions(+), 45 deletions(-) create mode 100644 web/src/lib/stores/localIndexPersistence.ts diff --git a/web/package-lock.json b/web/package-lock.json index 30f6a037..842cbb1d 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -25,6 +25,7 @@ "@tiptap/y-tiptap": "^3.0.3", "diff": "^9.0.0", "dompurify": "^3.4.2", + "idb": "^8.0.3", "lowlight": "^3.3.0", "mermaid": "^11.14.0", "qrcode": "^1.5.4", @@ -2437,6 +2438,12 @@ "node": ">=0.10.0" } }, + "node_modules/idb": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", + "integrity": "sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==", + "license": "ISC" + }, "node_modules/import-meta-resolve": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", diff --git a/web/package.json b/web/package.json index e5885687..2d3a237a 100644 --- a/web/package.json +++ b/web/package.json @@ -48,6 +48,7 @@ "@tiptap/y-tiptap": "^3.0.3", "diff": "^9.0.0", "dompurify": "^3.4.2", + "idb": "^8.0.3", "lowlight": "^3.3.0", "mermaid": "^11.14.0", "qrcode": "^1.5.4", diff --git a/web/src/lib/stores/localIndex.svelte.ts b/web/src/lib/stores/localIndex.svelte.ts index 9e4d62de..ebab0029 100644 --- a/web/src/lib/stores/localIndex.svelte.ts +++ b/web/src/lib/stores/localIndex.svelte.ts @@ -40,11 +40,31 @@ // `api.items.create` / `update`) cannot accidentally leak the rich // body into the local index. // -// All operations except `bootstrap` are synchronous — readers don't -// `await`, they just read. +// All read operations are synchronous — consumers don't `await`, +// they just read. `bootstrap` is async because it may hit IDB and +// the network; mutation methods (`upsert`, `applyDelta`, `remove`) +// are synchronous from the caller's perspective and write through +// to IDB in the background (fire-and-forget, never throwing — see +// `localIndexPersistence`). +// +// IDB persistence (TASK-1356): on bootstrap, hydrate from IDB FIRST +// for an immediate paint, then call `/items-changes?since=` +// to reconcile. On cache miss (IDB empty / unavailable), fall through +// to the cold-path `/items-index` fetch. Every mutation writes through +// to IDB so a reload picks up the latest state without a network +// round-trip. Storage failures are silently swallowed — the store +// keeps working in-memory. import { SvelteMap } from 'svelte/reactivity'; import { api } from '$lib/api/client'; +import { PadApiError } from '$lib/api/client'; +import { + hydrate as persistHydrate, + persistDelta, + persistRemovals, + persistUpserts, + wipe as persistWipe, +} from './localIndexPersistence'; import type { Item, ItemChangeRow, ItemIndexRow } from '$lib/types'; export type BootstrapState = 'cold' | 'loading' | 'ready' | 'error'; @@ -60,6 +80,29 @@ class WorkspaceState { items: SvelteMap = new SvelteMap(); cursor = $state('0'); bootstrapState = $state('cold'); + + // `userId` is captured on first bootstrap and used to scope the + // IDB database name. Null = anonymous (pre-auth bootstrap). A + // later bootstrap call with a different userId triggers a reset + // (see `bootstrap`) so we never mix caches across users. + userId: string | null = null; + + // `generation` is bumped on every `reset()`. Bootstrap captures + // the value at start; if it advances during an await, the + // in-flight bootstrap bails out instead of writing/reapplying + // rows that belong to a stale identity. Without this, a sign-out + // or 403 purge during a slow /items-index request would let the + // completed snapshot resurrect just-purged rows (Codex P1 + // round 3). + generation = 0; + + // `pendingResync` is true when the warm-cache path hydrated rows + // from IDB but the follow-up /items-changes reconcile didn't + // complete (transient network blip). The cache is usable — + // `bootstrapState` is `'ready'` so the UI renders — but a later + // `bootstrap()` call will retry the delta sync instead of + // no-opping. Cleared on successful sync. Codex P2 round 5. + pendingResync = $state(false); } // Outer map: reactive (SvelteMap) so consumers re-render when a fresh @@ -106,65 +149,274 @@ function cursorAsNum(c: string): number { return Number.isFinite(n) ? n : 0; } +/** + * Apply a single row to a workspace's items map with the per-row + * seq guard. Used by `bootstrap` (both warm and cold paths) and + * `upsert`/`applyDelta` indirectly via the existing inline logic. + * Returns true if the row was written, false if it was skipped as + * stale. + */ +function mergeRow(state: WorkspaceState, row: ItemIndexRow | Item): boolean { + const next = toSkinny(row); + const existing = state.items.get(next.id); + if ( + existing?.seq !== undefined && + next.seq !== undefined && + next.seq <= existing.seq + ) { + return false; + } + state.items.set(next.id, next); + return true; +} + export const localIndex = { /** - * Hydrate a workspace from `/items-index`. Idempotent: returns the - * same in-flight promise if already loading; resolves immediately - * if already `ready`. On error the state flips to `'error'` and - * the caller can retry by calling `bootstrap` again — the next - * call sees `bootstrapState === 'error'` and proceeds. Archived - * items are included in the snapshot (the store is the canonical - * read model for both live and archived rows; consumers filter - * via `{ includeArchived }`). + * Hydrate a workspace. Idempotent: returns the same in-flight + * promise if already loading; resolves immediately if already + * `'ready'`. On error the state flips to `'error'` and the caller + * can retry by calling `bootstrap` again. Archived items are + * included (the store is the canonical read model for both live + * and archived rows; consumers filter via `{ includeArchived }`). * - * Merge, don't clear. An optimistic `upsert()` or an SSE-driven - * write can land while the /items-index request is in flight; if - * we cleared and replaced, we'd silently regress those rows to - * the older snapshot value (Codex P2 round 3). Instead, we - * MERGE each response row through the same per-row seq guard - * `upsert` uses, and the cursor only advances forward. The - * cleared-state semantic isn't needed in practice because - * `reset()` is the explicit "drop everything" entry point. + * Two-stage flow (TASK-1356): + * + * 1. WARM PATH — hydrate from IDB. If the cache is populated, + * copy rows into the in-RAM store, set the cursor from the + * meta row, and flip `bootstrapState` to `'ready'` *before* + * any network IO. The UI paints instantly. Then kick off + * `/items-changes?since=` in the background and + * apply the deltas via `applyDelta` (which write-throughs to + * IDB on its own). A failed delta-sync doesn't move state + * back to `'loading'` — the UI keeps working off the cached + * data and the next reconnect retries. + * + * 2. COLD PATH — IDB miss / unavailable. Fall through to the + * classic `/items-index` snapshot, then write the result to + * IDB so the next visit is warm. + * + * Merge-not-clear semantics are preserved: in either path, rows + * are MERGED through the same per-row seq guard `upsert` uses, + * and the cursor only advances forward. An optimistic `upsert()` + * or SSE write that landed while bootstrap was in flight is + * never regressed. */ - async bootstrap(ws: string): Promise { + async bootstrap( + ws: string, + opts: { userId: string | null }, + ): Promise { + // User-mismatch reset BEFORE the early-return checks. Otherwise + // a different user signing into the same browser would inherit + // the previous user's `ready` state and in-flight promise + // (Codex P1 round 5). `ensureState` auto-creates a fresh + // WorkspaceState after the reset. + const prior = workspaces.get(ws); + if (prior && prior.bootstrapState !== 'cold' && prior.userId !== opts.userId) { + localIndex.reset(ws); + } + const state = ensureState(ws); - if (state.bootstrapState === 'ready') return; + // Early return for already-ready states ONLY when there's no + // outstanding resync work. A transient delta-sync failure + // (network blip) leaves `pendingResync = true`; the next + // bootstrap call must retry the reconcile, not no-op. Codex P2 + // round 5. + if (state.bootstrapState === 'ready' && !state.pendingResync) return; const pending = inflight.get(ws); if (pending) return pending; - state.bootstrapState = 'loading'; - const p = (async () => { + // `userId` is REQUIRED (not defaulted) so authenticated callers + // can't silently land their cache in the shared `anon` + // namespace. Pass null explicitly for pre-auth / public-share + // flows. Codex P? (round 4) caught the leak risk. + state.userId = opts.userId; + // Capture reentry BEFORE flipping bootstrapState to 'loading' + // — otherwise the `state.bootstrapState === 'ready'` check + // below always reads false and the warm IDB hydrate runs again + // on a pendingResync retry. That's bad because IDB writes are + // fire-and-forget; re-reading rows whose RAM copy was just + // removed but whose IDB delete hasn't landed would resurrect + // them. Codex P2 round 7. + const reentry = + state.bootstrapState === 'ready' && state.pendingResync; + // Only flip to 'loading' for first-time bootstrap. A reentry + // keeps state='ready' throughout so the UI never blanks while + // retrying the reconcile. + if (!reentry) state.bootstrapState = 'loading'; + // Capture the generation at start. If `reset()` runs during + // any await below, generation bumps; we then bail out before + // re-applying rows or writing the snapshot back, otherwise + // purged data could resurrect (Codex P1 round 3). + const bootstrapGen = state.generation; + const userId = state.userId; + const isStale = () => state.generation !== bootstrapGen; + + // `slot.p` holds the in-flight promise so the IIFE body's + // `finally` can do an identity check against it — see Codex + // round 7 P2. We need a level of indirection (the object) + // because a bare `const p = ...` puts `p` in the TDZ when the + // suspended async body resumes and references it. + const slot: { p: Promise | null } = { p: null }; + slot.p = (async () => { try { - const resp = await api.items.listIndex(ws, { includeArchived: true }); - for (const row of resp.items) { - const next = toSkinny(row); - const existing = state.items.get(row.id); - if ( - existing?.seq !== undefined && - next.seq !== undefined && - next.seq <= existing.seq - ) { - continue; + // Stage 1: warm path. Always try IDB first. Skip + // re-hydration when this is a `pendingResync` retry — + // the in-RAM state is already authoritative for this + // session; we just need to redo the reconcile. + const cached = reentry + ? { items: [], cursor: state.cursor } + : await persistHydrate(userId, ws); + if (isStale()) return; + // A populated cache is one we've successfully synced + // from before — either there are rows, or the cursor + // has moved off the "0" floor (empty workspaces / + // guests with item-level grants legitimately have + // zero rows but a real cursor). Both deserve the + // warm-path fast boot. Codex P2 round 8. + const cacheIsPopulated = + cached.items.length > 0 || cursorAsNum(cached.cursor) > 0; + const hasCache = reentry || cacheIsPopulated; + if (!reentry && cacheIsPopulated) { + for (const row of cached.items) { + mergeRow(state, row); } - state.items.set(row.id, next); + if (cursorAsNum(cached.cursor) > cursorAsNum(state.cursor)) { + state.cursor = cached.cursor; + } + // Flip to ready immediately — the UI paints from + // the cache while delta-sync runs. `pendingResync` + // stays true until the reconcile finishes. + state.bootstrapState = 'ready'; + state.pendingResync = true; } - // Cursor moves forward only. A fresh /items-index can - // occasionally return a cursor below the in-RAM one - // (e.g. an SSE delta advanced it during the request); - // don't backslide. - if (cursorAsNum(resp.cursor) > cursorAsNum(state.cursor)) { - state.cursor = resp.cursor; + + if (hasCache) { + // Reconcile cache against server via /items-changes. + // The endpoint is capped at DefaultItemChangesLimit + // (5000) per response, so we loop until the cursor + // stops advancing — otherwise a cache that's + // behind by more than one page would only catch + // up by 5000 rows and then `bootstrapState` would + // pin at `ready` forever, with no later trigger + // to fetch the rest (Codex P2 round 1). + // + // Auth/authz failures (403) must NOT be swallowed + // — the cache is stale-by-permission and showing + // it as live is a real correctness bug. Re-throw + // 403 so the registered access-revoked handler + // (TASK-1360) sees it; the cache reset is its job. + // Other network blips are non-fatal — the cache + // stands and the next reconnect retries. + try { + // Cap iterations defensively — a healthy server + // drains in < 10 pages even for huge gaps; if + // something pathological loops without cursor + // advance, give up after 50 and let the user's + // next visit retry. + let caughtUp = false; + for (let i = 0; i < 50; i++) { + const since = state.cursor; + const delta = await api.items.changes(ws, since); + if (isStale()) return; + if (delta.changes.length === 0 || delta.cursor === since) { + // Server returned no new rows AND no + // cursor advance — we're caught up. + caughtUp = true; + break; + } + localIndex.applyDelta(ws, delta.changes, delta.cursor); + if (delta.cursor === since) { + caughtUp = true; + break; + } + } + // Only mark fresh if the loop reached the + // server cursor. Hitting the 50-page cap + // without catching up leaves `pendingResync` + // true so a later bootstrap call resumes + // (Codex P2 round 6). 50 × 5000 = 250,000 + // rows; we don't expect to hit this in practice + // but it's the difference between "stale + // forever" and "next visit retries". + if (caughtUp) state.pendingResync = false; + } catch (err) { + if (isStale()) return; + // 401 (unauthorized — session expired) and 403 + // (forbidden — access revoked) both mean the + // cached rows are no longer ours to display. + // Drop the cache and re-throw so the caller's + // redirect / purge handler can react. Other + // errors stay transient — cache stands and the + // next bootstrap() call retries the reconcile + // because `pendingResync` is still true. + if ( + err instanceof PadApiError && + (err.code === 'forbidden' || err.code === 'unauthorized') + ) { + state.bootstrapState = 'error'; + state.pendingResync = false; + state.items.clear(); + state.cursor = '0'; + persistWipe(userId, ws).catch(() => undefined); + throw err; + } + // Transient network failure. Cache stands and + // state stays 'ready' so the UI keeps working. + // `pendingResync` remains true so the next + // bootstrap() call retries (Codex P2 round 5). + // Permission revocation that doesn't change + // row data is NOT covered here — TASK-1360 and + // DOC-1342 decision #3 explicitly punt that in + // favor of the 403-on-click purge path. + } + } else { + // Stage 2: cold path. /items-index full snapshot. + const resp = await api.items.listIndex(ws, { + includeArchived: true, + }); + if (isStale()) return; + for (const row of resp.items) { + mergeRow(state, row); + } + if (cursorAsNum(resp.cursor) > cursorAsNum(state.cursor)) { + state.cursor = resp.cursor; + } + state.bootstrapState = 'ready'; + // Cold path is a full snapshot — nothing pending. + state.pendingResync = false; + // Best-effort persist the cold snapshot to IDB so + // the next visit is warm. We persist the POST-MERGE + // in-memory rows (not raw `resp.items`), and use + // the same atomic rows+cursor write applyDelta does. + // Otherwise an SSE/applyDelta that landed during the + // in-flight /items-index request could overwrite a + // newer row in IDB while the cursor on disk pointed + // past the gap, leaving the cache permanently stale + // (Codex P1 round 2). Iterating state.items.values() + // yields exactly the merged, winning rows. + const snapshot: ItemIndexRow[] = []; + for (const row of state.items.values()) snapshot.push(row); + persistDelta(userId, ws, snapshot, state.cursor).catch( + () => undefined, + ); } - state.bootstrapState = 'ready'; } catch (err) { + if (isStale()) return; state.bootstrapState = 'error'; throw err; } finally { - inflight.delete(ws); + // Identity-checked cleanup: only clear inflight if + // THIS promise is the registered one. After a + // `reset()` mid-bootstrap, a fresh bootstrap call can + // re-occupy the slot before this stale promise's + // `finally` runs; deleting unconditionally would + // remove the new entry and let a duplicate bootstrap + // start (Codex P2 round 7). + if (slot.p && inflight.get(ws) === slot.p) inflight.delete(ws); } })(); - inflight.set(ws, p); - return p; + inflight.set(ws, slot.p); + return slot.p; }, /** @@ -253,6 +505,7 @@ export const localIndex = { // Guard 1: whole-batch drop on non-advancing cursor. if (newCursorNum <= startCursorNum) return; + const toPersist: ItemIndexRow[] = []; for (const change of changes) { if (change.seq !== undefined) { // Guard 2: row's seq vs. cursor floor. @@ -263,6 +516,14 @@ export const localIndex = { existing?.seq !== undefined && change.seq <= existing.seq ) { + // Existing wins in RAM. Include it in the + // persist set so the IDB cursor we're about + // to advance doesn't lap a row that may not + // be durable yet (upsert's fire-and-forget + // IDB write could still be pending / failed + // — Codex P? round 4). One redundant put is + // cheaper than a missing row on warm boot. + toPersist.push(existing); continue; } } @@ -274,10 +535,22 @@ export const localIndex = { // only manages the seq-ordered identity of the row. Hard // deletes (workspace GC / 403 purge) go through the // `remove()` method, not through this batch path. - const { deleted: _d, ...row } = change; - state.items.set(change.id, toSkinny(row as ItemIndexRow)); + const { deleted: _d, ...rest } = change; + const skinny = toSkinny(rest as ItemIndexRow); + state.items.set(change.id, skinny); + toPersist.push(skinny); } state.cursor = newCursor; + // Write-through to IDB. ATOMIC: rows + cursor land in a + // single transaction so the persisted cursor can never + // advance past rows that didn't make it to disk (Codex P2 + // round 1). Fire-and-forget; storage failures degrade to + // in-memory only and never break the read path. Routed + // through the workspace's captured `userId` so a different + // user signing into the same browser sees their own cache. + persistDelta(state.userId, ws, toPersist, newCursor).catch( + () => undefined, + ); }, /** @@ -307,6 +580,9 @@ export const localIndex = { return; } state.items.set(row.id, next); + // Write-through to IDB. Fire-and-forget; storage failures + // degrade silently. + persistUpserts(state.userId, ws, [next]).catch(() => undefined); }, /** @@ -318,6 +594,8 @@ export const localIndex = { const state = workspaces.get(ws); if (!state) return; state.items.delete(id); + // Write-through hard-delete to IDB. + persistRemovals(state.userId, ws, [id]).catch(() => undefined); }, /** Current cursor for a workspace, or "0" if unhydrated. */ @@ -342,8 +620,26 @@ export const localIndex = { * user's cache. After reset, `bootstrap(ws)` from cold. */ reset(ws: string): void { + const prior = workspaces.get(ws); + // Bump generation on the prior state object BEFORE deleting + // it from the map. Any in-flight bootstrap promise still holds + // a reference to `prior` — checking `state.generation !== + // bootstrapGen` after each await lets it bail out instead of + // writing or re-applying rows that belong to a stale identity + // (Codex P1 round 3). Without this, a sign-out / 403 purge + // during a slow /items-index request could resurrect just- + // purged rows when the snapshot resolved. + const priorUserId = prior?.userId ?? null; + if (prior) prior.generation += 1; + workspaces.delete(ws); inflight.delete(ws); + + // Drop the persisted cache for the workspace's last-known + // userId. If a different user later bootstraps the same + // workspace, their cache is in a different IDB namespace and + // remains untouched, by design. + persistWipe(priorUserId, ws).catch(() => undefined); }, /** Number of items currently held for a workspace. Test/debug aid. */ diff --git a/web/src/lib/stores/localIndexPersistence.ts b/web/src/lib/stores/localIndexPersistence.ts new file mode 100644 index 00000000..9817960f --- /dev/null +++ b/web/src/lib/stores/localIndexPersistence.ts @@ -0,0 +1,325 @@ +// localIndexPersistence — IndexedDB write-behind layer for localIndex +// (PLAN-1343 / TASK-1356). Per DOC-1342 design decision #4: the +// in-RAM Svelte store is canonical; IDB is a hydration source on +// cold/warm boot and a write-behind cache for every mutation. The +// reader path is unaffected — consumers always go through localIndex, +// never through this module directly. +// +// Database shape: one IDB database per (user, workspace) pair, named +// `pad-local-index-{userId}-{wsSlug}`. Scoping by user is required — +// the cache is what the user could see last sync, and if a different +// user signs into the same browser, exposing the previous user's +// view would be a real correctness/permission leak. Anonymous / +// bootstrap-time callers (with no user id yet) use the `anon` +// namespace; those caches are independent of any signed-in user. +// +// Two object stores: +// +// items (keyPath: 'id') — ItemIndexRow rows, keyed by item.id +// meta (keyPath: 'key') — { key: 'sync', cursor, schemaVersion } +// +// SCHEMA_VERSION is the local equivalent of the Yjs schemaVersion in +// `web/src/lib/collab/schemaVersion.ts`. Bump it whenever the +// `ItemIndexRow` wire shape or store layout changes incompatibly — +// hydrators will see the mismatch on open and drop the persisted +// data, forcing a full /items-index resync. The server is the +// source of truth, so dropping the cache is always safe. +// +// All public functions never throw. Storage failures (Safari private +// mode, browser eviction, quota exceeded) degrade silently to +// in-memory only operation; the next bootstrap hits /items-index +// the normal way and the warm-load fast path simply skips. +// +// SSR-safe: every IDB call is gated on `typeof indexedDB !== 'undefined'` +// so SvelteKit's prerender / SSR phase doesn't blow up. + +import { openDB, type IDBPDatabase } from 'idb'; +import type { ItemIndexRow } from '$lib/types'; + +/** + * SCHEMA_VERSION is the cache-shape contract. Bump it whenever the + * `ItemIndexRow` skinny projection or this module's IDB layout changes + * incompatibly — old clients reopening on a new build see the + * mismatch and wipe their store, then re-bootstrap from + * `/items-index`. Server truth (items.content) is never persisted + * here, so a cache wipe loses nothing. + */ +export const LOCAL_INDEX_SCHEMA_VERSION = 1; + +/** Result of a `hydrate()` call. Empty payload when there's no cache yet. */ +export interface HydrateResult { + items: ItemIndexRow[]; + cursor: string; +} + +/** Shape of the single row stored in the `meta` store. */ +interface MetaRow { + key: 'sync'; + cursor: string; + schemaVersion: number; +} + +// Open IDB connections are cached per (user, workspace) pair. The +// map key matches `dbName()` so cache slots can't collide across +// user namespaces. +const dbs = new Map(); + +function isSupported(): boolean { + return typeof indexedDB !== 'undefined'; +} + +/** + * Encode a value so it's safe to embed in an IDB database name. + * Names allow any UTF-16 string per spec, but URL-encoding keeps + * the on-disk handle predictable in dev tools and avoids surprises + * from exotic IDs. + */ +function safe(s: string): string { + return encodeURIComponent(s); +} + +function dbName(userId: string | null, ws: string): string { + const ns = userId ? safe(userId) : 'anon'; + return `pad-local-index-${ns}-${safe(ws)}`; +} + +function key(userId: string | null, ws: string): string { + return dbName(userId, ws); +} + +/** + * Open the workspace's IDB database for the given user, creating + * object stores on first run. Cached so subsequent calls reuse the + * connection. Returns null on any storage failure — callers must + * treat that as "no cache available, fall back to network". + */ +async function open( + userId: string | null, + ws: string, +): Promise { + if (!isSupported()) return null; + const k = key(userId, ws); + const cached = dbs.get(k); + if (cached) return cached; + + try { + // The version arg to openDB is the IDB-format version (used + // for migrations). We pin it to 1 and use our own + // `schemaVersion` row in `meta` for content-shape versioning. + // That keeps schema bumps decoupled from idb library quirks. + const db = await openDB(dbName(userId, ws), 1, { + upgrade(db) { + if (!db.objectStoreNames.contains('items')) { + db.createObjectStore('items', { keyPath: 'id' }); + } + if (!db.objectStoreNames.contains('meta')) { + db.createObjectStore('meta', { keyPath: 'key' }); + } + }, + }); + dbs.set(k, db); + return db; + } catch { + return null; + } +} + +/** + * Read everything from IDB for a workspace. Returns the persisted + * items + cursor, OR an empty result when: + * - IDB isn't supported (SSR, ancient browser), + * - the persisted schemaVersion doesn't match LOCAL_INDEX_SCHEMA_VERSION + * (in which case the store is also wiped as a side effect so the + * next persist write starts fresh), + * - opening or reading fails (storage error, quota issue). + * + * Never throws. The caller does the cold-path /items-index fetch when + * `items` is empty. + */ +export async function hydrate( + userId: string | null, + ws: string, +): Promise { + const empty: HydrateResult = { items: [], cursor: '0' }; + if (!isSupported()) return empty; + + const db = await open(userId, ws); + if (!db) return empty; + + try { + const tx = db.transaction(['items', 'meta'], 'readonly'); + const meta = (await tx.objectStore('meta').get('sync')) as + | MetaRow + | undefined; + + // Schema-version mismatch is the "your local cache is from a + // previous incompatible build" case. Drop everything and + // signal an empty cache. The next bootstrap fully resyncs. + if (meta && meta.schemaVersion !== LOCAL_INDEX_SCHEMA_VERSION) { + await tx.done.catch(() => undefined); + await wipe(userId, ws); + return empty; + } + + const items = (await tx.objectStore('items').getAll()) as ItemIndexRow[]; + await tx.done.catch(() => undefined); + return { + items: items ?? [], + cursor: meta?.cursor ?? '0', + }; + } catch { + return empty; + } +} + +/** + * Replace the cursor + schemaVersion meta row. Standalone variant + * used by the cold-path bootstrap when no row writes are happening + * alongside (the snapshot is persisted via `persistUpserts` and the + * cursor lands after). For per-delta writes prefer + * `persistDelta` which advances rows + cursor atomically. + */ +export async function persistCursor( + userId: string | null, + ws: string, + cursor: string, +): Promise { + if (!isSupported()) return; + const db = await open(userId, ws); + if (!db) return; + try { + await db.put('meta', { + key: 'sync', + cursor, + schemaVersion: LOCAL_INDEX_SCHEMA_VERSION, + } satisfies MetaRow); + } catch { + /* swallow — best-effort cache */ + } +} + +/** + * Upsert a batch of rows in a single transaction. Used by + * `applyDelta`/`bootstrap` write-through. Batching matters when an + * SSE flurry arrives — a single tx is much cheaper than N + * one-shot puts. + */ +export async function persistUpserts( + userId: string | null, + ws: string, + rows: ItemIndexRow[], +): Promise { + if (!isSupported() || rows.length === 0) return; + const db = await open(userId, ws); + if (!db) return; + try { + const tx = db.transaction('items', 'readwrite'); + const store = tx.objectStore('items'); + // Fire-and-forget per-row put; the .done promise waits for them all. + for (const row of rows) { + store.put(row).catch(() => undefined); + } + await tx.done; + } catch { + /* swallow — best-effort cache */ + } +} + +/** + * Atomically advance a delta — write upserted rows AND the new + * cursor in a single IDB transaction. If the tx fails or is aborted + * (browser eviction, quota, tab freeze), nothing is written so the + * persisted cursor never overshoots the persisted rows. The next + * warm hydrate sees a consistent floor and `/items-changes?since=` + * can pick up from there without skipping rows. Codex P2 (round 1) + * caught the divergence risk of separate row/cursor writes. + * + * Soft deletes flow through `rows` (as upserts with `deleted_at` + * populated); hard removals still go through `persistRemovals`. + */ +export async function persistDelta( + userId: string | null, + ws: string, + rows: ItemIndexRow[], + cursor: string, +): Promise { + if (!isSupported()) return; + const db = await open(userId, ws); + if (!db) return; + try { + const tx = db.transaction(['items', 'meta'], 'readwrite'); + const itemsStore = tx.objectStore('items'); + for (const row of rows) { + itemsStore.put(row).catch(() => undefined); + } + tx.objectStore('meta') + .put({ + key: 'sync', + cursor, + schemaVersion: LOCAL_INDEX_SCHEMA_VERSION, + } satisfies MetaRow) + .catch(() => undefined); + await tx.done; + } catch { + /* swallow — best-effort cache */ + } +} + +/** + * Delete rows by id (hard remove). Used by `localIndex.remove` for + * 403 purge (TASK-1360) and any other hard-delete path. Soft deletes + * stay in the cache as upserts with `deleted_at` populated — they + * flow through `persistUpserts`. + */ +export async function persistRemovals( + userId: string | null, + ws: string, + ids: string[], +): Promise { + if (!isSupported() || ids.length === 0) return; + const db = await open(userId, ws); + if (!db) return; + try { + const tx = db.transaction('items', 'readwrite'); + const store = tx.objectStore('items'); + for (const id of ids) { + store.delete(id).catch(() => undefined); + } + await tx.done; + } catch { + /* swallow — best-effort cache */ + } +} + +/** + * Drop the IDB database entirely. Used by `localIndex.reset` (403 + * full-workspace purge / sign-out) and internally by `hydrate` when + * the schemaVersion doesn't match. Closes the cached connection + * first so the delete request isn't blocked by a still-open handle. + */ +export async function wipe( + userId: string | null, + ws: string, +): Promise { + if (!isSupported()) return; + const k = key(userId, ws); + const existing = dbs.get(k); + if (existing) { + try { + existing.close(); + } catch { + /* swallow */ + } + dbs.delete(k); + } + try { + await new Promise((resolve) => { + const req = indexedDB.deleteDatabase(dbName(userId, ws)); + req.onsuccess = () => resolve(); + req.onerror = () => resolve(); + req.onblocked = () => resolve(); + }); + } catch { + /* swallow */ + } +}