From cb99879d53fb91870e1103070d3f069dc8bad4f9 Mon Sep 17 00:00:00 2001 From: xarmian Date: Wed, 30 Sep 2026 11:03:07 -0400 Subject: [PATCH] feat(materialize): recover unsaved editor sessions into items.content (TASK-2198 U4) (#1683) * feat(materialize): recover unsaved editor sessions into items.content in the background (TASK-2198 U4) Wires the U3 supervisor in. A collab room reclaimed at the end of its grace TTL with content-bearing rows above the flush watermark (T3), or a dormant pending item found by a one-minute sweep (T1), is queued; one worker goroutine materializes the item's whole content-bearing op-log with the whole-workspace link index and lands it through store.MaterializeFlush, under the collab item lock with no room open. The write is a flush: cursor == MAX(op-log id), rows pending, no set-aside rows, checked in the write's own transaction; watermark to the cursor only; seq bumped; one version row attributed to system/recovery; the op-log is never pruned. PAD_MATERIALIZE=off disables it entirely. A per-item in-memory failure budget (3 failures, doubling backoff, reset by a new op-log row) keeps untrusted input from being retried in a loop. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * test(store): recovery watermark is the cursor, not MAX at UPDATE time (TASK-2198 U4) Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * test(store): a second recovery still writes its own version row (TASK-2198 U4) Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * docs(mcp): op-log recovery bounds the stale-body window; tool surface v0.60 (TASK-2198 U4) content_state applied_pending_flush now clears on its own after the item's room closes, unless recovery fails or PAD_MATERIALIZE=off. Every sentence that called the window unbounded, or said nothing server-side moves op-log content into items.content, is corrected (CLAUDE.md, instructions.md, README, docs/deployment.md, docs/backup.md, code comments). BEHAVIOR bump to 0.60; goldens regenerated. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * fix(materialize): count ErrNoMemoryCap silently against the per-item budget (TASK-2198 U4) U3 now refuses every job with ErrNoMemoryCap when no memory cap can be established, and warns once itself. The recovery worker counts it like any failure but logs it at debug, not per item. Its log lines go through a logger seam so the test can tell. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * style: gofmt (TASK-2198 U4) Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * feat(materialize): one log line per budget exhaustion; count set-aside skips on the sweep line (TASK-2198 U4) An item that uses up its failure budget logs exactly one WARN naming the item, the failure count and the last error kind (DEBUG when the kind is no_memory_cap, which the supervisor already warned about once), and logs again only after op-log growth resets the budget and it is exhausted anew. The schema-version skip goes through the same line. Items skipped for set-aside rows are counted and reported on a new single sweep summary line, written only when it has news; never per item. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * fix(materialize): keyset-page the recovery sweep so no candidate starves (TASK-2198 U4, codex r1) The sweep read the oldest 200 candidates from the start every tick, and the room check and the failure budget filtered them afterwards. Once 200 sat in backoff or exhausted, every sweep returned the same 200 and the 201st was never queued: filtering above a bound is starvation. Each tick now reads one page strictly after an in-memory keyset cursor (newest op-log created_at, item id) and wraps to the start after a short page, so every candidate is offered within ceil(N/limit)+1 ticks, whatever sorts ahead of it. Keyset over NOT IN: no parameter-count limit, and it also covers the room check, which an exclusion list would not. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * test(materialize): the sweep cursor wraps; docs for the exhaustion line, the set-aside count and paging (TASK-2198 U4) Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * docs(deployment): reflow the recovery budget paragraph (TASK-2198 U4) Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * test(materialize): recovery strips the follows-title marker as a tab's save does (TASK-2198 U4, BUG-3315) Since #1688 a tab loads a link that follows its target's title with the follows-title marker in the link's title attribute, so the marker lives in Y.Docs that recovery replays. Three fixture cases (a follows-title link; the same link after a rename; a URL edited to an external address) are generated from documents that really hold the marker, and the existing 8 are unchanged. A new leg recovers each through the production path (whole-workspace index) and through the runner with a restricted and an empty index, and asserts no 'pad-follows-title' reaches the body. It goes red in all 7 ways when the bundle's marker strips are disabled. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC * test(store): a recovery version row carries #1691's history data (TASK-2198 U4) Rebased onto PLAN-2348 U2 (#1691), which adds user_id, line counts and a create marker to version rows. Recovery writes its row through the same UpdateItem insert, so it gets them automatically: no user_id (a system row), not a create, and line counts equal to diff.LineCounts of the stale and recovered bodies. The leg pins that. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC --- CLAUDE.md | 10 +- README.md | 4 +- cmd/pad/cmd_db.go | 24 +- cmd/pad/cmd_db_unflushed_test.go | 4 +- cmd/pad/cmd_item.go | 13 +- cmd/pad/cmd_materialize.go | 19 + cmd/pad/cmd_materialize_test.go | 51 ++ cmd/pad/cmd_server.go | 9 + .../testdata/mcp_wire/remote_initialize.json | 4 +- .../testdata/mcp_wire/stdio_initialize.json | 4 +- docs/backup.md | 14 +- docs/deployment.md | 53 +- internal/collab/manager.go | 68 +- internal/materialize/supervisor_config.go | 24 + internal/mcp/instructions.md | 4 +- internal/mcp/version.go | 16 +- internal/models/item.go | 25 +- internal/server/handlers_items.go | 5 + .../handlers_items_open_children_guard.go | 6 +- internal/server/materialize_recovery.go | 659 +++++++++++++++ .../server/materialize_recovery_links_test.go | 283 +++++++ internal/server/materialize_recovery_test.go | 778 ++++++++++++++++++ internal/server/server.go | 12 + .../server/testdata/materialize_links.json | 90 ++ internal/store/item_content_state_test.go | 6 +- internal/store/items.go | 13 +- internal/store/materialize_flush.go | 348 ++++++++ .../store/materialize_flush_internal_test.go | 64 ++ internal/store/materialize_flush_test.go | 346 ++++++++ .../materializerLinkFixture.test.ts | 91 ++ .../timeline/TimelineVersionCard.svelte | 18 +- .../TimelineVersionCard.svelte.test.ts | 27 + 32 files changed, 3036 insertions(+), 56 deletions(-) create mode 100644 internal/server/materialize_recovery.go create mode 100644 internal/server/materialize_recovery_links_test.go create mode 100644 internal/server/materialize_recovery_test.go create mode 100644 internal/server/testdata/materialize_links.json create mode 100644 internal/store/materialize_flush.go create mode 100644 internal/store/materialize_flush_internal_test.go create mode 100644 internal/store/materialize_flush_test.go create mode 100644 web/src/lib/collab/materializer/materializerLinkFixture.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 2e0a2d47b..55a5d9c4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -85,8 +85,8 @@ REST API at `/api/v1/`. Key endpoints: - **Field filters on the list:** any query parameter that is not a known list parameter filters on that field. A value containing a comma is an OR over its trimmed pieces (`?status=open,done`, which is what `pad item list --status open,done` sends). **Documented limit (BUG-3167):** over the query string, a literal value that itself contains a comma cannot be filtered for exactly, because the comma is the delimiter. A repeated-parameter form was declined, because an older server reads only the first value and silently narrows the result. The census when this was written found 0 comma-containing option values (2,025 select/multi_select options across 27 workspaces). In-process callers are unaffected: `store.ListItems` matches `ItemListParams.Fields` EXACTLY and takes an OR only through `FieldsAnyOf`, so the uniqueness check, playbook routing, and schema-driven completed-work and bootstrap queries match a comma value as itself - `GET/PATCH/DELETE /workspaces/{ws}/items/{slug}` — item by slug - **Write responses (create/update) may carry `warnings.undeclared_fields`** (BUG-2850) — field keys stored in the item's `fields` blob that the collection's schema does not declare. They are ACCEPTED, not refused: a census found 168 live values under 14 such keys, and refusing them would break read-modify-write on items nobody edited wrongly. The element is additive and `omitempty`, so a clean write is byte-identical to before; system-written metadata (`implementation_notes`, `decision_log`, `github_pr`, `convention`) is excluded. The CLI prints the same list to **stderr**, never stdout, so `--format json` stays parseable - - **Write responses may also carry `warnings.content_outcome`** (BUG-2995) — on a 200 it takes exactly one value, `applied_pending_flush`: the content went to the live collaborative document (a browser tab has the item open) rather than to `items.content`. It describes the WRITE, not the row's state at response time — a concurrent flush may already have updated the row, and the server does not re-read to find out. The response's `content` is then the markdown as **sent**, not as stored, and a read before a later `?source=collab-snapshot` flush lands answers with the PREVIOUS content — the lag, not a failed write. The stored form may also differ from the sent form: the markdown round-trips through the editor, so bullet markers, emphasis characters, list numbering and blank-line runs normalise, while already-canonical markdown survives unchanged (measured — `web/src/lib/collab/bug2995Roundtrip.svelte.test.ts`). No duration is claimed anywhere, and the flush is NOT guaranteed to happen at all (BUG-3000). The CLI prints this to **stderr**, like the other write warnings, and since TASK-3132 the LAST line of `pad item update`'s stdout also names it (`— body sent to the open editor … a following "pad item show" will return the OLD body`), so it survives `| tail -1` and a discarded stderr; that last line reads `body replaced, A → B bytes` / `body cleared` on every other content write - - **READ responses may carry `content_state`** (BUG-3000 / BUG-3033) — the read-side counterpart to `warnings.content_outcome` above, and its pending value is the same, `applied_pending_flush`. It has a second value no write produces, `superseded_set_aside` (BUG-3244): edits a collab schema-version rebuild set aside, which no tab can restore; see the set-aside bullet below. It wins when both apply, and a door that only marks staleness treats the two alike (`models.IsContentStateStale`, web `isBodyStale`) while any copy that names a remedy branches on it. A write warning says what a REQUEST did; this says what the ROW is: the item's op-log holds CONTENT-BEARING updates above `items.content_flushed_op_log_id`, so the body being served is BEHIND the live collaborative document. Since BUG-3124 rows that provably cannot change the document — y-protocols SyncStep1 frames, empty updates, and byte-identical re-sends of an earlier row (a SyncStep2 and an update differing only in their subtype byte count as identical since BUG-3136, which is what an open tab's answer to a joining tab is) — are persisted but marked `content_bearing = false` and not counted; they used to keep an item "pending" forever after its tab closed (measured: 470 pending items → 189 on the live DB). The classifier is envelope-only (no Yjs parse) and strict: a frame that does not parse exactly stays content-bearing. Additive and `omitempty`, and emitted only where a real body is — a summary or metadata projection that serves no content never claims its absent body is stale. The window is **unbounded**: nothing server-side moves that content into `items.content` (the op-log GC's flush-coverage guard deliberately RETAINS unflushed rows rather than flushing them), so the row catches up only when a browser tab next opens the item, which may never happen. Opening it is now enough even when nothing needs flushing (BUG-3124 unit B): a caught-up tab whose document still renders to the stored body (byte for byte, or, since BUG-3197, as the editor's own serialization of it) calls `POST /workspaces/{ws}/items/{slug}/collab-watermark {op_log_cursor, content_sha256}`, and the server advances the flush watermark iff `cursor == MAX(op-log id)` and the sha256 matches `items.content`, atomically, writing no content, version, `seq` or `updated_at` — before this, a view that changed nothing was deduped and left the item "pending" forever. Web-client only; no CLI/MCP surface. The content is not lost — the op-log is the durability layer — but the marker is a signal to RE-READ LATER, never to re-send. The predicate is broader than "an API write is pending flush" and deliberately so: a user typing in an open tab produces the same stale body, and the row does not record WHY the op-log is ahead. Carried by every door that serialises `models.Item`, and by the hand-built ones too — the playbook `run` response, bootstrap convention and bodies-mode include payloads, the exported artifact's `provenance` block, and a line above the body in the MCP item resource. The CLI prints it to **stderr** (`pad item show --format markdown`, `pad playbook show`, `pad playbook run`), like the write warnings — except `pad item show`'s default TABLE output, which since TASK-3132 opens with it on **stdout**; markdown stays stderr because its stdout is the body verbatim for `update --stdin` + - **Write responses may also carry `warnings.content_outcome`** (BUG-2995) — on a 200 it takes exactly one value, `applied_pending_flush`: the content went to the live collaborative document (a browser tab has the item open) rather than to `items.content`. It describes the WRITE, not the row's state at response time — a concurrent flush may already have updated the row, and the server does not re-read to find out. The response's `content` is then the markdown as **sent**, not as stored, and a read before a later `?source=collab-snapshot` flush lands answers with the PREVIOUS content — the lag, not a failed write. The stored form may also differ from the sent form: the markdown round-trips through the editor, so bullet markers, emphasis characters, list numbering and blank-line runs normalise, while already-canonical markdown survives unchanged (measured — `web/src/lib/collab/bug2995Roundtrip.svelte.test.ts`). No duration is claimed anywhere, and the flush is NOT guaranteed to happen at all (BUG-3000); since TASK-2198 U4 the server recovers the body itself after the tab's room closes, which bounds the window when recovery is on and the job succeeds, and nothing more. The CLI prints this to **stderr**, like the other write warnings, and since TASK-3132 the LAST line of `pad item update`'s stdout also names it (`— body sent to the open editor … a following "pad item show" will return the OLD body`), so it survives `| tail -1` and a discarded stderr; that last line reads `body replaced, A → B bytes` / `body cleared` on every other content write + - **READ responses may carry `content_state`** (BUG-3000 / BUG-3033) — the read-side counterpart to `warnings.content_outcome` above, and its pending value is the same, `applied_pending_flush`. It has a second value no write produces, `superseded_set_aside` (BUG-3244): edits a collab schema-version rebuild set aside, which no tab can restore; see the set-aside bullet below. It wins when both apply, and a door that only marks staleness treats the two alike (`models.IsContentStateStale`, web `isBodyStale`) while any copy that names a remedy branches on it. A write warning says what a REQUEST did; this says what the ROW is: the item's op-log holds CONTENT-BEARING updates above `items.content_flushed_op_log_id`, so the body being served is BEHIND the live collaborative document. Since BUG-3124 rows that provably cannot change the document — y-protocols SyncStep1 frames, empty updates, and byte-identical re-sends of an earlier row (a SyncStep2 and an update differing only in their subtype byte count as identical since BUG-3136, which is what an open tab's answer to a joining tab is) — are persisted but marked `content_bearing = false` and not counted; they used to keep an item "pending" forever after its tab closed (measured: 470 pending items → 189 on the live DB). The classifier is envelope-only (no Yjs parse) and strict: a frame that does not parse exactly stays content-bearing. Additive and `omitempty`, and emitted only where a real body is — a summary or metadata projection that serves no content never claims its absent body is stale. The window is **bounded by server-side recovery** (TASK-2198 U4), not by a tab: when the item's collab room closes (60s after its last tab leaves) with content-bearing rows above the watermark, or when a one-minute sweep finds such an item dormant for 2 minutes with no room, a background worker materializes the WHOLE content-bearing op-log in a capped child process (`pad __materialize-worker`, internal/materialize) and `store.MaterializeFlush` lands it: cursor == MAX(op-log id), rows pending and no set-aside rows, checked in the write's own transaction under the collab item lock with no room open; watermark to the cursor only, `seq` bumped, one version row attributed `system` / `recovery` ("recovered from an unsaved editor session"), op-log NOT pruned. It never runs on a request path, and it is NOT a guarantee: a job that fails 3 times in a row is not retried until the op-log changes, an item with set-aside rows or rows from another schema version is skipped, and with `PAD_MATERIALIZE=off` the window is unbounded as before (the op-log GC's flush-coverage guard still RETAINS unflushed rows), so the row catches up only when a browser tab next opens the item. Opening it is now enough even when nothing needs flushing (BUG-3124 unit B): a caught-up tab whose document still renders to the stored body (byte for byte, or, since BUG-3197, as the editor's own serialization of it) calls `POST /workspaces/{ws}/items/{slug}/collab-watermark {op_log_cursor, content_sha256}`, and the server advances the flush watermark iff `cursor == MAX(op-log id)` and the sha256 matches `items.content`, atomically, writing no content, version, `seq` or `updated_at` — before this, a view that changed nothing was deduped and left the item "pending" forever. Web-client only; no CLI/MCP surface. The content is not lost — the op-log is the durability layer — but the marker is a signal to RE-READ LATER, never to re-send. The predicate is broader than "an API write is pending flush" and deliberately so: a user typing in an open tab produces the same stale body, and the row does not record WHY the op-log is ahead. Carried by every door that serialises `models.Item`, and by the hand-built ones too — the playbook `run` response, bootstrap convention and bodies-mode include payloads, the exported artifact's `provenance` block, and a line above the body in the MCP item resource. The CLI prints it to **stderr** (`pad item show --format markdown`, `pad playbook show`, `pad playbook run`), like the write warnings — except `pad item show`'s default TABLE output, which since TASK-3132 opens with it on **stdout**; markdown stays stderr because its stdout is the body verbatim for `update --stdin` - **`github_pr` is written only through typed update members** (BUG-2696): `github_pr: {number, url, title?, state?, branch?, repo?, updated_at?}` (number > 0 and an absolute http(s) url required, else 400) and `clear_github_pr: true`, mutually exclusive with each other and with a full `fields` write. A `fields_patch` entry naming it is refused like the other three reserved keys. The point is VALIDITY, not sender identity: any caller may send the member, but only a well-formed PR object gets in. `pad github link` / `unlink` / `project reconcile` use it and verify from the response that it landed; a server older than the CLI ignores the member and answers 200, which the CLI reports as an error naming the likely skew. Not on the MCP catalog - **Reserved metadata keys are refused in create's `fields`, and a full `fields` update may only CARRY them** (BUG-3163): `implementation_notes`, `decision_log`, `github_pr` and `convention` in an item CREATE's `fields` answer 400 `validation_error` and nothing is created. Convention metadata goes in through a typed create member, `convention: {category?, trigger?, surfaces?, enforcement?, commands?}` (an empty object is refused), which library activation uses on every surface (CLI, remote MCP, web library and Conventions pages); the CLI verifies from the stored fields that it landed, because an older server ignores the member. A FULL `fields` blob on update may carry a stored reserved value unchanged (compared as decoded JSON, not bytes), but a differing value or an OMITTED stored key (which would delete it) is refused with 400, checked against the row under the write lock. Use `fields_patch` for ordinary writes; it leaves unnamed keys untouched - **`refuse_undeclared_fields: true` is a strict opt-in on item update** (BUG-3156): a write that would store a field key the collection does not declare is REFUSED with 400 `validation_error`, and nothing is written, instead of being accepted and named in `warnings.undeclared_fields`. It refuses EXACTLY the keys that warning would have named: on a `fields_patch` write the patched keys only; on a full `fields` write the whole blob, stray stored keys included. It is additive and omitted by default, so a write without it is unchanged. **Skew:** a server that predates it ignores the member, which is the accept-and-warn, so a strict caller confirms by the ABSENCE of the warning rather than trusting a 200. `pad item bulk-update` and the remote bulk-update dispatcher set it; `pad item update` / `pad_item.update` do not @@ -323,7 +323,7 @@ Collection names accept singular forms: `task`→`tasks`, `idea`→`ideas`, `doc ## MCP server -Pad runs as a local Model Context Protocol server so Claude Desktop / Cursor / Windsurf can call non-interactive `pad` commands as tools. The same catalog is also served as remote MCP over HTTP at `/mcp` (Streamable HTTP): always on Pad Cloud, and on a self-hosted install off by default until an admin turns it on (`GET/PUT /api/v1/admin/mcp` above; OAuth plus PATs when the issuer URL is https, PATs only on http; operator docs in `docs/deployment.md`). The tool surface is a **hand-curated catalog** (currently v0.59) in `internal/mcp/catalog_*.go` — one ToolDef per resource (`pad_item`, `pad_workspace`, `pad_collection`, `pad_project`, `pad_role`, `pad_search`, `pad_meta`, `pad_playbook`, `pad_library`, `pad_attachment`) with an `action` enum dispatching to underlying CLI commands. v0.33 (PLAN-2857 U4 / TASK-2999) adds a **`multi_relation`** field type — an ORDERED LIST of references, each element resolving through the same UUID→ref→exact-title ladder a scalar `relation` uses, declared through the DSL as `owners:multi_relation:people` (third part = target collection; a bare `owners:multi_relation` is refused at parse time rather than minting a field no write can satisfy). The TYPE is additive — every rule is reachable only through a field whose schema says `multi_relation`, which no existing schema can contain — so the bump is owed by the READ shape: `relation_targets[key]` now carries EITHER the scalar object v0.31 emitted OR a JSON **ARRAY** of those objects, in stored order and one per stored element (an `id`-only entry for an element that names nothing, so a position lines up with the same position in `fields`). A scalar entry is byte-identical to v0.31's — pinned against a literal, not against another call into the same code — so a consumer with no such fields sees nothing new, while one that acquires such a field must handle the array. ONE map, not a parallel `multi_relation_targets` key: the field's type already says which shape to expect. Value rules, all NEW rather than inherited because scalar `relation` had nothing coherent to inherit (see BUG-3028 — a required scalar relation WAS satisfied by `""`, with three stored spellings of "no target", until v0.40 converged it on this same key-absent rule): exactly ONE stored form for none, the key ABSENT, with `[]` normalising to it at the write door; an empty or whitespace-only ELEMENT REFUSED rather than skipped, naming its index; `required` meaning >=1 RESOLVED element; order part of the value; duplicates REFUSED with a new `duplicate_referent` reason, detected AFTER resolution because two elements naming one item are usually different strings. Any failing element refuses the WHOLE write — never a partial list — and a cross-workspace copy drops the value whole, reported once, so an import can never change an element count. **0.32 is RESERVED for PR #1337**, which claims it and was opened first; this unit took 0.33 rather than contest the number, so 0.32 may be skipped if #1337 is abandoned — a gap is cheaper than a collision resolved under merge pressure. v0.31 (PLAN-2857 U6 / TASK-2996) lets a `relation` value be an EXACT TITLE, scoped to the collection that field declares, alongside the UUID and ref v0.29 accepted; adds a `relation_targets` member to reads hydrating each stored id to `{id, ref, title}`; and makes the `fields` DSL take the target collection slug as a relation's third part. The first two are additive — a title is a THIRD spelling, and `relation_targets` is `omitempty` — so the bump is owed by the DSL: `fields="owner:relation"` used to be ACCEPTED and is now REFUSED, because the old parser put the third part into `Options` for every type and therefore built a relation with NO target, a field every subsequent write refused with `target_missing`. Scoping is the point: a title unique only workspace-wide refuses and names the collection searched, and two matches inside the declared collection get their own `ambiguous` reason rather than `not_found`, which would state the opposite of what happened. The ladder is UUID, then ref, then title, so an item literally TITLED `COLO-3` is unreachable by title while the ref resolves. An id-only `relation_targets` entry means the target is GONE **or** the caller may not SEE it — the two are made indistinguishable deliberately, matching the write side's `wrong_collection`→`not_found` collapse, so no consumer may render it as either. v0.30 (BUG-2870) makes one `--field key=value` entry mean ONE thing at every door. Six sites parsed that entry independently — `item create`, `item list`, `item update`, `item move` and `item copy` in `cmd/pad`, plus `ingestFieldKVP` on the remote door — in four spellings, so `field:[" effort=l"]` stored an undeclared field literally named `" effort"` through the CLI while the remote door trimmed and wrote `effort`: the same call storing two different keys, decided by the transport. All six now call `items.SplitFieldEntry`, under two deliberately asymmetric rules — a padded KEY is REFUSED everywhere (trimming silently retargets the write to a field the caller did not type), a VALUE is carried VERBATIM everywhere (trimming reinterprets a caller's bytes, and on a text field the padding is content). The catalog's conflict pass is re-grounded on the same change, since its rules were derived from that trimming: comparisons are RAW, a padded entry is refused in the pass rather than skipped, `parseFieldArray`'s refusal is PROPAGATED rather than swallowed (it had no second owner on the no-`fields` path, where four existing refusals were landing as successes), and canonicalization and the re-emission path are gone — nothing rewrites a caller's key any more. v0.29 (PLAN-2857 / TASK-2878) makes a `relation` field value have to NAME A LIVE ITEM in the collection that field declares. `internal/items` only ever checked the SHAPE of a relation ("must be a string") because that package is DB-free, so any string at all was accepted and stored and no client could render it honestly. Every write door now refuses a value that names nothing, names an item in the WRONG collection, sits in a field declaring no target collection, or is a SLUG (deliberate divergence from `ResolveItem` — a slug is neither an ID nor stable). A CARRIED value, already on the item and asserted by nobody, is never refused: within a workspace it resolves and survives; across a boundary it is dropped without a lookup and reported in `warnings.dropped_fields`. v0.28 (IDEA-2641) adds two ADDITIVE `pad_item` actions — `remind` (arm a one-shot reminder at an RFC3339 `remind_at` instant) and `ack-reminder` (acknowledge a fired one by `reminder_id`). Agents already RECEIVED reminders, since the poll surface is `pad_project.next` / `ready`; what was missing is the other half — deferring work is exactly when an agent knows it wants to be asked again. A bare date is refused rather than read as midnight. Re-arm and disarm stay CLI-only until a listing action exists to discover an id. v0.27 (BUG-2850) typed field values server-side, carried the `fields` object with its JSON types intact, named undeclared keys back in `warnings.undeclared_fields`, and replaced the per-site conflict guards with one check that refuses two names for one target in a single call. v0.26 (IDEA-2756) makes `pad_workspace.create` REFUSE with a 403 when the calling OAuth connection's grant carries `may_create_workspaces=false` — that consent checkbox previously gated only the post-creation auto-add, so a connection whose user declined it created workspaces anyway — invisible to it when the connection carried an explicit allow-list, visible when it carried the `all_current_workspaces` wildcard; the consent mismatch is the defect in both cases. The same gate covers `POST /workspaces/import` (a second door onto `store.ImportWorkspace` → `CreateWorkspace`, with no MCP action today). No escape-hatch param, deliberately: the gate expresses the USER's consent decision, so only the user can lift it — by re-authorizing, or by enabling the flag on the existing connection at `/console/connected-apps`. v0.25 (TASK-2657 / BUG-2702) makes `pad_library.activate` resolve its destination collection from the target's declared artifact kind rather than the literal `conventions` / `playbooks` slugs, and surfaces a lookup ERROR instead of falling back. v0.24 (#1066) makes the `pad_item` `fields` OBJECT a real write form on create/update — reads return `fields` as a native object (BUG-991 normalization), and writing that shape back was a silent no-op: not a declared param, no `additionalProperties`, so it was accepted, never mapped by `BuildCLIArgs`, and dropped while the PATCH still bumped `updated_at`. The alias merges into the same path as `field: ["key=value"]` / the dedicated params (`catalog_item_fields.go`), refusing the same key in two places with conflicting values; and input validation is now STRICT across all catalog tools — an undeclared top-level key fails with a structured `validation_failed` naming it, instead of being silently dropped (a small documented compat list survives: pad_item's v0.16 `assigned_user_id` / `agent_role_id` remote clear form). One bump covers both halves — they are one contract change. v0.23 (BUG-2627 part 2 + BUG-2675, PR #1166) refuses raw `field` setters naming system-metadata keys in `fields_patch` on every transport (`github_pr` exempt on UPDATE only — the sole remote writer, itself broken — until v0.47 closed it: BUG-2696) and adds the retry-hostile `stored_state_unreadable` error code. v0.22 (BUG-2674, PR #1165) makes reserved metadata survive a move and refuses `field` setters naming those keys on move/copy — see `internal/mcp/version.go` for both full entries. v0.21 (BUG-2608) bounds `pad_item.action=history`, which was unbounded on every surface: the `limit` param now covers it (default 50, max 300 — the NEWEST N versions, with no `offset`, because reverse-patch storage makes only a newest-end window cheap to reconstruct), applied in the CATALOG action so it lands on both transports, and summary mode now asks the server to skip patch resolution (`?summary=true`) instead of resolving every body and discarding it. Additive param bump — `limit` already existed and nothing changed shape. v0.20 (BUG-2302 + BUG-2305, one bump) adds explicit MCP tool annotations (`readOnlyHint`/`destructiveHint`/`idempotentHint` derived from the catalog's own write-shape knowledge, fixing read-only tools that advertised `destructiveHint:true`) and makes `pad_item.list` summary-shaped on the REMOTE /mcp transport too (the hand-written `dispatchItemList` projects via `cli.ToItemSummaries`; `full=true` opts back into complete bodies) — see `internal/mcp/version.go` for the authoritative per-version changelog. Post-0.20 without a bump (BUG-2304): `item backlinks` / `item history` / `project report` gained HTTP route coverage — they were advertised but answered "not yet implemented over HTTP transport" — and a catalog↔route parity test (`dispatch_http_parity_test.go`) now drives every catalog action and fails on any future advertised-but-unrouted action; no names, enums, or shapes changed, hence no bump. v0.19 adds a `clear_parent` boolean to `pad_item` — the canonical, schema-discoverable way to detach an item from its parent, backed by a new `--clear-parent` bareword flag on `pad item update` (BUG-2078). v0.18 adds `clear_assigned_user` / `clear_agent_role` booleans to `pad_item` — the canonical, schema-discoverable way to unassign, backed by new `--clear-assigned-user` / `--clear-agent-role` bareword flags on `pad item update` (IDEA-2584). Update-only, deliberately asymmetric with create. v0.17 carries the empty-string clear to the LOCAL STDIO transport, which shells out to the CLI — `cmd/pad/cmd_item.go` now lifts `assigned_user_id` / `agent_role_id` onto their columns instead of into the fields blob, on create and update (BUG-2583). v0.16 makes an empty-string `assigned_user_id` / `agent_role_id` CLEAR the assignment instead of being silently dropped, so an MCP agent can finally unassign an item (TASK-2571). v0.15 adds the `pad_item.list` `unparented` boolean, mutually exclusive with `parent`, for items with no parent or implements relationship (TASK-2096). v0.2 introduced the catalog (PLAN-969 / TASK-981); v0.3 added `pad_playbook`, `pad_meta.action: bootstrap`, `pad_set_workspace`'s embedded-bootstrap response, and the `pad://workspace/{ws}/bootstrap` resource (PLAN-1377 / TASK-1380); v0.4 trimmed the bootstrap payload by ~40% (PLAN-1410) — slim `BootstrapCollection` + `BootstrapRole` projections (no UUIDs/timestamps/settings; nested `schema` object; redundant labels omitted), removed top-level `recent_activity` duplicate, dropped convention `slug`, and added a `BootstrapDashboard` wrapper that caps five sub-arrays (`attention`, `recent_activity`, `active_items`, `active_plans`, `by_role`) at 5 entries each with parallel `*_overflow_count` fields. The pre-catalog v0.1 cmdhelp leaf walker is retired. +Pad runs as a local Model Context Protocol server so Claude Desktop / Cursor / Windsurf can call non-interactive `pad` commands as tools. The same catalog is also served as remote MCP over HTTP at `/mcp` (Streamable HTTP): always on Pad Cloud, and on a self-hosted install off by default until an admin turns it on (`GET/PUT /api/v1/admin/mcp` above; OAuth plus PATs when the issuer URL is https, PATs only on http; operator docs in `docs/deployment.md`). The tool surface is a **hand-curated catalog** (currently v0.60) in `internal/mcp/catalog_*.go` — one ToolDef per resource (`pad_item`, `pad_workspace`, `pad_collection`, `pad_project`, `pad_role`, `pad_search`, `pad_meta`, `pad_playbook`, `pad_library`, `pad_attachment`) with an `action` enum dispatching to underlying CLI commands. v0.33 (PLAN-2857 U4 / TASK-2999) adds a **`multi_relation`** field type — an ORDERED LIST of references, each element resolving through the same UUID→ref→exact-title ladder a scalar `relation` uses, declared through the DSL as `owners:multi_relation:people` (third part = target collection; a bare `owners:multi_relation` is refused at parse time rather than minting a field no write can satisfy). The TYPE is additive — every rule is reachable only through a field whose schema says `multi_relation`, which no existing schema can contain — so the bump is owed by the READ shape: `relation_targets[key]` now carries EITHER the scalar object v0.31 emitted OR a JSON **ARRAY** of those objects, in stored order and one per stored element (an `id`-only entry for an element that names nothing, so a position lines up with the same position in `fields`). A scalar entry is byte-identical to v0.31's — pinned against a literal, not against another call into the same code — so a consumer with no such fields sees nothing new, while one that acquires such a field must handle the array. ONE map, not a parallel `multi_relation_targets` key: the field's type already says which shape to expect. Value rules, all NEW rather than inherited because scalar `relation` had nothing coherent to inherit (see BUG-3028 — a required scalar relation WAS satisfied by `""`, with three stored spellings of "no target", until v0.40 converged it on this same key-absent rule): exactly ONE stored form for none, the key ABSENT, with `[]` normalising to it at the write door; an empty or whitespace-only ELEMENT REFUSED rather than skipped, naming its index; `required` meaning >=1 RESOLVED element; order part of the value; duplicates REFUSED with a new `duplicate_referent` reason, detected AFTER resolution because two elements naming one item are usually different strings. Any failing element refuses the WHOLE write — never a partial list — and a cross-workspace copy drops the value whole, reported once, so an import can never change an element count. **0.32 is RESERVED for PR #1337**, which claims it and was opened first; this unit took 0.33 rather than contest the number, so 0.32 may be skipped if #1337 is abandoned — a gap is cheaper than a collision resolved under merge pressure. v0.31 (PLAN-2857 U6 / TASK-2996) lets a `relation` value be an EXACT TITLE, scoped to the collection that field declares, alongside the UUID and ref v0.29 accepted; adds a `relation_targets` member to reads hydrating each stored id to `{id, ref, title}`; and makes the `fields` DSL take the target collection slug as a relation's third part. The first two are additive — a title is a THIRD spelling, and `relation_targets` is `omitempty` — so the bump is owed by the DSL: `fields="owner:relation"` used to be ACCEPTED and is now REFUSED, because the old parser put the third part into `Options` for every type and therefore built a relation with NO target, a field every subsequent write refused with `target_missing`. Scoping is the point: a title unique only workspace-wide refuses and names the collection searched, and two matches inside the declared collection get their own `ambiguous` reason rather than `not_found`, which would state the opposite of what happened. The ladder is UUID, then ref, then title, so an item literally TITLED `COLO-3` is unreachable by title while the ref resolves. An id-only `relation_targets` entry means the target is GONE **or** the caller may not SEE it — the two are made indistinguishable deliberately, matching the write side's `wrong_collection`→`not_found` collapse, so no consumer may render it as either. v0.30 (BUG-2870) makes one `--field key=value` entry mean ONE thing at every door. Six sites parsed that entry independently — `item create`, `item list`, `item update`, `item move` and `item copy` in `cmd/pad`, plus `ingestFieldKVP` on the remote door — in four spellings, so `field:[" effort=l"]` stored an undeclared field literally named `" effort"` through the CLI while the remote door trimmed and wrote `effort`: the same call storing two different keys, decided by the transport. All six now call `items.SplitFieldEntry`, under two deliberately asymmetric rules — a padded KEY is REFUSED everywhere (trimming silently retargets the write to a field the caller did not type), a VALUE is carried VERBATIM everywhere (trimming reinterprets a caller's bytes, and on a text field the padding is content). The catalog's conflict pass is re-grounded on the same change, since its rules were derived from that trimming: comparisons are RAW, a padded entry is refused in the pass rather than skipped, `parseFieldArray`'s refusal is PROPAGATED rather than swallowed (it had no second owner on the no-`fields` path, where four existing refusals were landing as successes), and canonicalization and the re-emission path are gone — nothing rewrites a caller's key any more. v0.29 (PLAN-2857 / TASK-2878) makes a `relation` field value have to NAME A LIVE ITEM in the collection that field declares. `internal/items` only ever checked the SHAPE of a relation ("must be a string") because that package is DB-free, so any string at all was accepted and stored and no client could render it honestly. Every write door now refuses a value that names nothing, names an item in the WRONG collection, sits in a field declaring no target collection, or is a SLUG (deliberate divergence from `ResolveItem` — a slug is neither an ID nor stable). A CARRIED value, already on the item and asserted by nobody, is never refused: within a workspace it resolves and survives; across a boundary it is dropped without a lookup and reported in `warnings.dropped_fields`. v0.28 (IDEA-2641) adds two ADDITIVE `pad_item` actions — `remind` (arm a one-shot reminder at an RFC3339 `remind_at` instant) and `ack-reminder` (acknowledge a fired one by `reminder_id`). Agents already RECEIVED reminders, since the poll surface is `pad_project.next` / `ready`; what was missing is the other half — deferring work is exactly when an agent knows it wants to be asked again. A bare date is refused rather than read as midnight. Re-arm and disarm stay CLI-only until a listing action exists to discover an id. v0.27 (BUG-2850) typed field values server-side, carried the `fields` object with its JSON types intact, named undeclared keys back in `warnings.undeclared_fields`, and replaced the per-site conflict guards with one check that refuses two names for one target in a single call. v0.26 (IDEA-2756) makes `pad_workspace.create` REFUSE with a 403 when the calling OAuth connection's grant carries `may_create_workspaces=false` — that consent checkbox previously gated only the post-creation auto-add, so a connection whose user declined it created workspaces anyway — invisible to it when the connection carried an explicit allow-list, visible when it carried the `all_current_workspaces` wildcard; the consent mismatch is the defect in both cases. The same gate covers `POST /workspaces/import` (a second door onto `store.ImportWorkspace` → `CreateWorkspace`, with no MCP action today). No escape-hatch param, deliberately: the gate expresses the USER's consent decision, so only the user can lift it — by re-authorizing, or by enabling the flag on the existing connection at `/console/connected-apps`. v0.25 (TASK-2657 / BUG-2702) makes `pad_library.activate` resolve its destination collection from the target's declared artifact kind rather than the literal `conventions` / `playbooks` slugs, and surfaces a lookup ERROR instead of falling back. v0.24 (#1066) makes the `pad_item` `fields` OBJECT a real write form on create/update — reads return `fields` as a native object (BUG-991 normalization), and writing that shape back was a silent no-op: not a declared param, no `additionalProperties`, so it was accepted, never mapped by `BuildCLIArgs`, and dropped while the PATCH still bumped `updated_at`. The alias merges into the same path as `field: ["key=value"]` / the dedicated params (`catalog_item_fields.go`), refusing the same key in two places with conflicting values; and input validation is now STRICT across all catalog tools — an undeclared top-level key fails with a structured `validation_failed` naming it, instead of being silently dropped (a small documented compat list survives: pad_item's v0.16 `assigned_user_id` / `agent_role_id` remote clear form). One bump covers both halves — they are one contract change. v0.23 (BUG-2627 part 2 + BUG-2675, PR #1166) refuses raw `field` setters naming system-metadata keys in `fields_patch` on every transport (`github_pr` exempt on UPDATE only — the sole remote writer, itself broken — until v0.47 closed it: BUG-2696) and adds the retry-hostile `stored_state_unreadable` error code. v0.22 (BUG-2674, PR #1165) makes reserved metadata survive a move and refuses `field` setters naming those keys on move/copy — see `internal/mcp/version.go` for both full entries. v0.21 (BUG-2608) bounds `pad_item.action=history`, which was unbounded on every surface: the `limit` param now covers it (default 50, max 300 — the NEWEST N versions, with no `offset`, because reverse-patch storage makes only a newest-end window cheap to reconstruct), applied in the CATALOG action so it lands on both transports, and summary mode now asks the server to skip patch resolution (`?summary=true`) instead of resolving every body and discarding it. Additive param bump — `limit` already existed and nothing changed shape. v0.20 (BUG-2302 + BUG-2305, one bump) adds explicit MCP tool annotations (`readOnlyHint`/`destructiveHint`/`idempotentHint` derived from the catalog's own write-shape knowledge, fixing read-only tools that advertised `destructiveHint:true`) and makes `pad_item.list` summary-shaped on the REMOTE /mcp transport too (the hand-written `dispatchItemList` projects via `cli.ToItemSummaries`; `full=true` opts back into complete bodies) — see `internal/mcp/version.go` for the authoritative per-version changelog. Post-0.20 without a bump (BUG-2304): `item backlinks` / `item history` / `project report` gained HTTP route coverage — they were advertised but answered "not yet implemented over HTTP transport" — and a catalog↔route parity test (`dispatch_http_parity_test.go`) now drives every catalog action and fails on any future advertised-but-unrouted action; no names, enums, or shapes changed, hence no bump. v0.19 adds a `clear_parent` boolean to `pad_item` — the canonical, schema-discoverable way to detach an item from its parent, backed by a new `--clear-parent` bareword flag on `pad item update` (BUG-2078). v0.18 adds `clear_assigned_user` / `clear_agent_role` booleans to `pad_item` — the canonical, schema-discoverable way to unassign, backed by new `--clear-assigned-user` / `--clear-agent-role` bareword flags on `pad item update` (IDEA-2584). Update-only, deliberately asymmetric with create. v0.17 carries the empty-string clear to the LOCAL STDIO transport, which shells out to the CLI — `cmd/pad/cmd_item.go` now lifts `assigned_user_id` / `agent_role_id` onto their columns instead of into the fields blob, on create and update (BUG-2583). v0.16 makes an empty-string `assigned_user_id` / `agent_role_id` CLEAR the assignment instead of being silently dropped, so an MCP agent can finally unassign an item (TASK-2571). v0.15 adds the `pad_item.list` `unparented` boolean, mutually exclusive with `parent`, for items with no parent or implements relationship (TASK-2096). v0.2 introduced the catalog (PLAN-969 / TASK-981); v0.3 added `pad_playbook`, `pad_meta.action: bootstrap`, `pad_set_workspace`'s embedded-bootstrap response, and the `pad://workspace/{ws}/bootstrap` resource (PLAN-1377 / TASK-1380); v0.4 trimmed the bootstrap payload by ~40% (PLAN-1410) — slim `BootstrapCollection` + `BootstrapRole` projections (no UUIDs/timestamps/settings; nested `schema` object; redundant labels omitted), removed top-level `recent_activity` duplicate, dropped convention `slug`, and added a `BootstrapDashboard` wrapper that caps five sub-arrays (`attention`, `recent_activity`, `active_items`, `active_plans`, `by_role`) at 5 entries each with parallel `*_overflow_count` fields. The pre-catalog v0.1 cmdhelp leaf walker is retired. cmdhelp is still consumed at dispatch time — `BuildCLIArgs` reads individual command schemas to translate the catalog's snake_case input map into CLI args. cmdhelp no longer drives tool naming or count. @@ -337,7 +337,7 @@ pad mcp status # Install state across supported clients ``` Surface: -- **Tools:** the v0.59 catalog — ten resource × action tools (`pad_item`, `pad_workspace`, `pad_collection`, `pad_project`, `pad_role`, `pad_search`, `pad_meta`, `pad_playbook`, `pad_library`, `pad_attachment`) plus `pad_set_workspace` (takes a `workspace` slug only — no action enum). The ten resource × action tools take `action: ` to choose what they do. `pad_item` (v0.19) exposes `clear_parent` as the canonical parent-detach (update only); (v0.18) exposes `clear_assigned_user` / `clear_agent_role` booleans as the canonical unassign (update only); (v0.17) treats an empty-string `assigned_user_id` / `agent_role_id` as a clear on BOTH transports via `field: ["assigned_user_id="]` — the direct param form is remote-only, since it isn't schema-declared and stdio's BuildCLIArgs drops unknown keys (IDEA-2584); v0.16 fixed remote only; (v0.15) adds the `unparented` list parameter; v0.14 added `history` + `expected_updated_at`. `pad_project` (v0.13) adds `ready` (actionable backlog) + `stale` (items needing attention); `pad_project.activity` (v0.12) is the non-streaming, bounded activity feed — catch up on what other agents/users changed since you last worked. `pad_attachment` is the read-only attachment-metadata surface — `list`/`show` (upload/download/view stay CLI-only). `pad_library` is the convention+playbook library surface — `list`/`get`/`activate`. `pad_playbook` is the playbook surface from PLAN-1377 — `list`/`get`/`run` mirror the CLI's `pad playbook` subcommands; `run` is side-effect-free and returns the body + bound args for the agent to execute. v0.4 (PLAN-1410) didn't change the tool/action surface; it trimmed the bootstrap JSON those tools/resources return — see the Stability contract subsection below for details. +- **Tools:** the v0.60 catalog — ten resource × action tools (`pad_item`, `pad_workspace`, `pad_collection`, `pad_project`, `pad_role`, `pad_search`, `pad_meta`, `pad_playbook`, `pad_library`, `pad_attachment`) plus `pad_set_workspace` (takes a `workspace` slug only — no action enum). The ten resource × action tools take `action: ` to choose what they do. `pad_item` (v0.19) exposes `clear_parent` as the canonical parent-detach (update only); (v0.18) exposes `clear_assigned_user` / `clear_agent_role` booleans as the canonical unassign (update only); (v0.17) treats an empty-string `assigned_user_id` / `agent_role_id` as a clear on BOTH transports via `field: ["assigned_user_id="]` — the direct param form is remote-only, since it isn't schema-declared and stdio's BuildCLIArgs drops unknown keys (IDEA-2584); v0.16 fixed remote only; (v0.15) adds the `unparented` list parameter; v0.14 added `history` + `expected_updated_at`. `pad_project` (v0.13) adds `ready` (actionable backlog) + `stale` (items needing attention); `pad_project.activity` (v0.12) is the non-streaming, bounded activity feed — catch up on what other agents/users changed since you last worked. `pad_attachment` is the read-only attachment-metadata surface — `list`/`show` (upload/download/view stay CLI-only). `pad_library` is the convention+playbook library surface — `list`/`get`/`activate`. `pad_playbook` is the playbook surface from PLAN-1377 — `list`/`get`/`run` mirror the CLI's `pad playbook` subcommands; `run` is side-effect-free and returns the body + bound args for the agent to execute. v0.4 (PLAN-1410) didn't change the tool/action surface; it trimmed the bootstrap JSON those tools/resources return — see the Stability contract subsection below for details. - **Resources:** `pad://workspace/{ws}/items/{ref}`, `pad://workspace/{ws}/items`, `pad://workspace/{ws}/dashboard`, `pad://workspace/{ws}/collections`, `pad://workspace/{ws}/attachments/{id}` (bounded base64 image via `thumb-md`; non-images and image bytes over 1 MiB (pre-base64) rejected), `pad://workspace/{ws}/bootstrap` (one-shot workspace overview — user + collections + always-on conventions + roles + playbook metadata + dashboard + recent activity), plus the server-wide `pad://_meta/version`. - **Prompts:** `pad_plan`, `pad_ideate`, `pad_retro`, `pad_onboard` — multi-step workflows lifted from `skills/pad/SKILL.md`. @@ -345,7 +345,7 @@ Surface: **Stability contract.** Two version constants live in `internal/mcp/version.go`, advertised in the handshake under `capabilities.experimental.padCmdhelp` and `capabilities.experimental.padToolSurface`: - `CmdhelpVersion` (currently `"0.1"`) — the cmdhelp CLI help-tree contract. Bump when CLI flag/arg schemas change incompatibly. -- `ToolSurfaceVersion` (currently `"0.59"`) — the MCP tool catalog contract. Bump when tool names, action enums, or parameter shapes change incompatibly. **v0.59** (BUG-3252, tombstone ruled over cascade) makes `pad_item.action=delete-comment` on a comment with replies SUCCEED, leaving a tombstone (body emptied, author and timestamps kept, additive `deleted: true`) its replies stay under; it is removed when its last reply is deleted. `edit-comment` on a tombstone answers 409 `comment_deleted` on every transport. BEHAVIOR bump on the v0.58 grounds; `comment_has_replies` stays classified for older servers. **v0.58** (BUG-3252) makes `pad_item.action=delete-comment` on a comment that still has replies answer 409 `comment_has_replies` (details `comment_id`, `reply_count`) on every transport instead of `server_error`: `comments.parent_id` has no ON DELETE, so the delete failed the FK inside the store. Nothing is deleted either way. Tombstone was then ruled over cascade; v0.59 replaced this refusal with it. BEHAVIOR bump on the v0.45 grounds. **v0.57** (TASK-2695) adds `pad_item` `edit-comment` (ref, comment_id, message) and `delete-comment` (ref, comment_id) on all three transports, backed by the item-scoped routes `PATCH/DELETE /workspaces/{ws}/items/{ref}/comments/{id}`, which answer 404 unless the comment is on `ref` and otherwise run the workspace routes' handlers (edit author-only, delete item-edit). CLI: `pad item comment-edit` / `comment-delete`, refused against a server without the `item_scoped_comment_writes` capability. Every serialised comment gains a derived `edited` bool (updated_at > created_at), and `pad item comments` prints each comment's id. ADDITIVE. **v0.56** (BUG-3244) gives `content_state` a second value, `superseded_set_aside`, on every MCP door that carries it (`pad_item` get / full list, `pad_playbook` run, the item resource's line above the body): edits an editor schema-version rebuild set aside, which opening the item will NOT restore, so the resource line and the `content_pending_flush` hint (chosen by `details.set_aside_rows`) name `pad item set-aside` and `overwrite_pending_edits` instead. ADDITIVE bump: no name, enum or param changed, but an agent branching on the one-word vocabulary meets a word whose remedy differs. Reading or discarding the rows alone is not on the catalog. **v0.55** (BUG-2659) sends `pad_item.action=search`'s `collection` to /search as typed, and /search resolves it exact-first per workspace in scope (the item routes' resolver). In a workspace holding both `task` and `tasks`, `collection: "task"` now answers from `task` instead of `tasks`; every other workspace is unchanged, because the shorthand still resolves server-side. Stdio reaches it through the CLI, which sends the name as typed only to a server advertising `search_collection_resolution`. BEHAVIOR bump on the v0.36 / v0.40 grounds. **v0.54** (BUG-3217) keeps a `fields` number's literal through MCP on both transports (mcp-go's argument decode rounded anything above 2^53, and stdio's `--field` stringify carried the rounded value), and REFUSES a `fields` number beside a `field` entry for the same key when they differ as exact numbers — two values that differ only above 2^53 used to collapse into one rounded write; a matching pair (3.0 beside n=3) still collapses. BEHAVIOR bump on the v0.48 grounds. **v0.53** (BUG-2819) adds `filename_source` (caller / normalised / substituted / derived / unknown) to every `pad_attachment.action=list` row on both transports, because a stored filename cannot say whether the caller sent it or the server substituted it. ADDITIVE, the v0.28 / v0.13 disposition; `show` does not carry it, and `mcp_audit_log.tool_name_source` reaches no catalog action. Rows written before it read `unknown`, never `caller`. **v0.52** (BUG-2367 item 4) refuses a move or copy (every migrate door) that would change an item between open, done and abandoned unless the caller names the destination done field. Only the STATE is compared, so two values that both mean done carry. Move answers 400 `state_change_requires_value` with details `{field, options, from, to}`; bulk move puts it in a `failed[]` row; the copy answers 400 `validation_error` with the same sentence; the preflight emits a needs_value row with reason `state_change`, which the web dialog picker renders. A destination with no done select field is exempt. BEHAVIOR bump. **v0.51** (BUG-2367) stops every migrate door (single move, bulk move, cross-workspace copy and its preflight) carrying a value into a field the destination marks `computed` (preflight drop reason `target_computed`) or a value that collides on a destination `unique_scope` field. Provenance decides, the TASK-2878 rule: a CARRIED collision is dropped and reported (`not_unique`), a supplied override or injected default that collides is refused 409 `conflict`, and a required field emptied by the drop is refused as required, which is the existing needs_value route. Before it, single move answered 500 on the indexed `invocation_slug`, bulk move leaked the SQL error, and an unindexed unique field stored a duplicate. Reporting is additive and omitempty: `warnings.not_unique` [{key, value, holder?, message}] on the move response, bulk `updated[]` rows and the copy response, and a `detail` sentence on the preflight's dropped row. `holder` is present only when the caller may see that item, so the drop is not an existence oracle. BEHAVIOR bump on the v0.43 / v0.29 grounds. **v0.50** (BUG-3014) adds an ADDITIVE `omitempty` `stored_as_text: true` to a `relation_targets` entry whose stored value is not UUID-shaped (a title a bundle import carried verbatim, or legacy free text). Such a value was never an id and used to hydrate as a plain id-only entry, which reads as "gone or hidden"; an id-only entry without the flag keeps that meaning, and a UUID-shaped value, resolved or not, is byte-identical to v0.49. The predicate is syntactic, decided from the stored bytes and never from a lookup, so it discloses nothing, and deliberately not coupled to the resolver ladder. The CLI prints `"" (text, not a reference)` and the web chip says the same. **v0.49** (BUG-2410) makes `pad_project.action=report` with no `collections` param leave out SYSTEM collections (`Collection.IsSystem`: Conventions and Playbooks), which were counted in Insights' created, WIP and status totals; naming one explicitly still includes it. BEHAVIOR bump on the v0.40 / v0.42 grounds: a default call's content changed for every caller (web Insights, print, `pad project report`, both MCP transports, all through the one store rule). **v0.48** (BUG-3163) makes `pad_item.action=create` REFUSE a `field` / `fields` entry naming any reserved metadata key (`implementation_notes`, `decision_log`, `github_pr`, `convention`) on every transport, with 400 `validation_error`, creating nothing: create was the last door that stored one, as a string no extractor reads. Library activation, its one system writer, moves to an additive typed `convention` create member (not on the catalog; `pad_library.activate` is unchanged), and the CLI verifies from the stored fields that it landed, because an older server ignores the member and answers 201. BEHAVIOR bump on the v0.47 grounds. **v0.47** (BUG-2696) makes `pad_item.action=update` REFUSE a field setter naming `github_pr` on every transport (400 `validation_error` naming `pad github link` / `unlink`), closing the exemption v0.23 left: that door stored the PR as an unreadable string, so no link ever rendered. The writer is a new additive, typed update member (`github_pr` object + `clear_github_pr`) used by `pad github link` / `unlink` / `project reconcile`, deliberately NOT on the catalog (a remote PR-link action waits on the catalog-trim decision). BEHAVIOR bump; item create is BUG-3163. **v0.46** (BUG-3156) makes `pad_item.action=bulk-update` over stdio and remote REFUSE, per item, a status or priority change on an item whose collection does not declare that field, writing nothing for that item: the answer WebMCP has given since v0.44, so the three transports agree. Rows keep each transport's shape (stdio `failed[]` with `validation_error`, remote `results[]` with `validation_failed` and the server message in `hint`). The mechanism is the update door's new `refuse_undeclared_fields` member, which both bulk-update transports set; `pad_item.update` does not, and still accepts-and-warns. A server that predates the member ignores it, which is the old accept-and-warn. BEHAVIOR bump on the v0.44 / v0.43 grounds. **v0.45** (BUG-2829) reports an HTTP 413 as `too_large` on BOTH transports, with the server's own code in `details.reason`, where it used to be `server_error`: three deliberate caps are reachable from the catalog (a title rename's link cascade, an oversized outbox event, an oversized artifact import), and the old code read as transient and invited a retry that fails identically. Remote maps 413 in `classifyHTTPStatusKind`; stdio gets it from a structured marker the CLI root writes for any 413, and `too_large` joins `allowedStructuredErrorCodes`, which a test now proves contains every code any CLI marker writer emits. BEHAVIOR bump on the v0.42 grounds; no name, enum or param shape changed. **v0.44** (BUG-3154) makes `pad_item.action=bulk-update` over the WebMCP browser transport REFUSE, per item, a status or priority change on an item whose collection does not declare that field: the item lands in the bulk response's `failed` array with `validation_error` naming the collection and field, nothing is written, and the rest of the call applies. The door is `POST /items/bulk` (`set-priority`, and `move` with a `status`, with or without a `collection`, where the target's schema decides), where the request carries an operation and a value and the SERVER chooses the key; it used to store an orphan field and report the item updated. BEHAVIOR bump on the v0.43 / v0.40 grounds. The transports disagreed after it, stated rather than implied: stdio and remote issued a per-item PATCH, the single-item update door, which ACCEPTED the key with `warnings.undeclared_fields` (v0.27), until v0.46 converged them. **v0.43** (BUG-2379) makes `pad_item.action=move` REFUSE a field override naming a field the destination schema does not declare, with 400 `malformed_override` — the same check, code and message the cross-workspace copy already used. BEHAVIOR bump on the v0.40 / v0.29 grounds: a write door refuses a call it used to accept, where it used to store the key as an orphan field no schema renders. Create and update still ACCEPT undeclared keys with `warnings.undeclared_fields` (v0.27), because callers round-trip the whole fields blob; an override is a fresh assertion about the destination schema, not round-tripped state. **v0.42** (BUG-3142 + BUG-3147) changes the error CODE a LOCAL STDIO caller receives for two failure families that used to arrive as retryable `server_error`: an argv refusal (cobra/pflag: unknown flag, arg count, …) is now `validation_failed` with its message kept, and a 429 on any command is now `rate_limited` with `details.retry_after_seconds`, via the structured marker line the CLI writes at its root. It also stops the refusal that caused most of the first family: `BuildCLIArgs` now emits flags first and the positionals behind `--`, so free text starting with `-` (`item search "-ship it"`) is no longer parsed as a flag. BEHAVIOR bump; no name, enum or param shape changed, and the remote transport is untouched. **v0.41** (PLAN-3114 U5 / TASK-3120) adds a `match` action to `pad_playbook`: given free `text`, a Choice question over the workspace's ACTIVE playbooks plus a reserved `"none"` option returns the best match's ref (or `"none"`), confidence, per-option probabilities and the model. No provider configured → 404 `decision_provider_unavailable`; a provider error or an out-of-set answer → 502 `decision_provider_error`. Pure ADDITIVE bump (new action + param, nothing existing moved), read-only and side-effect-free. **v0.40** (BUG-3028) gives a scalar `relation` ONE stored form for "no target", the key ABSENT, converging on multi_relation's rule. A blank (`""` or whitespace) a write SETS — create, `fields_patch`, a changed value in full `fields`, a move/copy override — is normalised to absent before validation, so a `required` relation is refused as required (a patch blanking it now gets the answer deleting it always got). A legacy blank a write only CARRIES — an untouched key under `fields_patch` (normalised by the store under its write lock, via `ItemUpdate.BlankRelationKeys`), an unchanged value in full `fields`, move, copy, artifact import — is normalised after validation and never refused. An empty list filter on a scalar relation key of a collection-scoped list (`owner=`) matches all three spellings. BEHAVIOR bump; no shape change. Workspace import (`store.ImportWorkspace`) still stores blobs verbatim; its legacy blanks normalise on the next write. **v0.39** (BUG-3133) refuses `pad_item.update` that sets content WITH a version token (`expected_seq` / `expected_updated_at`) while the item's op-log holds content-bearing rows above its flush watermark, with 409 `content_pending_flush` (details `ref`, `pending_rows`). BEHAVIOR bump on the v0.35 / v0.36 grounds, carrying one additive param: `overwrite_pending_edits` lifts it, on the API, the CLI (`--overwrite-pending-edits`; `pad item edit --force` sets it on the save) and MCP alike, because an agent must be able to clear the refusal without a browser. The token guards the ROW and a tab's unflushed typing is not in it, so the write used to replace those edits — through the applier's setContent with a tab open, and by the direct path's op-log prune, which DELETES them, with none. Its own code rather than `update_conflict` because re-reading returns the same row and seq. A write without a token, without content, or to an item with nothing pending is unchanged, except that a direct write now reports `warnings.pruned_pending_edits` (additive, omitempty) when its prune deleted such rows. The check runs in the write's own tx; on the direct path it is under the item lock and appendMu and exact, on the applier path a tab can still type in the one server hop between the check and setContent. (v0.37 and v0.38 — the `agent` projection on `pad_item.get` and `claim` / `release` — are in `internal/mcp/version.go`; this paragraph had stopped at v0.36.) **v0.36** (BUG-3082) stops a REF-SHAPED `relation` value falling back to matching by item NUMBER when its prefix names a collection that is LIVE in the workspace; such a value is now `not_found`. BEHAVIOR bump on the v0.35 / v0.30 / v0.29 / v0.16 grounds — no tool name, action enum or param shape changed, but a write door refuses a value it used to accept. It finishes v0.29: that entry closed free-text corruption ("red" resolving to whatever is slugged `red` today) and left the same corruption open for anything shaped like a REF, which is the spelling the docs tell callers to prefer. Item numbers are workspace-unique and sequential ACROSS collections, so `CONVE-1` and `SECRE-1` are never both real — the fallback dropped the prefix, matched on the number alone, and landed on an item that was usually inside the field's declared collection, so the wrong-collection check above it passed and NOTHING was raised: a caller wrote `CONVE-1`, the item stored a different item's UUID, and the write answered 201 with the substitution visible only in `relation_targets`. The predicate is about the PREFIX rather than the item because a collection RENAME changes its prefix and a relation already written as the old ref must keep resolving (BUG-2873) — that ref's prefix names nothing afterwards, so the fallback still fires for it, while a prefix that IS live and simply holds no item at that number is the caller naming a real collection that does not contain what they said. A SOFT-DELETED collection counts as absent IN THAT PREDICATE, for the same reason a renamed one does — a claim about the fallback only: the exact-prefix lookup ahead of it never filtered on the collection’s own `deleted_at`, so an item whose collection was soft-deleted still matches THERE and never reaches the predicate. That is left alone rather than made consistent, because it cannot retarget — the downstream collection check refuses it. RESIDUAL, stated rather than implied and pinned by a test: a ref pasted from ANOTHER workspace whose prefix also names no collection here still resolves by number, because nothing in the value distinguishes it from a rename's leftover. No escape hatch, for v0.29's reason. Out of scope and unchanged: `GetItemByRef`, the navigational read path, where landing on the moved item is the helpful answer. **v0.35** (BUG-3079) makes an injected schema DEFAULT take the same `validateFieldType` check a caller-supplied value takes; one that fails is DISCARDED and named in `warnings.dropped_fields` instead of being stored. BEHAVIOR bump on the v0.30 / v0.29 / v0.16 grounds — no tool name, action enum or param shape changed, but a write door stores something different from what it stored before. Unlike v0.29, which turned an accepted value into a refusal, this turns a stored value into a reported drop: a create that used to land WITH a bad field still lands, without it. What it closes is two doors disagreeing about one value, decided only by who put it there. It also widens `dropped_fields` from the single relation case v0.29 introduced to a field of ANY type. No escape hatch, for v0.29's reason. ONE exception, which is the rule finishing its sentence rather than a carve-out: dropping leaves a REQUIRED field absent, which is exactly what the required check reports, so such a write is refused — naming the default as the cause, because a bare "field is required" points its reader at a request that never mentioned the field. **v0.34** (BUG-3037) adds an `expected_seq` param to `pad_item.action=update` — the STRONG optimistic-concurrency token, preferred over `expected_updated_at`. ADDITIVE (the v0.13 / v0.11 / v0.8 disposition): nothing existing moved and `expected_updated_at` still works. It is needed rather than nice because `updated_at` is stored at ONE-SECOND resolution, so two writes to a row inside one second both match the token and NEITHER conflicts — the loser is accepted and silently overwrites the winner, both callers seeing success. That is the shape of agent traffic, so the token agents were offered could not refuse the race it exists to refuse. `seq` is bumped on every mutation of the row under the write lock. Raising `updated_at`'s resolution instead was measured and REJECTED: the column is TEXT on both dialects and compared LEXICALLY in SQL (the since-cursor read plus eight `ORDER BY` sites), and Go's `RFC3339Nano` omits trailing zeros, so a sub-second value sorts BEFORE a whole-second value of the same second and the cursor would start skipping rows. The read side is what makes the token reachable: `seq` now serialises without `omitempty` on `models.Item` AND on the item SUMMARY shape (`cli.ItemSummary`), which is what `pad_item.list` returns by default since v0.9 — a token absent from the shape a caller reads is a token that caller cannot send. A value below 1 is a 400, not a 409: the server never issues one, so a 409 would read as contention that never clears. **v0.30** (BUG-2870) is a BEHAVIOR bump on the v0.29/v0.27/v0.26/v0.16/v0.10/v0.9 grounds — no tool name, action enum or param shape changed. **Every door refuses a padded KEY now**, and each was accepting it differently: /mcp trimmed it and wrote the declared field, the CLI stored a ghost field beside it — so both refuse something they used to accept. What is /mcp-only is the VALUE half, which it used to trim and type and now passes through to the same validation the CLI has always applied. A caller writing canonical entries sees no difference at either door. A second, separately-noticeable fix rides with it: `detectFieldConflicts` swallowed `parseFieldArray`'s refusal as "the caller owns this error surface", which held only while the sole possible error was a shape error — `reshapeItemFields` returns early with no `fields` object, so on the no-`fields` path (this bug's own path) four existing refusals were landing as successes. No escape hatch, for v0.29's reason: there is no legitimate call this refuses, only calls whose two readings a door used to choose between silently. **v0.29** (PLAN-2857 U1 / TASK-2878) is a BEHAVIOR bump on the v0.27/v0.26/v0.16/v0.10/v0.9 grounds (NOT v0.28's, which was purely additive) — no tool name, action enum, or param shape changed, but a `relation` field value must now name a live item in the collection that field declares, so every write door refuses values it used to store. Refused: names nothing (`not_found`), names an item in the wrong collection, the field declares no target collection (`target_missing`), or the value is a SLUG — a deliberate divergence from `ResolveItem`'s UUID→ref→slug ladder, since a slug is neither an ID nor stable and free text like "red" resolving to whatever is slugged `red` today is exactly the corruption this closes. Ordinary `validation_error`, no new code and no new details key, because stdio classifies errors by matching CLI stderr prose. A CARRIED value — already on the item, asserted by nobody — is never refused, since refusing would make every legacy item un-updatable, un-movable and un-copyable: within a workspace it resolves and SURVIVES, across a workspace boundary it is dropped without a lookup (a source-workspace id cannot mean anything in the destination) and reported through the same `warnings.dropped_fields` channel BUG-2674 established. No escape hatch, deliberately: unlike v0.10's `allow_draft` there is no legitimate call this refuses, and the case with a real claim to leniency is already exempt by provenance rather than by a flag. **v0.28** (IDEA-2641 / GitHub #1010) adds two ADDITIVE `pad_item` actions and two optional params: `remind` arms a one-shot reminder at an RFC3339 `remind_at` INSTANT, and `ack-reminder` acknowledges a fired one by `reminder_id`. Purely additive — nothing existing moved, and a v0.27 consumer that enumerates neither action is unaffected; same disposition as v0.13 / v0.11 / v0.8, which likewise wired existing CLI verbs onto the catalog. Agents already RECEIVED reminders (the poll surface is `pad_project.next` / `ready`, long exposed); what was missing is the half where an agent that defers work can say when it wants to be asked again. `remind_at` REFUSES a bare date rather than reading it as midnight — the `date` schema type accepts `YYYY-MM-DD` so a caller will try it, but a bare date names a 24-hour span and choosing an hour inside it would fire at a time nobody picked. Re-arm and disarm stay CLI-only: both address a reminder by an id the agent would have to list first, and no listing action exists on this surface yet — a door with no handle. **v0.27** (BUG-2850) types field values SERVER-SIDE at all eight validate sites, carries the `fields` object to the remote door with its JSON types intact, accepts undeclared keys while NAMING them in `warnings.undeclared_fields`, and replaces five accreted conflict guards with one canonical pass; the merge refuses several ambiguities it used to resolve silently. (This entry was missing from CLAUDE.md — the 0.27 unit swept `instructions.md` and `README.md` and not this file.) **v0.27** (BUG-2850) types field values server-side at all eight `Validate*` call sites so a declared number/json field is writable from the remote transport at all, carries the `fields` OBJECT with its JSON types intact, names undeclared keys back in `warnings.undeclared_fields` (accepted rather than refused — a census of 1012 items found 14 such keys across 168 live values, so refusing would have broken read-modify-write on items nobody had edited wrongly), and replaces the accreted per-site conflict guards with ONE check over a canonical view of every source; that check refuses ambiguities v0.26 resolved silently, chiefly two names for one target in a single call (`parent`/`plan`, `assign`/`assigned_user_id`, `role`/`agent_role_id`), refused even when the values match because the names address one thing through incomparable vocabularies and the two doors resolved them differently. **v0.26** (IDEA-2756) is a BEHAVIOR bump on the v0.9/v0.16/v0.25 grounds — no tool name, action enum, or param shape changed, but `pad_workspace.create` now refuses a call it used to permit. Closest precedent is v0.10, which likewise turned a server-side gate into a structured refusal; unlike v0.10 there is no `allow_draft`-style override, because the gate encodes a decision the USER made at consent time and a bypass param would be the app overriding its own grant. `POST /workspaces/import` is gated by the same shared helper (import mints a workspace through `store.ImportWorkspace`), though it has no MCP action today. **v0.25** (TASK-2657 / BUG-2702) resolves `pad_library.activate`'s destination collection from the target's declared artifact kind rather than the literal `conventions` / `playbooks` slugs, so activating into a workspace that renamed either collection lands correctly; a lookup ERROR is surfaced rather than silently falling back. **v0.24** (#1066) adds the `fields` OBJECT param to `pad_item` create/update — an alias merging into the same path as `field`/the dedicated params, so the shape reads return is finally a valid write shape; the same key supplied twice with conflicting values is REFUSED (refuse-on-ambiguity, the v0.18/v0.19 disposition), equal duplicates collapse to one write, and non-writer actions refuse a `fields` param loudly. It also makes input validation STRICT for every catalog tool: undeclared top-level keys are rejected with a structured error naming them, instead of being accepted and silently dropped by `BuildCLIArgs` — which is the mechanism that made the `fields` object a session-scoped silent no-op in the first place. Compat carve-out: `pad_item`'s v0.16 `assigned_user_id` / `agent_role_id` remote-transport clear form stays accepted (documented, undeprecated, deliberately never schema-declared). The strict half changes behaviour for inputs that previously "succeeded", but that reliance was indistinguishable from a caller bug (the key never did anything), so the break is the fix; one bump covers both halves. **v0.23** (BUG-2627 part 2 + BUG-2675) refuses system-metadata keys through `fields_patch` on all three doors at once, `github_pr` exempt on update (move/copy still refuse it), and adds the retry-hostile `stored_state_unreadable` code. **v0.22** (BUG-2674) stops `pad_item.action=move` destroying system metadata and refuses `field` setters naming the reserved keys there. **v0.21** bounds `pad_item.action=history` (BUG-2608): the `limit` param now covers it, default 50 / max 300, applied in the CATALOG action so it reaches both transports (HTTP reads the input; stdio gets the CLI's new `--limit` via BuildCLIArgs). The window is the NEWEST N and there is deliberately no `offset` — versions are reverse patches, so only a newest-end window is cheap to reconstruct. Additive param bump; a v0.20 consumer sending no limit now receives the newest 50 rather than every version, which is the fix. Summary mode additionally asks the server to skip patch resolution rather than resolving bodies the dispatcher discards. **v0.19** adds a `clear_parent` boolean to `pad_item` (BUG-2078) — an ADDITIVE param bump, same grounds as v0.18; nothing existing changed shape. The server has supported clearing a parent since BUG-2013 (`extractParentLink` treats a present-but-empty `parent` key in `fields_patch` as detach), but neither client surface could reach it — `--parent ""` was a silent no-op on the CLI and the MCP `parent` param has the same "empty means not provided" convention every other declared string on the tool has. Boolean rather than overloading the empty string, same two reasons as v0.18: keeps that invariant intact for every other param, and only a boolean reaches LOCAL STDIO via `BuildCLIArgs`, mapping to a new `--clear-parent` bareword flag exactly as `clear_assigned_user` maps to `--clear-assigned-user`. Update-only, same asymmetry as v0.18. A simultaneous `parent` + `clear_parent` — including via `field: ["parent=..."]` or the `plan` alias `extractParentLink` also accepts — is REFUSED on both transports, not silently resolved (codex round 1). Also refused, not silently applied: `clear_parent` against a collection whose schema declares its own `parent`/`plan` field — `extractParentLink` skips hierarchy handling entirely for a schema-shadowed key and lets it fall through as an ordinary field write, so the wire shape `{"parent":""}` can no longer distinguish clear-hierarchy intent from a legitimate blank-a-real-field write once it reaches the server; the ambiguity is created at the client surface that accepted `clear_parent`, so that surface refuses rather than guessing (codex round 2). **v0.18** adds `clear_assigned_user` / `clear_agent_role` booleans to `pad_item` (IDEA-2584) — an ADDITIVE param bump (v0.5/v0.6 precedent); nothing existing changed shape and v0.16/v0.17's empty-string forms still work, undeprecated. v0.16 and v0.17 made the clear WORK; nothing advertised it, because the params that do it were never in the catalog, so an agent reading the schema reached for `assign: ""` (a no-op, and it stays one). Booleans rather than declaring the string params, for two reasons: an empty DECLARED string is inert everywhere else on the tool, so giving one a destructive meaning would let a param-padding client silently unassign everything; and only a boolean can reach LOCAL STDIO, since `BuildCLIArgs` emits the CLI's real flags and a param with no flag behind it is dropped — these map to new `--clear-assigned-user` / `--clear-agent-role` bareword flags, exactly as `allow_draft` maps to `--allow-draft`. Update-only, deliberately asymmetric with create (clearing at create has no honest behaviour but a no-op; a test fails if someone adds them there). Server-side it is wiring, not new semantics: `models.ItemUpdate.ClearAssignedUser`/`ClearAgentRole` already existed with store support since BUG-2566. **v0.17** closes the transport gap v0.16 documented: local stdio MCP shells out to the CLI, which wrote `--field assigned_user_id=` into the item's FIELDS BLOB while the column stayed stale and then printed "Updated TASK-9". `cmd/pad/cmd_item.go` now lifts `columnFieldKeys` onto the columns on create AND update, mirroring `liftFieldsToColumns` and its INVARIANT. Two compat changes, ruled separately: non-empty values move to the column and stop writing the blob key (relying on the old behaviour is relying on a shadowing defect), and empty values clear (falls out of the lift, inherits BUG-2566). Existing stray blob keys are left alone — the fix stops minting new ones. Another behaviour-only bump (BUG-2583). **v0.16** lets an MCP agent UNASSIGN an item over the REMOTE transport (TASK-2571). No tool/action/param shape changed — this is a BEHAVIOR bump on the same grounds as v0.9: an empty-string `assigned_user_id` / `agent_role_id`, passed at the top level or as `field: ["assigned_user_id="]`, was silently dropped by two dispatch-path filters (`mapItemUpdate`, `liftFieldsToColumns`) and is now forwarded as a clear-to-NULL. The store has had defined clear semantics for exactly these two columns since BUG-2566 and HTTP inherited them, so this is uniformity restoration — MCP was the only surface with no way to unassign. Compat posture accepted deliberately: today's `""` senders get a no-op, and a no-op is the surprising reading. The empty-string filter on `tags` at the same call site STAYS (codex #547 r3 P2) — `tags: ""` is a corrupt JSONB/TEXT write, not a clear; same-looking guard, opposite justification. `clear_assigned_user` / `clear_agent_role` schema flags (option (b)) deliberately skipped as additive sugar, though codex review reopened the case — the catalog exposes `assign` / `role`, NOT the ID params, so an agent reading the schema still can't discover the clear (IDEA-2584); an empty `assign` is deliberately left inert because every other schema-declared string on that mapper treats empty as not-provided. **Transport scope:** v0.16 fixed the REMOTE /mcp transport only; v0.17 (BUG-2583) closed the local-stdio half at the CLI. **v0.15** adds the `unparented` boolean to `pad_item.list`, mutually exclusive with `parent`, for structural loose-item filtering (TASK-2096). **v0.14** added a `history` action to `pad_item` (read-only item version history — newest-first metadata; content body omitted for token thrift) and an `expected_updated_at` param for optimistic concurrency on `update` (round-trip the `updated_at` you last read; a stale value fails with a structured 409 `code=update_conflict`). The `update` action's field writes are now a server-side field-level MERGE (only the keys you set change) rather than a full-blob replace, closing the concurrent-update lost-write race (IDEA-1480 / TASK-2022) — pure addition to the action enum + param vocabulary; existing `pad_item` actions/params are unchanged and backwards-compatible. **v0.13** adds `ready` + `stale` actions to `pad_project`, mirroring the existing CLI `pad project ready` / `pad project stale` (TASK-2019): `ready` (read-only) returns the actionable backlog — the query-oriented counterpart to `next`, reusing the dashboard's suggested-next logic; `stale` (read-only) lists items needing attention (stalled, blocked, overdue, or out of the active workflow). Both HTTP dispatchers already existed (`dispatch_http_project.go`); this just wires them onto the catalog. `pad project reconcile` stays CLI-only (shells out to `gh` for live PR state — a local-git dependency MCP agents lack). Pure addition of two read-only actions — existing actions unchanged; backwards-compatible for v0.12 consumers that don't enumerate the new actions. **v0.12** adds an `activity` action to `pad_project`, mirroring the new CLI `pad project activity [--limit N] [--actor user|agent] [--since DATE]` (TASK-2018) — the non-streaming, bounded query counterpart to the CLI-only `pad project watch` SSE stream. Read-only snapshot of the workspace's enriched activity feed (item refs, titles, field-level change details) backed by the existing `GET /workspaces/{ws}/activity` endpoint (previously web-UI-only, now extended with a server-side `since` date filter so `limit`/`actor`/`since` behave identically across CLI, stdio MCP, and cloud HTTP), so agents can catch up on what other agents/users did since they last worked. Adds `actor` + `limit` params to the `pad_project` vocabulary (`since` already existed for changelog); pure addition — existing actions unchanged; backwards-compatible for v0.11 consumers that don't enumerate the new action. **v0.11** adds the read-only `pad_attachment` tool (the tenth resource × action tool) with `list` + `show` actions, mirroring the CLI `pad attachment list` / `pad attachment show` (TASK-2017): `list` enumerates a workspace's attachments (optional filters: item / category / collection / attached / unattached / sort / limit / offset); `show` returns one attachment's metadata (MIME, size, filename, ETag, last-modified) via a HEAD request without transferring bytes. Both HTTP dispatchers already existed (`dispatch_http_attachments.go`); this just wires them onto the catalog. Upload / download / view stay CLI-only (filesystem-bound, excluded per the catalog's exclusion rules). Pure addition — existing tools/actions unchanged; backwards-compatible for v0.10 consumers that don't enumerate the new tool. The base64 image RESOURCE for multimodal agents (`pad://workspace/{ws}/attachments/{id}`) shipped later in TASK-2077 (PR #930) as a bounded, image-only resource; TASK-2101 brought it — and the full read-only resource set — to the remote /mcp transport via the in-process `HTTPResourceFetcher`, so resources are no longer local-stdio-only. **v0.10** enforces the draft-playbook gate server-side: `pad_playbook.run` (and the underlying `POST /playbooks/{ref}/run`) now refuses a playbook whose `status` isn't `active` with a structured `playbook_not_active` error, adds an `allow_draft` boolean param (bareword `--allow-draft` on the CLI) as the escape hatch, and echoes the playbook `status` on both the `run` and `get` responses (BUG-2020). **v0.9** makes `pad_item.list` summary-shaped by default (drops item `content`, adds a default result limit of 50 / hard max 300 on MCP; CLI `--full` restores the complete shape) — a behavior change to the tool's return shape, hence the bump, though tool names, action enums, and parameter shapes are unchanged (TASK-2000). **v0.8** adds `restore` + `deleted` actions to `pad_workspace`, mirroring the CLI `pad workspace restore` / `pad workspace deleted` (TASK-1972): `deleted` (read-only) lists the caller's soft-deleted workspaces still inside the 30-day restore window; `restore` (mutating, not destructive, owner-only) un-soft-deletes a workspace by `slug` while it's still restorable. Both reuse the existing `slug` param — no new params; pure addition. **v0.7** adds `export` + `import` actions to `pad_item`, mirroring the CLI `pad item export` / `pad item import` (covers playbooks AND conventions). `export` (read-only) takes `ref` and returns the portable artifact text — it forces the CLI's stdout sink (`-o -`) so the bytes come back as the result instead of a file. `import` (mutating, not destructive) takes a new `artifact` param (the full artifact text) and returns `{ref, slug, warnings}`; the ExecDispatcher can't pipe stdin, so it spills the artifact to a temp file and dispatches `item import `. v0.6 added the `pad_item.backlinks` action; v0.5 added `pad_library`. v0.3 (PLAN-1377 / TASK-1380) introduced `pad_meta.action: bootstrap`, `pad_set_workspace`'s embedded-bootstrap response, and the `pad://workspace/{ws}/bootstrap` resource. **v0.4 (PLAN-1410)** is a comprehensive bootstrap-payload trim — same tool catalog, slimmer JSON shape inside bootstrap responses: `BootstrapCollection` projection drops `id`/`workspace_id`/timestamps/`settings` and emits `schema` as a nested object; `BootstrapRole` projection drops UUIDs/timestamps/`tools`; convention `slug` dropped; top-level `recent_activity` (a duplicate of `dashboard.recent_activity`) removed; new `BootstrapDashboard` wrapper caps five sub-arrays (`attention`, `recent_activity`, `active_items`, `active_plans`, `by_role`) at 5 entries each with parallel `*_overflow_count` fields; redundant schema labels omitted when `label == TitleCase(key)`. Cumulative size reduction: ~40% on a representative workspace, ~54% on the fixture (see PLAN-1410's Result section for per-section deltas). Compatibility: most changes are subtractive (dropped fields) or additive (overflow counts), but **one type change is breaking**: `collections[].schema` went from a JSON-encoded string to a nested JSON object — clients that JSON.parse()'d the string need to consume it directly as an object now. The dropped fields (UUIDs, timestamps, settings, duplicate `recent_activity`, convention `slug`) have canonical alternatives (slugs for addressing; `pad collection list` / `pad role list` for the full models when needed). +- `ToolSurfaceVersion` (currently `"0.60"`) — the MCP tool catalog contract. Bump when tool names, action enums, or parameter shapes change incompatibly. **v0.60** (TASK-2198 U4) makes `content_state: "applied_pending_flush"` clear ON ITS OWN on every door that carries it: after the item's collab room closes (or a one-minute sweep finds it dormant) the server rebuilds the body from the op-log and stores it, moving `seq` (a held `expected_seq` then answers `update_conflict` instead of `content_pending_flush`) and adding a `system` / `recovery` version. BEHAVIOR bump on the v0.59 / v0.39 grounds; no name, enum or param changed. Not a guarantee (failures, set-aside rows, `PAD_MATERIALIZE=off`), so re-read later, never re-send, still stands. **v0.59** (BUG-3252, tombstone ruled over cascade) makes `pad_item.action=delete-comment` on a comment with replies SUCCEED, leaving a tombstone (body emptied, author and timestamps kept, additive `deleted: true`) its replies stay under; it is removed when its last reply is deleted. `edit-comment` on a tombstone answers 409 `comment_deleted` on every transport. BEHAVIOR bump on the v0.58 grounds; `comment_has_replies` stays classified for older servers. **v0.58** (BUG-3252) makes `pad_item.action=delete-comment` on a comment that still has replies answer 409 `comment_has_replies` (details `comment_id`, `reply_count`) on every transport instead of `server_error`: `comments.parent_id` has no ON DELETE, so the delete failed the FK inside the store. Nothing is deleted either way. Tombstone was then ruled over cascade; v0.59 replaced this refusal with it. BEHAVIOR bump on the v0.45 grounds. **v0.57** (TASK-2695) adds `pad_item` `edit-comment` (ref, comment_id, message) and `delete-comment` (ref, comment_id) on all three transports, backed by the item-scoped routes `PATCH/DELETE /workspaces/{ws}/items/{ref}/comments/{id}`, which answer 404 unless the comment is on `ref` and otherwise run the workspace routes' handlers (edit author-only, delete item-edit). CLI: `pad item comment-edit` / `comment-delete`, refused against a server without the `item_scoped_comment_writes` capability. Every serialised comment gains a derived `edited` bool (updated_at > created_at), and `pad item comments` prints each comment's id. ADDITIVE. **v0.56** (BUG-3244) gives `content_state` a second value, `superseded_set_aside`, on every MCP door that carries it (`pad_item` get / full list, `pad_playbook` run, the item resource's line above the body): edits an editor schema-version rebuild set aside, which opening the item will NOT restore, so the resource line and the `content_pending_flush` hint (chosen by `details.set_aside_rows`) name `pad item set-aside` and `overwrite_pending_edits` instead. ADDITIVE bump: no name, enum or param changed, but an agent branching on the one-word vocabulary meets a word whose remedy differs. Reading or discarding the rows alone is not on the catalog. **v0.55** (BUG-2659) sends `pad_item.action=search`'s `collection` to /search as typed, and /search resolves it exact-first per workspace in scope (the item routes' resolver). In a workspace holding both `task` and `tasks`, `collection: "task"` now answers from `task` instead of `tasks`; every other workspace is unchanged, because the shorthand still resolves server-side. Stdio reaches it through the CLI, which sends the name as typed only to a server advertising `search_collection_resolution`. BEHAVIOR bump on the v0.36 / v0.40 grounds. **v0.54** (BUG-3217) keeps a `fields` number's literal through MCP on both transports (mcp-go's argument decode rounded anything above 2^53, and stdio's `--field` stringify carried the rounded value), and REFUSES a `fields` number beside a `field` entry for the same key when they differ as exact numbers — two values that differ only above 2^53 used to collapse into one rounded write; a matching pair (3.0 beside n=3) still collapses. BEHAVIOR bump on the v0.48 grounds. **v0.53** (BUG-2819) adds `filename_source` (caller / normalised / substituted / derived / unknown) to every `pad_attachment.action=list` row on both transports, because a stored filename cannot say whether the caller sent it or the server substituted it. ADDITIVE, the v0.28 / v0.13 disposition; `show` does not carry it, and `mcp_audit_log.tool_name_source` reaches no catalog action. Rows written before it read `unknown`, never `caller`. **v0.52** (BUG-2367 item 4) refuses a move or copy (every migrate door) that would change an item between open, done and abandoned unless the caller names the destination done field. Only the STATE is compared, so two values that both mean done carry. Move answers 400 `state_change_requires_value` with details `{field, options, from, to}`; bulk move puts it in a `failed[]` row; the copy answers 400 `validation_error` with the same sentence; the preflight emits a needs_value row with reason `state_change`, which the web dialog picker renders. A destination with no done select field is exempt. BEHAVIOR bump. **v0.51** (BUG-2367) stops every migrate door (single move, bulk move, cross-workspace copy and its preflight) carrying a value into a field the destination marks `computed` (preflight drop reason `target_computed`) or a value that collides on a destination `unique_scope` field. Provenance decides, the TASK-2878 rule: a CARRIED collision is dropped and reported (`not_unique`), a supplied override or injected default that collides is refused 409 `conflict`, and a required field emptied by the drop is refused as required, which is the existing needs_value route. Before it, single move answered 500 on the indexed `invocation_slug`, bulk move leaked the SQL error, and an unindexed unique field stored a duplicate. Reporting is additive and omitempty: `warnings.not_unique` [{key, value, holder?, message}] on the move response, bulk `updated[]` rows and the copy response, and a `detail` sentence on the preflight's dropped row. `holder` is present only when the caller may see that item, so the drop is not an existence oracle. BEHAVIOR bump on the v0.43 / v0.29 grounds. **v0.50** (BUG-3014) adds an ADDITIVE `omitempty` `stored_as_text: true` to a `relation_targets` entry whose stored value is not UUID-shaped (a title a bundle import carried verbatim, or legacy free text). Such a value was never an id and used to hydrate as a plain id-only entry, which reads as "gone or hidden"; an id-only entry without the flag keeps that meaning, and a UUID-shaped value, resolved or not, is byte-identical to v0.49. The predicate is syntactic, decided from the stored bytes and never from a lookup, so it discloses nothing, and deliberately not coupled to the resolver ladder. The CLI prints `"" (text, not a reference)` and the web chip says the same. **v0.49** (BUG-2410) makes `pad_project.action=report` with no `collections` param leave out SYSTEM collections (`Collection.IsSystem`: Conventions and Playbooks), which were counted in Insights' created, WIP and status totals; naming one explicitly still includes it. BEHAVIOR bump on the v0.40 / v0.42 grounds: a default call's content changed for every caller (web Insights, print, `pad project report`, both MCP transports, all through the one store rule). **v0.48** (BUG-3163) makes `pad_item.action=create` REFUSE a `field` / `fields` entry naming any reserved metadata key (`implementation_notes`, `decision_log`, `github_pr`, `convention`) on every transport, with 400 `validation_error`, creating nothing: create was the last door that stored one, as a string no extractor reads. Library activation, its one system writer, moves to an additive typed `convention` create member (not on the catalog; `pad_library.activate` is unchanged), and the CLI verifies from the stored fields that it landed, because an older server ignores the member and answers 201. BEHAVIOR bump on the v0.47 grounds. **v0.47** (BUG-2696) makes `pad_item.action=update` REFUSE a field setter naming `github_pr` on every transport (400 `validation_error` naming `pad github link` / `unlink`), closing the exemption v0.23 left: that door stored the PR as an unreadable string, so no link ever rendered. The writer is a new additive, typed update member (`github_pr` object + `clear_github_pr`) used by `pad github link` / `unlink` / `project reconcile`, deliberately NOT on the catalog (a remote PR-link action waits on the catalog-trim decision). BEHAVIOR bump; item create is BUG-3163. **v0.46** (BUG-3156) makes `pad_item.action=bulk-update` over stdio and remote REFUSE, per item, a status or priority change on an item whose collection does not declare that field, writing nothing for that item: the answer WebMCP has given since v0.44, so the three transports agree. Rows keep each transport's shape (stdio `failed[]` with `validation_error`, remote `results[]` with `validation_failed` and the server message in `hint`). The mechanism is the update door's new `refuse_undeclared_fields` member, which both bulk-update transports set; `pad_item.update` does not, and still accepts-and-warns. A server that predates the member ignores it, which is the old accept-and-warn. BEHAVIOR bump on the v0.44 / v0.43 grounds. **v0.45** (BUG-2829) reports an HTTP 413 as `too_large` on BOTH transports, with the server's own code in `details.reason`, where it used to be `server_error`: three deliberate caps are reachable from the catalog (a title rename's link cascade, an oversized outbox event, an oversized artifact import), and the old code read as transient and invited a retry that fails identically. Remote maps 413 in `classifyHTTPStatusKind`; stdio gets it from a structured marker the CLI root writes for any 413, and `too_large` joins `allowedStructuredErrorCodes`, which a test now proves contains every code any CLI marker writer emits. BEHAVIOR bump on the v0.42 grounds; no name, enum or param shape changed. **v0.44** (BUG-3154) makes `pad_item.action=bulk-update` over the WebMCP browser transport REFUSE, per item, a status or priority change on an item whose collection does not declare that field: the item lands in the bulk response's `failed` array with `validation_error` naming the collection and field, nothing is written, and the rest of the call applies. The door is `POST /items/bulk` (`set-priority`, and `move` with a `status`, with or without a `collection`, where the target's schema decides), where the request carries an operation and a value and the SERVER chooses the key; it used to store an orphan field and report the item updated. BEHAVIOR bump on the v0.43 / v0.40 grounds. The transports disagreed after it, stated rather than implied: stdio and remote issued a per-item PATCH, the single-item update door, which ACCEPTED the key with `warnings.undeclared_fields` (v0.27), until v0.46 converged them. **v0.43** (BUG-2379) makes `pad_item.action=move` REFUSE a field override naming a field the destination schema does not declare, with 400 `malformed_override` — the same check, code and message the cross-workspace copy already used. BEHAVIOR bump on the v0.40 / v0.29 grounds: a write door refuses a call it used to accept, where it used to store the key as an orphan field no schema renders. Create and update still ACCEPT undeclared keys with `warnings.undeclared_fields` (v0.27), because callers round-trip the whole fields blob; an override is a fresh assertion about the destination schema, not round-tripped state. **v0.42** (BUG-3142 + BUG-3147) changes the error CODE a LOCAL STDIO caller receives for two failure families that used to arrive as retryable `server_error`: an argv refusal (cobra/pflag: unknown flag, arg count, …) is now `validation_failed` with its message kept, and a 429 on any command is now `rate_limited` with `details.retry_after_seconds`, via the structured marker line the CLI writes at its root. It also stops the refusal that caused most of the first family: `BuildCLIArgs` now emits flags first and the positionals behind `--`, so free text starting with `-` (`item search "-ship it"`) is no longer parsed as a flag. BEHAVIOR bump; no name, enum or param shape changed, and the remote transport is untouched. **v0.41** (PLAN-3114 U5 / TASK-3120) adds a `match` action to `pad_playbook`: given free `text`, a Choice question over the workspace's ACTIVE playbooks plus a reserved `"none"` option returns the best match's ref (or `"none"`), confidence, per-option probabilities and the model. No provider configured → 404 `decision_provider_unavailable`; a provider error or an out-of-set answer → 502 `decision_provider_error`. Pure ADDITIVE bump (new action + param, nothing existing moved), read-only and side-effect-free. **v0.40** (BUG-3028) gives a scalar `relation` ONE stored form for "no target", the key ABSENT, converging on multi_relation's rule. A blank (`""` or whitespace) a write SETS — create, `fields_patch`, a changed value in full `fields`, a move/copy override — is normalised to absent before validation, so a `required` relation is refused as required (a patch blanking it now gets the answer deleting it always got). A legacy blank a write only CARRIES — an untouched key under `fields_patch` (normalised by the store under its write lock, via `ItemUpdate.BlankRelationKeys`), an unchanged value in full `fields`, move, copy, artifact import — is normalised after validation and never refused. An empty list filter on a scalar relation key of a collection-scoped list (`owner=`) matches all three spellings. BEHAVIOR bump; no shape change. Workspace import (`store.ImportWorkspace`) still stores blobs verbatim; its legacy blanks normalise on the next write. **v0.39** (BUG-3133) refuses `pad_item.update` that sets content WITH a version token (`expected_seq` / `expected_updated_at`) while the item's op-log holds content-bearing rows above its flush watermark, with 409 `content_pending_flush` (details `ref`, `pending_rows`). BEHAVIOR bump on the v0.35 / v0.36 grounds, carrying one additive param: `overwrite_pending_edits` lifts it, on the API, the CLI (`--overwrite-pending-edits`; `pad item edit --force` sets it on the save) and MCP alike, because an agent must be able to clear the refusal without a browser. The token guards the ROW and a tab's unflushed typing is not in it, so the write used to replace those edits — through the applier's setContent with a tab open, and by the direct path's op-log prune, which DELETES them, with none. Its own code rather than `update_conflict` because re-reading returns the same row and seq. A write without a token, without content, or to an item with nothing pending is unchanged, except that a direct write now reports `warnings.pruned_pending_edits` (additive, omitempty) when its prune deleted such rows. The check runs in the write's own tx; on the direct path it is under the item lock and appendMu and exact, on the applier path a tab can still type in the one server hop between the check and setContent. (v0.37 and v0.38 — the `agent` projection on `pad_item.get` and `claim` / `release` — are in `internal/mcp/version.go`; this paragraph had stopped at v0.36.) **v0.36** (BUG-3082) stops a REF-SHAPED `relation` value falling back to matching by item NUMBER when its prefix names a collection that is LIVE in the workspace; such a value is now `not_found`. BEHAVIOR bump on the v0.35 / v0.30 / v0.29 / v0.16 grounds — no tool name, action enum or param shape changed, but a write door refuses a value it used to accept. It finishes v0.29: that entry closed free-text corruption ("red" resolving to whatever is slugged `red` today) and left the same corruption open for anything shaped like a REF, which is the spelling the docs tell callers to prefer. Item numbers are workspace-unique and sequential ACROSS collections, so `CONVE-1` and `SECRE-1` are never both real — the fallback dropped the prefix, matched on the number alone, and landed on an item that was usually inside the field's declared collection, so the wrong-collection check above it passed and NOTHING was raised: a caller wrote `CONVE-1`, the item stored a different item's UUID, and the write answered 201 with the substitution visible only in `relation_targets`. The predicate is about the PREFIX rather than the item because a collection RENAME changes its prefix and a relation already written as the old ref must keep resolving (BUG-2873) — that ref's prefix names nothing afterwards, so the fallback still fires for it, while a prefix that IS live and simply holds no item at that number is the caller naming a real collection that does not contain what they said. A SOFT-DELETED collection counts as absent IN THAT PREDICATE, for the same reason a renamed one does — a claim about the fallback only: the exact-prefix lookup ahead of it never filtered on the collection’s own `deleted_at`, so an item whose collection was soft-deleted still matches THERE and never reaches the predicate. That is left alone rather than made consistent, because it cannot retarget — the downstream collection check refuses it. RESIDUAL, stated rather than implied and pinned by a test: a ref pasted from ANOTHER workspace whose prefix also names no collection here still resolves by number, because nothing in the value distinguishes it from a rename's leftover. No escape hatch, for v0.29's reason. Out of scope and unchanged: `GetItemByRef`, the navigational read path, where landing on the moved item is the helpful answer. **v0.35** (BUG-3079) makes an injected schema DEFAULT take the same `validateFieldType` check a caller-supplied value takes; one that fails is DISCARDED and named in `warnings.dropped_fields` instead of being stored. BEHAVIOR bump on the v0.30 / v0.29 / v0.16 grounds — no tool name, action enum or param shape changed, but a write door stores something different from what it stored before. Unlike v0.29, which turned an accepted value into a refusal, this turns a stored value into a reported drop: a create that used to land WITH a bad field still lands, without it. What it closes is two doors disagreeing about one value, decided only by who put it there. It also widens `dropped_fields` from the single relation case v0.29 introduced to a field of ANY type. No escape hatch, for v0.29's reason. ONE exception, which is the rule finishing its sentence rather than a carve-out: dropping leaves a REQUIRED field absent, which is exactly what the required check reports, so such a write is refused — naming the default as the cause, because a bare "field is required" points its reader at a request that never mentioned the field. **v0.34** (BUG-3037) adds an `expected_seq` param to `pad_item.action=update` — the STRONG optimistic-concurrency token, preferred over `expected_updated_at`. ADDITIVE (the v0.13 / v0.11 / v0.8 disposition): nothing existing moved and `expected_updated_at` still works. It is needed rather than nice because `updated_at` is stored at ONE-SECOND resolution, so two writes to a row inside one second both match the token and NEITHER conflicts — the loser is accepted and silently overwrites the winner, both callers seeing success. That is the shape of agent traffic, so the token agents were offered could not refuse the race it exists to refuse. `seq` is bumped on every mutation of the row under the write lock. Raising `updated_at`'s resolution instead was measured and REJECTED: the column is TEXT on both dialects and compared LEXICALLY in SQL (the since-cursor read plus eight `ORDER BY` sites), and Go's `RFC3339Nano` omits trailing zeros, so a sub-second value sorts BEFORE a whole-second value of the same second and the cursor would start skipping rows. The read side is what makes the token reachable: `seq` now serialises without `omitempty` on `models.Item` AND on the item SUMMARY shape (`cli.ItemSummary`), which is what `pad_item.list` returns by default since v0.9 — a token absent from the shape a caller reads is a token that caller cannot send. A value below 1 is a 400, not a 409: the server never issues one, so a 409 would read as contention that never clears. **v0.30** (BUG-2870) is a BEHAVIOR bump on the v0.29/v0.27/v0.26/v0.16/v0.10/v0.9 grounds — no tool name, action enum or param shape changed. **Every door refuses a padded KEY now**, and each was accepting it differently: /mcp trimmed it and wrote the declared field, the CLI stored a ghost field beside it — so both refuse something they used to accept. What is /mcp-only is the VALUE half, which it used to trim and type and now passes through to the same validation the CLI has always applied. A caller writing canonical entries sees no difference at either door. A second, separately-noticeable fix rides with it: `detectFieldConflicts` swallowed `parseFieldArray`'s refusal as "the caller owns this error surface", which held only while the sole possible error was a shape error — `reshapeItemFields` returns early with no `fields` object, so on the no-`fields` path (this bug's own path) four existing refusals were landing as successes. No escape hatch, for v0.29's reason: there is no legitimate call this refuses, only calls whose two readings a door used to choose between silently. **v0.29** (PLAN-2857 U1 / TASK-2878) is a BEHAVIOR bump on the v0.27/v0.26/v0.16/v0.10/v0.9 grounds (NOT v0.28's, which was purely additive) — no tool name, action enum, or param shape changed, but a `relation` field value must now name a live item in the collection that field declares, so every write door refuses values it used to store. Refused: names nothing (`not_found`), names an item in the wrong collection, the field declares no target collection (`target_missing`), or the value is a SLUG — a deliberate divergence from `ResolveItem`'s UUID→ref→slug ladder, since a slug is neither an ID nor stable and free text like "red" resolving to whatever is slugged `red` today is exactly the corruption this closes. Ordinary `validation_error`, no new code and no new details key, because stdio classifies errors by matching CLI stderr prose. A CARRIED value — already on the item, asserted by nobody — is never refused, since refusing would make every legacy item un-updatable, un-movable and un-copyable: within a workspace it resolves and SURVIVES, across a workspace boundary it is dropped without a lookup (a source-workspace id cannot mean anything in the destination) and reported through the same `warnings.dropped_fields` channel BUG-2674 established. No escape hatch, deliberately: unlike v0.10's `allow_draft` there is no legitimate call this refuses, and the case with a real claim to leniency is already exempt by provenance rather than by a flag. **v0.28** (IDEA-2641 / GitHub #1010) adds two ADDITIVE `pad_item` actions and two optional params: `remind` arms a one-shot reminder at an RFC3339 `remind_at` INSTANT, and `ack-reminder` acknowledges a fired one by `reminder_id`. Purely additive — nothing existing moved, and a v0.27 consumer that enumerates neither action is unaffected; same disposition as v0.13 / v0.11 / v0.8, which likewise wired existing CLI verbs onto the catalog. Agents already RECEIVED reminders (the poll surface is `pad_project.next` / `ready`, long exposed); what was missing is the half where an agent that defers work can say when it wants to be asked again. `remind_at` REFUSES a bare date rather than reading it as midnight — the `date` schema type accepts `YYYY-MM-DD` so a caller will try it, but a bare date names a 24-hour span and choosing an hour inside it would fire at a time nobody picked. Re-arm and disarm stay CLI-only: both address a reminder by an id the agent would have to list first, and no listing action exists on this surface yet — a door with no handle. **v0.27** (BUG-2850) types field values SERVER-SIDE at all eight validate sites, carries the `fields` object to the remote door with its JSON types intact, accepts undeclared keys while NAMING them in `warnings.undeclared_fields`, and replaces five accreted conflict guards with one canonical pass; the merge refuses several ambiguities it used to resolve silently. (This entry was missing from CLAUDE.md — the 0.27 unit swept `instructions.md` and `README.md` and not this file.) **v0.27** (BUG-2850) types field values server-side at all eight `Validate*` call sites so a declared number/json field is writable from the remote transport at all, carries the `fields` OBJECT with its JSON types intact, names undeclared keys back in `warnings.undeclared_fields` (accepted rather than refused — a census of 1012 items found 14 such keys across 168 live values, so refusing would have broken read-modify-write on items nobody had edited wrongly), and replaces the accreted per-site conflict guards with ONE check over a canonical view of every source; that check refuses ambiguities v0.26 resolved silently, chiefly two names for one target in a single call (`parent`/`plan`, `assign`/`assigned_user_id`, `role`/`agent_role_id`), refused even when the values match because the names address one thing through incomparable vocabularies and the two doors resolved them differently. **v0.26** (IDEA-2756) is a BEHAVIOR bump on the v0.9/v0.16/v0.25 grounds — no tool name, action enum, or param shape changed, but `pad_workspace.create` now refuses a call it used to permit. Closest precedent is v0.10, which likewise turned a server-side gate into a structured refusal; unlike v0.10 there is no `allow_draft`-style override, because the gate encodes a decision the USER made at consent time and a bypass param would be the app overriding its own grant. `POST /workspaces/import` is gated by the same shared helper (import mints a workspace through `store.ImportWorkspace`), though it has no MCP action today. **v0.25** (TASK-2657 / BUG-2702) resolves `pad_library.activate`'s destination collection from the target's declared artifact kind rather than the literal `conventions` / `playbooks` slugs, so activating into a workspace that renamed either collection lands correctly; a lookup ERROR is surfaced rather than silently falling back. **v0.24** (#1066) adds the `fields` OBJECT param to `pad_item` create/update — an alias merging into the same path as `field`/the dedicated params, so the shape reads return is finally a valid write shape; the same key supplied twice with conflicting values is REFUSED (refuse-on-ambiguity, the v0.18/v0.19 disposition), equal duplicates collapse to one write, and non-writer actions refuse a `fields` param loudly. It also makes input validation STRICT for every catalog tool: undeclared top-level keys are rejected with a structured error naming them, instead of being accepted and silently dropped by `BuildCLIArgs` — which is the mechanism that made the `fields` object a session-scoped silent no-op in the first place. Compat carve-out: `pad_item`'s v0.16 `assigned_user_id` / `agent_role_id` remote-transport clear form stays accepted (documented, undeprecated, deliberately never schema-declared). The strict half changes behaviour for inputs that previously "succeeded", but that reliance was indistinguishable from a caller bug (the key never did anything), so the break is the fix; one bump covers both halves. **v0.23** (BUG-2627 part 2 + BUG-2675) refuses system-metadata keys through `fields_patch` on all three doors at once, `github_pr` exempt on update (move/copy still refuse it), and adds the retry-hostile `stored_state_unreadable` code. **v0.22** (BUG-2674) stops `pad_item.action=move` destroying system metadata and refuses `field` setters naming the reserved keys there. **v0.21** bounds `pad_item.action=history` (BUG-2608): the `limit` param now covers it, default 50 / max 300, applied in the CATALOG action so it reaches both transports (HTTP reads the input; stdio gets the CLI's new `--limit` via BuildCLIArgs). The window is the NEWEST N and there is deliberately no `offset` — versions are reverse patches, so only a newest-end window is cheap to reconstruct. Additive param bump; a v0.20 consumer sending no limit now receives the newest 50 rather than every version, which is the fix. Summary mode additionally asks the server to skip patch resolution rather than resolving bodies the dispatcher discards. **v0.19** adds a `clear_parent` boolean to `pad_item` (BUG-2078) — an ADDITIVE param bump, same grounds as v0.18; nothing existing changed shape. The server has supported clearing a parent since BUG-2013 (`extractParentLink` treats a present-but-empty `parent` key in `fields_patch` as detach), but neither client surface could reach it — `--parent ""` was a silent no-op on the CLI and the MCP `parent` param has the same "empty means not provided" convention every other declared string on the tool has. Boolean rather than overloading the empty string, same two reasons as v0.18: keeps that invariant intact for every other param, and only a boolean reaches LOCAL STDIO via `BuildCLIArgs`, mapping to a new `--clear-parent` bareword flag exactly as `clear_assigned_user` maps to `--clear-assigned-user`. Update-only, same asymmetry as v0.18. A simultaneous `parent` + `clear_parent` — including via `field: ["parent=..."]` or the `plan` alias `extractParentLink` also accepts — is REFUSED on both transports, not silently resolved (codex round 1). Also refused, not silently applied: `clear_parent` against a collection whose schema declares its own `parent`/`plan` field — `extractParentLink` skips hierarchy handling entirely for a schema-shadowed key and lets it fall through as an ordinary field write, so the wire shape `{"parent":""}` can no longer distinguish clear-hierarchy intent from a legitimate blank-a-real-field write once it reaches the server; the ambiguity is created at the client surface that accepted `clear_parent`, so that surface refuses rather than guessing (codex round 2). **v0.18** adds `clear_assigned_user` / `clear_agent_role` booleans to `pad_item` (IDEA-2584) — an ADDITIVE param bump (v0.5/v0.6 precedent); nothing existing changed shape and v0.16/v0.17's empty-string forms still work, undeprecated. v0.16 and v0.17 made the clear WORK; nothing advertised it, because the params that do it were never in the catalog, so an agent reading the schema reached for `assign: ""` (a no-op, and it stays one). Booleans rather than declaring the string params, for two reasons: an empty DECLARED string is inert everywhere else on the tool, so giving one a destructive meaning would let a param-padding client silently unassign everything; and only a boolean can reach LOCAL STDIO, since `BuildCLIArgs` emits the CLI's real flags and a param with no flag behind it is dropped — these map to new `--clear-assigned-user` / `--clear-agent-role` bareword flags, exactly as `allow_draft` maps to `--allow-draft`. Update-only, deliberately asymmetric with create (clearing at create has no honest behaviour but a no-op; a test fails if someone adds them there). Server-side it is wiring, not new semantics: `models.ItemUpdate.ClearAssignedUser`/`ClearAgentRole` already existed with store support since BUG-2566. **v0.17** closes the transport gap v0.16 documented: local stdio MCP shells out to the CLI, which wrote `--field assigned_user_id=` into the item's FIELDS BLOB while the column stayed stale and then printed "Updated TASK-9". `cmd/pad/cmd_item.go` now lifts `columnFieldKeys` onto the columns on create AND update, mirroring `liftFieldsToColumns` and its INVARIANT. Two compat changes, ruled separately: non-empty values move to the column and stop writing the blob key (relying on the old behaviour is relying on a shadowing defect), and empty values clear (falls out of the lift, inherits BUG-2566). Existing stray blob keys are left alone — the fix stops minting new ones. Another behaviour-only bump (BUG-2583). **v0.16** lets an MCP agent UNASSIGN an item over the REMOTE transport (TASK-2571). No tool/action/param shape changed — this is a BEHAVIOR bump on the same grounds as v0.9: an empty-string `assigned_user_id` / `agent_role_id`, passed at the top level or as `field: ["assigned_user_id="]`, was silently dropped by two dispatch-path filters (`mapItemUpdate`, `liftFieldsToColumns`) and is now forwarded as a clear-to-NULL. The store has had defined clear semantics for exactly these two columns since BUG-2566 and HTTP inherited them, so this is uniformity restoration — MCP was the only surface with no way to unassign. Compat posture accepted deliberately: today's `""` senders get a no-op, and a no-op is the surprising reading. The empty-string filter on `tags` at the same call site STAYS (codex #547 r3 P2) — `tags: ""` is a corrupt JSONB/TEXT write, not a clear; same-looking guard, opposite justification. `clear_assigned_user` / `clear_agent_role` schema flags (option (b)) deliberately skipped as additive sugar, though codex review reopened the case — the catalog exposes `assign` / `role`, NOT the ID params, so an agent reading the schema still can't discover the clear (IDEA-2584); an empty `assign` is deliberately left inert because every other schema-declared string on that mapper treats empty as not-provided. **Transport scope:** v0.16 fixed the REMOTE /mcp transport only; v0.17 (BUG-2583) closed the local-stdio half at the CLI. **v0.15** adds the `unparented` boolean to `pad_item.list`, mutually exclusive with `parent`, for structural loose-item filtering (TASK-2096). **v0.14** added a `history` action to `pad_item` (read-only item version history — newest-first metadata; content body omitted for token thrift) and an `expected_updated_at` param for optimistic concurrency on `update` (round-trip the `updated_at` you last read; a stale value fails with a structured 409 `code=update_conflict`). The `update` action's field writes are now a server-side field-level MERGE (only the keys you set change) rather than a full-blob replace, closing the concurrent-update lost-write race (IDEA-1480 / TASK-2022) — pure addition to the action enum + param vocabulary; existing `pad_item` actions/params are unchanged and backwards-compatible. **v0.13** adds `ready` + `stale` actions to `pad_project`, mirroring the existing CLI `pad project ready` / `pad project stale` (TASK-2019): `ready` (read-only) returns the actionable backlog — the query-oriented counterpart to `next`, reusing the dashboard's suggested-next logic; `stale` (read-only) lists items needing attention (stalled, blocked, overdue, or out of the active workflow). Both HTTP dispatchers already existed (`dispatch_http_project.go`); this just wires them onto the catalog. `pad project reconcile` stays CLI-only (shells out to `gh` for live PR state — a local-git dependency MCP agents lack). Pure addition of two read-only actions — existing actions unchanged; backwards-compatible for v0.12 consumers that don't enumerate the new actions. **v0.12** adds an `activity` action to `pad_project`, mirroring the new CLI `pad project activity [--limit N] [--actor user|agent] [--since DATE]` (TASK-2018) — the non-streaming, bounded query counterpart to the CLI-only `pad project watch` SSE stream. Read-only snapshot of the workspace's enriched activity feed (item refs, titles, field-level change details) backed by the existing `GET /workspaces/{ws}/activity` endpoint (previously web-UI-only, now extended with a server-side `since` date filter so `limit`/`actor`/`since` behave identically across CLI, stdio MCP, and cloud HTTP), so agents can catch up on what other agents/users did since they last worked. Adds `actor` + `limit` params to the `pad_project` vocabulary (`since` already existed for changelog); pure addition — existing actions unchanged; backwards-compatible for v0.11 consumers that don't enumerate the new action. **v0.11** adds the read-only `pad_attachment` tool (the tenth resource × action tool) with `list` + `show` actions, mirroring the CLI `pad attachment list` / `pad attachment show` (TASK-2017): `list` enumerates a workspace's attachments (optional filters: item / category / collection / attached / unattached / sort / limit / offset); `show` returns one attachment's metadata (MIME, size, filename, ETag, last-modified) via a HEAD request without transferring bytes. Both HTTP dispatchers already existed (`dispatch_http_attachments.go`); this just wires them onto the catalog. Upload / download / view stay CLI-only (filesystem-bound, excluded per the catalog's exclusion rules). Pure addition — existing tools/actions unchanged; backwards-compatible for v0.10 consumers that don't enumerate the new tool. The base64 image RESOURCE for multimodal agents (`pad://workspace/{ws}/attachments/{id}`) shipped later in TASK-2077 (PR #930) as a bounded, image-only resource; TASK-2101 brought it — and the full read-only resource set — to the remote /mcp transport via the in-process `HTTPResourceFetcher`, so resources are no longer local-stdio-only. **v0.10** enforces the draft-playbook gate server-side: `pad_playbook.run` (and the underlying `POST /playbooks/{ref}/run`) now refuses a playbook whose `status` isn't `active` with a structured `playbook_not_active` error, adds an `allow_draft` boolean param (bareword `--allow-draft` on the CLI) as the escape hatch, and echoes the playbook `status` on both the `run` and `get` responses (BUG-2020). **v0.9** makes `pad_item.list` summary-shaped by default (drops item `content`, adds a default result limit of 50 / hard max 300 on MCP; CLI `--full` restores the complete shape) — a behavior change to the tool's return shape, hence the bump, though tool names, action enums, and parameter shapes are unchanged (TASK-2000). **v0.8** adds `restore` + `deleted` actions to `pad_workspace`, mirroring the CLI `pad workspace restore` / `pad workspace deleted` (TASK-1972): `deleted` (read-only) lists the caller's soft-deleted workspaces still inside the 30-day restore window; `restore` (mutating, not destructive, owner-only) un-soft-deletes a workspace by `slug` while it's still restorable. Both reuse the existing `slug` param — no new params; pure addition. **v0.7** adds `export` + `import` actions to `pad_item`, mirroring the CLI `pad item export` / `pad item import` (covers playbooks AND conventions). `export` (read-only) takes `ref` and returns the portable artifact text — it forces the CLI's stdout sink (`-o -`) so the bytes come back as the result instead of a file. `import` (mutating, not destructive) takes a new `artifact` param (the full artifact text) and returns `{ref, slug, warnings}`; the ExecDispatcher can't pipe stdin, so it spills the artifact to a temp file and dispatches `item import `. v0.6 added the `pad_item.backlinks` action; v0.5 added `pad_library`. v0.3 (PLAN-1377 / TASK-1380) introduced `pad_meta.action: bootstrap`, `pad_set_workspace`'s embedded-bootstrap response, and the `pad://workspace/{ws}/bootstrap` resource. **v0.4 (PLAN-1410)** is a comprehensive bootstrap-payload trim — same tool catalog, slimmer JSON shape inside bootstrap responses: `BootstrapCollection` projection drops `id`/`workspace_id`/timestamps/`settings` and emits `schema` as a nested object; `BootstrapRole` projection drops UUIDs/timestamps/`tools`; convention `slug` dropped; top-level `recent_activity` (a duplicate of `dashboard.recent_activity`) removed; new `BootstrapDashboard` wrapper caps five sub-arrays (`attention`, `recent_activity`, `active_items`, `active_plans`, `by_role`) at 5 entries each with parallel `*_overflow_count` fields; redundant schema labels omitted when `label == TitleCase(key)`. Cumulative size reduction: ~40% on a representative workspace, ~54% on the fixture (see PLAN-1410's Result section for per-section deltas). Compatibility: most changes are subtractive (dropped fields) or additive (overflow counts), but **one type change is breaking**: `collections[].schema` went from a JSON-encoded string to a nested JSON object — clients that JSON.parse()'d the string need to consume it directly as an object now. The dropped fields (UUIDs, timestamps, settings, duplicate `recent_activity`, convention `slug`) have canonical alternatives (slugs for addressing; `pad collection list` / `pad role list` for the full models when needed). Post-0.30 without a bump (BUG-2995): a successful content write through the designated applier used to answer with the item's PREVIOUS content — on that path the markdown goes to a live tab's Y.Doc and the row write runs with `Content` nil, so the response described the item as it stood before the request. It now carries the content as SENT plus an additive `omitempty` `warnings.content_outcome: "applied_pending_flush"`. No bump, on the BUG-2304 rather than the v0.9 grounds: v0.9 bumped because list rows LOST fields consumers read, whereas here nothing is removed or retyped — what changed is the VALUE of one field on one path, which a caller does observe. The claim is that no DOCUMENTED OR SUPPORTED reliance breaks, not that none can exist: a consumer could have detected the applier path by noticing the response echoed something other than what it sent, and that stops working — but it was never documented, a genuinely lost write produced the same mismatch, and `warnings.content_outcome` answers that question properly in the same change. Reads are unchanged: during the window a `get` (or `list` with `full: true`) still answers from the row, so the warning means re-read later, not re-send. diff --git a/README.md b/README.md index 34ce70334..83ab0b8da 100644 --- a/README.md +++ b/README.md @@ -406,7 +406,7 @@ directory for `claude-code`, and an `[mcp_servers.pad]` table in project-scoped, it's install-on-request only — `--all` and `pad mcp status` cover the per-user clients (including Codex) and skip it. -**Tool catalog (v0.59)** — ten resource × action tools plus `pad_set_workspace` (eleven total), no flat verb explosion. Undeclared input keys are rejected with a structured error rather than silently dropped. `pad_item` create/update accept field values as a `fields` object (the same shape reads return) as an equivalent to the dedicated params / `field: ["key=value"]`, and its values keep their JSON types where the transport can carry them. Field values are typed against the collection schema server-side, so a declared number or json field is writable from the remote transport (which sends every value as a string). Keys the schema does not declare are stored and NAMED back in `warnings.undeclared_fields`. One key supplied through two doors is adjudicated once: differing values are refused, equal ones collapse, and two names for the same target — `parent`/`plan`, `assign`/`assigned_user_id`, `role`/`agent_role_id` — are refused even when the values match. `pad_item.get` accepts `agent: true` for the full body and work-relevant metadata without internal UUID plumbing; `pad_item.list` accepts `unparented: true` (mutually exclusive with `parent`) to select items with no parent or implements relationship, and is summary-shaped by default on both transports (`full: true` opts into complete content bodies): +**Tool catalog (v0.60)** — ten resource × action tools plus `pad_set_workspace` (eleven total), no flat verb explosion. Undeclared input keys are rejected with a structured error rather than silently dropped. `pad_item` create/update accept field values as a `fields` object (the same shape reads return) as an equivalent to the dedicated params / `field: ["key=value"]`, and its values keep their JSON types where the transport can carry them. Field values are typed against the collection schema server-side, so a declared number or json field is writable from the remote transport (which sends every value as a string). Keys the schema does not declare are stored and NAMED back in `warnings.undeclared_fields`. One key supplied through two doors is adjudicated once: differing values are refused, equal ones collapse, and two names for the same target — `parent`/`plan`, `assign`/`assigned_user_id`, `role`/`agent_role_id` — are refused even when the values match. `pad_item.get` accepts `agent: true` for the full body and work-relevant metadata without internal UUID plumbing; `pad_item.list` accepts `unparented: true` (mutually exclusive with `parent`) to select items with no parent or implements relationship, and is summary-shaped by default on both transports (`full: true` opts into complete content bodies): | Tool | Actions | |---|---| @@ -435,7 +435,7 @@ initialize handshake under `capabilities.experimental.padCmdhelp` and `pad://_meta/version`): - `cmdhelp_version: "0.1"` — CLI help-tree contract (used at dispatch time) -- `tool_surface_version: "0.59"` — MCP tool catalog contract (v0.5 added `pad_library`; v0.6 `pad_item.backlinks`; v0.7 `pad_item` `export`/`import`; v0.8 `pad_workspace` `deleted`/`restore`; v0.9 made `pad_item.list` summary-shaped by default with a default+max result cap; v0.10 enforced the draft-playbook gate server-side on `pad_playbook.run` with an `allow_draft` escape hatch; v0.11 added the read-only `pad_attachment` tool (`list`/`show`); v0.12 added `pad_project.activity` (agent-accessible non-streaming activity feed); v0.13 added `pad_project` `ready`/`stale` (agent-oriented backlog + attention queries); v0.14 added `pad_item` `history` + optimistic concurrency (TASK-2022); v0.15 added the `pad_item.list` `unparented` parameter (TASK-2096); v0.16 made an empty-string `assigned_user_id` / `agent_role_id` CLEAR the assignment instead of being silently dropped, so an agent can finally unassign an item (TASK-2571); v0.17 carried that to the LOCAL STDIO transport by teaching the CLI to lift those keys onto their columns instead of into the fields blob (BUG-2583); v0.18 added `clear_assigned_user` / `clear_agent_role` booleans — the canonical, schema-discoverable way to unassign, backed by new `--clear-assigned-user` / `--clear-agent-role` flags on `pad item update` (IDEA-2584); v0.19 added a `clear_parent` boolean — the canonical, schema-discoverable way to detach an item from its parent, backed by a new `--clear-parent` flag on `pad item update` (BUG-2078); v0.20 gave every tool an explicit annotation block derived from the catalog’s read-only knowledge — fully-read-only tools advertise `readOnlyHint: true` / `destructiveHint: false`, all-additive-write tools (`pad_workspace`, `pad_library`) drop `destructiveHint`, overwrite/delete-capable tools stay conservatively destructive, `openWorldHint: false` everywhere — replacing mcp-go’s defaults that marked every tool destructive (BUG-2302), and made `pad_item.list` summary-shaped on the remote HTTP transport too, with a declared `full` boolean as the opt-in for complete bodies on both transports (BUG-2305); v0.21 bounded `pad_item.history`, which was unbounded on every surface — `limit` now covers it (default 50, max 300, the NEWEST N; no `offset`, because reverse-patch storage makes only a newest-end window cheap), applied in the catalog action so it lands on both transports, and summary mode now asks the server to skip patch resolution rather than resolving bodies the dispatcher discards (BUG-2608); v0.22 stopped `pad_item.move` destroying an item’s system metadata — implementation notes, decision log, linked PR and convention data now survive a move, any field the destination schema has no home for is REPORTED in the move’s activity entry rather than vanishing, and a `field` setter naming one of those reserved keys is refused with `malformed_override` instead of writing it (BUG-2674); v0.23 closed the same door on the ordinary update — a `field` setter naming `implementation_notes`, `decision_log` or `convention` is now refused on every transport at once (`validation_error` on HTTP, surfaced to MCP clients as `validation_failed`); the one gate covers the CLI, remote MCP and stdio MCP at once because all three lower a `field` setter into the same `fields_patch`; `github_pr` was exempt on update until v0.47 refused it too, because that door stored the PR as an unreadable string (BUG-2696), and `pad github link` / `unlink` now write it through a typed, validated update member; item CREATE stayed open until v0.48 refused the keys there too (BUG-3163), once library activation, the one system writer using that door, moved to a typed `convention` create member. v0.23 also added the retry-hostile `stored_state_unreadable` error code so an agent told its target item’s stored data is unreadable stops instead of retrying a permanent failure (BUG-2627 / BUG-2675); v0.24 made the `pad_item` `fields` object a real write form on create/update — reads return `fields` as a native object, and writing that shape back was a silent no-op (accepted, never mapped, dropped while the PATCH still bumped `updated_at`) — merging it into the same path as `field`/the dedicated params with conflicting duplicate keys refused, and made input validation strict across all catalog tools: undeclared top-level keys now fail with a structured error instead of being silently dropped (#1066); v0.25 made `pad_library.activate` resolve its DESTINATION collection from the target’s declared artifact kind (SPEC-5 collection traits) rather than the literal `conventions` / `playbooks` slugs, so activating into a workspace that renamed either collection lands correctly instead of failing not-found with the collection sitting right there (BUG-2702); a lookup ERROR is now surfaced rather than silently falling back to the canonical slug, because falling back on an error means writing to a slug nothing was confirmed about (TASK-2657); v0.26 made `pad_workspace.create` REFUSE with a 403 when the calling OAuth connection's grant has `may_create_workspaces=false` — that checkbox previously gated only the post-creation auto-add, so a connection whose user declined it could still create workspaces — and on a connection with an explicit workspace allow-list, could not then see them (a wildcard `all_current_workspaces` connection could, which is why the consent mismatch rather than the invisibility is the defect); the same gate covers `POST /workspaces/import`, which mints a workspace through a second door. There is deliberately no escape-hatch parameter: the gate expresses the USER's consent decision, so only the user can lift it — by re-authorizing, or by enabling the flag on the existing connection at `/console/connected-apps` (IDEA-2756); v0.27 typed field values server-side so a declared number/json field is writable from the remote transport at all, carried the `fields` object with its JSON types intact, named undeclared keys back in `warnings.undeclared_fields` (accepted rather than refused — a census of 1012 items found 14 such keys across 168 live values, so refusing would have broken read-modify-write on items nobody had edited wrongly), and replaced the accreted per-site conflict guards with ONE check over a canonical view of every source; that check refuses several ambiguities v0.26 resolved silently, chiefly two names for one target in a single call (`parent`/`plan`, `assign`/`assigned_user_id`, `role`/`agent_role_id`), refused even when the values match because the names address one thing through incomparable vocabularies and the two doors resolved them differently (BUG-2850); v0.28 added two ADDITIVE `pad_item` actions — `remind`, which arms a one-shot reminder at an RFC3339 instant (`remind_at`), and `ack-reminder`, which acknowledges a fired one by id (`reminder_id`); a bare `YYYY-MM-DD` is refused rather than read as midnight, since a date names a 24-hour span and picking an hour inside it would be the server choosing a time nobody did (IDEA-2641); v0.29 made a `relation` field value have to NAME A LIVE ITEM in the collection that field declares — `internal/items` only ever checked the SHAPE of a relation ("must be a string"), because deciding whether a string names an item is a database question and that package is DB-free, so any string at all was accepted and stored and no client could render it honestly; every write door now refuses a value that names nothing, names an item in the WRONG collection, sits in a field whose schema declares no target collection, or is a SLUG (a deliberate divergence from `ResolveItem`: a slug is neither an ID nor stable, and free text like "red" resolving to whatever is slugged `red` today is exactly the corruption this closes). A CARRIED value — one already on the item, asserted by nobody — is never refused, because refusing would make every legacy item un-updatable, un-movable and un-copyable: within a workspace it resolves and survives, across a workspace boundary it is dropped without a lookup and reported in `warnings.dropped_fields`, so `pad_item.action=copy` now names a drop where v0.28 silently landed a dangling reference (PLAN-2857 / TASK-2878); v0.30 made one `--field key=value` entry mean ONE thing at every door — six sites parsed it independently (`item create`, `item list`, `item update`, `item move`, `item copy`, and the remote door’s own ingest) in four spellings, so `field:[" effort=l"]` stored an undeclared field literally named " effort" through the CLI and wrote `effort` through /mcp, the same call storing two different keys depending only on the transport. All six now share one parse, with two deliberately asymmetric rules: a padded KEY is REFUSED everywhere rather than trimmed anywhere (trimming silently retargets the write to a field the caller did not type), and a VALUE is carried VERBATIM everywhere (trimming reinterprets a caller’s bytes, and on a text field the padding is content) — a padded value against a typed field is refused one layer down by validation, naming the field, which is the same answer at both doors since v0.27 types declared fields server-side. Both doors refuse something they used to accept, and they were accepting it differently — /mcp trimmed the padded key and wrote the declared field, the CLI stored a ghost field beside it; what is /mcp-only is the value half, which it used to trim and type and now passes through to the same validation the CLI has always applied. A caller writing canonical entries sees no difference at either door (BUG-2870); v0.31 let a `relation` field value be an EXACT TITLE, scoped to the collection that field declares, alongside the UUID and ref v0.29 accepted — a THIRD accepted spelling, so nothing a v0.30 caller sends stops working — added a `relation_targets` member to reads hydrating each stored id to `{id, ref, title}` so a client can render a relation without a request per value, and made the `fields` DSL take the target collection slug as a relation's third part. The bump is owed by that last part: `fields="owner:relation"` used to be ACCEPTED and is now REFUSED, because the old parser put the third part into Options for every type and so built a relation with no target — a field every subsequent write then refused with `target_missing`, which is to say a field that could never accept a value. Title resolution is COLLECTION-SCOPED on purpose: a title unique only workspace-wide refuses and names the collection searched, and two matches inside the declared collection get their own `ambiguous` reason rather than `not_found`, which would state the opposite of what happened. The ladder is UUID, then ref, then title, so an item literally TITLED "COLO-3" is unreachable by title while the ref resolves. An id-only `relation_targets` entry means the target is gone OR the caller may not see it, and consumers must not render it as either — the two are made indistinguishable deliberately (PLAN-2857 / TASK-2996); v0.33 added a `multi_relation` field type — an ORDERED LIST of references, each element resolving through the same UUID → ref → exact-title ladder a scalar relation uses, declared through the DSL as `owners:multi_relation:people` with the target collection as its third part (a bare `owners:multi_relation` is refused at parse time rather than minting a field no write can satisfy, exactly as v0.31 did for `relation`). The type itself is additive — every rule it introduces is reachable only through a field whose schema says `multi_relation`, which no existing schema can contain — but the READ shape is why this is a bump: `relation_targets[key]` now carries EITHER the scalar object v0.31 emitted OR a JSON ARRAY of those objects, in stored order and one per stored element, when the key names a `multi_relation`. A scalar entry is byte-identical to v0.31's, so a consumer with no such fields sees nothing new; one that acquires such a field must handle the array, and a strictly-typed client will fail to decode it. Value rules, all NEW rather than inherited because scalar `relation` had nothing coherent to inherit (BUG-3028): exactly ONE stored form for "none", the key being ABSENT, with `[]` normalising to it; an empty or whitespace-only ELEMENT refused rather than skipped; `required` meaning at least one RESOLVED element; order part of the value; and duplicates refused with a new `duplicate_referent` reason, detected AFTER resolution because two elements naming one item are usually different strings — a UUID and a ref, a ref and a title. Any failing element refuses the WHOLE write, and a cross-workspace copy drops the value whole and reports it once, so an import can never change an element count (PLAN-2857 / TASK-2999); v0.34 added an `expected_seq` param to `pad_item.action=update` — the STRONG optimistic-concurrency token, preferred over `expected_updated_at`, which is stored at one-second resolution and therefore cannot tell two writes inside one second apart: both match it, neither conflicts, and the loser silently overwrites the winner. `seq` is bumped on every mutation of the row under the write lock, and is now serialised on every item AND every item summary (without `omitempty`) so the shape a caller reads always carries the token it must send back. Raising `updated_at`'s resolution instead was measured and rejected: the column is TEXT on both dialects and compared lexically in SQL, and Go's RFC3339Nano omits trailing zeros, so a sub-second value sorts BEFORE a whole-second value of the same second and the since-cursor read would start skipping rows (BUG-3037); ; v0.35 made an injected schema DEFAULT take the same `validateFieldType` check a caller-supplied value takes. It used to be assigned and skipped past that check, so the SAME BYTES were refused through one door and stored through the other, decided only by who put them there: on a `status` retyped to `multi_select` whose scalar default survived the retype, `{"status":"open"}` answered 400 and `{}` answered 201 storing `{"status":"open"}` — a scalar sitting in a list field, which is the corruption BUG-3016, BUG-3057, BUG-3067, BUG-3068 and BUG-3074 have each been defending one surface at a time against. A failing default is now DISCARDED and named in `warnings.dropped_fields` rather than stored, which widens that key from the single relation case v0.29 introduced to a field of any type. Dropped rather than refused because nobody in the request typed the value and refusing would make every write into that collection fail on a schema defect its author must fix elsewhere; the one exception is a REQUIRED field, where dropping leaves it absent and the write is refused as required, with a message naming the default as the cause rather than a bare "field is required" that points at a request which never mentioned it (BUG-3079); v0.36 stopped a REF-SHAPED `relation` value falling back to matching by item NUMBER when its prefix names a collection that is LIVE in the workspace. Item numbers are workspace-unique and sequential ACROSS collections, so `CONVE-1` and `SECRE-1` are never both real — and the fallback dropped the prefix, matched on the number alone, and found an item that was usually inside the field’s declared collection, so the wrong-collection check above it passed and nothing was raised: a caller wrote `CONVE-1`, the item stored a different item’s UUID, and the write answered 201 with the substitution visible only in `relation_targets`. That is the corruption v0.29 closed for free text, left open for the spelling the docs tell callers to prefer. The predicate is about the PREFIX rather than the item because a collection RENAME changes its prefix and a relation already written as the old ref has to keep resolving (BUG-2873): that ref’s prefix names nothing afterwards, so the fallback still fires for it, while a prefix that IS live and simply has no item at that number is the caller naming a real collection that does not contain what they said, and is refused. Residual, stated rather than implied: a ref pasted from ANOTHER workspace whose prefix also names no collection here still resolves by number, because nothing in the value distinguishes it from a rename’s leftover. `GetItemByRef`, the navigational read path, is unchanged (BUG-3082); v0.37 added the opt-in `agent` projection to `pad_item.get`; v0.38 added two ADDITIVE `pad_item` actions from PR #1238 (b4rk13) — `claim`, an atomic lease on an item (`holder`, `ttl` as a Go duration; 409 `lease_held` carrying the holder and expiry when another live holder has it, a re-claim by the live holder extends the expiry), and `release`, which drops it idempotently; a `ttl` under one second is refused at the mapper because whole seconds cross the wire and a positive sub-second value truncated to the `ttl_seconds: 0` the server reads as absent (IDEA-2815); v0.39 refuses a `pad_item.update` that sets content WITH a version token (`expected_seq` / `expected_updated_at`) while a browser tab holds edits not yet saved back, with 409 `content_pending_flush` — the token guards the stored row and those edits are not in it, so the write used to replace them and, with no tab open, delete them outright. Re-reading does not clear it; the new `overwrite_pending_edits` boolean does, and a write without a token is unchanged apart from `warnings.pruned_pending_edits` naming how many such edits it deleted (BUG-3133); v0.40 gives a scalar `relation` field one stored form for "no target", the key absent: a blank a write SETS is normalised to absent before validation, so a REQUIRED relation can no longer be satisfied by `""`, while a legacy blank a write only carries is normalised without refusing it, and an empty list filter on a relation (`owner=`) matches every spelling of none (BUG-3028); v0.41 adds a `match` action to `pad_playbook` — given free `text`, a Choice question over the workspace's ACTIVE playbooks (ref/title/summary/trigger/invocation_slug) plus a reserved `"none"` option returns the best match's ref (or `"none"`), confidence, per-option probabilities and the model; no provider configured answers 404 `decision_provider_unavailable`, a provider error or an out-of-set answer answers 502 `decision_provider_error`; pure ADDITIVE bump, read-only and side-effect-free (PLAN-3114 / TASK-3120); v0.42 changes the error code a LOCAL STDIO caller receives: an argv refusal is `validation_failed` with its message, a 429 is `rate_limited` with `details.retry_after_seconds`, and free text starting with `-` is no longer parsed as a flag, since positionals now follow `--` (BUG-3142 / BUG-3147); v0.43 makes `pad_item.action=move` refuse a field override the destination schema does not declare, with 400 `malformed_override`, as the copy does (BUG-2379); v0.44 makes `pad_item.action=bulk-update` over WebMCP refuse, per item into `failed`, a status or priority change on an item whose collection does not declare that field, while stdio and remote, which issue a per-item PATCH, still accepted it with `warnings.undeclared_fields` (BUG-3154) until v0.46 made them refuse it too, per item, via the update door's new `refuse_undeclared_fields` member (BUG-3156); v0.45 reports an HTTP 413 as `too_large` on both transports, with the server's own code in `details.reason`, where it used to be `server_error` (BUG-2829); v0.48 makes `pad_item.action=create` refuse a `field` naming any reserved metadata key, as update already did (BUG-3163); v0.49 makes `pad_project.action=report` with no `collections` param leave out system collections (Conventions, Playbooks), keyed on `is_system`, while naming one still includes it (BUG-2410); v0.50 adds an additive `stored_as_text: true` to a `relation_targets` entry whose stored value is not UUID-shaped (an imported title, legacy text), so it renders as the text it is rather than as an unavailable target, while a UUID-shaped entry is byte-identical to v0.49 (BUG-3014); v0.51 stops a move or copy carrying a value into a COMPUTED field or a value that collides on a destination unique field — a carried collision is dropped and named in `warnings.not_unique`, a supplied or default one is refused 409 (BUG-2367); v0.52 refuses a move or copy that would change an item between open, done and abandoned unless the destination done field is named (`state_change_requires_value`; a `state_change` needs_value row on the preflight) (BUG-2367); v0.53 adds `filename_source` to every `pad_attachment.list` row (BUG-2819); v0.54 keeps a `fields` number's literal through MCP, so an integer above 2^53 is no longer rounded, and refuses a `fields` number beside a `field` entry for the same key when the two differ as exact numbers — a pair that used to collapse into one rounded write (BUG-3217); v0.55 resolves `pad_item.search`'s `collection` exact-first per workspace on the server, so where both `task` and `tasks` exist `task` means `task`, and every other workspace answers as before (BUG-2659); v0.56 `content_state` gains `superseded_set_aside`, edits an editor upgrade set aside that opening the item will not restore (BUG-3244); v0.57 `pad_item` `edit-comment` / `delete-comment` and an `edited` flag on every comment (TASK-2695); v0.58 `delete-comment` on a comment with replies answers `comment_has_replies` instead of `server_error` (BUG-3252); v0.59 makes that delete leave a tombstone (`deleted: true`, empty body) its replies stay under, and refuses an edit of one with `comment_deleted` (BUG-3252); see `internal/mcp/version.go` for the full changelog) +- `tool_surface_version: "0.60"` — MCP tool catalog contract (v0.5 added `pad_library`; v0.6 `pad_item.backlinks`; v0.7 `pad_item` `export`/`import`; v0.8 `pad_workspace` `deleted`/`restore`; v0.9 made `pad_item.list` summary-shaped by default with a default+max result cap; v0.10 enforced the draft-playbook gate server-side on `pad_playbook.run` with an `allow_draft` escape hatch; v0.11 added the read-only `pad_attachment` tool (`list`/`show`); v0.12 added `pad_project.activity` (agent-accessible non-streaming activity feed); v0.13 added `pad_project` `ready`/`stale` (agent-oriented backlog + attention queries); v0.14 added `pad_item` `history` + optimistic concurrency (TASK-2022); v0.15 added the `pad_item.list` `unparented` parameter (TASK-2096); v0.16 made an empty-string `assigned_user_id` / `agent_role_id` CLEAR the assignment instead of being silently dropped, so an agent can finally unassign an item (TASK-2571); v0.17 carried that to the LOCAL STDIO transport by teaching the CLI to lift those keys onto their columns instead of into the fields blob (BUG-2583); v0.18 added `clear_assigned_user` / `clear_agent_role` booleans — the canonical, schema-discoverable way to unassign, backed by new `--clear-assigned-user` / `--clear-agent-role` flags on `pad item update` (IDEA-2584); v0.19 added a `clear_parent` boolean — the canonical, schema-discoverable way to detach an item from its parent, backed by a new `--clear-parent` flag on `pad item update` (BUG-2078); v0.20 gave every tool an explicit annotation block derived from the catalog’s read-only knowledge — fully-read-only tools advertise `readOnlyHint: true` / `destructiveHint: false`, all-additive-write tools (`pad_workspace`, `pad_library`) drop `destructiveHint`, overwrite/delete-capable tools stay conservatively destructive, `openWorldHint: false` everywhere — replacing mcp-go’s defaults that marked every tool destructive (BUG-2302), and made `pad_item.list` summary-shaped on the remote HTTP transport too, with a declared `full` boolean as the opt-in for complete bodies on both transports (BUG-2305); v0.21 bounded `pad_item.history`, which was unbounded on every surface — `limit` now covers it (default 50, max 300, the NEWEST N; no `offset`, because reverse-patch storage makes only a newest-end window cheap), applied in the catalog action so it lands on both transports, and summary mode now asks the server to skip patch resolution rather than resolving bodies the dispatcher discards (BUG-2608); v0.22 stopped `pad_item.move` destroying an item’s system metadata — implementation notes, decision log, linked PR and convention data now survive a move, any field the destination schema has no home for is REPORTED in the move’s activity entry rather than vanishing, and a `field` setter naming one of those reserved keys is refused with `malformed_override` instead of writing it (BUG-2674); v0.23 closed the same door on the ordinary update — a `field` setter naming `implementation_notes`, `decision_log` or `convention` is now refused on every transport at once (`validation_error` on HTTP, surfaced to MCP clients as `validation_failed`); the one gate covers the CLI, remote MCP and stdio MCP at once because all three lower a `field` setter into the same `fields_patch`; `github_pr` was exempt on update until v0.47 refused it too, because that door stored the PR as an unreadable string (BUG-2696), and `pad github link` / `unlink` now write it through a typed, validated update member; item CREATE stayed open until v0.48 refused the keys there too (BUG-3163), once library activation, the one system writer using that door, moved to a typed `convention` create member. v0.23 also added the retry-hostile `stored_state_unreadable` error code so an agent told its target item’s stored data is unreadable stops instead of retrying a permanent failure (BUG-2627 / BUG-2675); v0.24 made the `pad_item` `fields` object a real write form on create/update — reads return `fields` as a native object, and writing that shape back was a silent no-op (accepted, never mapped, dropped while the PATCH still bumped `updated_at`) — merging it into the same path as `field`/the dedicated params with conflicting duplicate keys refused, and made input validation strict across all catalog tools: undeclared top-level keys now fail with a structured error instead of being silently dropped (#1066); v0.25 made `pad_library.activate` resolve its DESTINATION collection from the target’s declared artifact kind (SPEC-5 collection traits) rather than the literal `conventions` / `playbooks` slugs, so activating into a workspace that renamed either collection lands correctly instead of failing not-found with the collection sitting right there (BUG-2702); a lookup ERROR is now surfaced rather than silently falling back to the canonical slug, because falling back on an error means writing to a slug nothing was confirmed about (TASK-2657); v0.26 made `pad_workspace.create` REFUSE with a 403 when the calling OAuth connection's grant has `may_create_workspaces=false` — that checkbox previously gated only the post-creation auto-add, so a connection whose user declined it could still create workspaces — and on a connection with an explicit workspace allow-list, could not then see them (a wildcard `all_current_workspaces` connection could, which is why the consent mismatch rather than the invisibility is the defect); the same gate covers `POST /workspaces/import`, which mints a workspace through a second door. There is deliberately no escape-hatch parameter: the gate expresses the USER's consent decision, so only the user can lift it — by re-authorizing, or by enabling the flag on the existing connection at `/console/connected-apps` (IDEA-2756); v0.27 typed field values server-side so a declared number/json field is writable from the remote transport at all, carried the `fields` object with its JSON types intact, named undeclared keys back in `warnings.undeclared_fields` (accepted rather than refused — a census of 1012 items found 14 such keys across 168 live values, so refusing would have broken read-modify-write on items nobody had edited wrongly), and replaced the accreted per-site conflict guards with ONE check over a canonical view of every source; that check refuses several ambiguities v0.26 resolved silently, chiefly two names for one target in a single call (`parent`/`plan`, `assign`/`assigned_user_id`, `role`/`agent_role_id`), refused even when the values match because the names address one thing through incomparable vocabularies and the two doors resolved them differently (BUG-2850); v0.28 added two ADDITIVE `pad_item` actions — `remind`, which arms a one-shot reminder at an RFC3339 instant (`remind_at`), and `ack-reminder`, which acknowledges a fired one by id (`reminder_id`); a bare `YYYY-MM-DD` is refused rather than read as midnight, since a date names a 24-hour span and picking an hour inside it would be the server choosing a time nobody did (IDEA-2641); v0.29 made a `relation` field value have to NAME A LIVE ITEM in the collection that field declares — `internal/items` only ever checked the SHAPE of a relation ("must be a string"), because deciding whether a string names an item is a database question and that package is DB-free, so any string at all was accepted and stored and no client could render it honestly; every write door now refuses a value that names nothing, names an item in the WRONG collection, sits in a field whose schema declares no target collection, or is a SLUG (a deliberate divergence from `ResolveItem`: a slug is neither an ID nor stable, and free text like "red" resolving to whatever is slugged `red` today is exactly the corruption this closes). A CARRIED value — one already on the item, asserted by nobody — is never refused, because refusing would make every legacy item un-updatable, un-movable and un-copyable: within a workspace it resolves and survives, across a workspace boundary it is dropped without a lookup and reported in `warnings.dropped_fields`, so `pad_item.action=copy` now names a drop where v0.28 silently landed a dangling reference (PLAN-2857 / TASK-2878); v0.30 made one `--field key=value` entry mean ONE thing at every door — six sites parsed it independently (`item create`, `item list`, `item update`, `item move`, `item copy`, and the remote door’s own ingest) in four spellings, so `field:[" effort=l"]` stored an undeclared field literally named " effort" through the CLI and wrote `effort` through /mcp, the same call storing two different keys depending only on the transport. All six now share one parse, with two deliberately asymmetric rules: a padded KEY is REFUSED everywhere rather than trimmed anywhere (trimming silently retargets the write to a field the caller did not type), and a VALUE is carried VERBATIM everywhere (trimming reinterprets a caller’s bytes, and on a text field the padding is content) — a padded value against a typed field is refused one layer down by validation, naming the field, which is the same answer at both doors since v0.27 types declared fields server-side. Both doors refuse something they used to accept, and they were accepting it differently — /mcp trimmed the padded key and wrote the declared field, the CLI stored a ghost field beside it; what is /mcp-only is the value half, which it used to trim and type and now passes through to the same validation the CLI has always applied. A caller writing canonical entries sees no difference at either door (BUG-2870); v0.31 let a `relation` field value be an EXACT TITLE, scoped to the collection that field declares, alongside the UUID and ref v0.29 accepted — a THIRD accepted spelling, so nothing a v0.30 caller sends stops working — added a `relation_targets` member to reads hydrating each stored id to `{id, ref, title}` so a client can render a relation without a request per value, and made the `fields` DSL take the target collection slug as a relation's third part. The bump is owed by that last part: `fields="owner:relation"` used to be ACCEPTED and is now REFUSED, because the old parser put the third part into Options for every type and so built a relation with no target — a field every subsequent write then refused with `target_missing`, which is to say a field that could never accept a value. Title resolution is COLLECTION-SCOPED on purpose: a title unique only workspace-wide refuses and names the collection searched, and two matches inside the declared collection get their own `ambiguous` reason rather than `not_found`, which would state the opposite of what happened. The ladder is UUID, then ref, then title, so an item literally TITLED "COLO-3" is unreachable by title while the ref resolves. An id-only `relation_targets` entry means the target is gone OR the caller may not see it, and consumers must not render it as either — the two are made indistinguishable deliberately (PLAN-2857 / TASK-2996); v0.33 added a `multi_relation` field type — an ORDERED LIST of references, each element resolving through the same UUID → ref → exact-title ladder a scalar relation uses, declared through the DSL as `owners:multi_relation:people` with the target collection as its third part (a bare `owners:multi_relation` is refused at parse time rather than minting a field no write can satisfy, exactly as v0.31 did for `relation`). The type itself is additive — every rule it introduces is reachable only through a field whose schema says `multi_relation`, which no existing schema can contain — but the READ shape is why this is a bump: `relation_targets[key]` now carries EITHER the scalar object v0.31 emitted OR a JSON ARRAY of those objects, in stored order and one per stored element, when the key names a `multi_relation`. A scalar entry is byte-identical to v0.31's, so a consumer with no such fields sees nothing new; one that acquires such a field must handle the array, and a strictly-typed client will fail to decode it. Value rules, all NEW rather than inherited because scalar `relation` had nothing coherent to inherit (BUG-3028): exactly ONE stored form for "none", the key being ABSENT, with `[]` normalising to it; an empty or whitespace-only ELEMENT refused rather than skipped; `required` meaning at least one RESOLVED element; order part of the value; and duplicates refused with a new `duplicate_referent` reason, detected AFTER resolution because two elements naming one item are usually different strings — a UUID and a ref, a ref and a title. Any failing element refuses the WHOLE write, and a cross-workspace copy drops the value whole and reports it once, so an import can never change an element count (PLAN-2857 / TASK-2999); v0.34 added an `expected_seq` param to `pad_item.action=update` — the STRONG optimistic-concurrency token, preferred over `expected_updated_at`, which is stored at one-second resolution and therefore cannot tell two writes inside one second apart: both match it, neither conflicts, and the loser silently overwrites the winner. `seq` is bumped on every mutation of the row under the write lock, and is now serialised on every item AND every item summary (without `omitempty`) so the shape a caller reads always carries the token it must send back. Raising `updated_at`'s resolution instead was measured and rejected: the column is TEXT on both dialects and compared lexically in SQL, and Go's RFC3339Nano omits trailing zeros, so a sub-second value sorts BEFORE a whole-second value of the same second and the since-cursor read would start skipping rows (BUG-3037); ; v0.35 made an injected schema DEFAULT take the same `validateFieldType` check a caller-supplied value takes. It used to be assigned and skipped past that check, so the SAME BYTES were refused through one door and stored through the other, decided only by who put them there: on a `status` retyped to `multi_select` whose scalar default survived the retype, `{"status":"open"}` answered 400 and `{}` answered 201 storing `{"status":"open"}` — a scalar sitting in a list field, which is the corruption BUG-3016, BUG-3057, BUG-3067, BUG-3068 and BUG-3074 have each been defending one surface at a time against. A failing default is now DISCARDED and named in `warnings.dropped_fields` rather than stored, which widens that key from the single relation case v0.29 introduced to a field of any type. Dropped rather than refused because nobody in the request typed the value and refusing would make every write into that collection fail on a schema defect its author must fix elsewhere; the one exception is a REQUIRED field, where dropping leaves it absent and the write is refused as required, with a message naming the default as the cause rather than a bare "field is required" that points at a request which never mentioned it (BUG-3079); v0.36 stopped a REF-SHAPED `relation` value falling back to matching by item NUMBER when its prefix names a collection that is LIVE in the workspace. Item numbers are workspace-unique and sequential ACROSS collections, so `CONVE-1` and `SECRE-1` are never both real — and the fallback dropped the prefix, matched on the number alone, and found an item that was usually inside the field’s declared collection, so the wrong-collection check above it passed and nothing was raised: a caller wrote `CONVE-1`, the item stored a different item’s UUID, and the write answered 201 with the substitution visible only in `relation_targets`. That is the corruption v0.29 closed for free text, left open for the spelling the docs tell callers to prefer. The predicate is about the PREFIX rather than the item because a collection RENAME changes its prefix and a relation already written as the old ref has to keep resolving (BUG-2873): that ref’s prefix names nothing afterwards, so the fallback still fires for it, while a prefix that IS live and simply has no item at that number is the caller naming a real collection that does not contain what they said, and is refused. Residual, stated rather than implied: a ref pasted from ANOTHER workspace whose prefix also names no collection here still resolves by number, because nothing in the value distinguishes it from a rename’s leftover. `GetItemByRef`, the navigational read path, is unchanged (BUG-3082); v0.37 added the opt-in `agent` projection to `pad_item.get`; v0.38 added two ADDITIVE `pad_item` actions from PR #1238 (b4rk13) — `claim`, an atomic lease on an item (`holder`, `ttl` as a Go duration; 409 `lease_held` carrying the holder and expiry when another live holder has it, a re-claim by the live holder extends the expiry), and `release`, which drops it idempotently; a `ttl` under one second is refused at the mapper because whole seconds cross the wire and a positive sub-second value truncated to the `ttl_seconds: 0` the server reads as absent (IDEA-2815); v0.39 refuses a `pad_item.update` that sets content WITH a version token (`expected_seq` / `expected_updated_at`) while a browser tab holds edits not yet saved back, with 409 `content_pending_flush` — the token guards the stored row and those edits are not in it, so the write used to replace them and, with no tab open, delete them outright. Re-reading does not clear it; the new `overwrite_pending_edits` boolean does, and a write without a token is unchanged apart from `warnings.pruned_pending_edits` naming how many such edits it deleted (BUG-3133); v0.40 gives a scalar `relation` field one stored form for "no target", the key absent: a blank a write SETS is normalised to absent before validation, so a REQUIRED relation can no longer be satisfied by `""`, while a legacy blank a write only carries is normalised without refusing it, and an empty list filter on a relation (`owner=`) matches every spelling of none (BUG-3028); v0.41 adds a `match` action to `pad_playbook` — given free `text`, a Choice question over the workspace's ACTIVE playbooks (ref/title/summary/trigger/invocation_slug) plus a reserved `"none"` option returns the best match's ref (or `"none"`), confidence, per-option probabilities and the model; no provider configured answers 404 `decision_provider_unavailable`, a provider error or an out-of-set answer answers 502 `decision_provider_error`; pure ADDITIVE bump, read-only and side-effect-free (PLAN-3114 / TASK-3120); v0.42 changes the error code a LOCAL STDIO caller receives: an argv refusal is `validation_failed` with its message, a 429 is `rate_limited` with `details.retry_after_seconds`, and free text starting with `-` is no longer parsed as a flag, since positionals now follow `--` (BUG-3142 / BUG-3147); v0.43 makes `pad_item.action=move` refuse a field override the destination schema does not declare, with 400 `malformed_override`, as the copy does (BUG-2379); v0.44 makes `pad_item.action=bulk-update` over WebMCP refuse, per item into `failed`, a status or priority change on an item whose collection does not declare that field, while stdio and remote, which issue a per-item PATCH, still accepted it with `warnings.undeclared_fields` (BUG-3154) until v0.46 made them refuse it too, per item, via the update door's new `refuse_undeclared_fields` member (BUG-3156); v0.45 reports an HTTP 413 as `too_large` on both transports, with the server's own code in `details.reason`, where it used to be `server_error` (BUG-2829); v0.48 makes `pad_item.action=create` refuse a `field` naming any reserved metadata key, as update already did (BUG-3163); v0.49 makes `pad_project.action=report` with no `collections` param leave out system collections (Conventions, Playbooks), keyed on `is_system`, while naming one still includes it (BUG-2410); v0.50 adds an additive `stored_as_text: true` to a `relation_targets` entry whose stored value is not UUID-shaped (an imported title, legacy text), so it renders as the text it is rather than as an unavailable target, while a UUID-shaped entry is byte-identical to v0.49 (BUG-3014); v0.51 stops a move or copy carrying a value into a COMPUTED field or a value that collides on a destination unique field — a carried collision is dropped and named in `warnings.not_unique`, a supplied or default one is refused 409 (BUG-2367); v0.52 refuses a move or copy that would change an item between open, done and abandoned unless the destination done field is named (`state_change_requires_value`; a `state_change` needs_value row on the preflight) (BUG-2367); v0.53 adds `filename_source` to every `pad_attachment.list` row (BUG-2819); v0.54 keeps a `fields` number's literal through MCP, so an integer above 2^53 is no longer rounded, and refuses a `fields` number beside a `field` entry for the same key when the two differ as exact numbers — a pair that used to collapse into one rounded write (BUG-3217); v0.55 resolves `pad_item.search`'s `collection` exact-first per workspace on the server, so where both `task` and `tasks` exist `task` means `task`, and every other workspace answers as before (BUG-2659); v0.56 `content_state` gains `superseded_set_aside`, edits an editor upgrade set aside that opening the item will not restore (BUG-3244); v0.57 `pad_item` `edit-comment` / `delete-comment` and an `edited` flag on every comment (TASK-2695); v0.58 `delete-comment` on a comment with replies answers `comment_has_replies` instead of `server_error` (BUG-3252); v0.59 makes that delete leave a tombstone (`deleted: true`, empty body) its replies stay under, and refuses an edit of one with `comment_deleted` (BUG-3252); v0.60 makes `content_state: "applied_pending_flush"` clear on its own: the server rebuilds the body from the op-log after the item's last tab leaves, bumping `seq` and recording a `system` / `recovery` version, unless recovery fails or is turned off (TASK-2198 U4); see `internal/mcp/version.go` for the full changelog) v0.40 refuses a blank value on a REQUIRED relation field and stores an empty relation as an absent key; see `internal/mcp/version.go` for the full diff --git a/cmd/pad/cmd_db.go b/cmd/pad/cmd_db.go index b10d2864a..23a193074 100644 --- a/cmd/pad/cmd_db.go +++ b/cmd/pad/cmd_db.go @@ -606,13 +606,13 @@ Steps: // per item rather than a scan of pre-built bundles, which would mean // holding every workspace in memory to read one field. // - // The remedy is the only one there is: a browser tab. No Go code in - // this repo can decode a Yjs update payload (the dumb-relay design), - // so nothing server-side can move op-log content into items.content - // — and the two server-side paths that touch the op-log, - // PruneAndApply and ForceRefreshRoom, write CALLER-supplied content - // and prune, which would DISCARD exactly these edits. That is why - // the escape hatch is named for what it does to the data. + // The remedy is a browser tab, or a RUNNING server's op-log recovery + // (TASK-2198 U4, internal/materialize), which this command cannot + // use: it runs with the server stopped. The two other server-side + // paths that touch the op-log, PruneAndApply and ForceRefreshRoom, + // write CALLER-supplied content and prune, which would DISCARD + // exactly these edits. That is why the escape hatch is named for + // what it does to the data. pendingByWorkspace := map[string][]store.PendingFlushItem{} pendingTotal := 0 for _, ws := range workspaces { @@ -933,11 +933,11 @@ func gateUnflushedEdits(workspaces []models.Workspace, pending map[string][]stor fmt.Fprintf(&b, " %-12s %s\n", it.Ref, it.Title) } } - // The only remedy there is. No Go code in this repo can decode a Yjs update - // payload (the dumb-relay design), so nothing server-side can move op-log - // content into items.content — and the two server-side paths that touch the - // op-log, PruneAndApply and ForceRefreshRoom, write CALLER-supplied content - // and prune, which would DISCARD exactly these edits. + // The remedy a stopped server leaves. A running server's op-log recovery + // (TASK-2198 U4) also moves this content into items.content, but this + // command runs with the server stopped; the other two server-side paths + // that touch the op-log, PruneAndApply and ForceRefreshRoom, write + // CALLER-supplied content and prune, which would DISCARD exactly these edits. fmt.Fprint(&b, "\nOpen each of those items in the web UI so the tab flushes its pending edits\n") fmt.Fprint(&b, "into the database, then re-run this command. To migrate anyway and lose those\n") fmt.Fprint(&b, "edits, re-run with --discard-unflushed-edits.\n") diff --git a/cmd/pad/cmd_db_unflushed_test.go b/cmd/pad/cmd_db_unflushed_test.go index 7c8b705bf..d5c65b751 100644 --- a/cmd/pad/cmd_db_unflushed_test.go +++ b/cmd/pad/cmd_db_unflushed_test.go @@ -73,8 +73,8 @@ func TestGateUnflushedEditsRefusesAndNamesEveryAffectedItem(t *testing.T) { t.Errorf("refusal does not end by promising nothing was migrated:\n%s", msg) } // The remedy and the escape hatch both have to be IN the refusal: a refusal - // with no way forward is a trap, and per BUG-3000 the flush may never happen - // on its own. + // with no way forward is a trap, and the command runs with the server stopped, + // so the server's op-log recovery (TASK-2198 U4) cannot help it. if !strings.Contains(msg, "web UI") { t.Errorf("refusal does not name the only remedy (open the item so a tab flushes):\n%s", msg) } diff --git a/cmd/pad/cmd_item.go b/cmd/pad/cmd_item.go index 07e43268a..a08841349 100644 --- a/cmd/pad/cmd_item.go +++ b/cmd/pad/cmd_item.go @@ -4427,12 +4427,13 @@ func warnStaleEditSeed(item *models.Item) { // scripts and a warning on stdout would corrupt the JSON they parse. // // It deliberately states no duration, and BUG-3000's day-64 measurement is why -// that must stay: the flush is NOT guaranteed to land at all. Nothing server-side -// moves applier content into items.content — the op-log GC's flush-coverage guard -// deliberately retains those rows rather than flushing them — so the row catches up -// only when a tab next opens the item. Any number here would be a claim this -// command cannot support. (This comment previously said the question was "not -// established"; it now is.) +// that must stay: the flush is NOT guaranteed to land at all. The tab flushes it, +// or, once the item's room has closed, the server's op-log recovery does +// (TASK-2198 U4); that recovery can fail, and an operator can turn it off +// (PAD_MATERIALIZE=off), in which case the row catches up only when a tab next +// opens the item. Any number here would be a claim this command cannot +// support. (This comment previously said the question was "not established"; +// it now is.) func warnContentPendingFlush(item *models.Item) { if item == nil || item.Warnings == nil { return diff --git a/cmd/pad/cmd_materialize.go b/cmd/pad/cmd_materialize.go index a7ca704be..f20aa34ee 100644 --- a/cmd/pad/cmd_materialize.go +++ b/cmd/pad/cmd_materialize.go @@ -1,10 +1,12 @@ package main import ( + "log/slog" "os" pad "github.com/PerpetualSoftware/pad" "github.com/PerpetualSoftware/pad/internal/materialize" + "github.com/PerpetualSoftware/pad/internal/server" "github.com/spf13/cobra" ) @@ -33,3 +35,20 @@ func materializeWorkerCmd() *cobra.Command { }, } } + +// newMaterializerFromEnv builds the op-log recovery worker's Materializer +// (TASK-2198 U4): a materialize.Supervisor configured from +// PAD_MATERIALIZE_TIMEOUT / PAD_MATERIALIZE_MEM_LIMIT. It returns nil, and +// never calls build, when PAD_MATERIALIZE=off: then no Supervisor exists and +// no worker process is ever spawned. build is NewSupervisor in production. +func newMaterializerFromEnv(getenv func(string) string, logger *slog.Logger, build func(materialize.SupervisorConfig) server.Materializer) server.Materializer { + if !materialize.Enabled(getenv, logger) { + logger.Info("op-log recovery disabled (" + materialize.EnvSwitch + "=off)") + return nil + } + return build(materialize.ConfigFromEnv(getenv, logger)) +} + +func newSupervisorMaterializer(cfg materialize.SupervisorConfig) server.Materializer { + return materialize.NewSupervisor(cfg) +} diff --git a/cmd/pad/cmd_materialize_test.go b/cmd/pad/cmd_materialize_test.go index c1c136f1b..bea093da0 100644 --- a/cmd/pad/cmd_materialize_test.go +++ b/cmd/pad/cmd_materialize_test.go @@ -2,10 +2,15 @@ package main import ( "encoding/json" + "errors" + "log/slog" + "os/exec" "strings" "testing" "github.com/PerpetualSoftware/pad/internal/cmdhelp" + "github.com/PerpetualSoftware/pad/internal/materialize" + "github.com/PerpetualSoftware/pad/internal/server" ) // The materializer worker (TASK-2198) is an internal process entry point: it @@ -29,3 +34,49 @@ func TestMaterializeWorkerCmdHiddenAndOffCmdhelp(t *testing.T) { t.Fatal("the cmdhelp document mentions the materializer worker") } } + +// PAD_MATERIALIZE=off (TASK-2198 U4) must leave the server with no +// Materializer at all: the Supervisor is never built, so no worker process can +// be spawned and StartMaterializeRecovery installs no trigger. +func TestMaterializerDisabledBuildsNothing(t *testing.T) { + for _, v := range []string{"off", "OFF", " off ", "0", "false", "no", "disabled"} { + built := 0 + m := newMaterializerFromEnv(func(k string) string { + if k == materialize.EnvSwitch { + return v + } + return "" + }, slog.Default(), func(materialize.SupervisorConfig) server.Materializer { + built++ + return materialize.NewSupervisor(materialize.SupervisorConfig{}) + }) + if m != nil || built != 0 { + t.Errorf("%s=%q: materializer %v, builds %d; want nil, 0", materialize.EnvSwitch, v, m, built) + } + } + for _, v := range []string{"", "on", "1", "true", "yes", "something-else"} { + built, spawned := 0, 0 + m := newMaterializerFromEnv(func(k string) string { + if k == materialize.EnvSwitch { + return v + } + return "" + }, slog.Default(), func(cfg materialize.SupervisorConfig) server.Materializer { + built++ + cfg.Command = func() (*exec.Cmd, error) { + spawned++ + return nil, errors.New("test: no spawn") + } + return materialize.NewSupervisor(cfg) + }) + if m == nil || built != 1 { + t.Errorf("%s=%q: materializer %v, builds %d; want one", materialize.EnvSwitch, v, m, built) + } + if spawned != 0 { + t.Errorf("%s=%q: building the supervisor spawned %d workers; it must start none until a job", materialize.EnvSwitch, v, spawned) + } + if c, ok := m.(interface{ Close() error }); ok { + _ = c.Close() + } + } +} diff --git a/cmd/pad/cmd_server.go b/cmd/pad/cmd_server.go index e493aa18d..1e4d07188 100644 --- a/cmd/pad/cmd_server.go +++ b/cmd/pad/cmd_server.go @@ -705,6 +705,15 @@ func serveCmd() *cobra.Command { } srv.StartOpLogGC() + // Op-log recovery (TASK-2198 U4). Rebuilds items.content from the + // collaborative op-log when a tab went away without flushing, + // in a capped child process (`pad __materialize-worker`) started + // on the first job, not here. PAD_MATERIALIZE=off disables it + // entirely; PAD_MATERIALIZE_TIMEOUT / PAD_MATERIALIZE_MEM_LIMIT + // tune the worker. docs/deployment.md, "Op-log materializer worker". + srv.SetMaterializer(newMaterializerFromEnv(os.Getenv, slog.Default(), newSupervisorMaterializer)) + srv.StartMaterializeRecovery() + // Token reaper (PLAN-1933 DR-5 / TASK-1936). Periodic sweep // that deletes expired/used email-verification tokens, // password-reset tokens, sessions, CLI-auth sessions and expired diff --git a/cmd/pad/testdata/mcp_wire/remote_initialize.json b/cmd/pad/testdata/mcp_wire/remote_initialize.json index a3d4b8dcb..20698319e 100644 --- a/cmd/pad/testdata/mcp_wire/remote_initialize.json +++ b/cmd/pad/testdata/mcp_wire/remote_initialize.json @@ -8,7 +8,7 @@ }, "padToolSurface": { "tool_surface_stable": true, - "version": "0.59" + "version": "0.60" } }, "prompts": {}, @@ -21,5 +21,5 @@ "name": "pad-mcp", "version": "" }, - "instructions": "Pad is a project tracker for developers and AI agents — issues (TASK, BUG), plans (PLAN), ideas (IDEA), docs (DOC), conventions, comments, and dependencies. Use this server when a user mentions:\n\n- Issue refs like `TASK-5`, `BUG-12`, `PLAN-3`, `IDEA-8` — they are stable, human-readable IDs and the canonical way to address items. Use them, never slugs; they are what audit trails and PR titles show.\n- Tasks / issues / items / plans / progress / \"what's on my plate\" / \"what to work on next\" / standup / changelog / retrospective.\n- Project conventions, decision records, or \"how should this team do X.\"\n\nIf the user is asking general code questions with no project-management thread, you don't need this server.\n\n## Tool surface (v0.59)\n\nTen resource × action tools, plus `pad_set_workspace`. Eleven tools total.\n\nInputs are validated strictly: an undeclared top-level key is rejected with a structured `validation_failed` naming it, never accepted and silently dropped.\n\n- `pad_item` — Items: create / update / delete / get / list / move / restore / link / unlink / deps / star / unstar / starred / comment / list-comments / edit-comment / delete-comment / backlinks / bulk-update / note / decide / export / import / history / remind / ack-reminder / claim / release. On create/update, field values may be passed as a `fields` OBJECT (the same shape reads return, e.g. `{\"status\":\"done\",\"effort\":\"l\"}`) — it merges into the same path as the dedicated params and `field: [\"key=value\"]`; the same key in two places with CONFLICTING values is refused, not silently resolved. `list` accepts `unparented: true` to keep items with no parent or implements relationship (mutually exclusive with `parent`). `list` results are SUMMARY-shaped by default on both transports — no content bodies; pass `full: true` for complete bodies (token-expensive), or prefer `get` for a single item's body. `update` field writes are a server-side field-level merge (only the keys you set change); pass `expected_seq` for optimistic concurrency — the `seq` you last read, which is on every item AND every list summary — and a stale value fails with a structured 409 `update_conflict` whose details carry `actual_seq` to retry with. PREFER IT over `expected_updated_at`, which cannot tell two writes inside one second apart; that one still works, and if you send both, seq decides. A token cannot see a browser tab's unsaved edits, so `content` sent WITH one is refused with 409 `content_pending_flush` while any exist; re-reading will not clear it. Wait for the editor to save, or pass `overwrite_pending_edits: true` to replace them. `content_state: \"superseded_set_aside\"` is different and wins when both apply: an editor upgrade set aside edits no tab can restore, so EVERY content write is refused (token or not; `details.set_aside_rows`) and waiting never clears it. Tell the user, or pass `overwrite_pending_edits: true` to discard them. Writing `content` while a browser tab has the item open routes the markdown through that live document, and the stored copy is updated only by a later collab-snapshot flush — usually from the tab that applied it, though any such write updates the row, and nothing guarantees one happens: the 200 echoes the content you SENT and carries `warnings.content_outcome: \"applied_pending_flush\"`, which describes that WRITE — the content went to the document, not the row — rather than being a live reading of the row, which a concurrent flush may already have updated. A `get` (or a `list` with `full: true` — a default list carries no content at all) before such a flush lands reads the stored copy and answers with the PREVIOUS content, which is the lag rather than a failed write, so do not re-send on the strength of it. You do not have to infer that from the write, and you will also see it when someone ELSE's tab holds the unflushed edits and you wrote nothing: a read that serves a body the server knows is behind the live document carries `content_state: \"applied_pending_flush\"` — same value as the write warning, different question (what the ROW is, not what your REQUEST did). Treat it as re-read later, never as re-send, and do not wait on it in a loop: nothing server-side moves that content into the stored copy, so the row catches up only when a tab next opens the item, which may never happen. The stored form may also not be byte-identical to what you sent, since the markdown round-trips through the editor (bullet markers, emphasis characters, list numbering and blank-line runs normalise; already-canonical markdown survives unchanged), so compare on meaning rather than bytes. `move` changes an item's COLLECTION within its workspace: system metadata (implementation notes, decision log, linked PR) survives it, and any field value the target schema has no home for is dropped AND reported in the move's activity entry — check there rather than assuming a move is lossless. A carried unique-field collision is dropped, named in `warnings.not_unique`. Three system keys — `implementation_notes`, `decision_log`, `convention` — cannot be set through `field` on update or move; the call is refused with `validation_failed`, naming the key and the write path that does maintain it (`note`, `decide`, and library activation respectively). `github_pr` is refused as well (BUG-2696): a `field` value arrived as a string, so the PR data was stored double-encoded and no link appeared. Its writer is `pad github link` / `unlink`, which needs a local git checkout, and there is no PR-link action on this surface yet, so treat linking a PR as something to hand to a human rather than a call to retry. On `create` they are refused too (BUG-3163), with the same `validation_failed`: create the item first, then use `note` / `decide` on it; convention metadata comes from library activation. `history` returns read-only item version metadata (newest-first), bounded to the NEWEST 50 versions by default (max 300 — pass `limit` to change the window); pass `full: true` to include each version's resolved content body (token-expensive). There is no `offset`: versions are stored as reverse patches, so only a newest-end window is cheap to reconstruct. Unassign or detach with the `clear_assigned_user` / `clear_agent_role` / `clear_parent` booleans; an empty `assign` / `role` / `parent` does NOT clear. `remind` / `ack-reminder`: a fired reminder shows in `pad_project` next / ready (the delivery path on an instance with no webhook) until acknowledged, and one on a completed item is hidden there. `claim` / `release`: a loser's 409 is `lease_held`, and a lease bumps neither `updated_at` nor `seq`. `edit-comment` / `delete-comment` take a `comment_id` from `list-comments` (whose rows carry `edited`); only a comment's author may edit it. Correct a comment in place rather than appending a correction. Deleting a comment that has replies leaves a placeholder (`deleted: true`, empty body) so the replies keep their parent; editing, replying to or reacting to it answers 409 `comment_deleted`. (Two older forms still work and are not deprecated: `field: [\"assigned_user_id=\"]` on either transport, and a direct `assigned_user_id: \"\"` param over remote `/mcp` only — prefer the boolean, which is the only one this schema advertises.)\n\n**Relation fields (v0.31).** A `relation` value may be a UUID, an issue ref (`COLO-3`), or the target item's EXACT TITLE — in that order, so an item literally titled `COLO-3` is unreachable by title while the ref resolves. Title matching is scoped to the collection the field declares: a title that is unique only workspace-wide is REFUSED, naming the collection searched, and a title matching two or more items inside the declared collection is refused as `ambiguous` rather than `not_found` — it matched too much, not too little. Only items YOU can see count towards any of that, so a match you have no access to never changes your answer.\n\nReads carry a `relation_targets` member beside `fields`: field key → `{id, ref, title}`, so you can render a relation without a request per value. `fields` still holds the canonical id, which is what you write back. An entry with an `id` and NO `ref`/`title` means the target is gone **or** you may not see it — the two are deliberately indistinguishable, so do not render it as either \"deleted\" or \"hidden\"; \"unavailable\" is the honest word. An entry with `stored_as_text: true` differs: the stored value is not UUID-shaped, so it was never an id (an imported title, legacy text). Render it as text, not as unavailable.\n\n**`multi_relation` fields (v0.33).** A field declared `multi_relation` holds an ORDERED LIST of references — `[\"\u003cuuid\u003e\", \"COLO-3\", \"Red\"]` — and every element resolves through the same UUID → ref → exact-title ladder, with the same collection scoping. Declare one through the `fields` DSL as `owners:multi_relation:people`: the third part is the TARGET COLLECTION, and omitting it is refused at parse time rather than building a field no write can ever satisfy.\n\nFour rules differ from a scalar `relation`, and none of them is guessable from the scalar behaviour:\n\n- **There is exactly one stored form for \"no references\": the key ABSENT.** Send `[]` and the write normalises it to that. A `required` `multi_relation` means at least one element that RESOLVED, so `[]` and an absent key both refuse.\n- **An empty or whitespace-only ELEMENT is refused**, naming its index — it is not skipped the way an empty scalar value is. Remove the element rather than blanking it.\n- **Order is part of the value.** `[\"a\",\"b\"]` and `[\"b\",\"a\"]` are different values, and a write preserves the order you sent.\n- **Duplicates are refused**, with reason `duplicate_referent`, naming the SECOND occurrence. Two elements can be different strings naming one item — a UUID and a ref, a ref and a title — so this is decided after resolution, and the two forms you sent may look nothing alike.\n\nIf ANY element fails, the whole write is refused and nothing is stored — never a partial list. On a cross-workspace copy the value is dropped WHOLE and reported once in `warnings.dropped_fields`, so an import can never change an element count.\n\nIn `relation_targets`, a `multi_relation` key carries a JSON **ARRAY** of the same `{id, ref, title}` objects, in stored order and one per stored element — including an `id`-only entry for an element that names nothing, so a position in the list lines up with the same position in `fields`. A scalar key is unchanged: still a single object, byte-identical to v0.31. Decide which shape to expect from the field's declared type, not by probing.\n\n- `pad_workspace` — Workspaces: list / members / invite / storage / audit-log / create / claim / deleted / restore.\n- `pad_collection` — Collections: list / create / update / delete.\n- `pad_project` — Project intelligence: dashboard / next / ready / stale / standup / changelog / report / activity. Use `ready` for the actionable backlog and `stale` for items needing attention; `activity` to catch up on what other agents/users changed since you last worked (non-streaming feed with item refs + change details).\n- `pad_role` — Agent roles: list / create / update / delete.\n- `pad_search` — Full-text search across items: query.\n- `pad_playbook` — Invokable procedures: list / get / run / match. Use `run` to bind args against a playbook's declared spec and get the rendered body back; side-effect-free. `run` refuses a playbook whose status isn't `active` (a draft still being authored) with a `playbook_not_active` error — pass `allow_draft: true` to override. Both `run` and `get` echo the playbook's `status`. Both may also carry `content_state` (`\"applied_pending_flush\"`, or `\"superseded_set_aside\"` as under `pad_item`), meaning the BODY you are about to execute is behind the item's live collaborative document — an editor holds unflushed edits, so the steps may be superseded. It is a reason to re-read or to ask, not a reason to refuse to run; the body is still the best the server has. `match` finds a playbook from text.\n- `pad_library` — Convention + playbook library (the global catalog of pre-built entries workspaces activate): list / get / activate.\n- `pad_attachment` — Read-only attachment metadata: list / show. `list` enumerates a workspace's attachments (filter by item / category / collection / attached / unattached); `show` returns one attachment's MIME, size, filename, and ETag via a HEAD request. Uploading and general file downloads stay CLI-only; bounded image bytes are available through the attachment resource below.\n- `pad_meta` — Server introspection: server-info / version / tool-surface / bootstrap. The `bootstrap` action returns one-shot workspace context (user + collections + always-on conventions + a metadata-only `convention_index` of every active convention + roles + playbook metadata + dashboard + recent activity).\n- `pad_set_workspace` — Load workspace context; response embeds the bootstrap blob so you load context in one call. Whether it also pins a session default: see Workspace context below. Takes `workspace: \u003cslug\u003e` only (no `action`).\n\nFor the ten resource × action tools, always pass `action` as a top-level field. Per-action required parameters are documented in each tool's description.\n\n## Resources are cheaper than tool calls\n\nRead these directly when you need workspace state:\n\n- `pad://workspace/{ws}/dashboard` — computed project overview (active items, plans, attention, suggested next).\n- `pad://workspace/{ws}/collections` — collection types + schemas.\n- `pad://workspace/{ws}/items` — list of all items (use `pad_item.action: list` for filtering).\n- `pad://workspace/{ws}/items/{ref}` — single item rendered as markdown.\n- `pad://workspace/{ws}/attachments/{id}` — image attachment as a bounded base64 `thumb-md` resource; rejects non-images and image bytes over 1 MiB (pre-base64).\n- `pad://workspace/{ws}/bootstrap` — one-shot workspace context (same payload as `pad_meta.action: bootstrap` and `pad_set_workspace`'s embedded response).\n- `pad://_meta/version` — server version + stability tiers.\n\nResources support host-side prefetch — if the host can fetch them once at session start, you don't pay per turn.\n\nLoad the bootstrap resource once per workspace and reuse its collections, conventions, roles, and playbook metadata. Refresh it after switching workspaces, after changing those configuration surfaces, when Pad reports stale schema/context, or when the user asks. For changing work state, read the affected item or use a targeted project tool instead of fetching the whole bootstrap again.\n\n## Workspace context\n\nEvery action that operates within a workspace accepts an optional `workspace` parameter. Resolution order:\n\n1. Explicit `workspace` argument on the call (always wins).\n2. On a single-user local server only: the session default set via `pad_set_workspace`.\n3. On a single-user local server only: the CWD-linked workspace from `.pad.toml`.\n\nA multi-user/remote server does **not** persist a session default — pass `workspace` explicitly on every call. If none resolves, the action returns a structured `no_workspace` error with `available_workspaces`.\n\n## Update flow: read first, then patch\n\nFor `pad_item.action: update`, the server merges your patch with the item's current state. Pass only the fields you want to change. When changing `status`, ALWAYS include a `comment` explaining why — it builds the audit trail that helps the team understand history.\n\n## One error code you must not retry\n\nErrors come back as a structured envelope with a `code`. Most failures are worth a retry after you fix something, and a `server_error` may be transient. `stored_state_unreadable` is neither.\n\nIt means the ITEM'S STORED DATA cannot be decoded — your input was fine. Today it fires when `note` / `decide` would append to an `implementation_notes` / `decision_log` value that is not a list of entries. The operation is refused precisely because completing it would overwrite that value. The refusal is fully deterministic, so retrying — immediately or after a backoff — fails identically every time and changes nothing.\n\nWhen you see it: tell the user which item is affected, and stop. Whether you can inspect the bad value depends on which layer is broken — if the item's whole `fields` blob fails to parse, `action: get` hands it back as a raw string; if one structured key is at fault, you won't see it at all, because the blob is normalized on this surface and a value that doesn't decode is dropped from the top-level arrays. A human can always read it with `pad item show \u003cref\u003e --format json`. Do not route around the refusal by writing the field some other way; that is the write that caused it.\n\n## Project conventions\n\nWorkspaces can declare conventions (e.g. \"run `make test` before PR\", \"use conventional commit format\"). The bootstrap blob gives you two views:\n\n- `conventions` — full bodies of the always-on (`trigger=always`) rules. Follow these unconditionally.\n- `convention_index` — METADATA ONLY (`ref`, `title`, `trigger`, `role`; no bodies) for **every** active convention, including the triggered ones whose bodies are NOT in `conventions`. This is your map of what triggered rules exist.\n\nBefore performing meaningful work with a specific trigger (e.g. `on-implement` before writing code), consult `convention_index`: if it lists entries for that trigger, pull their bodies on demand; if it lists none, skip the query.\n\n```\npad_item.action: list, collection: \"conventions\", status: \"active\"\n```\n\nFilter by trigger (`always`, `on-implement`, `on-task-complete`, etc.) when relevant — the `convention_index` triggers tell you which filters are worth running.\n\n`\"conventions\"` above is the DEFAULT collection slug and a workspace may have renamed it, in which case that literal returns nothing while `convention_index` still lists entries — the bootstrap payload resolves by declaration, not by name. When the two disagree, address the items by the `ref`s the index gave you, or find the collection with `pad_collection.action: list`. The same applies to `\"playbooks\"`; `pad_playbook` resolves by declaration and is unaffected.\n\n## Pushes and the session monitor (security note)\n\nPad can push an item into a Claude Code session (`pad push`, or the web item view's push composer). That is terminal instruction injection by design, so receiving is **consent-gated** (PLAN-2613, since v0.15.0): a session receives pushes only after it has connected (`/pad:connect` in the plugin, or a repo-level `push.auto_arm = true` in `.pad.toml` — which a per-user `auto_arm = false` in `~/.pad/config.toml` vetoes), and pushes are self-addressed only — the sender is always the receiving session's own user. This MCP connection is not a session: it cannot arm one, receive a push, or run the monitor, and the `pad_item` / `pad_project` tools expose no push action. If a user asks you to \"push\" something to their terminal, tell them it happens from their own terminal (`pad push \u003cref\u003e -m \"...\"` — the sending session need not be connected, the receiving ones must have consented); do not try to emulate it by writing comments or instructions into items.\n\n## Adding a workspace to this connection\n\nIf the user references a workspace this connection can't see (you'll get a 403 from workspace tools, or the workspace won't appear in `pad_workspace.list`), tell the user you can't see that workspace with your current permissions, then walk them through how to grant access: open Pad in their browser → switch to that workspace → avatar menu → \"Connect project...\" A 6-digit claim code will appear. Have them paste it back in chat, then call `pad_workspace.claim` with `{workspace: \"\u003cslug\u003e\", code: \"\u003c6 digits\u003e\"}`. The workspace joins this connection's allow-list and stays until the user revokes it via `/console/connected-apps`. No re-auth required.\n\nFor brand-new workspaces, `pad_workspace.create` with `{name: \"\u003cname\u003e\"}` (and optional `template`) creates the workspace AND auto-adds it to this connection's allow-list in one call — no claim code needed. Only works when the connection currently carries the \"may create workspaces\" grant — granted at consent time, or enabled later on the connections page. If the grant is present and set to false, the create is **refused with a 403 and no workspace is made** — retrying won't help and neither will the claim flow, since there is nothing to claim. Tell the user this connection isn't permitted to create workspaces, and offer them the two ways forward: create the workspace themselves in the browser and grant access with a claim code, or re-authorize this connection with \"Let this app create new workspaces\" enabled (`/console/connected-apps` can also flip it on an existing connection).\n\n## New workspace: offer to set it up\n\nThe bootstrap blob carries `needs_onboarding: bool` — true when the workspace has zero user-created items (template seeds don't count). When it's true, **lead with an active offer before anything else**: *\"This workspace is brand new and isn't set up yet. Want me to set it up? I'll ask a few quick questions and adapt it to your project.\"*\n\nThis is an **offer, not an auto-run** — wait for the user to say yes before running onboarding. If they accept, run the `onboard` playbook (use the `pad_onboard` prompt, or load the body via `pad_playbook` `action: get`, `ref: onboard`). If they decline, or already declined earlier in the session, respect that and skip the offer. The flag flips to false the moment any item exists, so it won't nag past first setup.\n\n## Multi-step workflows\n\nFour prompts ship with the server: `pad_plan`, `pad_ideate`, `pad_retro`, `pad_onboard`. Use them when the user wants help planning, brainstorming, retrospecting, or onboarding into a workspace — they encode the multi-step Pad-aware playbook for each.\n" + "instructions": "Pad is a project tracker for developers and AI agents — issues (TASK, BUG), plans (PLAN), ideas (IDEA), docs (DOC), conventions, comments, and dependencies. Use this server when a user mentions:\n\n- Issue refs like `TASK-5`, `BUG-12`, `PLAN-3`, `IDEA-8` — they are stable, human-readable IDs and the canonical way to address items. Use them, never slugs; they are what audit trails and PR titles show.\n- Tasks / issues / items / plans / progress / \"what's on my plate\" / \"what to work on next\" / standup / changelog / retrospective.\n- Project conventions, decision records, or \"how should this team do X.\"\n\nIf the user is asking general code questions with no project-management thread, you don't need this server.\n\n## Tool surface (v0.60)\n\nTen resource × action tools, plus `pad_set_workspace`. Eleven tools total.\n\nInputs are validated strictly: an undeclared top-level key is rejected with a structured `validation_failed` naming it, never accepted and silently dropped.\n\n- `pad_item` — Items: create / update / delete / get / list / move / restore / link / unlink / deps / star / unstar / starred / comment / list-comments / edit-comment / delete-comment / backlinks / bulk-update / note / decide / export / import / history / remind / ack-reminder / claim / release. On create/update, field values may be passed as a `fields` OBJECT (the same shape reads return, e.g. `{\"status\":\"done\",\"effort\":\"l\"}`) — it merges into the same path as the dedicated params and `field: [\"key=value\"]`; the same key in two places with CONFLICTING values is refused, not silently resolved. `list` accepts `unparented: true` to keep items with no parent or implements relationship (mutually exclusive with `parent`). `list` results are SUMMARY-shaped by default on both transports — no content bodies; pass `full: true` for complete bodies (token-expensive), or prefer `get` for a single item's body. `update` field writes are a server-side field-level merge (only the keys you set change); pass `expected_seq` for optimistic concurrency — the `seq` you last read, which is on every item AND every list summary — and a stale value fails with a structured 409 `update_conflict` whose details carry `actual_seq` to retry with. PREFER IT over `expected_updated_at`, which cannot tell two writes inside one second apart; that one still works, and if you send both, seq decides. A token cannot see a browser tab's unsaved edits, so `content` sent WITH one is refused with 409 `content_pending_flush` while any exist; re-reading will not clear it. Wait for the editor to save, or pass `overwrite_pending_edits: true` to replace them. `content_state: \"superseded_set_aside\"` is different and wins when both apply: an editor upgrade set aside edits no tab can restore, so EVERY content write is refused (token or not; `details.set_aside_rows`) and waiting never clears it. Tell the user, or pass `overwrite_pending_edits: true` to discard them. Writing `content` while a browser tab has the item open routes the markdown through that live document, and the stored copy is updated only by a later collab-snapshot flush — usually from the tab that applied it, though any such write updates the row, and nothing guarantees one happens: the 200 echoes the content you SENT and carries `warnings.content_outcome: \"applied_pending_flush\"`, which describes that WRITE — the content went to the document, not the row — rather than being a live reading of the row, which a concurrent flush may already have updated. A `get` (or a `list` with `full: true` — a default list carries no content at all) before such a flush lands reads the stored copy and answers with the PREVIOUS content, which is the lag rather than a failed write, so do not re-send on the strength of it. You do not have to infer that from the write, and you will also see it when someone ELSE's tab holds the unflushed edits and you wrote nothing: a read that serves a body the server knows is behind the live document carries `content_state: \"applied_pending_flush\"` — same value as the write warning, different question (what the ROW is, not what your REQUEST did). Treat it as re-read later, never as re-send, and do not wait on it in a loop: the server normally recovers the stored copy itself about a minute after the last tab leaves the item, but that is not guaranteed (a recovery can fail, and an operator can turn it off), and otherwise the row catches up only when a tab next opens the item. The stored form may also not be byte-identical to what you sent, since the markdown round-trips through the editor (bullet markers, emphasis characters, list numbering and blank-line runs normalise; already-canonical markdown survives unchanged), so compare on meaning rather than bytes. `move` changes an item's COLLECTION within its workspace: system metadata (implementation notes, decision log, linked PR) survives it, and any field value the target schema has no home for is dropped AND reported in the move's activity entry — check there rather than assuming a move is lossless. A carried unique-field collision is dropped, named in `warnings.not_unique`. Three system keys — `implementation_notes`, `decision_log`, `convention` — cannot be set through `field` on update or move; the call is refused with `validation_failed`, naming the key and the write path that does maintain it (`note`, `decide`, and library activation respectively). `github_pr` is refused as well (BUG-2696): a `field` value arrived as a string, so the PR data was stored double-encoded and no link appeared. Its writer is `pad github link` / `unlink`, which needs a local git checkout, and there is no PR-link action on this surface yet, so treat linking a PR as something to hand to a human rather than a call to retry. On `create` they are refused too (BUG-3163), with the same `validation_failed`: create the item first, then use `note` / `decide` on it; convention metadata comes from library activation. `history` returns read-only item version metadata (newest-first), bounded to the NEWEST 50 versions by default (max 300 — pass `limit` to change the window); pass `full: true` to include each version's resolved content body (token-expensive). There is no `offset`: versions are stored as reverse patches, so only a newest-end window is cheap to reconstruct. Unassign or detach with the `clear_assigned_user` / `clear_agent_role` / `clear_parent` booleans; an empty `assign` / `role` / `parent` does NOT clear. `remind` / `ack-reminder`: a fired reminder shows in `pad_project` next / ready (the delivery path on an instance with no webhook) until acknowledged, and one on a completed item is hidden there. `claim` / `release`: a loser's 409 is `lease_held`, and a lease bumps neither `updated_at` nor `seq`. `edit-comment` / `delete-comment` take a `comment_id` from `list-comments` (whose rows carry `edited`); only a comment's author may edit it. Correct a comment in place rather than appending a correction. Deleting a comment that has replies leaves a placeholder (`deleted: true`, empty body) so the replies keep their parent; editing, replying to or reacting to it answers 409 `comment_deleted`. (Two older forms still work and are not deprecated: `field: [\"assigned_user_id=\"]` on either transport, and a direct `assigned_user_id: \"\"` param over remote `/mcp` only — prefer the boolean, which is the only one this schema advertises.)\n\n**Relation fields (v0.31).** A `relation` value may be a UUID, an issue ref (`COLO-3`), or the target item's EXACT TITLE — in that order, so an item literally titled `COLO-3` is unreachable by title while the ref resolves. Title matching is scoped to the collection the field declares: a title that is unique only workspace-wide is REFUSED, naming the collection searched, and a title matching two or more items inside the declared collection is refused as `ambiguous` rather than `not_found` — it matched too much, not too little. Only items YOU can see count towards any of that, so a match you have no access to never changes your answer.\n\nReads carry a `relation_targets` member beside `fields`: field key → `{id, ref, title}`, so you can render a relation without a request per value. `fields` still holds the canonical id, which is what you write back. An entry with an `id` and NO `ref`/`title` means the target is gone **or** you may not see it — the two are deliberately indistinguishable, so do not render it as either \"deleted\" or \"hidden\"; \"unavailable\" is the honest word. An entry with `stored_as_text: true` differs: the stored value is not UUID-shaped, so it was never an id (an imported title, legacy text). Render it as text, not as unavailable.\n\n**`multi_relation` fields (v0.33).** A field declared `multi_relation` holds an ORDERED LIST of references — `[\"\u003cuuid\u003e\", \"COLO-3\", \"Red\"]` — and every element resolves through the same UUID → ref → exact-title ladder, with the same collection scoping. Declare one through the `fields` DSL as `owners:multi_relation:people`: the third part is the TARGET COLLECTION, and omitting it is refused at parse time rather than building a field no write can ever satisfy.\n\nFour rules differ from a scalar `relation`, and none of them is guessable from the scalar behaviour:\n\n- **There is exactly one stored form for \"no references\": the key ABSENT.** Send `[]` and the write normalises it to that. A `required` `multi_relation` means at least one element that RESOLVED, so `[]` and an absent key both refuse.\n- **An empty or whitespace-only ELEMENT is refused**, naming its index — it is not skipped the way an empty scalar value is. Remove the element rather than blanking it.\n- **Order is part of the value.** `[\"a\",\"b\"]` and `[\"b\",\"a\"]` are different values, and a write preserves the order you sent.\n- **Duplicates are refused**, with reason `duplicate_referent`, naming the SECOND occurrence. Two elements can be different strings naming one item — a UUID and a ref, a ref and a title — so this is decided after resolution, and the two forms you sent may look nothing alike.\n\nIf ANY element fails, the whole write is refused and nothing is stored — never a partial list. On a cross-workspace copy the value is dropped WHOLE and reported once in `warnings.dropped_fields`, so an import can never change an element count.\n\nIn `relation_targets`, a `multi_relation` key carries a JSON **ARRAY** of the same `{id, ref, title}` objects, in stored order and one per stored element — including an `id`-only entry for an element that names nothing, so a position in the list lines up with the same position in `fields`. A scalar key is unchanged: still a single object, byte-identical to v0.31. Decide which shape to expect from the field's declared type, not by probing.\n\n- `pad_workspace` — Workspaces: list / members / invite / storage / audit-log / create / claim / deleted / restore.\n- `pad_collection` — Collections: list / create / update / delete.\n- `pad_project` — Project intelligence: dashboard / next / ready / stale / standup / changelog / report / activity. Use `ready` for the actionable backlog and `stale` for items needing attention; `activity` to catch up on what other agents/users changed since you last worked (non-streaming feed with item refs + change details).\n- `pad_role` — Agent roles: list / create / update / delete.\n- `pad_search` — Full-text search across items: query.\n- `pad_playbook` — Invokable procedures: list / get / run / match. Use `run` to bind args against a playbook's declared spec and get the rendered body back; side-effect-free. `run` refuses a playbook whose status isn't `active` (a draft still being authored) with a `playbook_not_active` error — pass `allow_draft: true` to override. Both `run` and `get` echo the playbook's `status`. Both may also carry `content_state` (`\"applied_pending_flush\"`, or `\"superseded_set_aside\"` as under `pad_item`), meaning the BODY you are about to execute is behind the item's live collaborative document — an editor holds unflushed edits, so the steps may be superseded. It is a reason to re-read or to ask, not a reason to refuse to run; the body is still the best the server has. `match` finds a playbook from text.\n- `pad_library` — Convention + playbook library (the global catalog of pre-built entries workspaces activate): list / get / activate.\n- `pad_attachment` — Read-only attachment metadata: list / show. `list` enumerates a workspace's attachments (filter by item / category / collection / attached / unattached); `show` returns one attachment's MIME, size, filename, and ETag via a HEAD request. Uploading and general file downloads stay CLI-only; bounded image bytes are available through the attachment resource below.\n- `pad_meta` — Server introspection: server-info / version / tool-surface / bootstrap. The `bootstrap` action returns one-shot workspace context (user + collections + always-on conventions + a metadata-only `convention_index` of every active convention + roles + playbook metadata + dashboard + recent activity).\n- `pad_set_workspace` — Load workspace context; response embeds the bootstrap blob so you load context in one call. Whether it also pins a session default: see Workspace context below. Takes `workspace: \u003cslug\u003e` only (no `action`).\n\nFor the ten resource × action tools, always pass `action` as a top-level field. Per-action required parameters are documented in each tool's description.\n\n## Resources are cheaper than tool calls\n\nRead these directly when you need workspace state:\n\n- `pad://workspace/{ws}/dashboard` — computed project overview (active items, plans, attention, suggested next).\n- `pad://workspace/{ws}/collections` — collection types + schemas.\n- `pad://workspace/{ws}/items` — list of all items (use `pad_item.action: list` for filtering).\n- `pad://workspace/{ws}/items/{ref}` — single item rendered as markdown.\n- `pad://workspace/{ws}/attachments/{id}` — image attachment as a bounded base64 `thumb-md` resource; rejects non-images and image bytes over 1 MiB (pre-base64).\n- `pad://workspace/{ws}/bootstrap` — one-shot workspace context (same payload as `pad_meta.action: bootstrap` and `pad_set_workspace`'s embedded response).\n- `pad://_meta/version` — server version + stability tiers.\n\nResources support host-side prefetch — if the host can fetch them once at session start, you don't pay per turn.\n\nLoad the bootstrap resource once per workspace and reuse its collections, conventions, roles, and playbook metadata. Refresh it after switching workspaces, after changing those configuration surfaces, when Pad reports stale schema/context, or when the user asks. For changing work state, read the affected item or use a targeted project tool instead of fetching the whole bootstrap again.\n\n## Workspace context\n\nEvery action that operates within a workspace accepts an optional `workspace` parameter. Resolution order:\n\n1. Explicit `workspace` argument on the call (always wins).\n2. On a single-user local server only: the session default set via `pad_set_workspace`.\n3. On a single-user local server only: the CWD-linked workspace from `.pad.toml`.\n\nA multi-user/remote server does **not** persist a session default — pass `workspace` explicitly on every call. If none resolves, the action returns a structured `no_workspace` error with `available_workspaces`.\n\n## Update flow: read first, then patch\n\nFor `pad_item.action: update`, the server merges your patch with the item's current state. Pass only the fields you want to change. When changing `status`, ALWAYS include a `comment` explaining why — it builds the audit trail that helps the team understand history.\n\n## One error code you must not retry\n\nErrors come back as a structured envelope with a `code`. Most failures are worth a retry after you fix something, and a `server_error` may be transient. `stored_state_unreadable` is neither.\n\nIt means the ITEM'S STORED DATA cannot be decoded — your input was fine. Today it fires when `note` / `decide` would append to an `implementation_notes` / `decision_log` value that is not a list of entries. The operation is refused precisely because completing it would overwrite that value. The refusal is fully deterministic, so retrying — immediately or after a backoff — fails identically every time and changes nothing.\n\nWhen you see it: tell the user which item is affected, and stop. Whether you can inspect the bad value depends on which layer is broken — if the item's whole `fields` blob fails to parse, `action: get` hands it back as a raw string; if one structured key is at fault, you won't see it at all, because the blob is normalized on this surface and a value that doesn't decode is dropped from the top-level arrays. A human can always read it with `pad item show \u003cref\u003e --format json`. Do not route around the refusal by writing the field some other way; that is the write that caused it.\n\n## Project conventions\n\nWorkspaces can declare conventions (e.g. \"run `make test` before PR\", \"use conventional commit format\"). The bootstrap blob gives you two views:\n\n- `conventions` — full bodies of the always-on (`trigger=always`) rules. Follow these unconditionally.\n- `convention_index` — METADATA ONLY (`ref`, `title`, `trigger`, `role`; no bodies) for **every** active convention, including the triggered ones whose bodies are NOT in `conventions`. This is your map of what triggered rules exist.\n\nBefore performing meaningful work with a specific trigger (e.g. `on-implement` before writing code), consult `convention_index`: if it lists entries for that trigger, pull their bodies on demand; if it lists none, skip the query.\n\n```\npad_item.action: list, collection: \"conventions\", status: \"active\"\n```\n\nFilter by trigger (`always`, `on-implement`, `on-task-complete`, etc.) when relevant — the `convention_index` triggers tell you which filters are worth running.\n\n`\"conventions\"` above is the DEFAULT collection slug and a workspace may have renamed it, in which case that literal returns nothing while `convention_index` still lists entries — the bootstrap payload resolves by declaration, not by name. When the two disagree, address the items by the `ref`s the index gave you, or find the collection with `pad_collection.action: list`. The same applies to `\"playbooks\"`; `pad_playbook` resolves by declaration and is unaffected.\n\n## Pushes and the session monitor (security note)\n\nPad can push an item into a Claude Code session (`pad push`, or the web item view's push composer). That is terminal instruction injection by design, so receiving is **consent-gated** (PLAN-2613, since v0.15.0): a session receives pushes only after it has connected (`/pad:connect` in the plugin, or a repo-level `push.auto_arm = true` in `.pad.toml` — which a per-user `auto_arm = false` in `~/.pad/config.toml` vetoes), and pushes are self-addressed only — the sender is always the receiving session's own user. This MCP connection is not a session: it cannot arm one, receive a push, or run the monitor, and the `pad_item` / `pad_project` tools expose no push action. If a user asks you to \"push\" something to their terminal, tell them it happens from their own terminal (`pad push \u003cref\u003e -m \"...\"` — the sending session need not be connected, the receiving ones must have consented); do not try to emulate it by writing comments or instructions into items.\n\n## Adding a workspace to this connection\n\nIf the user references a workspace this connection can't see (you'll get a 403 from workspace tools, or the workspace won't appear in `pad_workspace.list`), tell the user you can't see that workspace with your current permissions, then walk them through how to grant access: open Pad in their browser → switch to that workspace → avatar menu → \"Connect project...\" A 6-digit claim code will appear. Have them paste it back in chat, then call `pad_workspace.claim` with `{workspace: \"\u003cslug\u003e\", code: \"\u003c6 digits\u003e\"}`. The workspace joins this connection's allow-list and stays until the user revokes it via `/console/connected-apps`. No re-auth required.\n\nFor brand-new workspaces, `pad_workspace.create` with `{name: \"\u003cname\u003e\"}` (and optional `template`) creates the workspace AND auto-adds it to this connection's allow-list in one call — no claim code needed. Only works when the connection currently carries the \"may create workspaces\" grant — granted at consent time, or enabled later on the connections page. If the grant is present and set to false, the create is **refused with a 403 and no workspace is made** — retrying won't help and neither will the claim flow, since there is nothing to claim. Tell the user this connection isn't permitted to create workspaces, and offer them the two ways forward: create the workspace themselves in the browser and grant access with a claim code, or re-authorize this connection with \"Let this app create new workspaces\" enabled (`/console/connected-apps` can also flip it on an existing connection).\n\n## New workspace: offer to set it up\n\nThe bootstrap blob carries `needs_onboarding: bool` — true when the workspace has zero user-created items (template seeds don't count). When it's true, **lead with an active offer before anything else**: *\"This workspace is brand new and isn't set up yet. Want me to set it up? I'll ask a few quick questions and adapt it to your project.\"*\n\nThis is an **offer, not an auto-run** — wait for the user to say yes before running onboarding. If they accept, run the `onboard` playbook (use the `pad_onboard` prompt, or load the body via `pad_playbook` `action: get`, `ref: onboard`). If they decline, or already declined earlier in the session, respect that and skip the offer. The flag flips to false the moment any item exists, so it won't nag past first setup.\n\n## Multi-step workflows\n\nFour prompts ship with the server: `pad_plan`, `pad_ideate`, `pad_retro`, `pad_onboard`. Use them when the user wants help planning, brainstorming, retrospecting, or onboarding into a workspace — they encode the multi-step Pad-aware playbook for each.\n" } diff --git a/cmd/pad/testdata/mcp_wire/stdio_initialize.json b/cmd/pad/testdata/mcp_wire/stdio_initialize.json index a3d4b8dcb..20698319e 100644 --- a/cmd/pad/testdata/mcp_wire/stdio_initialize.json +++ b/cmd/pad/testdata/mcp_wire/stdio_initialize.json @@ -8,7 +8,7 @@ }, "padToolSurface": { "tool_surface_stable": true, - "version": "0.59" + "version": "0.60" } }, "prompts": {}, @@ -21,5 +21,5 @@ "name": "pad-mcp", "version": "" }, - "instructions": "Pad is a project tracker for developers and AI agents — issues (TASK, BUG), plans (PLAN), ideas (IDEA), docs (DOC), conventions, comments, and dependencies. Use this server when a user mentions:\n\n- Issue refs like `TASK-5`, `BUG-12`, `PLAN-3`, `IDEA-8` — they are stable, human-readable IDs and the canonical way to address items. Use them, never slugs; they are what audit trails and PR titles show.\n- Tasks / issues / items / plans / progress / \"what's on my plate\" / \"what to work on next\" / standup / changelog / retrospective.\n- Project conventions, decision records, or \"how should this team do X.\"\n\nIf the user is asking general code questions with no project-management thread, you don't need this server.\n\n## Tool surface (v0.59)\n\nTen resource × action tools, plus `pad_set_workspace`. Eleven tools total.\n\nInputs are validated strictly: an undeclared top-level key is rejected with a structured `validation_failed` naming it, never accepted and silently dropped.\n\n- `pad_item` — Items: create / update / delete / get / list / move / restore / link / unlink / deps / star / unstar / starred / comment / list-comments / edit-comment / delete-comment / backlinks / bulk-update / note / decide / export / import / history / remind / ack-reminder / claim / release. On create/update, field values may be passed as a `fields` OBJECT (the same shape reads return, e.g. `{\"status\":\"done\",\"effort\":\"l\"}`) — it merges into the same path as the dedicated params and `field: [\"key=value\"]`; the same key in two places with CONFLICTING values is refused, not silently resolved. `list` accepts `unparented: true` to keep items with no parent or implements relationship (mutually exclusive with `parent`). `list` results are SUMMARY-shaped by default on both transports — no content bodies; pass `full: true` for complete bodies (token-expensive), or prefer `get` for a single item's body. `update` field writes are a server-side field-level merge (only the keys you set change); pass `expected_seq` for optimistic concurrency — the `seq` you last read, which is on every item AND every list summary — and a stale value fails with a structured 409 `update_conflict` whose details carry `actual_seq` to retry with. PREFER IT over `expected_updated_at`, which cannot tell two writes inside one second apart; that one still works, and if you send both, seq decides. A token cannot see a browser tab's unsaved edits, so `content` sent WITH one is refused with 409 `content_pending_flush` while any exist; re-reading will not clear it. Wait for the editor to save, or pass `overwrite_pending_edits: true` to replace them. `content_state: \"superseded_set_aside\"` is different and wins when both apply: an editor upgrade set aside edits no tab can restore, so EVERY content write is refused (token or not; `details.set_aside_rows`) and waiting never clears it. Tell the user, or pass `overwrite_pending_edits: true` to discard them. Writing `content` while a browser tab has the item open routes the markdown through that live document, and the stored copy is updated only by a later collab-snapshot flush — usually from the tab that applied it, though any such write updates the row, and nothing guarantees one happens: the 200 echoes the content you SENT and carries `warnings.content_outcome: \"applied_pending_flush\"`, which describes that WRITE — the content went to the document, not the row — rather than being a live reading of the row, which a concurrent flush may already have updated. A `get` (or a `list` with `full: true` — a default list carries no content at all) before such a flush lands reads the stored copy and answers with the PREVIOUS content, which is the lag rather than a failed write, so do not re-send on the strength of it. You do not have to infer that from the write, and you will also see it when someone ELSE's tab holds the unflushed edits and you wrote nothing: a read that serves a body the server knows is behind the live document carries `content_state: \"applied_pending_flush\"` — same value as the write warning, different question (what the ROW is, not what your REQUEST did). Treat it as re-read later, never as re-send, and do not wait on it in a loop: nothing server-side moves that content into the stored copy, so the row catches up only when a tab next opens the item, which may never happen. The stored form may also not be byte-identical to what you sent, since the markdown round-trips through the editor (bullet markers, emphasis characters, list numbering and blank-line runs normalise; already-canonical markdown survives unchanged), so compare on meaning rather than bytes. `move` changes an item's COLLECTION within its workspace: system metadata (implementation notes, decision log, linked PR) survives it, and any field value the target schema has no home for is dropped AND reported in the move's activity entry — check there rather than assuming a move is lossless. A carried unique-field collision is dropped, named in `warnings.not_unique`. Three system keys — `implementation_notes`, `decision_log`, `convention` — cannot be set through `field` on update or move; the call is refused with `validation_failed`, naming the key and the write path that does maintain it (`note`, `decide`, and library activation respectively). `github_pr` is refused as well (BUG-2696): a `field` value arrived as a string, so the PR data was stored double-encoded and no link appeared. Its writer is `pad github link` / `unlink`, which needs a local git checkout, and there is no PR-link action on this surface yet, so treat linking a PR as something to hand to a human rather than a call to retry. On `create` they are refused too (BUG-3163), with the same `validation_failed`: create the item first, then use `note` / `decide` on it; convention metadata comes from library activation. `history` returns read-only item version metadata (newest-first), bounded to the NEWEST 50 versions by default (max 300 — pass `limit` to change the window); pass `full: true` to include each version's resolved content body (token-expensive). There is no `offset`: versions are stored as reverse patches, so only a newest-end window is cheap to reconstruct. Unassign or detach with the `clear_assigned_user` / `clear_agent_role` / `clear_parent` booleans; an empty `assign` / `role` / `parent` does NOT clear. `remind` / `ack-reminder`: a fired reminder shows in `pad_project` next / ready (the delivery path on an instance with no webhook) until acknowledged, and one on a completed item is hidden there. `claim` / `release`: a loser's 409 is `lease_held`, and a lease bumps neither `updated_at` nor `seq`. `edit-comment` / `delete-comment` take a `comment_id` from `list-comments` (whose rows carry `edited`); only a comment's author may edit it. Correct a comment in place rather than appending a correction. Deleting a comment that has replies leaves a placeholder (`deleted: true`, empty body) so the replies keep their parent; editing, replying to or reacting to it answers 409 `comment_deleted`. (Two older forms still work and are not deprecated: `field: [\"assigned_user_id=\"]` on either transport, and a direct `assigned_user_id: \"\"` param over remote `/mcp` only — prefer the boolean, which is the only one this schema advertises.)\n\n**Relation fields (v0.31).** A `relation` value may be a UUID, an issue ref (`COLO-3`), or the target item's EXACT TITLE — in that order, so an item literally titled `COLO-3` is unreachable by title while the ref resolves. Title matching is scoped to the collection the field declares: a title that is unique only workspace-wide is REFUSED, naming the collection searched, and a title matching two or more items inside the declared collection is refused as `ambiguous` rather than `not_found` — it matched too much, not too little. Only items YOU can see count towards any of that, so a match you have no access to never changes your answer.\n\nReads carry a `relation_targets` member beside `fields`: field key → `{id, ref, title}`, so you can render a relation without a request per value. `fields` still holds the canonical id, which is what you write back. An entry with an `id` and NO `ref`/`title` means the target is gone **or** you may not see it — the two are deliberately indistinguishable, so do not render it as either \"deleted\" or \"hidden\"; \"unavailable\" is the honest word. An entry with `stored_as_text: true` differs: the stored value is not UUID-shaped, so it was never an id (an imported title, legacy text). Render it as text, not as unavailable.\n\n**`multi_relation` fields (v0.33).** A field declared `multi_relation` holds an ORDERED LIST of references — `[\"\u003cuuid\u003e\", \"COLO-3\", \"Red\"]` — and every element resolves through the same UUID → ref → exact-title ladder, with the same collection scoping. Declare one through the `fields` DSL as `owners:multi_relation:people`: the third part is the TARGET COLLECTION, and omitting it is refused at parse time rather than building a field no write can ever satisfy.\n\nFour rules differ from a scalar `relation`, and none of them is guessable from the scalar behaviour:\n\n- **There is exactly one stored form for \"no references\": the key ABSENT.** Send `[]` and the write normalises it to that. A `required` `multi_relation` means at least one element that RESOLVED, so `[]` and an absent key both refuse.\n- **An empty or whitespace-only ELEMENT is refused**, naming its index — it is not skipped the way an empty scalar value is. Remove the element rather than blanking it.\n- **Order is part of the value.** `[\"a\",\"b\"]` and `[\"b\",\"a\"]` are different values, and a write preserves the order you sent.\n- **Duplicates are refused**, with reason `duplicate_referent`, naming the SECOND occurrence. Two elements can be different strings naming one item — a UUID and a ref, a ref and a title — so this is decided after resolution, and the two forms you sent may look nothing alike.\n\nIf ANY element fails, the whole write is refused and nothing is stored — never a partial list. On a cross-workspace copy the value is dropped WHOLE and reported once in `warnings.dropped_fields`, so an import can never change an element count.\n\nIn `relation_targets`, a `multi_relation` key carries a JSON **ARRAY** of the same `{id, ref, title}` objects, in stored order and one per stored element — including an `id`-only entry for an element that names nothing, so a position in the list lines up with the same position in `fields`. A scalar key is unchanged: still a single object, byte-identical to v0.31. Decide which shape to expect from the field's declared type, not by probing.\n\n- `pad_workspace` — Workspaces: list / members / invite / storage / audit-log / create / claim / deleted / restore.\n- `pad_collection` — Collections: list / create / update / delete.\n- `pad_project` — Project intelligence: dashboard / next / ready / stale / standup / changelog / report / activity. Use `ready` for the actionable backlog and `stale` for items needing attention; `activity` to catch up on what other agents/users changed since you last worked (non-streaming feed with item refs + change details).\n- `pad_role` — Agent roles: list / create / update / delete.\n- `pad_search` — Full-text search across items: query.\n- `pad_playbook` — Invokable procedures: list / get / run / match. Use `run` to bind args against a playbook's declared spec and get the rendered body back; side-effect-free. `run` refuses a playbook whose status isn't `active` (a draft still being authored) with a `playbook_not_active` error — pass `allow_draft: true` to override. Both `run` and `get` echo the playbook's `status`. Both may also carry `content_state` (`\"applied_pending_flush\"`, or `\"superseded_set_aside\"` as under `pad_item`), meaning the BODY you are about to execute is behind the item's live collaborative document — an editor holds unflushed edits, so the steps may be superseded. It is a reason to re-read or to ask, not a reason to refuse to run; the body is still the best the server has. `match` finds a playbook from text.\n- `pad_library` — Convention + playbook library (the global catalog of pre-built entries workspaces activate): list / get / activate.\n- `pad_attachment` — Read-only attachment metadata: list / show. `list` enumerates a workspace's attachments (filter by item / category / collection / attached / unattached); `show` returns one attachment's MIME, size, filename, and ETag via a HEAD request. Uploading and general file downloads stay CLI-only; bounded image bytes are available through the attachment resource below.\n- `pad_meta` — Server introspection: server-info / version / tool-surface / bootstrap. The `bootstrap` action returns one-shot workspace context (user + collections + always-on conventions + a metadata-only `convention_index` of every active convention + roles + playbook metadata + dashboard + recent activity).\n- `pad_set_workspace` — Load workspace context; response embeds the bootstrap blob so you load context in one call. Whether it also pins a session default: see Workspace context below. Takes `workspace: \u003cslug\u003e` only (no `action`).\n\nFor the ten resource × action tools, always pass `action` as a top-level field. Per-action required parameters are documented in each tool's description.\n\n## Resources are cheaper than tool calls\n\nRead these directly when you need workspace state:\n\n- `pad://workspace/{ws}/dashboard` — computed project overview (active items, plans, attention, suggested next).\n- `pad://workspace/{ws}/collections` — collection types + schemas.\n- `pad://workspace/{ws}/items` — list of all items (use `pad_item.action: list` for filtering).\n- `pad://workspace/{ws}/items/{ref}` — single item rendered as markdown.\n- `pad://workspace/{ws}/attachments/{id}` — image attachment as a bounded base64 `thumb-md` resource; rejects non-images and image bytes over 1 MiB (pre-base64).\n- `pad://workspace/{ws}/bootstrap` — one-shot workspace context (same payload as `pad_meta.action: bootstrap` and `pad_set_workspace`'s embedded response).\n- `pad://_meta/version` — server version + stability tiers.\n\nResources support host-side prefetch — if the host can fetch them once at session start, you don't pay per turn.\n\nLoad the bootstrap resource once per workspace and reuse its collections, conventions, roles, and playbook metadata. Refresh it after switching workspaces, after changing those configuration surfaces, when Pad reports stale schema/context, or when the user asks. For changing work state, read the affected item or use a targeted project tool instead of fetching the whole bootstrap again.\n\n## Workspace context\n\nEvery action that operates within a workspace accepts an optional `workspace` parameter. Resolution order:\n\n1. Explicit `workspace` argument on the call (always wins).\n2. On a single-user local server only: the session default set via `pad_set_workspace`.\n3. On a single-user local server only: the CWD-linked workspace from `.pad.toml`.\n\nA multi-user/remote server does **not** persist a session default — pass `workspace` explicitly on every call. If none resolves, the action returns a structured `no_workspace` error with `available_workspaces`.\n\n## Update flow: read first, then patch\n\nFor `pad_item.action: update`, the server merges your patch with the item's current state. Pass only the fields you want to change. When changing `status`, ALWAYS include a `comment` explaining why — it builds the audit trail that helps the team understand history.\n\n## One error code you must not retry\n\nErrors come back as a structured envelope with a `code`. Most failures are worth a retry after you fix something, and a `server_error` may be transient. `stored_state_unreadable` is neither.\n\nIt means the ITEM'S STORED DATA cannot be decoded — your input was fine. Today it fires when `note` / `decide` would append to an `implementation_notes` / `decision_log` value that is not a list of entries. The operation is refused precisely because completing it would overwrite that value. The refusal is fully deterministic, so retrying — immediately or after a backoff — fails identically every time and changes nothing.\n\nWhen you see it: tell the user which item is affected, and stop. Whether you can inspect the bad value depends on which layer is broken — if the item's whole `fields` blob fails to parse, `action: get` hands it back as a raw string; if one structured key is at fault, you won't see it at all, because the blob is normalized on this surface and a value that doesn't decode is dropped from the top-level arrays. A human can always read it with `pad item show \u003cref\u003e --format json`. Do not route around the refusal by writing the field some other way; that is the write that caused it.\n\n## Project conventions\n\nWorkspaces can declare conventions (e.g. \"run `make test` before PR\", \"use conventional commit format\"). The bootstrap blob gives you two views:\n\n- `conventions` — full bodies of the always-on (`trigger=always`) rules. Follow these unconditionally.\n- `convention_index` — METADATA ONLY (`ref`, `title`, `trigger`, `role`; no bodies) for **every** active convention, including the triggered ones whose bodies are NOT in `conventions`. This is your map of what triggered rules exist.\n\nBefore performing meaningful work with a specific trigger (e.g. `on-implement` before writing code), consult `convention_index`: if it lists entries for that trigger, pull their bodies on demand; if it lists none, skip the query.\n\n```\npad_item.action: list, collection: \"conventions\", status: \"active\"\n```\n\nFilter by trigger (`always`, `on-implement`, `on-task-complete`, etc.) when relevant — the `convention_index` triggers tell you which filters are worth running.\n\n`\"conventions\"` above is the DEFAULT collection slug and a workspace may have renamed it, in which case that literal returns nothing while `convention_index` still lists entries — the bootstrap payload resolves by declaration, not by name. When the two disagree, address the items by the `ref`s the index gave you, or find the collection with `pad_collection.action: list`. The same applies to `\"playbooks\"`; `pad_playbook` resolves by declaration and is unaffected.\n\n## Pushes and the session monitor (security note)\n\nPad can push an item into a Claude Code session (`pad push`, or the web item view's push composer). That is terminal instruction injection by design, so receiving is **consent-gated** (PLAN-2613, since v0.15.0): a session receives pushes only after it has connected (`/pad:connect` in the plugin, or a repo-level `push.auto_arm = true` in `.pad.toml` — which a per-user `auto_arm = false` in `~/.pad/config.toml` vetoes), and pushes are self-addressed only — the sender is always the receiving session's own user. This MCP connection is not a session: it cannot arm one, receive a push, or run the monitor, and the `pad_item` / `pad_project` tools expose no push action. If a user asks you to \"push\" something to their terminal, tell them it happens from their own terminal (`pad push \u003cref\u003e -m \"...\"` — the sending session need not be connected, the receiving ones must have consented); do not try to emulate it by writing comments or instructions into items.\n\n## Adding a workspace to this connection\n\nIf the user references a workspace this connection can't see (you'll get a 403 from workspace tools, or the workspace won't appear in `pad_workspace.list`), tell the user you can't see that workspace with your current permissions, then walk them through how to grant access: open Pad in their browser → switch to that workspace → avatar menu → \"Connect project...\" A 6-digit claim code will appear. Have them paste it back in chat, then call `pad_workspace.claim` with `{workspace: \"\u003cslug\u003e\", code: \"\u003c6 digits\u003e\"}`. The workspace joins this connection's allow-list and stays until the user revokes it via `/console/connected-apps`. No re-auth required.\n\nFor brand-new workspaces, `pad_workspace.create` with `{name: \"\u003cname\u003e\"}` (and optional `template`) creates the workspace AND auto-adds it to this connection's allow-list in one call — no claim code needed. Only works when the connection currently carries the \"may create workspaces\" grant — granted at consent time, or enabled later on the connections page. If the grant is present and set to false, the create is **refused with a 403 and no workspace is made** — retrying won't help and neither will the claim flow, since there is nothing to claim. Tell the user this connection isn't permitted to create workspaces, and offer them the two ways forward: create the workspace themselves in the browser and grant access with a claim code, or re-authorize this connection with \"Let this app create new workspaces\" enabled (`/console/connected-apps` can also flip it on an existing connection).\n\n## New workspace: offer to set it up\n\nThe bootstrap blob carries `needs_onboarding: bool` — true when the workspace has zero user-created items (template seeds don't count). When it's true, **lead with an active offer before anything else**: *\"This workspace is brand new and isn't set up yet. Want me to set it up? I'll ask a few quick questions and adapt it to your project.\"*\n\nThis is an **offer, not an auto-run** — wait for the user to say yes before running onboarding. If they accept, run the `onboard` playbook (use the `pad_onboard` prompt, or load the body via `pad_playbook` `action: get`, `ref: onboard`). If they decline, or already declined earlier in the session, respect that and skip the offer. The flag flips to false the moment any item exists, so it won't nag past first setup.\n\n## Multi-step workflows\n\nFour prompts ship with the server: `pad_plan`, `pad_ideate`, `pad_retro`, `pad_onboard`. Use them when the user wants help planning, brainstorming, retrospecting, or onboarding into a workspace — they encode the multi-step Pad-aware playbook for each.\n" + "instructions": "Pad is a project tracker for developers and AI agents — issues (TASK, BUG), plans (PLAN), ideas (IDEA), docs (DOC), conventions, comments, and dependencies. Use this server when a user mentions:\n\n- Issue refs like `TASK-5`, `BUG-12`, `PLAN-3`, `IDEA-8` — they are stable, human-readable IDs and the canonical way to address items. Use them, never slugs; they are what audit trails and PR titles show.\n- Tasks / issues / items / plans / progress / \"what's on my plate\" / \"what to work on next\" / standup / changelog / retrospective.\n- Project conventions, decision records, or \"how should this team do X.\"\n\nIf the user is asking general code questions with no project-management thread, you don't need this server.\n\n## Tool surface (v0.60)\n\nTen resource × action tools, plus `pad_set_workspace`. Eleven tools total.\n\nInputs are validated strictly: an undeclared top-level key is rejected with a structured `validation_failed` naming it, never accepted and silently dropped.\n\n- `pad_item` — Items: create / update / delete / get / list / move / restore / link / unlink / deps / star / unstar / starred / comment / list-comments / edit-comment / delete-comment / backlinks / bulk-update / note / decide / export / import / history / remind / ack-reminder / claim / release. On create/update, field values may be passed as a `fields` OBJECT (the same shape reads return, e.g. `{\"status\":\"done\",\"effort\":\"l\"}`) — it merges into the same path as the dedicated params and `field: [\"key=value\"]`; the same key in two places with CONFLICTING values is refused, not silently resolved. `list` accepts `unparented: true` to keep items with no parent or implements relationship (mutually exclusive with `parent`). `list` results are SUMMARY-shaped by default on both transports — no content bodies; pass `full: true` for complete bodies (token-expensive), or prefer `get` for a single item's body. `update` field writes are a server-side field-level merge (only the keys you set change); pass `expected_seq` for optimistic concurrency — the `seq` you last read, which is on every item AND every list summary — and a stale value fails with a structured 409 `update_conflict` whose details carry `actual_seq` to retry with. PREFER IT over `expected_updated_at`, which cannot tell two writes inside one second apart; that one still works, and if you send both, seq decides. A token cannot see a browser tab's unsaved edits, so `content` sent WITH one is refused with 409 `content_pending_flush` while any exist; re-reading will not clear it. Wait for the editor to save, or pass `overwrite_pending_edits: true` to replace them. `content_state: \"superseded_set_aside\"` is different and wins when both apply: an editor upgrade set aside edits no tab can restore, so EVERY content write is refused (token or not; `details.set_aside_rows`) and waiting never clears it. Tell the user, or pass `overwrite_pending_edits: true` to discard them. Writing `content` while a browser tab has the item open routes the markdown through that live document, and the stored copy is updated only by a later collab-snapshot flush — usually from the tab that applied it, though any such write updates the row, and nothing guarantees one happens: the 200 echoes the content you SENT and carries `warnings.content_outcome: \"applied_pending_flush\"`, which describes that WRITE — the content went to the document, not the row — rather than being a live reading of the row, which a concurrent flush may already have updated. A `get` (or a `list` with `full: true` — a default list carries no content at all) before such a flush lands reads the stored copy and answers with the PREVIOUS content, which is the lag rather than a failed write, so do not re-send on the strength of it. You do not have to infer that from the write, and you will also see it when someone ELSE's tab holds the unflushed edits and you wrote nothing: a read that serves a body the server knows is behind the live document carries `content_state: \"applied_pending_flush\"` — same value as the write warning, different question (what the ROW is, not what your REQUEST did). Treat it as re-read later, never as re-send, and do not wait on it in a loop: the server normally recovers the stored copy itself about a minute after the last tab leaves the item, but that is not guaranteed (a recovery can fail, and an operator can turn it off), and otherwise the row catches up only when a tab next opens the item. The stored form may also not be byte-identical to what you sent, since the markdown round-trips through the editor (bullet markers, emphasis characters, list numbering and blank-line runs normalise; already-canonical markdown survives unchanged), so compare on meaning rather than bytes. `move` changes an item's COLLECTION within its workspace: system metadata (implementation notes, decision log, linked PR) survives it, and any field value the target schema has no home for is dropped AND reported in the move's activity entry — check there rather than assuming a move is lossless. A carried unique-field collision is dropped, named in `warnings.not_unique`. Three system keys — `implementation_notes`, `decision_log`, `convention` — cannot be set through `field` on update or move; the call is refused with `validation_failed`, naming the key and the write path that does maintain it (`note`, `decide`, and library activation respectively). `github_pr` is refused as well (BUG-2696): a `field` value arrived as a string, so the PR data was stored double-encoded and no link appeared. Its writer is `pad github link` / `unlink`, which needs a local git checkout, and there is no PR-link action on this surface yet, so treat linking a PR as something to hand to a human rather than a call to retry. On `create` they are refused too (BUG-3163), with the same `validation_failed`: create the item first, then use `note` / `decide` on it; convention metadata comes from library activation. `history` returns read-only item version metadata (newest-first), bounded to the NEWEST 50 versions by default (max 300 — pass `limit` to change the window); pass `full: true` to include each version's resolved content body (token-expensive). There is no `offset`: versions are stored as reverse patches, so only a newest-end window is cheap to reconstruct. Unassign or detach with the `clear_assigned_user` / `clear_agent_role` / `clear_parent` booleans; an empty `assign` / `role` / `parent` does NOT clear. `remind` / `ack-reminder`: a fired reminder shows in `pad_project` next / ready (the delivery path on an instance with no webhook) until acknowledged, and one on a completed item is hidden there. `claim` / `release`: a loser's 409 is `lease_held`, and a lease bumps neither `updated_at` nor `seq`. `edit-comment` / `delete-comment` take a `comment_id` from `list-comments` (whose rows carry `edited`); only a comment's author may edit it. Correct a comment in place rather than appending a correction. Deleting a comment that has replies leaves a placeholder (`deleted: true`, empty body) so the replies keep their parent; editing, replying to or reacting to it answers 409 `comment_deleted`. (Two older forms still work and are not deprecated: `field: [\"assigned_user_id=\"]` on either transport, and a direct `assigned_user_id: \"\"` param over remote `/mcp` only — prefer the boolean, which is the only one this schema advertises.)\n\n**Relation fields (v0.31).** A `relation` value may be a UUID, an issue ref (`COLO-3`), or the target item's EXACT TITLE — in that order, so an item literally titled `COLO-3` is unreachable by title while the ref resolves. Title matching is scoped to the collection the field declares: a title that is unique only workspace-wide is REFUSED, naming the collection searched, and a title matching two or more items inside the declared collection is refused as `ambiguous` rather than `not_found` — it matched too much, not too little. Only items YOU can see count towards any of that, so a match you have no access to never changes your answer.\n\nReads carry a `relation_targets` member beside `fields`: field key → `{id, ref, title}`, so you can render a relation without a request per value. `fields` still holds the canonical id, which is what you write back. An entry with an `id` and NO `ref`/`title` means the target is gone **or** you may not see it — the two are deliberately indistinguishable, so do not render it as either \"deleted\" or \"hidden\"; \"unavailable\" is the honest word. An entry with `stored_as_text: true` differs: the stored value is not UUID-shaped, so it was never an id (an imported title, legacy text). Render it as text, not as unavailable.\n\n**`multi_relation` fields (v0.33).** A field declared `multi_relation` holds an ORDERED LIST of references — `[\"\u003cuuid\u003e\", \"COLO-3\", \"Red\"]` — and every element resolves through the same UUID → ref → exact-title ladder, with the same collection scoping. Declare one through the `fields` DSL as `owners:multi_relation:people`: the third part is the TARGET COLLECTION, and omitting it is refused at parse time rather than building a field no write can ever satisfy.\n\nFour rules differ from a scalar `relation`, and none of them is guessable from the scalar behaviour:\n\n- **There is exactly one stored form for \"no references\": the key ABSENT.** Send `[]` and the write normalises it to that. A `required` `multi_relation` means at least one element that RESOLVED, so `[]` and an absent key both refuse.\n- **An empty or whitespace-only ELEMENT is refused**, naming its index — it is not skipped the way an empty scalar value is. Remove the element rather than blanking it.\n- **Order is part of the value.** `[\"a\",\"b\"]` and `[\"b\",\"a\"]` are different values, and a write preserves the order you sent.\n- **Duplicates are refused**, with reason `duplicate_referent`, naming the SECOND occurrence. Two elements can be different strings naming one item — a UUID and a ref, a ref and a title — so this is decided after resolution, and the two forms you sent may look nothing alike.\n\nIf ANY element fails, the whole write is refused and nothing is stored — never a partial list. On a cross-workspace copy the value is dropped WHOLE and reported once in `warnings.dropped_fields`, so an import can never change an element count.\n\nIn `relation_targets`, a `multi_relation` key carries a JSON **ARRAY** of the same `{id, ref, title}` objects, in stored order and one per stored element — including an `id`-only entry for an element that names nothing, so a position in the list lines up with the same position in `fields`. A scalar key is unchanged: still a single object, byte-identical to v0.31. Decide which shape to expect from the field's declared type, not by probing.\n\n- `pad_workspace` — Workspaces: list / members / invite / storage / audit-log / create / claim / deleted / restore.\n- `pad_collection` — Collections: list / create / update / delete.\n- `pad_project` — Project intelligence: dashboard / next / ready / stale / standup / changelog / report / activity. Use `ready` for the actionable backlog and `stale` for items needing attention; `activity` to catch up on what other agents/users changed since you last worked (non-streaming feed with item refs + change details).\n- `pad_role` — Agent roles: list / create / update / delete.\n- `pad_search` — Full-text search across items: query.\n- `pad_playbook` — Invokable procedures: list / get / run / match. Use `run` to bind args against a playbook's declared spec and get the rendered body back; side-effect-free. `run` refuses a playbook whose status isn't `active` (a draft still being authored) with a `playbook_not_active` error — pass `allow_draft: true` to override. Both `run` and `get` echo the playbook's `status`. Both may also carry `content_state` (`\"applied_pending_flush\"`, or `\"superseded_set_aside\"` as under `pad_item`), meaning the BODY you are about to execute is behind the item's live collaborative document — an editor holds unflushed edits, so the steps may be superseded. It is a reason to re-read or to ask, not a reason to refuse to run; the body is still the best the server has. `match` finds a playbook from text.\n- `pad_library` — Convention + playbook library (the global catalog of pre-built entries workspaces activate): list / get / activate.\n- `pad_attachment` — Read-only attachment metadata: list / show. `list` enumerates a workspace's attachments (filter by item / category / collection / attached / unattached); `show` returns one attachment's MIME, size, filename, and ETag via a HEAD request. Uploading and general file downloads stay CLI-only; bounded image bytes are available through the attachment resource below.\n- `pad_meta` — Server introspection: server-info / version / tool-surface / bootstrap. The `bootstrap` action returns one-shot workspace context (user + collections + always-on conventions + a metadata-only `convention_index` of every active convention + roles + playbook metadata + dashboard + recent activity).\n- `pad_set_workspace` — Load workspace context; response embeds the bootstrap blob so you load context in one call. Whether it also pins a session default: see Workspace context below. Takes `workspace: \u003cslug\u003e` only (no `action`).\n\nFor the ten resource × action tools, always pass `action` as a top-level field. Per-action required parameters are documented in each tool's description.\n\n## Resources are cheaper than tool calls\n\nRead these directly when you need workspace state:\n\n- `pad://workspace/{ws}/dashboard` — computed project overview (active items, plans, attention, suggested next).\n- `pad://workspace/{ws}/collections` — collection types + schemas.\n- `pad://workspace/{ws}/items` — list of all items (use `pad_item.action: list` for filtering).\n- `pad://workspace/{ws}/items/{ref}` — single item rendered as markdown.\n- `pad://workspace/{ws}/attachments/{id}` — image attachment as a bounded base64 `thumb-md` resource; rejects non-images and image bytes over 1 MiB (pre-base64).\n- `pad://workspace/{ws}/bootstrap` — one-shot workspace context (same payload as `pad_meta.action: bootstrap` and `pad_set_workspace`'s embedded response).\n- `pad://_meta/version` — server version + stability tiers.\n\nResources support host-side prefetch — if the host can fetch them once at session start, you don't pay per turn.\n\nLoad the bootstrap resource once per workspace and reuse its collections, conventions, roles, and playbook metadata. Refresh it after switching workspaces, after changing those configuration surfaces, when Pad reports stale schema/context, or when the user asks. For changing work state, read the affected item or use a targeted project tool instead of fetching the whole bootstrap again.\n\n## Workspace context\n\nEvery action that operates within a workspace accepts an optional `workspace` parameter. Resolution order:\n\n1. Explicit `workspace` argument on the call (always wins).\n2. On a single-user local server only: the session default set via `pad_set_workspace`.\n3. On a single-user local server only: the CWD-linked workspace from `.pad.toml`.\n\nA multi-user/remote server does **not** persist a session default — pass `workspace` explicitly on every call. If none resolves, the action returns a structured `no_workspace` error with `available_workspaces`.\n\n## Update flow: read first, then patch\n\nFor `pad_item.action: update`, the server merges your patch with the item's current state. Pass only the fields you want to change. When changing `status`, ALWAYS include a `comment` explaining why — it builds the audit trail that helps the team understand history.\n\n## One error code you must not retry\n\nErrors come back as a structured envelope with a `code`. Most failures are worth a retry after you fix something, and a `server_error` may be transient. `stored_state_unreadable` is neither.\n\nIt means the ITEM'S STORED DATA cannot be decoded — your input was fine. Today it fires when `note` / `decide` would append to an `implementation_notes` / `decision_log` value that is not a list of entries. The operation is refused precisely because completing it would overwrite that value. The refusal is fully deterministic, so retrying — immediately or after a backoff — fails identically every time and changes nothing.\n\nWhen you see it: tell the user which item is affected, and stop. Whether you can inspect the bad value depends on which layer is broken — if the item's whole `fields` blob fails to parse, `action: get` hands it back as a raw string; if one structured key is at fault, you won't see it at all, because the blob is normalized on this surface and a value that doesn't decode is dropped from the top-level arrays. A human can always read it with `pad item show \u003cref\u003e --format json`. Do not route around the refusal by writing the field some other way; that is the write that caused it.\n\n## Project conventions\n\nWorkspaces can declare conventions (e.g. \"run `make test` before PR\", \"use conventional commit format\"). The bootstrap blob gives you two views:\n\n- `conventions` — full bodies of the always-on (`trigger=always`) rules. Follow these unconditionally.\n- `convention_index` — METADATA ONLY (`ref`, `title`, `trigger`, `role`; no bodies) for **every** active convention, including the triggered ones whose bodies are NOT in `conventions`. This is your map of what triggered rules exist.\n\nBefore performing meaningful work with a specific trigger (e.g. `on-implement` before writing code), consult `convention_index`: if it lists entries for that trigger, pull their bodies on demand; if it lists none, skip the query.\n\n```\npad_item.action: list, collection: \"conventions\", status: \"active\"\n```\n\nFilter by trigger (`always`, `on-implement`, `on-task-complete`, etc.) when relevant — the `convention_index` triggers tell you which filters are worth running.\n\n`\"conventions\"` above is the DEFAULT collection slug and a workspace may have renamed it, in which case that literal returns nothing while `convention_index` still lists entries — the bootstrap payload resolves by declaration, not by name. When the two disagree, address the items by the `ref`s the index gave you, or find the collection with `pad_collection.action: list`. The same applies to `\"playbooks\"`; `pad_playbook` resolves by declaration and is unaffected.\n\n## Pushes and the session monitor (security note)\n\nPad can push an item into a Claude Code session (`pad push`, or the web item view's push composer). That is terminal instruction injection by design, so receiving is **consent-gated** (PLAN-2613, since v0.15.0): a session receives pushes only after it has connected (`/pad:connect` in the plugin, or a repo-level `push.auto_arm = true` in `.pad.toml` — which a per-user `auto_arm = false` in `~/.pad/config.toml` vetoes), and pushes are self-addressed only — the sender is always the receiving session's own user. This MCP connection is not a session: it cannot arm one, receive a push, or run the monitor, and the `pad_item` / `pad_project` tools expose no push action. If a user asks you to \"push\" something to their terminal, tell them it happens from their own terminal (`pad push \u003cref\u003e -m \"...\"` — the sending session need not be connected, the receiving ones must have consented); do not try to emulate it by writing comments or instructions into items.\n\n## Adding a workspace to this connection\n\nIf the user references a workspace this connection can't see (you'll get a 403 from workspace tools, or the workspace won't appear in `pad_workspace.list`), tell the user you can't see that workspace with your current permissions, then walk them through how to grant access: open Pad in their browser → switch to that workspace → avatar menu → \"Connect project...\" A 6-digit claim code will appear. Have them paste it back in chat, then call `pad_workspace.claim` with `{workspace: \"\u003cslug\u003e\", code: \"\u003c6 digits\u003e\"}`. The workspace joins this connection's allow-list and stays until the user revokes it via `/console/connected-apps`. No re-auth required.\n\nFor brand-new workspaces, `pad_workspace.create` with `{name: \"\u003cname\u003e\"}` (and optional `template`) creates the workspace AND auto-adds it to this connection's allow-list in one call — no claim code needed. Only works when the connection currently carries the \"may create workspaces\" grant — granted at consent time, or enabled later on the connections page. If the grant is present and set to false, the create is **refused with a 403 and no workspace is made** — retrying won't help and neither will the claim flow, since there is nothing to claim. Tell the user this connection isn't permitted to create workspaces, and offer them the two ways forward: create the workspace themselves in the browser and grant access with a claim code, or re-authorize this connection with \"Let this app create new workspaces\" enabled (`/console/connected-apps` can also flip it on an existing connection).\n\n## New workspace: offer to set it up\n\nThe bootstrap blob carries `needs_onboarding: bool` — true when the workspace has zero user-created items (template seeds don't count). When it's true, **lead with an active offer before anything else**: *\"This workspace is brand new and isn't set up yet. Want me to set it up? I'll ask a few quick questions and adapt it to your project.\"*\n\nThis is an **offer, not an auto-run** — wait for the user to say yes before running onboarding. If they accept, run the `onboard` playbook (use the `pad_onboard` prompt, or load the body via `pad_playbook` `action: get`, `ref: onboard`). If they decline, or already declined earlier in the session, respect that and skip the offer. The flag flips to false the moment any item exists, so it won't nag past first setup.\n\n## Multi-step workflows\n\nFour prompts ship with the server: `pad_plan`, `pad_ideate`, `pad_retro`, `pad_onboard`. Use them when the user wants help planning, brainstorming, retrospecting, or onboarding into a workspace — they encode the multi-step Pad-aware playbook for each.\n" } diff --git a/docs/backup.md b/docs/backup.md index e81abd1a6..ff2ab99f2 100644 --- a/docs/backup.md +++ b/docs/backup.md @@ -251,11 +251,15 @@ since otherwise no server is running to make one — that workspace is refused t That refusal does **not** claim nothing has been migrated: earlier workspaces in the run are already in PostgreSQL, and it says so with the count. -Opening the item in the web UI is the only remedy, and that is structural rather -than an omission: Pad's collaboration server is a dumb relay that stores opaque -Yjs updates without parsing them, so nothing server-side can turn those updates -back into markdown — only a client can. This is also why the escape hatch is -named for what it does to the data: +There are two remedies. Opening each item in the web UI lets the tab flush it. +Or start the Pad server with op-log recovery on (the default; see +[Op-log materializer worker](deployment.md#op-log-materializer-worker)) and +leave it running for a few minutes: it rebuilds each such body from the op-log +in the background, then stop it and re-run the migration. The migration itself +never recovers anything, because it runs with the server stopped, and a +recovery can fail or be turned off (`PAD_MATERIALIZE=off`), so the refusal +stays. This is also why the escape hatch is named for what it does to the +data: ```bash pad db migrate-to-pg --discard-unflushed-edits diff --git a/docs/deployment.md b/docs/deployment.md index 1a8001254..ac1f0403d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1224,18 +1224,58 @@ address inside `PAD_TRUSTED_PROXIES` (the proxy sends no `X-Forwarded-For` or ### Op-log materializer worker -**Not active yet.** The worker and the two variables below take effect once -the materializer is wired into the server (TASK-2198 U4); until then they are -inert, and setting them changes nothing. - When a browser tab closes without saving, its last edits are only in the -item's collaborative op-log. Pad can turn that op-log back into the item's -markdown on the server by running the editor's own JavaScript. It does this in +item's collaborative op-log, and the item's stored body (what the API, the +CLI, MCP and search read, marked `content_state: applied_pending_flush`) is +behind them. Pad turns that op-log back into the item's markdown on the server +by running the editor's own JavaScript, and stores the result. It does this in a **separate worker process** (the same `pad` binary, started as `pad __materialize-worker`), never inside the server: a crafted update can make that JavaScript use gigabytes of memory, and in a separate process that kills one worker instead of the server. +**When recovery happens.** Only in the background, never while a request +waits: + +- **When the item's editing room closes.** A room stays open while any tab has + the item open, and for 60 seconds after the last one leaves. When it closes + with unsaved edits in the op-log, the item is queued. A tab that closed + without saving is therefore normally recovered about a minute after it went + away, plus the time the jobs queued ahead of it take. +- **A sweep every minute** picks up items the first trigger could not: unsaved + edits whose newest op-log row is more than 2 minutes old and whose item has + no open room (for example after a server restart). It takes 200 at a time + and continues where the last sweep stopped, so a large backlog is worked + through in turn rather than the same 200 being retried. + +A recovery writes the body only if the op-log has not changed since the job +was built and no tab has the item open; otherwise it does nothing and the next +trigger tries again. It bumps the item's `seq`, so a writer holding an older +`expected_seq` is refused rather than overwriting the recovered text, and it +records one version, attributed to `system` with source `recovery` and the +summary "recovered from an unsaved editor session" (the op-log records no +author, so none is guessed). It does not delete the op-log; the next tab to +open the item replays it as before. Items holding edits set aside by an editor +upgrade (`content_state: superseded_set_aside`) are not touched. + +A job that fails (timeout, memory limit, a worker death, an editor error) is +retried after 1 minute, then 2; after 3 consecutive failures the item is not +tried again until new edits arrive for it. Each failure is logged with its +kind, and an item that gives up logs one warning, `item exhausted its failure +budget`, with its id, failure count and `last_error_kind`, so a stuck item can +be found in the log. Items skipped because they hold set-aside edits are not +logged one by one; the sweep's summary line (`op-log recovery sweep`, written +only when something changed) carries their running count as +`set_aside_skipped_total`. This budget is kept in memory, so a restart grants +each item its 3 attempts again. An item whose stored edits were written under +another editor schema version is skipped, and logs the same one warning. + +With `PAD_MATERIALIZE=off` none of this runs: no worker is ever started, and a +tab that closed without saving catches up only when a tab next opens the item, +which may never happen. The same is true while no memory cap can be +established (see the end of this section): each pending item's job is then +refused, counted against its 3 attempts, and not logged per item. + - The worker starts on the first job, not at server start, and then stays up while jobs keep coming (loading it takes a few seconds). Jobs run one at a time. @@ -1260,6 +1300,7 @@ kills one worker instead of the server. | Variable | Default | Description | |----------|---------|-------------| +| `PAD_MATERIALIZE` | `on` | `off` (or `0`, `false`, `no`) disables op-log recovery entirely: no worker process is started and no trigger runs. Any other value leaves it on; an unrecognised one is logged. | | `PAD_MATERIALIZE_TIMEOUT` | `2s` | Per-job time limit, in Go duration syntax (`2s`, `1500ms`). Clamped to 250ms–60s. | | `PAD_MATERIALIZE_IDLE_TIMEOUT` | `5m` | How long the worker may sit with no job before it is stopped (it holds about 200 MB resident while it runs), in Go duration syntax. `0` means never stop it. Clamped to 30s–24h. | | `PAD_MATERIALIZE_MEM_LIMIT` | `2GiB` | How much memory one job may add to the loaded worker (see below): a whole number of bytes, or a whole number followed by `KiB`, `MiB` or `GiB` with no space (`2048MiB`, `2GiB`, `2147483648`). Decimal units such as `GB` are refused. Clamped to 256MiB–16GiB. | diff --git a/internal/collab/manager.go b/internal/collab/manager.go index b6748850f..2267f0ceb 100644 --- a/internal/collab/manager.go +++ b/internal/collab/manager.go @@ -146,6 +146,65 @@ type RoomManager struct { // column read in the common (same-process) case. lastRestoreSeqMu sync.Mutex lastRestoreSeqs map[string]int64 + + // idleHook, when set, is called with the item's id after a room has been + // reclaimed at the end of its grace TTL (TASK-2198 U4: the op-log + // recovery trigger). Called outside m.mu, from the grace timer's + // goroutine; it must not block. + idleHookMu sync.Mutex + idleHook func(itemID string) +} + +// SetIdleHook installs fn as the room-reclaimed callback (see idleHook). Nil +// removes it. +func (m *RoomManager) SetIdleHook(fn func(itemID string)) { + m.idleHookMu.Lock() + m.idleHook = fn + m.idleHookMu.Unlock() +} + +// ErrRoomOpen is returned by UnderItemLockIfNoRoom when the item has a room. +var ErrRoomOpen = errors.New("collab: a room is open for the item") + +// UnderItemLockIfNoRoom runs fn under the per-item setup lock, and only when +// no room exists for the item; otherwise it returns ErrRoomOpen without +// calling fn. A room counts from its creation until its grace TTL reclaims +// it, whether or not a connection is attached. +// +// While fn runs nothing can append to the item's op-log: every append +// happens through a room (a connection's readLoop, or the designated +// applier), and Join creates a room only while holding this same lock. The op-log +// recovery write (TASK-2198 U4) relies on that; it also re-checks the op-log +// inside its own transaction. +func (m *RoomManager) UnderItemLockIfNoRoom(itemID string, fn func() error) error { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return errManagerClosed + } + m.mu.Unlock() + lock := m.itemLock(itemID) + lock.Lock() + defer lock.Unlock() + m.mu.Lock() + closed, open := m.closed, m.rooms[itemID] != nil + m.mu.Unlock() + if closed { + return errManagerClosed + } + if open { + return ErrRoomOpen + } + return fn() +} + +// HasRoom reports whether a room exists for the item right now. Racy by +// nature; a caller that acts on the answer re-checks under the lock +// (UnderItemLockIfNoRoom). +func (m *RoomManager) HasRoom(itemID string) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.rooms[itemID] != nil } // SetRestoreBoundary records a restore boundary for itemID, keeping the @@ -841,9 +900,16 @@ func (m *RoomManager) getOrCreate(itemID string) *Room { // Join mints a fresh Room. func (m *RoomManager) markRoomGone(itemID string) { m.mu.Lock() - defer m.mu.Unlock() delete(m.rooms, itemID) + m.mu.Unlock() slog.Debug("collab: room reclaimed after grace TTL", "item_id", itemID) + + m.idleHookMu.Lock() + hook := m.idleHook + m.idleHookMu.Unlock() + if hook != nil { + hook(itemID) + } } // RoomCount is a test/debug accessor. Production code shouldn't make diff --git a/internal/materialize/supervisor_config.go b/internal/materialize/supervisor_config.go index 51e39b039..6e31a4fdb 100644 --- a/internal/materialize/supervisor_config.go +++ b/internal/materialize/supervisor_config.go @@ -29,6 +29,30 @@ const ( EnvIdleTimeout = "PAD_MATERIALIZE_IDLE_TIMEOUT" ) +// EnvSwitch turns op-log recovery off entirely: PAD_MATERIALIZE=off (or 0, +// false, no; case-insensitive) and the server constructs no Supervisor, +// installs no trigger and never spawns a worker. Unset, or any other value, +// leaves it on; a value that is neither a recognised on nor off word is +// logged and read as on. +const EnvSwitch = "PAD_MATERIALIZE" + +// Enabled reads EnvSwitch through getenv. +func Enabled(getenv func(string) string, logger *slog.Logger) bool { + if logger == nil { + logger = slog.Default() + } + v := strings.ToLower(strings.TrimSpace(getenv(EnvSwitch))) + switch v { + case "off", "0", "false", "no", "disabled": + return false + case "", "on", "1", "true", "yes", "enabled": + return true + } + logger.Warn("materialize: unrecognised "+EnvSwitch+" value; op-log recovery stays ON (set it to off to disable)", + "value", getenv(EnvSwitch)) + return true +} + const ( DefaultTimeout = 2 * time.Second MinTimeout = 250 * time.Millisecond diff --git a/internal/mcp/instructions.md b/internal/mcp/instructions.md index e61997a76..d101c8771 100644 --- a/internal/mcp/instructions.md +++ b/internal/mcp/instructions.md @@ -6,13 +6,13 @@ Pad is a project tracker for developers and AI agents — issues (TASK, BUG), pl If the user is asking general code questions with no project-management thread, you don't need this server. -## Tool surface (v0.59) +## Tool surface (v0.60) Ten resource × action tools, plus `pad_set_workspace`. Eleven tools total. Inputs are validated strictly: an undeclared top-level key is rejected with a structured `validation_failed` naming it, never accepted and silently dropped. -- `pad_item` — Items: create / update / delete / get / list / move / restore / link / unlink / deps / star / unstar / starred / comment / list-comments / edit-comment / delete-comment / backlinks / bulk-update / note / decide / export / import / history / remind / ack-reminder / claim / release. On create/update, field values may be passed as a `fields` OBJECT (the same shape reads return, e.g. `{"status":"done","effort":"l"}`) — it merges into the same path as the dedicated params and `field: ["key=value"]`; the same key in two places with CONFLICTING values is refused, not silently resolved. `list` accepts `unparented: true` to keep items with no parent or implements relationship (mutually exclusive with `parent`). `list` results are SUMMARY-shaped by default on both transports — no content bodies; pass `full: true` for complete bodies (token-expensive), or prefer `get` for a single item's body. `update` field writes are a server-side field-level merge (only the keys you set change); pass `expected_seq` for optimistic concurrency — the `seq` you last read, which is on every item AND every list summary — and a stale value fails with a structured 409 `update_conflict` whose details carry `actual_seq` to retry with. PREFER IT over `expected_updated_at`, which cannot tell two writes inside one second apart; that one still works, and if you send both, seq decides. A token cannot see a browser tab's unsaved edits, so `content` sent WITH one is refused with 409 `content_pending_flush` while any exist; re-reading will not clear it. Wait for the editor to save, or pass `overwrite_pending_edits: true` to replace them. `content_state: "superseded_set_aside"` is different and wins when both apply: an editor upgrade set aside edits no tab can restore, so EVERY content write is refused (token or not; `details.set_aside_rows`) and waiting never clears it. Tell the user, or pass `overwrite_pending_edits: true` to discard them. Writing `content` while a browser tab has the item open routes the markdown through that live document, and the stored copy is updated only by a later collab-snapshot flush — usually from the tab that applied it, though any such write updates the row, and nothing guarantees one happens: the 200 echoes the content you SENT and carries `warnings.content_outcome: "applied_pending_flush"`, which describes that WRITE — the content went to the document, not the row — rather than being a live reading of the row, which a concurrent flush may already have updated. A `get` (or a `list` with `full: true` — a default list carries no content at all) before such a flush lands reads the stored copy and answers with the PREVIOUS content, which is the lag rather than a failed write, so do not re-send on the strength of it. You do not have to infer that from the write, and you will also see it when someone ELSE's tab holds the unflushed edits and you wrote nothing: a read that serves a body the server knows is behind the live document carries `content_state: "applied_pending_flush"` — same value as the write warning, different question (what the ROW is, not what your REQUEST did). Treat it as re-read later, never as re-send, and do not wait on it in a loop: nothing server-side moves that content into the stored copy, so the row catches up only when a tab next opens the item, which may never happen. The stored form may also not be byte-identical to what you sent, since the markdown round-trips through the editor (bullet markers, emphasis characters, list numbering and blank-line runs normalise; already-canonical markdown survives unchanged), so compare on meaning rather than bytes. `move` changes an item's COLLECTION within its workspace: system metadata (implementation notes, decision log, linked PR) survives it, and any field value the target schema has no home for is dropped AND reported in the move's activity entry — check there rather than assuming a move is lossless. A carried unique-field collision is dropped, named in `warnings.not_unique`. Three system keys — `implementation_notes`, `decision_log`, `convention` — cannot be set through `field` on update or move; the call is refused with `validation_failed`, naming the key and the write path that does maintain it (`note`, `decide`, and library activation respectively). `github_pr` is refused as well (BUG-2696): a `field` value arrived as a string, so the PR data was stored double-encoded and no link appeared. Its writer is `pad github link` / `unlink`, which needs a local git checkout, and there is no PR-link action on this surface yet, so treat linking a PR as something to hand to a human rather than a call to retry. On `create` they are refused too (BUG-3163), with the same `validation_failed`: create the item first, then use `note` / `decide` on it; convention metadata comes from library activation. `history` returns read-only item version metadata (newest-first), bounded to the NEWEST 50 versions by default (max 300 — pass `limit` to change the window); pass `full: true` to include each version's resolved content body (token-expensive). There is no `offset`: versions are stored as reverse patches, so only a newest-end window is cheap to reconstruct. Unassign or detach with the `clear_assigned_user` / `clear_agent_role` / `clear_parent` booleans; an empty `assign` / `role` / `parent` does NOT clear. `remind` / `ack-reminder`: a fired reminder shows in `pad_project` next / ready (the delivery path on an instance with no webhook) until acknowledged, and one on a completed item is hidden there. `claim` / `release`: a loser's 409 is `lease_held`, and a lease bumps neither `updated_at` nor `seq`. `edit-comment` / `delete-comment` take a `comment_id` from `list-comments` (whose rows carry `edited`); only a comment's author may edit it. Correct a comment in place rather than appending a correction. Deleting a comment that has replies leaves a placeholder (`deleted: true`, empty body) so the replies keep their parent; editing, replying to or reacting to it answers 409 `comment_deleted`. (Two older forms still work and are not deprecated: `field: ["assigned_user_id="]` on either transport, and a direct `assigned_user_id: ""` param over remote `/mcp` only — prefer the boolean, which is the only one this schema advertises.) +- `pad_item` — Items: create / update / delete / get / list / move / restore / link / unlink / deps / star / unstar / starred / comment / list-comments / edit-comment / delete-comment / backlinks / bulk-update / note / decide / export / import / history / remind / ack-reminder / claim / release. On create/update, field values may be passed as a `fields` OBJECT (the same shape reads return, e.g. `{"status":"done","effort":"l"}`) — it merges into the same path as the dedicated params and `field: ["key=value"]`; the same key in two places with CONFLICTING values is refused, not silently resolved. `list` accepts `unparented: true` to keep items with no parent or implements relationship (mutually exclusive with `parent`). `list` results are SUMMARY-shaped by default on both transports — no content bodies; pass `full: true` for complete bodies (token-expensive), or prefer `get` for a single item's body. `update` field writes are a server-side field-level merge (only the keys you set change); pass `expected_seq` for optimistic concurrency — the `seq` you last read, which is on every item AND every list summary — and a stale value fails with a structured 409 `update_conflict` whose details carry `actual_seq` to retry with. PREFER IT over `expected_updated_at`, which cannot tell two writes inside one second apart; that one still works, and if you send both, seq decides. A token cannot see a browser tab's unsaved edits, so `content` sent WITH one is refused with 409 `content_pending_flush` while any exist; re-reading will not clear it. Wait for the editor to save, or pass `overwrite_pending_edits: true` to replace them. `content_state: "superseded_set_aside"` is different and wins when both apply: an editor upgrade set aside edits no tab can restore, so EVERY content write is refused (token or not; `details.set_aside_rows`) and waiting never clears it. Tell the user, or pass `overwrite_pending_edits: true` to discard them. Writing `content` while a browser tab has the item open routes the markdown through that live document, and the stored copy is updated only by a later collab-snapshot flush — usually from the tab that applied it, though any such write updates the row, and nothing guarantees one happens: the 200 echoes the content you SENT and carries `warnings.content_outcome: "applied_pending_flush"`, which describes that WRITE — the content went to the document, not the row — rather than being a live reading of the row, which a concurrent flush may already have updated. A `get` (or a `list` with `full: true` — a default list carries no content at all) before such a flush lands reads the stored copy and answers with the PREVIOUS content, which is the lag rather than a failed write, so do not re-send on the strength of it. You do not have to infer that from the write, and you will also see it when someone ELSE's tab holds the unflushed edits and you wrote nothing: a read that serves a body the server knows is behind the live document carries `content_state: "applied_pending_flush"` — same value as the write warning, different question (what the ROW is, not what your REQUEST did). Treat it as re-read later, never as re-send, and do not wait on it in a loop: the server normally recovers the stored copy itself about a minute after the last tab leaves the item, but that is not guaranteed (a recovery can fail, and an operator can turn it off), and otherwise the row catches up only when a tab next opens the item. The stored form may also not be byte-identical to what you sent, since the markdown round-trips through the editor (bullet markers, emphasis characters, list numbering and blank-line runs normalise; already-canonical markdown survives unchanged), so compare on meaning rather than bytes. `move` changes an item's COLLECTION within its workspace: system metadata (implementation notes, decision log, linked PR) survives it, and any field value the target schema has no home for is dropped AND reported in the move's activity entry — check there rather than assuming a move is lossless. A carried unique-field collision is dropped, named in `warnings.not_unique`. Three system keys — `implementation_notes`, `decision_log`, `convention` — cannot be set through `field` on update or move; the call is refused with `validation_failed`, naming the key and the write path that does maintain it (`note`, `decide`, and library activation respectively). `github_pr` is refused as well (BUG-2696): a `field` value arrived as a string, so the PR data was stored double-encoded and no link appeared. Its writer is `pad github link` / `unlink`, which needs a local git checkout, and there is no PR-link action on this surface yet, so treat linking a PR as something to hand to a human rather than a call to retry. On `create` they are refused too (BUG-3163), with the same `validation_failed`: create the item first, then use `note` / `decide` on it; convention metadata comes from library activation. `history` returns read-only item version metadata (newest-first), bounded to the NEWEST 50 versions by default (max 300 — pass `limit` to change the window); pass `full: true` to include each version's resolved content body (token-expensive). There is no `offset`: versions are stored as reverse patches, so only a newest-end window is cheap to reconstruct. Unassign or detach with the `clear_assigned_user` / `clear_agent_role` / `clear_parent` booleans; an empty `assign` / `role` / `parent` does NOT clear. `remind` / `ack-reminder`: a fired reminder shows in `pad_project` next / ready (the delivery path on an instance with no webhook) until acknowledged, and one on a completed item is hidden there. `claim` / `release`: a loser's 409 is `lease_held`, and a lease bumps neither `updated_at` nor `seq`. `edit-comment` / `delete-comment` take a `comment_id` from `list-comments` (whose rows carry `edited`); only a comment's author may edit it. Correct a comment in place rather than appending a correction. Deleting a comment that has replies leaves a placeholder (`deleted: true`, empty body) so the replies keep their parent; editing, replying to or reacting to it answers 409 `comment_deleted`. (Two older forms still work and are not deprecated: `field: ["assigned_user_id="]` on either transport, and a direct `assigned_user_id: ""` param over remote `/mcp` only — prefer the boolean, which is the only one this schema advertises.) **Relation fields (v0.31).** A `relation` value may be a UUID, an issue ref (`COLO-3`), or the target item's EXACT TITLE — in that order, so an item literally titled `COLO-3` is unreachable by title while the ref resolves. Title matching is scoped to the collection the field declares: a title that is unique only workspace-wide is REFUSED, naming the collection searched, and a title matching two or more items inside the declared collection is refused as `ambiguous` rather than `not_found` — it matched too much, not too little. Only items YOU can see count towards any of that, so a match you have no access to never changes your answer. diff --git a/internal/mcp/version.go b/internal/mcp/version.go index 28e61b563..1962d7bc3 100644 --- a/internal/mcp/version.go +++ b/internal/mcp/version.go @@ -1651,6 +1651,20 @@ const CmdhelpVersion = "0.1" // this one either, and it is not a promise that the row will catch // up at all. // +// 0.60 — TASK-2198 U4, op-log recovery. BEHAVIOR bump on the v0.59 / +// v0.39 grounds: no tool name, action enum or param shape changed, but +// `content_state: "applied_pending_flush"` — on `pad_item` get / full +// list, `pad_playbook` run and the item resource's line above the body — +// now CLEARS ON ITS OWN. When the item's collab room closes (60s after +// its last tab leaves) or a one-minute sweep finds it dormant, the server +// rebuilds the body from the op-log in a worker process and stores it: +// the body changes, `seq` moves (a held `expected_seq` then answers +// `update_conflict`, where it used to answer `content_pending_flush`), +// and the history gains a version by `system` / `recovery`. Before it the +// row caught up only when a tab next opened the item. Not a guarantee: a +// failed job, an item with set-aside rows and PAD_MATERIALIZE=off all +// leave the old behaviour, so "re-read later, never re-send" stands. +// // 0.59 — BUG-3252, the tombstone (ruled over cascade). BEHAVIOR bump // on the v0.58 / v0.45 grounds: no name, enum or param shape changed, // but `pad_item.action=delete-comment` on a comment that still has @@ -1705,7 +1719,7 @@ const CmdhelpVersion = "0.1" // this surface can receive it; the entry exists so a future action does // not collapse it to permission_denied. When an action that can reach // it is added, that addition is the contract change and owns the bump. -const ToolSurfaceVersion = "0.59" +const ToolSurfaceVersion = "0.60" // MetaVersionURI is the canonical URI of the queryable version document. // Lives outside the pad://workspace/{ws}/... namespace because it's a diff --git a/internal/models/item.go b/internal/models/item.go index e64e947d4..0ce879147 100644 --- a/internal/models/item.go +++ b/internal/models/item.go @@ -272,6 +272,17 @@ const ContentStatePendingFlush = ContentOutcomeAppliedPendingFlush // stale should ask IsContentStateStale instead. const ContentStateSetAside = "superseded_set_aside" +// The version-row attribution of an op-log recovery (ItemUpdate.Recovered, +// TASK-2198 U4). item_versions.created_by and .source carry no CHECK +// constraint on either dialect; "system" is already the created_by of +// template-seeded items. The web timeline labels them "System" / "Recovered"; +// `pad item history` prints them as stored. +const ( + VersionCreatedBySystem = "system" + VersionSourceRecovery = "recovery" + RecoveryChangeSummary = "recovered from an unsaved editor session" +) + // IsContentStateStale reports whether an Item.ContentState value says the // stored body is behind edits that exist elsewhere, for any reason. It is the // one helper for doors that mark a body stale without saying what to do. @@ -1332,8 +1343,18 @@ type ItemUpdate struct { // boundary is what Join reads to force_refresh a client whose ?content_seq // seed predates the restore — surviving a server restart, unlike the // in-memory fast-path. Internal-only (`json:"-"`); no HTTP client can set it. - MarkRestoreBoundary bool `json:"-"` - Comment *string `json:"comment,omitempty"` + MarkRestoreBoundary bool `json:"-"` + // Recovered marks the op-log materializer's write (TASK-2198 U4): the + // server rebuilt items.content from the item's collaborative op-log after + // the tab holding those edits went away without flushing them. The store + // attributes the version row to the system (VersionCreatedBySystem / + // VersionSourceRecovery) rather than to a guessed author, because op-log + // rows carry no user id, and advances the flush watermark only to + // OpLogCursor and only when that cursor is still MAX(op-log id), exactly + // as a collab-snapshot flush does. Internal-only (`json:"-"`); no HTTP + // client can set it. + Recovered bool `json:"-"` + Comment *string `json:"comment,omitempty"` // OpLogCursor is the highest item_yjs_updates.id the calling client // has applied into its local Y.Doc (TASK-1319). Used by the // collab-snapshot flush PATCH to advance the op-log GC watermark diff --git a/internal/server/handlers_items.go b/internal/server/handlers_items.go index 6756ba8a3..f51e46fda 100644 --- a/internal/server/handlers_items.go +++ b/internal/server/handlers_items.go @@ -1990,6 +1990,11 @@ func (s *Server) handleUpdateItem(w http.ResponseWriter, r *http.Request) { // items.source must not move on an update. _, input.VersionSource = actorFromRequest(r) } + // TASK-2198 U4: "recovery" labels the server's own op-log recovery write + // in the version history. A client may not claim it. + if input.VersionSource == models.VersionSourceRecovery { + input.VersionSource = "" + } // BUG-2542: stamp the writer on single-item updates. Bulk ops already do // this (handlers_items_bulk.go), but this path did not, so an agent's diff --git a/internal/server/handlers_items_open_children_guard.go b/internal/server/handlers_items_open_children_guard.go index 4d05f0756..055b1fb6f 100644 --- a/internal/server/handlers_items_open_children_guard.go +++ b/internal/server/handlers_items_open_children_guard.go @@ -468,8 +468,10 @@ func writeContentNotAppliedError(w http.ResponseWriter, ref string, landedFields // carrying a version token while the item's op-log holds unflushed // collaborative edits. 409, but NOT update_conflict — re-reading returns the // same row and the same seq, so a read-and-retry loop would spin. The message -// says what clears it, because nothing server-side does: an open tab flushes -// within seconds; with no tab, only opening the item or the override. +// says what clears it: an open tab flushes within seconds; with no tab, opening +// the item, the override, or the server's own op-log recovery once the item's +// room has closed (TASK-2198 U4), which can fail or be turned off, so the +// message does not promise it. // // BUG-3244: when the item holds rows a schema rebuild set aside, the refusal // covers every content write and the message must NOT offer opening the item, diff --git a/internal/server/materialize_recovery.go b/internal/server/materialize_recovery.go new file mode 100644 index 000000000..63adbe2c2 --- /dev/null +++ b/internal/server/materialize_recovery.go @@ -0,0 +1,659 @@ +package server + +import ( + "context" + "errors" + "fmt" + "log/slog" + "runtime/debug" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/PerpetualSoftware/pad/internal/collab" + "github.com/PerpetualSoftware/pad/internal/materialize" + "github.com/PerpetualSoftware/pad/internal/models" + "github.com/PerpetualSoftware/pad/internal/store" +) + +// Op-log recovery (TASK-2198 U4): the triggers and the worker. +// +// A tab that dies without flushing leaves its last typing only in the +// op-log, and items.content (what REST, the CLI, MCP and search read) stays +// behind it until a tab next opens the item — before this, possibly never. +// This file closes that window in the background: +// +// - T3, room idle: when a collab room is reclaimed at the end of its grace +// TTL (collab.DefaultGraceTTL, 60s after the last connection left) and the +// item still has content-bearing rows above its flush watermark, the item +// is queued. +// - T1, sweep: every materializeSweepInterval the store is asked for live +// items whose op-log holds such rows, whose newest row is older than +// materializeDormancy, and which have no set-aside rows +// (store.ListMaterializeCandidates); those without an open room are +// queued. It catches what T3 cannot see: a server restart, a room that +// closed while the worker was failing, rows left by an older server. +// +// One goroutine drains a de-duplicated queue into the Materializer (in +// production the single materialize.Supervisor, which runs the job in a +// capped child process). Nothing here runs on a request path. +// +// FAILURE BUDGET. The input is untrusted (any editor can write op-log rows), +// so a job is never retried in a loop: a failed item waits +// materializeBackoffBase, doubling per consecutive failure, and after +// materializeMaxFailures consecutive failures it is not tried again until its +// op-log changes (MAX(id) moves past the value recorded at the last failure). +// The budget is kept in memory, bounded at materializeBudgetCap entries: a +// restart re-grants each item its K attempts, which bounds a poison item at K +// worker deaths per process lifetime. Each failure is logged with its typed +// error (materialize.Err*). + +const ( + materializeSweepInterval = time.Minute + materializeDormancy = 2 * time.Minute + materializeSweepLimit = 200 + materializeMaxFailures = 3 + materializeBackoffBase = time.Minute + materializeBudgetCap = 10000 + materializeQueueCap = 10000 +) + +// Materializer runs one materialization. *materialize.Supervisor implements it +// (and *materialize.Runner, which tests use in-process). +type Materializer interface { + Materialize(ctx context.Context, job materialize.Job) (string, error) +} + +type materializeBudget struct { + failures int + opLogMax int64 + nextAttempt time.Time + lastErr string + lastKind string + // exhaustionLogged: the one exhaustion line for this budget has been + // written. A fresh budget (after the op-log grows) starts false again. + exhaustionLogged bool +} + +type materializeRecovery struct { + s *Server + m Materializer + cfg materializeRecoveryConfig + + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + started bool + stopped bool + queued map[string]bool + queue []string + wake chan struct{} + budget map[string]*materializeBudget + + // setAsideSkipped counts items the worker skipped because they hold rows a + // schema rebuild set aside (BUG-3244), over the process lifetime; reported + // by the sweep's summary line, never per item. setAsideLogged is the value + // the last summary line carried. + setAsideSkipped atomic.Int64 + setAsideLogged int64 + + // sweepCursor is where the next sweep page starts (sweep goroutine only). + sweepCursor store.MaterializeCursor + + // processed, when set by a test, receives each processed item and its + // result after the item is done. + processed func(itemID string, r materializeResult) +} + +type materializeRecoveryConfig struct { + sweepInterval time.Duration + dormancy time.Duration + maxFailures int + backoffBase time.Duration + sweepLimit int + now func() time.Time + // logger receives the worker's log lines. Nil: slog.Default(). + logger *slog.Logger +} + +// materializeResult is what one processing pass did with an item. +type materializeResult string + +const ( + mrApplied materializeResult = "applied" + mrStamped materializeResult = "stamped" + mrNothing materializeResult = "nothing_pending" + mrRoomOpen materializeResult = "room_open" + mrSetAside materializeResult = "set_aside" + mrCursorMoved materializeResult = "cursor_moved" + mrGone materializeResult = "gone" + mrBudgetExceeded materializeResult = "budget_exhausted" + mrBackoff materializeResult = "backoff" + mrSchemaVersion materializeResult = "schema_version" + mrFailed materializeResult = "failed" + mrStopped materializeResult = "stopped" +) + +// SetMaterializer installs the op-log recovery worker's Materializer. Nil (the +// default, and what PAD_MATERIALIZE=off leaves) disables recovery entirely: +// StartMaterializeRecovery then installs no hook and starts no goroutine. Must +// be called before StartMaterializeRecovery. If m also has a Close() error +// method, Stop() calls it after the worker has drained. +func (s *Server) SetMaterializer(m Materializer) { + if m == nil { + s.materializeRecovery = nil + return + } + s.materializeRecovery = newMaterializeRecovery(s, m, materializeRecoveryConfig{}) +} + +func newMaterializeRecovery(s *Server, m Materializer, cfg materializeRecoveryConfig) *materializeRecovery { + if cfg.sweepInterval <= 0 { + cfg.sweepInterval = materializeSweepInterval + } + if cfg.dormancy <= 0 { + cfg.dormancy = materializeDormancy + } + if cfg.maxFailures <= 0 { + cfg.maxFailures = materializeMaxFailures + } + if cfg.backoffBase <= 0 { + cfg.backoffBase = materializeBackoffBase + } + if cfg.sweepLimit <= 0 { + cfg.sweepLimit = materializeSweepLimit + } + if cfg.now == nil { + cfg.now = time.Now + } + if cfg.logger == nil { + cfg.logger = slog.Default() + } + ctx, cancel := context.WithCancel(context.Background()) + return &materializeRecovery{ + s: s, m: m, cfg: cfg, ctx: ctx, cancel: cancel, + queued: map[string]bool{}, + wake: make(chan struct{}, 1), + budget: map[string]*materializeBudget{}, + } +} + +// StartMaterializeRecovery starts the op-log recovery worker and its two +// triggers. A no-op without a Materializer (SetMaterializer) or without a +// collab room manager (SetCollabRoomManager), and idempotent. +func (s *Server) StartMaterializeRecovery() { + r := s.materializeRecovery + if r == nil || s.collab == nil { + return + } + r.mu.Lock() + if r.started || r.stopped { + r.mu.Unlock() + return + } + r.started = true + r.mu.Unlock() + + s.collab.SetIdleHook(r.onRoomIdle) + r.cfg.logger.Info("op-log recovery started", + "sweep_interval", r.cfg.sweepInterval.String(), + "dormancy", r.cfg.dormancy.String(), + "max_failures", r.cfg.maxFailures) + + s.bg.Add(2) + go func() { + defer s.bg.Done() + r.workLoop() + }() + go func() { + defer s.bg.Done() + defer s.recoverSweeper("materialize-sweep") + t := time.NewTicker(r.cfg.sweepInterval) + defer t.Stop() + for { + select { + case <-r.ctx.Done(): + return + case <-t.C: + r.sweep() + } + } + }() +} + +func (s *Server) stopMaterializeRecovery() { + r := s.materializeRecovery + if r == nil { + return + } + r.mu.Lock() + r.stopped = true + r.mu.Unlock() + if s.collab != nil { + s.collab.SetIdleHook(nil) + } + r.cancel() +} + +// closeMaterializer closes the Materializer when it has a Close method (the +// Supervisor: kills its worker process). Called by Stop after bg.Wait. +func (s *Server) closeMaterializer() { + r := s.materializeRecovery + if r == nil { + return + } + if c, ok := r.m.(interface{ Close() error }); ok { + if err := c.Close(); err != nil { + r.cfg.logger.Warn("op-log recovery: closing the materializer failed", "error", err) + } + } +} + +// onRoomIdle is T3: the collab room for itemID was reclaimed. Runs on the +// grace timer's goroutine; one indexed read, then a non-blocking enqueue. +func (r *materializeRecovery) onRoomIdle(itemID string) { + pending, err := r.s.store.ItemHasPendingContent(itemID) + if err != nil { + r.cfg.logger.Warn("op-log recovery: pending check on room idle failed", "item_id", itemID, "error", err) + return + } + if pending { + r.enqueue(itemID) + } +} + +// sweep is T1. +func (r *materializeRecovery) sweep() { + // One page per tick, continuing from where the last tick stopped and + // wrapping to the start after a short page: every candidate is offered + // within ceil(candidates / sweepLimit) + 1 ticks, however many of those + // ahead of it the room check or the failure budget turn away (see + // ListMaterializeCandidates). + page, err := r.s.store.ListMaterializeCandidates(r.cfg.now().Add(-r.cfg.dormancy), r.sweepCursor, r.cfg.sweepLimit) + if err != nil { + r.cfg.logger.Warn("op-log recovery: sweep query failed", "error", err) + return + } + if len(page) < r.cfg.sweepLimit { + r.sweepCursor = store.MaterializeCursor{} + } else { + last := page[len(page)-1] + r.sweepCursor = store.MaterializeCursor{LastAt: last.LastAt, ItemID: last.ItemID} + } + ids := make([]string, 0, len(page)) + queued := 0 + for _, c := range page { + ids = append(ids, c.ItemID) + if r.s.collab != nil && r.s.collab.HasRoom(c.ItemID) { + continue + } + if r.enqueue(c.ItemID) { + queued++ + } + } + // ONE summary line per sweep, and only when it has news: items queued, or + // a set-aside skip since the last line. Set-aside items never come from + // this query (it excludes them); they reach the worker through the room + // trigger or a race, and are counted there, not logged per item. + skipped := r.setAsideSkipped.Load() + if queued > 0 || skipped != r.setAsideLogged { + r.setAsideLogged = skipped + r.cfg.logger.Info("op-log recovery sweep", + "candidates", len(ids), "queued", queued, "set_aside_skipped_total", skipped) + } +} + +// enqueue adds itemID unless it is already queued. Never blocks; a full queue +// drops the item, which the next sweep will offer again. +func (r *materializeRecovery) enqueue(itemID string) bool { + r.mu.Lock() + defer r.mu.Unlock() + if r.stopped || r.queued[itemID] { + return false + } + if len(r.queue) >= materializeQueueCap { + return false + } + r.queued[itemID] = true + r.queue = append(r.queue, itemID) + select { + case r.wake <- struct{}{}: + default: + } + return true +} + +func (r *materializeRecovery) dequeue() (string, bool) { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.queue) == 0 { + return "", false + } + id := r.queue[0] + r.queue = r.queue[1:] + delete(r.queued, id) + return id, true +} + +func (r *materializeRecovery) workLoop() { + for { + for { + if r.ctx.Err() != nil { + return + } + id, ok := r.dequeue() + if !ok { + break + } + res := r.processSafely(id) + if r.processed != nil { + r.processed(id, res) + } + } + select { + case <-r.ctx.Done(): + return + case <-r.wake: + } + } +} + +// processSafely is process with a panic fence, so one bad item cannot end the +// worker goroutine. +func (r *materializeRecovery) processSafely(itemID string) (res materializeResult) { + defer func() { + if p := recover(); p != nil { + r.cfg.logger.Error("op-log recovery: panic processing item", + "item_id", itemID, "panic", p, "stack", string(debug.Stack())) + res = mrFailed + } + }() + return r.process(itemID) +} + +func (r *materializeRecovery) process(itemID string) materializeResult { + s := r.s + if s.collab == nil { + return mrStopped + } + if s.collab.HasRoom(itemID) { + return mrRoomOpen + } + in, err := s.store.LoadMaterializeInput(itemID) + if err != nil { + r.cfg.logger.Warn("op-log recovery: reading the op-log failed", "item_id", itemID, "error", err) + return mrFailed + } + if in == nil { + r.forget(itemID) + return mrGone + } + if in.SetAside > 0 { + r.setAsideSkipped.Add(1) + return mrSetAside + } + if in.Pending == 0 { + r.forget(itemID) + return mrNothing + } + if res, ok := r.admit(itemID, in.Cursor); !ok { + return res + } + for _, v := range in.SchemaVersions { + if v != collab.DefaultSchemaVersion { + r.exhaust(itemID, in.Cursor, fmt.Sprintf("op-log schema version %q, server %q", v, collab.DefaultSchemaVersion)) + return mrSchemaVersion + } + } + + ws, err := s.store.GetWorkspaceByID(in.WorkspaceID) + if err != nil || ws == nil { + r.cfg.logger.Warn("op-log recovery: reading the workspace failed", "item_id", itemID, "error", err) + return mrFailed + } + index, err := s.materializeLinkIndex(in.WorkspaceID) + if err != nil { + r.cfg.logger.Warn("op-log recovery: building the link index failed", "item_id", itemID, "error", err) + return mrFailed + } + + job := materialize.Job{ + Rows: in.Rows, + SchemaVersion: collab.DefaultSchemaVersion, + LinkIndex: index, + WorkspaceSlug: ws.Slug, + } + started := time.Now() + md, err := r.m.Materialize(r.ctx, job) + if err != nil { + if r.ctx.Err() != nil { + return mrStopped + } + r.recordFailure(itemID, in.Cursor, err) + return mrFailed + } + + var ( + outcome store.MaterializeOutcome + updated *models.Item + ) + err = s.collab.UnderItemLockIfNoRoom(itemID, func() error { + var ferr error + outcome, updated, ferr = s.store.MaterializeFlush(itemID, in.Cursor, md) + return ferr + }) + if errors.Is(err, collab.ErrRoomOpen) { + return mrRoomOpen + } + if err != nil { + if r.ctx.Err() != nil { + return mrStopped + } + r.cfg.logger.Warn("op-log recovery: writing the recovered body failed", "item_id", itemID, "error", err) + return mrFailed + } + r.forget(itemID) + switch outcome { + case store.MaterializeApplied: + r.cfg.logger.Info("op-log recovery: recovered an item's unsaved editor session", + "item_id", itemID, "op_log_cursor", in.Cursor, "rows", len(in.Rows), + "pending_rows", in.Pending, "bytes", len(md), "elapsed", time.Since(started).String()) + if updated != nil { + s.publishItemEventWithName(sseItemUpdated, updated.WorkspaceID, updated.ID, updated.Title, + updated.CollectionSlug, models.VersionCreatedBySystem, "", models.VersionSourceRecovery, updated.Seq) + } + return mrApplied + case store.MaterializeStamped: + return mrStamped + case store.MaterializeCursorMoved: + return mrCursorMoved + case store.MaterializeNothingPending: + return mrNothing + case store.MaterializeSetAside: + r.setAsideSkipped.Add(1) + return mrSetAside + default: + return mrGone + } +} + +// admit applies the failure budget. The budget resets when the op-log has +// moved past the MAX(id) recorded at the last failure. +func (r *materializeRecovery) admit(itemID string, cursor int64) (materializeResult, bool) { + r.mu.Lock() + defer r.mu.Unlock() + b := r.budget[itemID] + if b == nil { + return "", true + } + if cursor > b.opLogMax { + delete(r.budget, itemID) + return "", true + } + if b.failures >= r.cfg.maxFailures { + return mrBudgetExceeded, false + } + if r.cfg.now().Before(b.nextAttempt) { + return mrBackoff, false + } + return "", true +} + +func (r *materializeRecovery) forget(itemID string) { + r.mu.Lock() + delete(r.budget, itemID) + r.mu.Unlock() +} + +func (r *materializeRecovery) budgetSlotLocked(itemID string) *materializeBudget { + b := r.budget[itemID] + if b == nil { + if len(r.budget) >= materializeBudgetCap { + for k := range r.budget { // evict one; it re-earns its K attempts + delete(r.budget, k) + break + } + } + b = &materializeBudget{} + r.budget[itemID] = b + } + return b +} + +func (r *materializeRecovery) recordFailure(itemID string, cursor int64, err error) { + r.mu.Lock() + b := r.budgetSlotLocked(itemID) + kind := materializeErrorKind(err) + b.failures++ + b.opLogMax = cursor + b.lastErr = err.Error() + b.lastKind = kind + delay := r.cfg.backoffBase << (b.failures - 1) + b.nextAttempt = r.cfg.now().Add(delay) + failures := b.failures + logExhaustion := r.claimExhaustionLogLocked(b) + r.mu.Unlock() + + attrs := []any{"item_id", itemID, "op_log_cursor", cursor, "failures", failures, + "max_failures", r.cfg.maxFailures, "kind", kind, "error", err} + // No memory cap could be established, so the supervisor refused the job + // without sending it anywhere, and has already warned once for the whole + // worker. It counts against the budget like any failure, but a line per + // item would repeat that warning for every pending item on every pass — + // so even its exhaustion line is DEBUG. + if errors.Is(err, materialize.ErrNoMemoryCap) { + if logExhaustion { + r.logExhaustion(slog.LevelDebug, itemID, failures, kind, err.Error()) + } else { + r.cfg.logger.Debug("op-log recovery: job refused, no memory cap", attrs...) + } + return + } + if logExhaustion { + r.logExhaustion(slog.LevelWarn, itemID, failures, kind, err.Error()) + return + } + r.cfg.logger.Warn("op-log recovery: materialization failed; will retry after a backoff", + append(attrs, "retry_after", delay.String())...) +} + +// claimExhaustionLogLocked reports whether b has just used up the budget and +// its exhaustion line has not been written yet, and marks it written. Once per +// budget: admit refuses an exhausted item without reaching recordFailure, and +// only op-log growth (which deletes the budget) can start a new one. +func (r *materializeRecovery) claimExhaustionLogLocked(b *materializeBudget) bool { + if b.failures < r.cfg.maxFailures || b.exhaustionLogged { + return false + } + b.exhaustionLogged = true + return true +} + +// logExhaustion is the ONE line a stuck item leaves: which item, how many +// failures, and the kind of the last one. WARN, because from here on the item's +// newest edits stay out of its stored body until someone edits it again, which +// an operator should be able to find with a grep and no database column; it +// cannot become noise, since it is written once per exhaustion. +func (r *materializeRecovery) logExhaustion(level slog.Level, itemID string, failures int, kind, lastErr string) { + r.cfg.logger.Log(r.ctx, level, "op-log recovery: item exhausted its failure budget; not retried until its op-log changes", + "item_id", itemID, "failures", failures, "last_error_kind", kind, "last_error", lastErr) +} + +// exhaust marks the item as not to be tried until its op-log changes. +func (r *materializeRecovery) exhaust(itemID string, cursor int64, why string) { + r.mu.Lock() + b := r.budgetSlotLocked(itemID) + b.failures = r.cfg.maxFailures + b.opLogMax = cursor + b.lastErr = why + b.lastKind = "schema_version" + logExhaustion := r.claimExhaustionLogLocked(b) + r.mu.Unlock() + if logExhaustion { + r.logExhaustion(slog.LevelWarn, itemID, r.cfg.maxFailures, "schema_version", why) + } +} + +// materializeErrorKind names the typed materialize error, for the log line. +func materializeErrorKind(err error) string { + for _, k := range []struct { + err error + name string + }{ + {materialize.ErrNoMemoryCap, "no_memory_cap"}, + {materialize.ErrMemoryLimit, "memory_limit"}, + {materialize.ErrDeadline, "deadline"}, + {materialize.ErrChildDied, "worker_died"}, + {materialize.ErrProtocol, "protocol"}, + {materialize.ErrStart, "start"}, + {materialize.ErrJobFailed, "job_failed"}, + {materialize.ErrJobTooLarge, "too_large"}, + {materialize.ErrSchemaVersion, "schema_version"}, + {materialize.ErrClosed, "closed"}, + {materialize.ErrInterrupted, "interrupted"}, + } { + if errors.Is(err, k.err) { + return k.name + } + } + return "other" +} + +// materializeLinkIndex is the link index a recovery job converts internal +// links against: EVERY live item of the workspace (Dave's ruling), not the +// set any one viewer can see. A tab's index is its viewer's localIndex, but a +// recovery has no viewer, and an index missing an item a link points at would +// leave that link as a raw URL instead of the [[REF]] a full tab stores. +// Titles in the output come only from the document itself: markdownToWikiLinks +// compares an item's title with the link text and never emits it (proven by +// TestMaterializeLinkIndexContributesNoTitle). +// +// ORDER mirrors localIndex.getAll (web/src/lib/stores/localIndex.svelte.ts): +// updated_at DESC, then id ASC, because markdownToWikiLinks takes the FIRST +// match. Sorted here rather than trusting ORDER BY, because Postgres compares +// TEXT under the database collation and the tab compares UTF-16 code units; +// Go's byte order is the tab's order for the ASCII ids the store mints. +func (s *Server) materializeLinkIndex(workspaceID string) ([]materialize.LinkEntry, error) { + items, err := s.store.ListItemsIndex(workspaceID, store.ItemIndexParams{}) + if err != nil { + return nil, err + } + sort.SliceStable(items, func(i, j int) bool { + a, b := items[i], items[j] + if !a.UpdatedAt.Equal(b.UpdatedAt) { + return a.UpdatedAt.After(b.UpdatedAt) + } + return a.ID < b.ID + }) + out := make([]materialize.LinkEntry, 0, len(items)) + for _, it := range items { + e := materialize.LinkEntry{Slug: it.Slug, Title: it.Title, CollectionPrefix: it.CollectionPrefix} + if it.ItemNumber != nil { + e.ItemNumber = *it.ItemNumber + } + out = append(out, e) + } + return out, nil +} diff --git a/internal/server/materialize_recovery_links_test.go b/internal/server/materialize_recovery_links_test.go new file mode 100644 index 000000000..d2f4f6a80 --- /dev/null +++ b/internal/server/materialize_recovery_links_test.go @@ -0,0 +1,283 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/PerpetualSoftware/pad/internal/collab" + "github.com/PerpetualSoftware/pad/internal/materialize" + "github.com/PerpetualSoftware/pad/internal/models" +) + +// TASK-2198 U4, legs (i) and (ii) of the link-index ruling. The recovery +// converts internal links against the WHOLE workspace's index (every live +// item, no viewer), so the question is whether that index can put text into a +// recovered body — above all an item's TITLE, which a restricted viewer of the +// body may not be entitled to see. +// +// The documents are in testdata/materialize_links.json, written by +// web/src/lib/collab/materializer/materializerLinkFixture.test.ts: each is a +// real Y.Doc holding a link as a tab stores it after loading. `raw` is the +// same document through the flush pipeline with NO index, i.e. the text the +// document itself contains. + +type linkFixtureCase struct { + Name string `json:"name"` + Markdown string `json:"markdown"` + Rows []string `json:"rows"` + Raw string `json:"raw"` +} + +func loadLinkFixture(t *testing.T) []linkFixtureCase { + t.Helper() + b, err := os.ReadFile("testdata/materialize_links.json") + if err != nil { + t.Fatal(err) + } + var cases []linkFixtureCase + if err := json.Unmarshal(b, &cases); err != nil { + t.Fatal(err) + } + if len(cases) < 8 { + t.Fatalf("fixture has %d cases", len(cases)) + } + return cases +} + +const secretTitle = "Secret Launch Codename" + +// linkWorld: workspace "ws" holding X = SECR-1 (secretTitle) in "secrets", a +// collection a restricted member or guest would not see, and PUB-2 in +// "public". Item numbers are workspace-sequential, so the order of creation +// fixes both refs; the test checks them. +func newLinkWorld(t *testing.T) (*recoveryFixture, *models.Item, *models.Item) { + t.Helper() + f := newRecoveryFixture(t, recoveryRunner(t), time.Minute, materializeRecoveryConfig{}) + secrets, err := f.srv.store.CreateCollection(f.ws.ID, models.CollectionCreate{Name: "Secrets", Prefix: "SECR", Schema: `{"fields":[]}`}) + if err != nil { + t.Fatal(err) + } + public, err := f.srv.store.CreateCollection(f.ws.ID, models.CollectionCreate{Name: "Public", Prefix: "PUB", Schema: `{"fields":[]}`}) + if err != nil { + t.Fatal(err) + } + x, err := f.srv.store.CreateItem(f.ws.ID, secrets.ID, models.ItemCreate{Title: secretTitle, Fields: `{}`}) + if err != nil { + t.Fatal(err) + } + p, err := f.srv.store.CreateItem(f.ws.ID, public.ID, models.ItemCreate{Title: "Public Plan", Fields: `{}`}) + if err != nil { + t.Fatal(err) + } + x, _ = f.srv.store.GetItem(x.ID) + p, _ = f.srv.store.GetItem(p.ID) + if x.Ref != "SECR-1" || x.Slug != "secret-launch-codename" || x.CollectionSlug != "secrets" || p.Ref != "PUB-2" { + t.Fatalf("fixture refs: X %s/%s/%s, P %s", x.Ref, x.Slug, x.CollectionSlug, p.Ref) + } + return f, x, p +} + +// recoverCase inserts the case's op-log over a stale body and runs one +// recovery pass through the production path (index builder included). +func recoverCase(t *testing.T, f *recoveryFixture, c linkFixtureCase) string { + t.Helper() + it := f.item(t, "Recovering "+c.Name, "stale") + f.appendRows(t, it.ID, decodeRows(t, c.Rows)...) + if res := f.r.process(it.ID); res != mrApplied { + t.Fatalf("%s: process = %q", c.Name, res) + } + got, _ := f.srv.store.GetItem(it.ID) + return got.Content +} + +// LEG (i). The STOP condition: no index-sourced TITLE may appear in a +// recovered body. Each form markdownToWikiLinks can emit is covered: +// +// [[REF]] link text == the item's CURRENT title: the title was in +// the document and is REMOVED, not added. +// [[REF|text]] any other text: `text` comes from the document. +// [[Title]] the legacy form for an item with no ref: `text` again. +// (Every item the store mints has a ref, so the legacy +// branch is reached only for rows with a NULL item_number; +// it emits the document's link text either way.) +// [[ws::REF|text]] cross-workspace: the index is not consulted. +// +// What the index CAN contribute is the REF, and only for a link addressed by +// SLUG (`/owner/ws/coll/` becomes `[[SECR-1|text]]`). The slug itself is +// in the document; the ref is a number the item already exposes in its URL +// wherever it is linked by ref. That is the whole of the index's contribution. +func TestMaterializeLinkIndexContributesNoTitle(t *testing.T) { + f, _, _ := newLinkWorld(t) + index, err := f.srv.materializeLinkIndex(f.ws.ID) + if err != nil { + t.Fatal(err) + } + + want := map[string]string{ + "ref-link-other-text": "See [[SECR-1|shown text]] here.", + "ref-link-old-title": "See [[SECR-1|Old Codename]] here.", + "slug-link-slug-text": "See [[SECR-1|the plan]] here.", + "text-equals-current-title": "See [[SECR-1]] here.", + "broken-link": "See [[gone text]] here.", + "cross-workspace": "See [[other::SECR-1|xw text]] here.", + "typed-wiki-literal": "Typed [[SECR-1]] literal.", + "public-ref-link": "See [[PUB-2|the public plan]] here.", + // BUG-3315 U2: a follows-title link follows the title, renamed or not. + "marker-follows-title": "See [[SECR-1]] here.", + "marker-renamed": "See [[SECR-1]] here.", + "marker-external-href": "See [Old Codename](https://example.com/x) here.", + } + for _, c := range loadLinkFixture(t) { + got := recoverCase(t, f, c) + if w, ok := want[c.Name]; !ok || got != w { + t.Errorf("%s: recovered %q, want %q", c.Name, got, w) + } + // The general claim, over EVERY title in the index the job was given: + // a title in the output must already be in the document. + for _, e := range index { + if e.Title != "" && strings.Contains(got, e.Title) && !strings.Contains(c.Raw, e.Title) { + t.Errorf("STOP: %s: title %q of %s-%d is in the recovered body but not in the document\nbody: %q\ndoc: %q", + c.Name, e.Title, e.CollectionPrefix, e.ItemNumber, got, c.Raw) + } + } + if strings.Contains(got, secretTitle) { + t.Errorf("STOP: %s: X's title reached the recovered body: %q", c.Name, got) + } + } +} + +// LEG (ii). A link to an item a restricted viewer cannot see survives the +// recovery as the same [[REF|text]] a full member's tab would store, because +// the index is the whole workspace. With a VIEWER-RESTRICTED index (what a +// restricted tab's localIndex holds) the same document stores a raw URL +// instead, bound to the owner and workspace slugs, which is the degradation +// the whole-workspace ruling exists to prevent. +func TestMaterializeLinkToRestrictedItemSurvives(t *testing.T) { + f, x, _ := newLinkWorld(t) + var c linkFixtureCase + for _, k := range loadLinkFixture(t) { + if k.Name == "ref-link-other-text" { + c = k + } + } + got := recoverCase(t, f, c) + if got != "See [[SECR-1|shown text]] here." { + t.Fatalf("recovered %q: the link to the restricted item did not survive as [[SECR-1|shown text]]", got) + } + + // The counterfactual, through the same bundle: drop X's collection. + full, err := f.srv.materializeLinkIndex(f.ws.ID) + if err != nil { + t.Fatal(err) + } + var restricted []materialize.LinkEntry + for _, e := range full { + if e.CollectionPrefix != "SECR" { + restricted = append(restricted, e) + } + } + if len(restricted) != len(full)-1 { + t.Fatalf("restricted index %d of %d; X %s not singled out", len(restricted), len(full), x.Ref) + } + md, err := recoveryRunner(t).Materialize(context.Background(), materialize.Job{ + Rows: decodeRows(t, c.Rows), SchemaVersion: collab.DefaultSchemaVersion, LinkIndex: restricted, WorkspaceSlug: "ws", + }) + if err != nil { + t.Fatal(err) + } + if md != "See [shown text](/alice/ws/secrets/SECR-1) here." { + t.Fatalf("restricted-index counterfactual = %q", md) + } +} + +// BUG-3315 U2: a tab since #1688 loads a link that follows its target's title +// with the follows-title marker, and the marker lives in the Y.Doc. Recovery +// replays that document, so it must strip the marker exactly as a tab's save +// does: on the production path (whole-workspace index), with a restricted +// index (the target unseen), and with none. The marker must never reach +// items.content. +func TestMaterializeRecoveryStripsFollowsTitleMarker(t *testing.T) { + f, _, _ := newLinkWorld(t) + full, err := f.srv.materializeLinkIndex(f.ws.ID) + if err != nil { + t.Fatal(err) + } + var restricted []materialize.LinkEntry + for _, e := range full { + if e.CollectionPrefix != "SECR" { + restricted = append(restricted, e) + } + } + n := 0 + for _, c := range loadLinkFixture(t) { + if !strings.HasPrefix(c.Name, "marker-") { + continue + } + n++ + // Guard: the op-log really carries the marker. + carries := false + for _, r := range decodeRows(t, c.Rows) { + carries = carries || bytes.Contains(r, []byte("pad-follows-title")) + } + if !carries { + t.Fatalf("%s: the fixture's op-log does not carry the marker", c.Name) + } + if got := recoverCase(t, f, c); strings.Contains(got, "pad-follows-title") { + t.Errorf("%s: production recovery stored the marker: %q", c.Name, got) + } + for name, idx := range map[string][]materialize.LinkEntry{"restricted": restricted, "empty": {}} { + md, err := recoveryRunner(t).Materialize(context.Background(), materialize.Job{ + Rows: decodeRows(t, c.Rows), SchemaVersion: collab.DefaultSchemaVersion, LinkIndex: idx, WorkspaceSlug: "ws", + }) + if err != nil { + t.Fatal(err) + } + if strings.Contains(md, "pad-follows-title") { + t.Errorf("%s, %s index: recovery kept the marker: %q", c.Name, name, md) + } + } + } + if n != 3 { + t.Fatalf("%d marker cases in the fixture, want 3", n) + } +} + +// The index is every live item, ordered as localIndex.getAll orders a tab's: +// updated_at DESC, then id ASC. +func TestMaterializeLinkIndexOrderAndScope(t *testing.T) { + f, x, p := newLinkWorld(t) + gone := f.item(t, "Archived one", "") + if err := f.srv.store.DeleteItem(gone.ID); err != nil { + t.Fatal(err) + } + // Make X the most recently updated. + title := secretTitle + if _, err := f.srv.store.UpdateItem(x.ID, models.ItemUpdate{Title: &title, Content: strPtr("touched")}); err != nil { + t.Fatal(err) + } + time.Sleep(1100 * time.Millisecond) // updated_at has one-second resolution + body := "later" + if _, err := f.srv.store.UpdateItem(p.ID, models.ItemUpdate{Content: &body}); err != nil { + t.Fatal(err) + } + index, err := f.srv.materializeLinkIndex(f.ws.ID) + if err != nil { + t.Fatal(err) + } + if len(index) != 2 { + t.Fatalf("index = %+v, want the two live items (archived excluded)", index) + } + if index[0].Title != "Public Plan" || index[1].Title != secretTitle { + t.Fatalf("order = [%q, %q], want most recently updated first", index[0].Title, index[1].Title) + } + if index[1].ItemNumber != 1 || index[1].CollectionPrefix != "SECR" || index[1].Slug != "secret-launch-codename" { + t.Fatalf("entry = %+v", index[1]) + } +} + +func strPtr(s string) *string { return &s } diff --git a/internal/server/materialize_recovery_test.go b/internal/server/materialize_recovery_test.go new file mode 100644 index 000000000..08c51df97 --- /dev/null +++ b/internal/server/materialize_recovery_test.go @@ -0,0 +1,778 @@ +package server + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + pad "github.com/PerpetualSoftware/pad" + "github.com/PerpetualSoftware/pad/internal/collab" + "github.com/PerpetualSoftware/pad/internal/materialize" + "github.com/PerpetualSoftware/pad/internal/models" + "github.com/PerpetualSoftware/pad/internal/store" +) + +// TASK-2198 U4: the op-log recovery triggers, worker and failure budget, and +// the end-to-end recovery through the real materializer bundle. + +// ---------------------------------------------------------------- fixtures + +// fakeMaterializer answers every job with fn and counts calls. +type fakeMaterializer struct { + calls atomic.Int64 + mu sync.Mutex + jobs []materialize.Job + fn func(job materialize.Job) (string, error) +} + +func (f *fakeMaterializer) Materialize(_ context.Context, job materialize.Job) (string, error) { + f.calls.Add(1) + f.mu.Lock() + f.jobs = append(f.jobs, job) + f.mu.Unlock() + if f.fn == nil { + return "recovered body", nil + } + return f.fn(job) +} + +// recoveryRunner is ONE in-process Runner over the embedded bundle; loading +// costs about a second. It satisfies Materializer, so the server drives the +// real JavaScript without a child process. +var ( + recoveryRunnerOnce sync.Once + recoveryRunnerVal *materialize.Runner + recoveryRunnerErr error +) + +func recoveryRunner(t *testing.T) *materialize.Runner { + t.Helper() + recoveryRunnerOnce.Do(func() { recoveryRunnerVal, recoveryRunnerErr = materialize.New(pad.MaterializerJS) }) + if recoveryRunnerErr != nil { + t.Fatalf("load the materializer bundle: %v", recoveryRunnerErr) + } + return recoveryRunnerVal +} + +type recoveryFixture struct { + srv *Server + rm *collab.RoomManager + r *materializeRecovery + ws *models.Workspace + coll *models.Collection + + mu sync.Mutex + processed []string + results map[string]materializeResult + done chan string +} + +// newRecoveryFixture: a test server with a room manager (grace TTL grace) and +// an op-log recovery worker over m, NOT started. cfg zero fields default. +func newRecoveryFixture(t *testing.T, m Materializer, grace time.Duration, cfg materializeRecoveryConfig) *recoveryFixture { + t.Helper() + srv := testServer(t) + bus := collab.NewMemoryOpBus() + t.Cleanup(bus.Close) + rm := collab.NewRoomManagerWithConfig(srv.store, bus, collab.RoomManagerConfig{GraceTTL: grace}) + t.Cleanup(rm.Close) + srv.SetCollabRoomManager(rm) + f := &recoveryFixture{srv: srv, rm: rm, results: map[string]materializeResult{}, done: make(chan string, 256)} + if m != nil { + f.r = newMaterializeRecovery(srv, m, cfg) + f.r.processed = func(id string, res materializeResult) { + f.mu.Lock() + f.processed = append(f.processed, id) + f.results[id] = res + f.mu.Unlock() + f.done <- id + } + srv.materializeRecovery = f.r + } + ws, err := srv.store.CreateWorkspace(models.WorkspaceCreate{Name: "Recovery WS", Slug: "ws"}) + if err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + coll, err := srv.store.CreateCollection(ws.ID, models.CollectionCreate{Name: "Notes", Prefix: "NOTE", Schema: `{"fields":[]}`}) + if err != nil { + t.Fatalf("CreateCollection: %v", err) + } + f.ws, f.coll = ws, coll + return f +} + +func (f *recoveryFixture) item(t *testing.T, title, content string) *models.Item { + t.Helper() + it, err := f.srv.store.CreateItem(f.ws.ID, f.coll.ID, models.ItemCreate{Title: title, Content: content, Fields: `{}`}) + if err != nil { + t.Fatalf("CreateItem: %v", err) + } + return it +} + +func (f *recoveryFixture) appendRows(t *testing.T, itemID string, rows ...[]byte) int64 { + t.Helper() + var last int64 + for _, r := range rows { + id, err := f.srv.store.AppendYjsUpdate(itemID, r, collab.DefaultSchemaVersion) + if err != nil { + t.Fatalf("AppendYjsUpdate: %v", err) + } + last = id + } + return last +} + +func (f *recoveryFixture) waitProcessed(t *testing.T, itemID string, within time.Duration) materializeResult { + t.Helper() + deadline := time.After(within) + for { + f.mu.Lock() + res, ok := f.results[itemID] + f.mu.Unlock() + if ok { + return res + } + select { + case <-f.done: + case <-deadline: + t.Fatalf("item %s was not processed within %s", itemID, within) + } + } +} + +func (f *recoveryFixture) queueSnapshot() []string { + f.r.mu.Lock() + defer f.r.mu.Unlock() + return append([]string(nil), f.r.queue...) +} + +// contentFrame is a content-bearing Update frame, distinct per n. +func contentFrame(n byte) []byte { return []byte{0x00, 0x02, 0x05, 0x01, n, 0x00, 0x7F, 0x00} } + +func dialAndClose(t *testing.T, baseURL, itemID string) { + t.Helper() + c, resp, err := dialCollab(t, baseURL, itemID, nil, "") + if err != nil { + st := "" + if resp != nil { + st = resp.Status + } + t.Fatalf("dial %s: %v %s", itemID, err, st) + } + // Read until the op_log_cursor text frame: the join has registered. + _ = c.SetReadDeadline(time.Now().Add(3 * time.Second)) + for { + mt, _, rerr := c.ReadMessage() + if rerr != nil { + t.Fatalf("read: %v", rerr) + } + if mt == 1 { // websocket.TextMessage + break + } + } + c.Close() +} + +// ---------------------------------------------------------------- T3 + +// T3's decision, unit level: a reclaimed room queues its item only when the +// item still has content-bearing rows above its watermark. +func TestMaterializeRoomIdleEnqueuesOnlyWithPendingRows(t *testing.T) { + f := newRecoveryFixture(t, &fakeMaterializer{}, time.Minute, materializeRecoveryConfig{}) + pending := f.item(t, "Pending", "stale") + f.appendRows(t, pending.ID, contentFrame(1)) + flushed := f.item(t, "Flushed", "fine") + c := f.appendRows(t, flushed.ID, contentFrame(2)) + if err := f.srv.store.SetItemContentFlushedOpLogIDForTesting(flushed.ID, c); err != nil { + t.Fatal(err) + } + empty := f.item(t, "No op-log", "fine") + + f.r.onRoomIdle(pending.ID) + f.r.onRoomIdle(flushed.ID) + f.r.onRoomIdle(empty.ID) + q := f.queueSnapshot() + if len(q) != 1 || q[0] != pending.ID { + t.Fatalf("queue after room idle = %v, want only %s", q, pending.ID) + } + // De-duplicated. + f.r.onRoomIdle(pending.ID) + if q := f.queueSnapshot(); len(q) != 1 { + t.Fatalf("queue after a second idle = %v, want one entry", q) + } +} + +// T3 end to end: a real collab room is reclaimed at the end of its grace TTL, +// the hook installed by StartMaterializeRecovery fires, and the pending item +// is recovered while the clean one is never handed to the materializer. +func TestMaterializeRoomIdleRecoversThroughTheHook(t *testing.T) { + fake := &fakeMaterializer{} + f := newRecoveryFixture(t, fake, 50*time.Millisecond, materializeRecoveryConfig{sweepInterval: time.Hour}) + ts := httptest.NewServer(f.srv) + t.Cleanup(ts.Close) + + pending := f.item(t, "Pending", "stale") + f.appendRows(t, pending.ID, contentFrame(1)) + clean := f.item(t, "Clean", "fine") + + f.srv.StartMaterializeRecovery() + dialAndClose(t, ts.URL, clean.ID) + dialAndClose(t, ts.URL, pending.ID) + + if res := f.waitProcessed(t, pending.ID, 5*time.Second); res != mrApplied { + t.Fatalf("pending item result %q, want applied", res) + } + got, _ := f.srv.store.GetItem(pending.ID) + if got.Content != "recovered body" || got.ContentState != "" { + t.Fatalf("after room idle: content %q state %q", got.Content, got.ContentState) + } + // The clean item's room is reclaimed too (both dials closed before the + // pending one was processed); it must not have been queued. + deadline := time.Now().Add(2 * time.Second) + for f.rm.RoomCount() > 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.results[clean.ID]; ok { + t.Fatalf("the clean item was queued on room idle") + } + if n := fake.calls.Load(); n != 1 { + t.Fatalf("materializer calls = %d, want 1", n) + } +} + +// ---------------------------------------------------------------- T1 + +func TestMaterializeSweepQueuesDormantPendingItemsWithoutARoom(t *testing.T) { + f := newRecoveryFixture(t, &fakeMaterializer{}, time.Minute, materializeRecoveryConfig{ + // The sweep's clock is an hour ahead, so rows written now are dormant. + now: func() time.Time { return time.Now().Add(time.Hour) }, + }) + ts := httptest.NewServer(f.srv) + t.Cleanup(ts.Close) + + dormant := f.item(t, "Dormant", "stale") + f.appendRows(t, dormant.ID, contentFrame(1)) + withRoom := f.item(t, "Open room", "stale") + f.appendRows(t, withRoom.ID, contentFrame(2)) + flushed := f.item(t, "Flushed", "fine") + c := f.appendRows(t, flushed.ID, contentFrame(3)) + if err := f.srv.store.SetItemContentFlushedOpLogIDForTesting(flushed.ID, c); err != nil { + t.Fatal(err) + } + + // Hold a room open on withRoom (the connection stays up). + conn, _, err := dialCollab(t, ts.URL, withRoom.ID, nil, "") + if err != nil { + t.Fatalf("dial: %v", err) + } + t.Cleanup(func() { conn.Close() }) + deadline := time.Now().Add(3 * time.Second) + for !f.rm.HasRoom(withRoom.ID) && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if !f.rm.HasRoom(withRoom.ID) { + t.Fatal("no room for the open connection") + } + + f.r.sweep() + q := f.queueSnapshot() + if len(q) != 1 || q[0] != dormant.ID { + t.Fatalf("sweep queued %v, want only the dormant item %s", q, dormant.ID) + } + + // The worker re-checks: even if an item with a room is queued, it is not + // materialized. + if res := f.r.process(withRoom.ID); res != mrRoomOpen { + t.Fatalf("process(item with a room) = %q, want room_open", res) + } +} + +// ---------------------------------------------------------------- budget + +func TestMaterializeFailureBudgetStopsAfterKAndResumesOnNewRows(t *testing.T) { + clock := time.Now() + var clockMu sync.Mutex + now := func() time.Time { clockMu.Lock(); defer clockMu.Unlock(); return clock } + advance := func(d time.Duration) { clockMu.Lock(); clock = clock.Add(d); clockMu.Unlock() } + + fake := &fakeMaterializer{fn: func(materialize.Job) (string, error) { + return "", fmt.Errorf("%w: exit status 2", materialize.ErrChildDied) + }} + f := newRecoveryFixture(t, fake, time.Minute, materializeRecoveryConfig{now: now, maxFailures: 3, backoffBase: time.Minute}) + it := f.item(t, "Poison", "stale") + f.appendRows(t, it.ID, contentFrame(1)) + + want := []struct { + res materializeResult + calls int64 + after time.Duration + }{ + {mrFailed, 1, 0}, + {mrBackoff, 1, 0}, // inside the 1m backoff + {mrFailed, 2, time.Minute + 1}, // backoff over + {mrBackoff, 2, time.Minute}, // 2m backoff now + {mrFailed, 3, time.Minute + 1}, // K reached + {mrBudgetExceeded, 3, 24 * time.Hour}, // no time heals it + } + for i, w := range want { + advance(w.after) + if res := f.r.process(it.ID); res != w.res || fake.calls.Load() != w.calls { + t.Fatalf("step %d: result %q calls %d, want %q calls %d", i, res, fake.calls.Load(), w.res, w.calls) + } + } + + // A new op-log row resets the budget. + f.appendRows(t, it.ID, contentFrame(2)) + if res := f.r.process(it.ID); res != mrFailed || fake.calls.Load() != 4 { + t.Fatalf("after a new row: result %q calls %d, want failed, 4", res, fake.calls.Load()) + } + // And a success clears it. + fake.fn = nil + advance(time.Hour) + if res := f.r.process(it.ID); res != mrApplied { + t.Fatalf("after recovery of the input: %q, want applied", res) + } + f.r.mu.Lock() + _, left := f.r.budget[it.ID] + f.r.mu.Unlock() + if left { + t.Fatal("budget entry survived a success") + } +} + +// Rows written under another editor schema are skipped, not materialized. +func TestMaterializeSkipsForeignSchemaVersion(t *testing.T) { + fake := &fakeMaterializer{} + f := newRecoveryFixture(t, fake, time.Minute, materializeRecoveryConfig{}) + it := f.item(t, "Old schema", "stale") + if _, err := f.srv.store.AppendYjsUpdate(it.ID, contentFrame(1), "0-old"); err != nil { + t.Fatal(err) + } + if res := f.r.process(it.ID); res != mrSchemaVersion || fake.calls.Load() != 0 { + t.Fatalf("process = %q calls %d, want schema_version, 0", res, fake.calls.Load()) + } + if res := f.r.process(it.ID); res != mrBudgetExceeded { + t.Fatalf("second pass = %q, want budget_exhausted until the op-log changes", res) + } +} + +// ---------------------------------------------------------------- disabled + +// With no Materializer (PAD_MATERIALIZE=off leaves it nil) nothing is +// installed: a room going idle over a pending item leaves it pending. +func TestMaterializeRecoveryDisabledInstallsNothing(t *testing.T) { + f := newRecoveryFixture(t, nil, 30*time.Millisecond, materializeRecoveryConfig{}) + ts := httptest.NewServer(f.srv) + t.Cleanup(ts.Close) + f.srv.SetMaterializer(nil) + f.srv.StartMaterializeRecovery() + if f.srv.materializeRecovery != nil { + t.Fatal("a recovery worker exists with no materializer") + } + it := f.item(t, "Pending", "stale") + f.appendRows(t, it.ID, contentFrame(1)) + dialAndClose(t, ts.URL, it.ID) + deadline := time.Now().Add(2 * time.Second) + for f.rm.RoomCount() > 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + got, _ := f.srv.store.GetItem(it.ID) + if got.Content != "stale" || got.ContentState != models.ContentStatePendingFlush { + t.Fatalf("disabled: content %q state %q, want untouched and pending", got.Content, got.ContentState) + } +} + +// ---------------------------------------------------------------- end to end + +type corpusCaseE2E struct { + Name string `json:"name"` + Rows []string `json:"rows"` + LinkIndex []materialize.LinkEntry `json:"link_index"` + WorkspaceSlug string `json:"workspace_slug"` + Expected string `json:"expected"` +} + +func decodeRows(t *testing.T, rows []string) [][]byte { + t.Helper() + out := make([][]byte, len(rows)) + for i, s := range rows { + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + t.Fatal(err) + } + out[i] = b + } + return out +} + +// TestMaterializeRecoveryEndToEnd: real y-protocols op-logs from the U2 +// corpus (recorded from two live Editor.svelte mounts) are inserted as items' +// op-logs over a stale items.content; the REAL sweep loop, worker and bundle +// recover them. The stored body must be the live editor's flush output, the +// item must read clean on GET, and a writer still holding the pre-recovery +// seq must be refused rather than overwrite the recovered text. +// +// Only corpus cases recorded with an EMPTY link index are used: the server +// builds its own index from the workspace, and none of these documents link +// to an item that exists here, so the flush pipeline's link conversion +// matches nothing, exactly as with the empty index the expected output was +// recorded with. +func TestMaterializeRecoveryEndToEnd(t *testing.T) { + raw, err := os.ReadFile("../materialize/testdata/corpus.json") + if err != nil { + t.Fatal(err) + } + var corpus []corpusCaseE2E + if err := json.Unmarshal(raw, &corpus); err != nil { + t.Fatal(err) + } + + f := newRecoveryFixture(t, recoveryRunner(t), time.Minute, materializeRecoveryConfig{ + sweepInterval: 20 * time.Millisecond, + now: func() time.Time { return time.Now().Add(time.Hour) }, + }) + + type seeded struct { + c corpusCaseE2E + item *models.Item + } + var cases []seeded + for _, c := range corpus { + // A cross-workspace link (/-/r/...) is rewritten to [[ws::REF]] + // whenever the index is non-empty, whatever it holds, and a real + // workspace's index always holds at least the item itself; the + // corpus's empty-index cases were recorded with the conversion + // skipped. Cases containing one are left out rather than papered over. + if len(c.LinkIndex) != 0 || strings.Contains(c.Expected, "](/-/r/") { + continue + } + if c.WorkspaceSlug != f.ws.Slug { + t.Fatalf("%s: corpus workspace %q, fixture %q", c.Name, c.WorkspaceSlug, f.ws.Slug) + } + it := f.item(t, "E2E "+c.Name, "stale body before recovery") + f.appendRows(t, it.ID, decodeRows(t, c.Rows)...) + cases = append(cases, seeded{c, it}) + } + if len(cases) < 4 { + t.Fatalf("only %d empty-index corpus cases; the instrument needs more", len(cases)) + } + + // Pre-recovery state, and the refusal a token-guarded write gets while + // the rows are pending (BUG-3133), for contrast with the one after. + first := cases[0].item + pre, _ := f.srv.store.GetItem(first.ID) + if pre.ContentState != models.ContentStatePendingFlush { + t.Fatalf("precondition: content_state %q", pre.ContentState) + } + preSeq := pre.Seq + + f.srv.StartMaterializeRecovery() + for _, c := range cases { + if res := f.waitProcessed(t, c.item.ID, 60*time.Second); res != mrApplied { + t.Fatalf("%s: result %q, want applied", c.c.Name, res) + } + } + + for _, c := range cases { + rr := doRequest(f.srv, http.MethodGet, "/api/v1/workspaces/ws/items/"+c.item.Slug, nil) + if rr.Code != http.StatusOK { + t.Fatalf("%s: GET %d %s", c.c.Name, rr.Code, rr.Body.String()) + } + var got models.Item + parseJSON(t, rr, &got) + if got.Content != c.c.Expected { + t.Errorf("%s: recovered body differs from the live editor's flush\n--- got ---\n%s\n--- want ---\n%s", c.c.Name, got.Content, c.c.Expected) + } + if got.ContentState == models.ContentStatePendingFlush { + t.Errorf("%s: content_state still %q after recovery", c.c.Name, got.ContentState) + } + } + + body := map[string]any{"content": "a caller's body from before the recovery", "expected_seq": preSeq} + rr := doRequest(f.srv, http.MethodPatch, "/api/v1/workspaces/ws/items/"+first.Slug, body) + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "update_conflict") { + t.Fatalf("stale expected_seq PATCH = %d %s, want 409 update_conflict", rr.Code, rr.Body.String()) + } + after, _ := f.srv.store.GetItem(first.ID) + if after.Content != cases[0].c.Expected { + t.Fatalf("the refused PATCH changed the body") + } + t.Logf("%d corpus op-logs recovered end to end", len(cases)) +} + +// The version history shows the recovery as the system's, on the wire. +func TestMaterializeRecoveryVersionOnTheWire(t *testing.T) { + f := newRecoveryFixture(t, &fakeMaterializer{}, time.Minute, materializeRecoveryConfig{}) + it := f.item(t, "Versioned", "stale") + f.appendRows(t, it.ID, contentFrame(1)) + if res := f.r.process(it.ID); res != mrApplied { + t.Fatalf("process = %q", res) + } + rr := doRequest(f.srv, http.MethodGet, "/api/v1/workspaces/ws/items/"+it.Slug+"/versions", nil) + if rr.Code != http.StatusOK { + t.Fatalf("versions: %d %s", rr.Code, rr.Body.String()) + } + var vs []models.Version + parseJSON(t, rr, &vs) + if len(vs) == 0 || vs[0].CreatedBy != models.VersionCreatedBySystem || vs[0].Source != models.VersionSourceRecovery || + vs[0].ChangeSummary != models.RecoveryChangeSummary { + t.Fatalf("newest version = %+v", vs) + } +} + +// A client cannot claim the recovery label for its own write. +func TestClientCannotClaimRecoveryVersionSource(t *testing.T) { + f := newRecoveryFixture(t, nil, time.Minute, materializeRecoveryConfig{}) + it := f.item(t, "Spoof", "before") + rr := doRequest(f.srv, http.MethodPatch, "/api/v1/workspaces/ws/items/"+it.Slug, + map[string]any{"content": "after", "version_source": models.VersionSourceRecovery}) + if rr.Code != http.StatusOK { + t.Fatalf("PATCH %d %s", rr.Code, rr.Body.String()) + } + vs, err := f.srv.store.ListItemVersions(it.ID) + if err != nil || len(vs) == 0 { + t.Fatalf("versions: %v %v", vs, err) + } + if vs[0].Source == models.VersionSourceRecovery { + t.Fatal("a client write was labelled as a recovery") + } +} + +// ErrNoMemoryCap (U3: no cap could be established, so the supervisor refused +// the job and warned once itself) counts against the budget like any failure, +// but is not logged per item. The control is ErrChildDied, which is. +func TestMaterializeNoMemoryCapCountsSilently(t *testing.T) { + for _, tc := range []struct { + err error + wantWarn bool + }{ + {fmt.Errorf("%w: no cap mechanism on plan9", materialize.ErrNoMemoryCap), false}, + {fmt.Errorf("%w: exit status 2", materialize.ErrChildDied), true}, + } { + var buf bytes.Buffer + clock := time.Now() + fake := &fakeMaterializer{fn: func(materialize.Job) (string, error) { return "", tc.err }} + f := newRecoveryFixture(t, fake, time.Minute, materializeRecoveryConfig{ + maxFailures: 3, backoffBase: time.Minute, + now: func() time.Time { return clock }, + logger: slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})), + }) + it := f.item(t, "Uncapped", "stale") + f.appendRows(t, it.ID, contentFrame(1)) + for i := 0; i < 3; i++ { + if res := f.r.process(it.ID); res != mrFailed { + t.Fatalf("%v: pass %d = %q, want failed", tc.err, i, res) + } + clock = clock.Add(time.Hour) + } + if res := f.r.process(it.ID); res != mrBudgetExceeded || fake.calls.Load() != 3 { + t.Fatalf("%v: after K failures %q with %d calls, want budget_exhausted, 3", tc.err, res, fake.calls.Load()) + } + if got := strings.Contains(buf.String(), "level=WARN"); got != tc.wantWarn { + t.Fatalf("%v: per-item WARN logged = %v, want %v\n%s", tc.err, got, tc.wantWarn, buf.String()) + } + } +} + +// An item that uses up its budget leaves exactly ONE line naming it, its +// failure count and the last error kind; a budget reset (op-log growth) and a +// second exhaustion leave exactly one more. +func TestMaterializeExhaustionLogsOncePerExhaustion(t *testing.T) { + var buf bytes.Buffer + clock := time.Now() + fake := &fakeMaterializer{fn: func(materialize.Job) (string, error) { + return "", fmt.Errorf("%w: exit status 2", materialize.ErrChildDied) + }} + f := newRecoveryFixture(t, fake, time.Minute, materializeRecoveryConfig{ + maxFailures: 3, backoffBase: time.Minute, + now: func() time.Time { return clock }, + logger: slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})), + }) + it := f.item(t, "Stuck", "stale") + f.appendRows(t, it.ID, contentFrame(1)) + const msg = "exhausted its failure budget" + exhaustLines := func() []string { + var out []string + for _, l := range strings.Split(buf.String(), "\n") { + if strings.Contains(l, msg) { + out = append(out, l) + } + } + return out + } + drive := func(passes int) { + for i := 0; i < passes; i++ { + f.r.process(it.ID) + clock = clock.Add(time.Hour) + } + } + + drive(3 + 5) // K failures, then five refused passes + lines := exhaustLines() + if len(lines) != 1 { + t.Fatalf("exhaustion lines after one exhaustion = %d, want 1\n%s", len(lines), buf.String()) + } + for _, want := range []string{"level=WARN", "item_id=" + it.ID, "failures=3", "last_error_kind=worker_died"} { + if !strings.Contains(lines[0], want) { + t.Fatalf("exhaustion line lacks %q: %s", want, lines[0]) + } + } + + f.appendRows(t, it.ID, contentFrame(2)) // op-log grows: the budget resets + drive(3 + 5) + if n := len(exhaustLines()); n != 2 { + t.Fatalf("exhaustion lines after a reset and a second exhaustion = %d, want 2\n%s", n, buf.String()) + } + if fake.calls.Load() != 6 { + t.Fatalf("materializer calls %d, want 6", fake.calls.Load()) + } + + // The schema-version skip is an exhaustion too, logged once. + buf.Reset() + old := f.item(t, "Old schema", "stale") + if _, err := f.srv.store.AppendYjsUpdate(old.ID, contentFrame(3), "0-old"); err != nil { + t.Fatal(err) + } + f.r.process(old.ID) + f.r.process(old.ID) + if l := exhaustLines(); len(l) != 1 || !strings.Contains(l[0], "last_error_kind=schema_version") { + t.Fatalf("schema-version exhaustion lines = %v", l) + } +} + +// Items skipped for set-aside rows are counted, reported on the sweep's one +// summary line, and never logged per item. +func TestMaterializeSetAsideSkipsAreCountedNotLogged(t *testing.T) { + var buf bytes.Buffer + fake := &fakeMaterializer{} + f := newRecoveryFixture(t, fake, time.Minute, materializeRecoveryConfig{ + logger: slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})), + }) + it := f.item(t, "Aside", "stale") + f.appendRows(t, it.ID, contentFrame(1)) + if _, _, err := f.srv.store.SetAsideAndClearOpLog(it.ID); err != nil { + t.Fatal(err) + } + f.appendRows(t, it.ID, contentFrame(2)) // pending again, over set-aside rows + + f.r.onRoomIdle(it.ID) // the room trigger does queue it: its op-log is pending + if q := f.queueSnapshot(); len(q) != 1 { + t.Fatalf("queue = %v", q) + } + for i := 0; i < 3; i++ { + if res := f.r.process(it.ID); res != mrSetAside { + t.Fatalf("process = %q, want set_aside", res) + } + } + if n := f.r.setAsideSkipped.Load(); n != 3 { + t.Fatalf("set-aside skip count = %d, want 3", n) + } + if fake.calls.Load() != 0 { + t.Fatal("a set-aside item reached the materializer") + } + if strings.Contains(buf.String(), it.ID) { + t.Fatalf("set-aside skips logged per item:\n%s", buf.String()) + } + + f.r.sweep() + f.r.sweep() // no news: no second line + var sweepLines []string + for _, l := range strings.Split(buf.String(), "\n") { + if strings.Contains(l, "op-log recovery sweep") { + sweepLines = append(sweepLines, l) + } + } + if len(sweepLines) != 1 || !strings.Contains(sweepLines[0], "set_aside_skipped_total=3") { + t.Fatalf("sweep summary lines = %v, want one carrying set_aside_skipped_total=3", sweepLines) + } +} + +// STARVATION (codex r1): the sweep reads a bounded page, and the room check +// and the failure budget filter it afterwards. With materializeSweepLimit+1 +// dormant pending items of which the first materializeSweepLimit fail every +// time, the last one must still be recovered, within +// ceil(N / limit) + 1 = 3 sweeps, at the REAL page size. +func TestMaterializeSweepReachesEveryCandidate(t *testing.T) { + clock := time.Now().Add(time.Hour) // rows written now are dormant + var targetFrame, healedFrame []byte + fake := &fakeMaterializer{fn: func(job materialize.Job) (string, error) { + if len(job.Rows) == 1 && (bytes.Equal(job.Rows[0], targetFrame) || bytes.Equal(job.Rows[0], healedFrame)) { + return "recovered body", nil + } + return "", fmt.Errorf("%w: exit status 2", materialize.ErrChildDied) + }} + f := newRecoveryFixture(t, fake, time.Minute, materializeRecoveryConfig{ + now: func() time.Time { return clock }, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + n := materializeSweepLimit + 1 + frameOf := map[string][]byte{} + for i := 0; i < n; i++ { + it := f.item(t, fmt.Sprintf("Backlog %03d", i), "stale") + fr := []byte{0x00, 0x02, 0x05, 0x01, byte(i >> 8), byte(i), 0x7F, 0x00} + f.appendRows(t, it.ID, fr) + frameOf[it.ID] = fr + } + all, err := f.srv.store.ListMaterializeCandidates(clock.Add(-materializeDormancy), store.MaterializeCursor{}, 10*n) + if err != nil || len(all) != n { + t.Fatalf("candidates %d (%v), want %d", len(all), err, n) + } + target := all[n-1].ItemID // the one every page read from the start leaves out + targetFrame = frameOf[target] + + // runUntil sweeps up to 3 times, draining the queue after each, and + // reports the sweep on which want was recovered (0: never). + runUntil := func(want string) int { + for tick := 1; tick <= 3; tick++ { + f.r.sweep() + hit := false + for { + id, ok := f.r.dequeue() + if !ok { + break + } + if res := f.r.process(id); id == want && res == mrApplied { + hit = true + } + } + clock = clock.Add(time.Hour) // every backoff expires between ticks + if hit { + return tick + } + } + return 0 + } + tick := runUntil(target) + if tick == 0 { + t.Fatalf("the %dth candidate was never recovered in 3 sweeps", n) + } + t.Logf("the %dth candidate was recovered on sweep %d", n, tick) + + // And the walk WRAPS: the first candidate starts succeeding (its op-log, + // and so its place in the order, unchanged; two failures, so still inside + // its budget) and is reached again after the cursor has passed it. + first := all[0].ItemID + healedFrame = frameOf[first] + if tick := runUntil(first); tick == 0 { + t.Fatal("after the cursor passed it, the first candidate was never offered again") + } else { + t.Logf("the first candidate was recovered on sweep %d after the wrap", tick) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 59dce2bf8..4056f785e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -264,6 +264,12 @@ type Server struct { // StartOpLogGC; Stop() signals the loop via stopOpLogGC. opLogGC opLogGCConfig + // materializeRecovery is the op-log recovery worker (TASK-2198 U4): it + // rebuilds items.content from an item's collaborative op-log after the tab + // holding those edits went away without flushing them. Nil until + // SetMaterializer; started by StartMaterializeRecovery; Stop() stops it. + materializeRecovery *materializeRecovery + // tokenReaper holds the periodic-sweep config + lifecycle for the // short-lived-credential reaper (PLAN-1933 DR-5 / TASK-1936). // Mirrors orphanGC/opLogGC. Configured via SetTokenReaperConfig + @@ -553,6 +559,10 @@ func (s *Server) Stop() { // Yjs op-log prune sweeper (TASK-1309). Same lifecycle pattern; // signals BEFORE Wait() so the goroutine sees the close and exits. s.stopOpLogGC() + // Op-log recovery worker (TASK-2198 U4). Signalled BEFORE collab.Close and + // bg.Wait: cancelling its context kills a job in flight (the supervisor + // kills the worker process), so the goroutine drains promptly. + s.stopMaterializeRecovery() // Short-lived-credential reaper (PLAN-1933 DR-5 / TASK-1936). Same // lifecycle pattern; signal BEFORE Wait() so the goroutine exits. s.stopTokenReaper() @@ -589,6 +599,8 @@ func (s *Server) Stop() { s.collab.Close() } s.bg.Wait() + // The worker PROCESS goes after bg.Wait, when nothing can hand it a job. + s.closeMaterializer() // Watch/nudge bus (BUG-2651). Closed AFTER bg.Wait() so a background // producer cannot publish into a bus that is already tearing down; both // implementations are safe if one does anyway (MemoryBus finds no diff --git a/internal/server/testdata/materialize_links.json b/internal/server/testdata/materialize_links.json new file mode 100644 index 000000000..5d722f68e --- /dev/null +++ b/internal/server/testdata/materialize_links.json @@ -0,0 +1,90 @@ +[ + { + "name": "ref-link-other-text", + "markdown": "See [shown text](/alice/ws/secrets/SECR-1) here.", + "rows": [ + "AAKxAQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua1R7ImhyZWYiOiIvYWxpY2Uvd3Mvc2VjcmV0cy9TRUNSLTEiLCJ0YXJnZXQiOm51bGwsInJlbCI6bnVsbCwiY2xhc3MiOiIiLCJ0aXRsZSI6bnVsbH2ElhEGCnNob3duIHRleHSGlhEQBGxpbmsEbnVsbISWEREGIGhlcmUuAA==" + ], + "raw": "See [shown text](/alice/ws/secrets/SECR-1) here." + }, + { + "name": "ref-link-old-title", + "markdown": "See [Old Codename](/alice/ws/secrets/SECR-1) here.", + "rows": [ + "AAKzAQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua1R7ImhyZWYiOiIvYWxpY2Uvd3Mvc2VjcmV0cy9TRUNSLTEiLCJ0YXJnZXQiOm51bGwsInJlbCI6bnVsbCwiY2xhc3MiOiIiLCJ0aXRsZSI6bnVsbH2ElhEGDE9sZCBDb2RlbmFtZYaWERIEbGluawRudWxshJYREwYgaGVyZS4A" + ], + "raw": "See [Old Codename](/alice/ws/secrets/SECR-1) here." + }, + { + "name": "slug-link-slug-text", + "markdown": "See [the plan](/alice/ws/secrets/secret-launch-codename) here.", + "rows": [ + "AAK/AQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua2R7ImhyZWYiOiIvYWxpY2Uvd3Mvc2VjcmV0cy9zZWNyZXQtbGF1bmNoLWNvZGVuYW1lIiwidGFyZ2V0IjpudWxsLCJyZWwiOm51bGwsImNsYXNzIjoiIiwidGl0bGUiOm51bGx9hJYRBgh0aGUgcGxhboaWEQ4EbGluawRudWxshJYRDwYgaGVyZS4A" + ], + "raw": "See [the plan](/alice/ws/secrets/secret-launch-codename) here." + }, + { + "name": "text-equals-current-title", + "markdown": "See [Secret Launch Codename](/alice/ws/secrets/SECR-1) here.", + "rows": [ + "AAK9AQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua1R7ImhyZWYiOiIvYWxpY2Uvd3Mvc2VjcmV0cy9TRUNSLTEiLCJ0YXJnZXQiOm51bGwsInJlbCI6bnVsbCwiY2xhc3MiOiIiLCJ0aXRsZSI6bnVsbH2ElhEGFlNlY3JldCBMYXVuY2ggQ29kZW5hbWWGlhEcBGxpbmsEbnVsbISWER0GIGhlcmUuAA==" + ], + "raw": "See [Secret Launch Codename](/alice/ws/secrets/SECR-1) here." + }, + { + "name": "broken-link", + "markdown": "See [gone text](broken) here.", + "rows": [ + "AAKeAQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua0J7ImhyZWYiOiJicm9rZW4iLCJ0YXJnZXQiOm51bGwsInJlbCI6bnVsbCwiY2xhc3MiOiIiLCJ0aXRsZSI6bnVsbH2ElhEGCWdvbmUgdGV4dIaWEQ8EbGluawRudWxshJYREAYgaGVyZS4A" + ], + "raw": "See [[gone text]] here." + }, + { + "name": "cross-workspace", + "markdown": "See [xw text](/-/r/other/SECR-1) here.", + "rows": [ + "AAKnAQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua017ImhyZWYiOiIvLS9yL290aGVyL1NFQ1ItMSIsInRhcmdldCI6bnVsbCwicmVsIjpudWxsLCJjbGFzcyI6IiIsInRpdGxlIjpudWxsfYSWEQYHeHcgdGV4dIaWEQ0EbGluawRudWxshJYRDgYgaGVyZS4A" + ], + "raw": "See [xw text](/-/r/other/SECR-1) here." + }, + { + "name": "typed-wiki-literal", + "markdown": "Typed \\[\\[SECR-1\\]\\] literal.", + "rows": [ + "AAJAAQOWEQAHAQdkZWZhdWx0AwlwYXJhZ3JhcGgHAJYRAAYEAJYRARlUeXBlZCBbW1NFQ1ItMV1dIGxpdGVyYWwuAA==" + ], + "raw": "Typed [[SECR-1]] literal." + }, + { + "name": "public-ref-link", + "markdown": "See [the public plan](/alice/ws/public/PUB-2) here.", + "rows": [ + "AAK0AQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua1J7ImhyZWYiOiIvYWxpY2Uvd3MvcHVibGljL1BVQi0yIiwidGFyZ2V0IjpudWxsLCJyZWwiOm51bGwsImNsYXNzIjoiIiwidGl0bGUiOm51bGx9hJYRBg90aGUgcHVibGljIHBsYW6GlhEVBGxpbmsEbnVsbISWERYGIGhlcmUuAA==" + ], + "raw": "See [the public plan](/alice/ws/public/PUB-2) here." + }, + { + "name": "marker-follows-title", + "markdown": "See [Secret Launch Codename](/alice/ws/secrets/SECR-1 \"pad-follows-title:Secret Launch Codename\") here.", + "rows": [ + "AALjAQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua3p7ImhyZWYiOiIvYWxpY2Uvd3Mvc2VjcmV0cy9TRUNSLTEiLCJ0YXJnZXQiOm51bGwsInJlbCI6bnVsbCwiY2xhc3MiOiIiLCJ0aXRsZSI6InBhZC1mb2xsb3dzLXRpdGxlOlNlY3JldCBMYXVuY2ggQ29kZW5hbWUifYSWEQYWU2VjcmV0IExhdW5jaCBDb2RlbmFtZYaWERwEbGluawRudWxshJYRHQYgaGVyZS4A" + ], + "raw": "See [Secret Launch Codename](/alice/ws/secrets/SECR-1) here." + }, + { + "name": "marker-renamed", + "markdown": "See [Old Codename](/alice/ws/secrets/SECR-1 \"pad-follows-title:Old Codename\") here.", + "rows": [ + "AALPAQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua3B7ImhyZWYiOiIvYWxpY2Uvd3Mvc2VjcmV0cy9TRUNSLTEiLCJ0YXJnZXQiOm51bGwsInJlbCI6bnVsbCwiY2xhc3MiOiIiLCJ0aXRsZSI6InBhZC1mb2xsb3dzLXRpdGxlOk9sZCBDb2RlbmFtZSJ9hJYRBgxPbGQgQ29kZW5hbWWGlhESBGxpbmsEbnVsbISWERMGIGhlcmUuAA==" + ], + "raw": "See [Old Codename](/alice/ws/secrets/SECR-1) here." + }, + { + "name": "marker-external-href", + "markdown": "See [Old Codename](https://example.com/x \"pad-follows-title:Old Codename\") here.", + "rows": [ + "AALMAQEHlhEABwEHZGVmYXVsdAMJcGFyYWdyYXBoBwCWEQAGBACWEQEEU2VlIIaWEQUEbGlua217ImhyZWYiOiJodHRwczovL2V4YW1wbGUuY29tL3giLCJ0YXJnZXQiOm51bGwsInJlbCI6bnVsbCwiY2xhc3MiOiIiLCJ0aXRsZSI6InBhZC1mb2xsb3dzLXRpdGxlOk9sZCBDb2RlbmFtZSJ9hJYRBgxPbGQgQ29kZW5hbWWGlhESBGxpbmsEbnVsbISWERMGIGhlcmUuAA==" + ], + "raw": "See [Old Codename](https://example.com/x) here." + } +] diff --git a/internal/store/item_content_state_test.go b/internal/store/item_content_state_test.go index f2f286263..a8703be8e 100644 --- a/internal/store/item_content_state_test.go +++ b/internal/store/item_content_state_test.go @@ -12,8 +12,10 @@ import ( // BUG-3000: a read that serves items.content must say so when that content is // behind the item's live collaborative document. // -// The window is UNBOUNDED, which is what makes the marker worth having rather than -// a documentation note: nothing server-side moves applier content into the row. +// The window was UNBOUNDED when this was written, which is what made the marker +// worth having rather than a documentation note. Since TASK-2198 U4 the server's +// op-log recovery closes it after the item's room closes, when it is on and the +// job succeeds; the op-log GC below still never does. // PruneSweep's candidate query requires MAX(op-log.id) <= content_flushed_op_log_id, // so an item in exactly this state is EXCLUDED from the sweep — its op-log rows are // retained (the content is durable) and the row stays behind until a tab next opens diff --git a/internal/store/items.go b/internal/store/items.go index 4fc7e98d8..08d008c92 100644 --- a/internal/store/items.go +++ b/internal/store/items.go @@ -2806,6 +2806,12 @@ func (s *Store) updateItemWithParentLinkOnce( if source == "" { source = "web" } + // TASK-2198 U4: an op-log recovery is the system's write, not a + // guessed author's (op-log rows carry no user id). + if input.Recovered { + createdBy = models.VersionCreatedBySystem + source = models.VersionSourceRecovery + } // ForceVersion (e.g. a version restore) and a title change both bypass // the per-(actor, source) throttle so a bracketing snapshot is always @@ -2916,7 +2922,12 @@ func (s *Store) updateItemWithParentLinkOnce( // cannot delete them. When the cursor is missing (older // clients, malformed bodies) we behave as before — no // advancement. - if input.VersionSource != "collab-snapshot" { + // + // An op-log RECOVERY (input.Recovered, TASK-2198 U4) takes the + // cursor-gated arm too: its markdown was rebuilt from the rows up to + // its cursor and no further, so stamping MAX would claim a row that + // committed after the job was built. + if input.VersionSource != "collab-snapshot" && !input.Recovered { sets = append(sets, "content_flushed_op_log_id = (SELECT COALESCE(MAX(id), 0) FROM item_yjs_updates WHERE item_id = ?)") args = append(args, id) } else if input.OpLogCursor != nil { diff --git a/internal/store/materialize_flush.go b/internal/store/materialize_flush.go new file mode 100644 index 000000000..c0fad8372 --- /dev/null +++ b/internal/store/materialize_flush.go @@ -0,0 +1,348 @@ +package store + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/PerpetualSoftware/pad/internal/models" +) + +// Op-log recovery (TASK-2198 U4): the store half. +// +// When a tab dies uncleanly its last typing exists only in item_yjs_updates: +// content-bearing rows above items.content_flushed_op_log_id, which +// content_state reports as applied_pending_flush. The server materializes +// those rows into markdown in a worker process (internal/materialize) and +// writes the result here. This file holds the reads that build the job and the +// write that lands it; internal/server/materialize_recovery.go holds the +// triggers and the worker, and is the only caller. +// +// THE WRITE IS A FLUSH, NOT AN EDIT. It carries the same proof a tab's +// collab-snapshot flush carries — the markdown was rebuilt from every op-log +// row up to a cursor, and the cursor is still MAX(op-log id) — so it advances +// the watermark to that cursor and to nothing beyond it. It never prunes the +// op-log: the next tab to open the item replays the op-log, which already holds +// everything this write stored. + +// MaterializeInput is what a recovery job is built from, read in ONE statement +// so the rows and the cursor describe the same op-log. +type MaterializeInput struct { + ItemID string + WorkspaceID string + // Rows are the item's CONTENT-BEARING op-log rows, oldest first, over the + // WHOLE op-log, not only those above the watermark: replay rebuilds the + // document from nothing, as a tab's replay (since=0) does. Rows marked + // non-content-bearing provably cannot change the document (BUG-3124), so + // leaving them out gives the same document (U2's parity set). + Rows [][]byte + // Cursor is MAX(id) over ALL of the item's op-log rows, content-bearing or + // not, from the same statement as Rows. + Cursor int64 + // SchemaVersions are the distinct schema_version stamps over all rows. + SchemaVersions []string + // Pending is how many content-bearing rows sit above the watermark. + Pending int + // SetAside is how many rows a schema rebuild set aside (BUG-3244). + SetAside int +} + +// LoadMaterializeInput reads the item's op-log for a recovery job. It returns +// nil (no error) when the item is missing, soft-deleted, or has no op-log. +func (s *Store) LoadMaterializeInput(itemID string) (*MaterializeInput, error) { + if itemID == "" { + return nil, errors.New("LoadMaterializeInput: itemID is required") + } + var workspaceID string + if err := s.db.QueryRow(s.q(`SELECT workspace_id FROM items WHERE id = ? AND deleted_at IS NULL`), itemID).Scan(&workspaceID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("materialize input (item): %w", err) + } + + // ONE statement: the rows, their stamps and the watermark come from the + // same snapshot on both dialects, so Cursor can never name a row whose + // bytes are not in Rows. Non-content-bearing rows contribute their id and + // stamp but not their bytes. + rows, err := s.db.Query(s.q(` + SELECT u.id, u.content_bearing, u.schema_version, + CASE WHEN u.content_bearing = TRUE THEN u.update_data END, + COALESCE(i.content_flushed_op_log_id, 0) + FROM item_yjs_updates u + JOIN items i ON i.id = u.item_id + WHERE u.item_id = ? + ORDER BY u.id ASC`), itemID) + if err != nil { + return nil, fmt.Errorf("materialize input (op-log): %w", err) + } + defer rows.Close() + + in := &MaterializeInput{ItemID: itemID, WorkspaceID: workspaceID} + seen := map[string]bool{} + for rows.Next() { + var ( + id int64 + bearing bool + version string + data []byte + watermark int64 + ) + if err := rows.Scan(&id, &bearing, &version, &data, &watermark); err != nil { + return nil, fmt.Errorf("materialize input (scan): %w", err) + } + in.Cursor = id + if !seen[version] { + seen[version] = true + in.SchemaVersions = append(in.SchemaVersions, version) + } + if bearing { + in.Rows = append(in.Rows, data) + if id > watermark { + in.Pending++ + } + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("materialize input (rows): %w", err) + } + if in.Cursor == 0 { + return nil, nil + } + if err := s.db.QueryRow(s.q(`SELECT COUNT(*) FROM item_yjs_updates_set_aside WHERE item_id = ?`), itemID).Scan(&in.SetAside); err != nil { + return nil, fmt.Errorf("materialize input (set-aside): %w", err) + } + return in, nil +} + +// ItemHasPendingContent reports whether the item's op-log holds content-bearing +// rows above its flush watermark: content_state's applied_pending_flush +// predicate, for a live item. +func (s *Store) ItemHasPendingContent(itemID string) (bool, error) { + var n int + err := s.db.QueryRow(s.q(` + SELECT COUNT(*) FROM items i + WHERE i.id = ? AND i.deleted_at IS NULL AND `+pendingFlushExistsSQLFor("i")), itemID).Scan(&n) + if err != nil { + return false, fmt.Errorf("item pending content: %w", err) + } + return n > 0, nil +} + +// MaterializeCandidate is one row of the recovery sweep's backstop set, with +// the keyset position (LastAt, ItemID) it sorts at. +type MaterializeCandidate struct { + ItemID string + // LastAt is the item's newest op-log created_at, exactly as stored. + LastAt string +} + +// MaterializeCursor is a keyset position in the sweep's order. The zero value +// is the start. +type MaterializeCursor struct { + LastAt string + ItemID string +} + +// ListMaterializeCandidates returns up to limit live items whose op-log holds +// content-bearing rows above the flush watermark, whose NEWEST op-log row is +// older than before, and which hold no set-aside rows: the recovery sweep's +// backstop set (the shape of ListDormantOpLogItemsBefore, with the flush +// predicate inverted). The caller still checks for an open room. +// +// KEYSET-PAGED, strictly after `after`, in (newest op-log created_at, item id) +// order. The caller filters what comes back (open rooms, and the worker's +// per-item failure budget), so a page read from the start every time would +// hand back the same items forever once `limit` of them sit filtered, and +// every candidate behind them would starve. Walking the set page by page and +// wrapping at the end reaches every candidate however many ineligible ones +// sort ahead of it. +func (s *Store) ListMaterializeCandidates(before time.Time, after MaterializeCursor, limit int) ([]MaterializeCandidate, error) { + if limit <= 0 { + limit = 100 + } + cutoff := before.UTC().Format(time.RFC3339) + rows, err := s.db.Query(s.q(` + SELECT u.item_id, MAX(u.created_at) + FROM item_yjs_updates u + JOIN items i ON i.id = u.item_id + WHERE i.deleted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM item_yjs_updates_set_aside sa WHERE sa.item_id = u.item_id) + GROUP BY u.item_id, i.content_flushed_op_log_id + HAVING MAX(u.created_at) < ? + AND SUM(CASE WHEN u.content_bearing = TRUE AND u.id > COALESCE(i.content_flushed_op_log_id, 0) THEN 1 ELSE 0 END) > 0 + AND (MAX(u.created_at) > ? OR (MAX(u.created_at) = ? AND u.item_id > ?)) + ORDER BY MAX(u.created_at) ASC, u.item_id ASC + LIMIT ?`), cutoff, after.LastAt, after.LastAt, after.ItemID, limit) + if err != nil { + return nil, fmt.Errorf("list materialize candidates: %w", err) + } + defer rows.Close() + var out []MaterializeCandidate + for rows.Next() { + var c MaterializeCandidate + if err := rows.Scan(&c.ItemID, &c.LastAt); err != nil { + return nil, fmt.Errorf("scan materialize candidate: %w", err) + } + out = append(out, c) + } + return out, rows.Err() +} + +// MaterializeOutcome is what MaterializeFlush did. +type MaterializeOutcome string + +const ( + // MaterializeApplied: items.content now holds the markdown; seq and + // updated_at were bumped, one version row was written (when the body + // changed) and the watermark is the cursor. + MaterializeApplied MaterializeOutcome = "applied" + // MaterializeStamped: the body already was the markdown, so only the + // watermark moved (StampContentWatermarkIfCaughtUp): no seq, no version. + MaterializeStamped MaterializeOutcome = "stamped" + // MaterializeCursorMoved: the op-log changed after the job was built + // (MAX(id) is not the cursor). Nothing was written. + MaterializeCursorMoved MaterializeOutcome = "cursor_moved" + // MaterializeNothingPending: no content-bearing row is above the watermark + // any more (a tab flushed, or a direct write replaced the op-log). + MaterializeNothingPending MaterializeOutcome = "nothing_pending" + // MaterializeSetAside: the item holds set-aside rows (BUG-3244). They are + // left untouched and the item is not written. + MaterializeSetAside MaterializeOutcome = "set_aside" + // MaterializeGone: the item is missing or soft-deleted. + MaterializeGone MaterializeOutcome = "gone" +) + +type materializeAbort struct{ outcome MaterializeOutcome } + +func (e *materializeAbort) Error() string { return "materialize flush: " + string(e.outcome) } + +// materializeFlushPrecheckHook, when set by a test, runs inside the write's +// transaction immediately before the checks; materializeFlushAfterCheckHook +// runs after they have passed, before the UPDATE: the window a row committed +// by another connection lands in on Postgres (READ COMMITTED). +var ( + materializeFlushPrecheckHook func(itemID string) + materializeFlushAfterCheckHook func(itemID string) +) + +// MaterializeFlush writes a recovered body to items.content (TASK-2198 U4). +// +// CALLER CONTRACT: hold the collab per-item setup lock with no room open for +// the item (collab.RoomManager.UnderItemLockIfNoRoom). Every op-log append +// goes through a room, and a room can only be created under that lock, so +// nothing can append while this runs. The checks below do not depend on that: +// they run inside the write's own transaction, under UpdateItem's locks, and +// the watermark moves through the cursor-gated SQL a collab-snapshot flush +// uses, so a row committing after the check is left above the watermark. +// +// It proceeds only when, in the write's transaction: MAX(op-log id) equals +// cursor, at least one content-bearing row is above the watermark, and the +// item holds no set-aside rows. Otherwise nothing is written and the outcome +// says which check failed; that is not an error. The version row is +// attributed to the system (models.ItemUpdate.Recovered). The op-log is never +// pruned. +func (s *Store) MaterializeFlush(itemID string, cursor int64, markdown string) (MaterializeOutcome, *models.Item, error) { + if itemID == "" || cursor < 1 { + return "", nil, errors.New("MaterializeFlush: itemID and a positive cursor are required") + } + + var current string + if err := s.db.QueryRow(s.q(`SELECT content FROM items WHERE id = ? AND deleted_at IS NULL`), itemID).Scan(¤t); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return MaterializeGone, nil, nil + } + return "", nil, fmt.Errorf("materialize flush (read): %w", err) + } + + check := func(q Queryer, tx *sql.Tx) (MaterializeOutcome, error) { + if materializeFlushPrecheckHook != nil { + materializeFlushPrecheckHook(itemID) + } + var maxID int64 + if err := q.QueryRow(s.q(`SELECT COALESCE(MAX(id), 0) FROM item_yjs_updates WHERE item_id = ?`), itemID).Scan(&maxID); err != nil { + return "", fmt.Errorf("materialize flush (max): %w", err) + } + if maxID != cursor { + return MaterializeCursorMoved, nil + } + var setAside int + if err := q.QueryRow(s.q(`SELECT COUNT(*) FROM item_yjs_updates_set_aside WHERE item_id = ?`), itemID).Scan(&setAside); err != nil { + return "", fmt.Errorf("materialize flush (set-aside): %w", err) + } + if setAside > 0 { + return MaterializeSetAside, nil + } + var pending int + if tx != nil { + n, err := s.CountPendingContentRowsTx(tx, itemID) + if err != nil { + return "", err + } + pending = n + } else if err := q.QueryRow(s.q(` + SELECT COUNT(*) FROM item_yjs_updates u JOIN items i ON i.id = u.item_id + WHERE u.item_id = ? AND u.id > COALESCE(i.content_flushed_op_log_id, 0) AND u.content_bearing = TRUE`), itemID).Scan(&pending); err != nil { + return "", fmt.Errorf("materialize flush (pending): %w", err) + } + if pending == 0 { + return MaterializeNothingPending, nil + } + return "", nil + } + + // The body already is the document: move the watermark only, as a tab's + // watermark stamp would (BUG-3124 unit B). A version row and a seq bump + // for a write that changes nothing would conflict every token-holder for + // no reason. + if current == markdown { + if outcome, err := check(s.db, nil); err != nil || outcome != "" { + return outcome, nil, err + } + sum := sha256.Sum256([]byte(markdown)) + moved, err := s.StampContentWatermarkIfCaughtUp(itemID, cursor, hex.EncodeToString(sum[:])) + if err != nil { + return "", nil, err + } + if !moved { + return MaterializeCursorMoved, nil, nil + } + return MaterializeStamped, nil, nil + } + + c := cursor + input := models.ItemUpdate{ + Content: &markdown, + ChangeSummary: models.RecoveryChangeSummary, + ForceVersion: true, + Recovered: true, + OpLogCursor: &c, + } + updated, err := s.UpdateItemWithPreCheck(itemID, input, func(tx *sql.Tx, _ *models.Item) error { + outcome, err := check(tx, tx) + if err != nil { + return err + } + if outcome != "" { + return &materializeAbort{outcome: outcome} + } + if materializeFlushAfterCheckHook != nil { + materializeFlushAfterCheckHook(itemID) + } + return nil + }) + if err != nil { + var abort *materializeAbort + if errors.As(err, &abort) { + return abort.outcome, nil, nil + } + return "", nil, err + } + if updated == nil { + return MaterializeGone, nil, nil + } + return MaterializeApplied, updated, nil +} diff --git a/internal/store/materialize_flush_internal_test.go b/internal/store/materialize_flush_internal_test.go new file mode 100644 index 000000000..7ab4d44e6 --- /dev/null +++ b/internal/store/materialize_flush_internal_test.go @@ -0,0 +1,64 @@ +package store + +import ( + "testing" + + "github.com/PerpetualSoftware/pad/internal/models" +) + +// TASK-2198 U4: the recovery write's watermark is its CURSOR, never MAX at +// UPDATE time. On Postgres (READ COMMITTED) a row another connection commits +// after the write's checks is visible to the UPDATE's own subqueries, so a +// write that stamped MAX would claim a row its markdown never saw, and +// content_state would read clean over an unrecovered edit. +// +// Postgres only: on SQLite the hook's append would wait on the write +// transaction's lock, so the window does not exist there. +func TestMaterializeFlushWatermarkIsTheCursorNotMax(t *testing.T) { + s := testStore(t) + if s.dialect.Driver() != DriverPostgres { + t.Skip("the READ COMMITTED window this pins exists on Postgres only (make test-pg)") + } + ws := createTestWorkspace(t, s, "Recovery race") + coll, err := s.CreateCollection(ws.ID, models.CollectionCreate{Name: "Notes", Schema: `{"fields":[]}`}) + if err != nil { + t.Fatal(err) + } + it, err := s.CreateItem(ws.ID, coll.ID, models.ItemCreate{Title: "Racy", Content: "stale", Fields: `{}`}) + if err != nil { + t.Fatal(err) + } + cursor, err := s.AppendYjsUpdate(it.ID, []byte{0x00, 0x02, 0x05, 0x01, 0x01, 0x00, 0x7F, 0x00}, "1") + if err != nil { + t.Fatal(err) + } + var late int64 + materializeFlushAfterCheckHook = func(itemID string) { + materializeFlushAfterCheckHook = nil + var aerr error + late, aerr = s.AppendYjsUpdate(itemID, []byte{0x00, 0x02, 0x05, 0x01, 0x02, 0x00, 0x7F, 0x00}, "1") + if aerr != nil { + t.Errorf("late append: %v", aerr) + } + } + t.Cleanup(func() { materializeFlushAfterCheckHook = nil }) + + outcome, _, err := s.MaterializeFlush(it.ID, cursor, "recovered") + if err != nil || outcome != MaterializeApplied { + t.Fatalf("MaterializeFlush = %q, %v", outcome, err) + } + if late <= cursor { + t.Fatalf("the late row %d did not land after the cursor %d", late, cursor) + } + wm, _, err := s.GetItemContentFlushedOpLogID(it.ID) + if err != nil { + t.Fatal(err) + } + if wm == late { + t.Fatalf("watermark %d covers the late row the recovered body never saw", wm) + } + got, _ := s.GetItem(it.ID) + if got.ContentState != models.ContentStatePendingFlush { + t.Fatalf("content_state %q, want still pending for the late row", got.ContentState) + } +} diff --git a/internal/store/materialize_flush_test.go b/internal/store/materialize_flush_test.go new file mode 100644 index 000000000..157b9466e --- /dev/null +++ b/internal/store/materialize_flush_test.go @@ -0,0 +1,346 @@ +package store_test + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/PerpetualSoftware/pad/internal/diff" + "github.com/PerpetualSoftware/pad/internal/models" + "github.com/PerpetualSoftware/pad/internal/store" +) + +// TASK-2198 U4, store half, on both dialects: the op-log recovery write +// (MaterializeFlush) and the reads that build its job. Postgres legs skip +// unless PAD_TEST_POSTGRES_URL is set (make test-pg). + +// recoveryFrame is a content-bearing y-protocols sync Update frame, distinct +// per n (the store never parses past the envelope; an identical frame would be +// classified as a re-send and not count). +func recoveryFrame(n byte) []byte { + return []byte{0x00, 0x02, 0x05, 0x01, n, 0x00, 0x7F, 0x00} +} + +// syncStep1Frame is a SyncStep1 (an empty state vector): persisted, but never +// content-bearing (BUG-3124). +var syncStep1Frame = []byte{0x00, 0x00, 0x01, 0x00} + +func appendFrames(t *testing.T, s *store.Store, itemID string, frames ...[]byte) int64 { + t.Helper() + var last int64 + for _, f := range frames { + id, err := s.AppendYjsUpdate(itemID, f, "1") + if err != nil { + t.Fatalf("AppendYjsUpdate: %v", err) + } + last = id + } + return last +} + +func versionCount(t *testing.T, s *store.Store, itemID string) int { + t.Helper() + vs, err := s.ListItemVersions(itemID) + if err != nil { + t.Fatal(err) + } + return len(vs) +} + +func TestMaterializeFlushAppliesWhenCaughtUpAndPending(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + _, _, item := seedStaleItem(t, s) + cursor := appendFrames(t, s, item.ID, recoveryFrame(1), recoveryFrame(2)) + + before, err := s.GetItem(item.ID) + if err != nil { + t.Fatal(err) + } + if before.ContentState != models.ContentStatePendingFlush { + t.Fatalf("precondition: content_state %q, want %q", before.ContentState, models.ContentStatePendingFlush) + } + versionsBefore := versionCount(t, s, item.ID) + + outcome, updated, err := s.MaterializeFlush(item.ID, cursor, "recovered body") + if err != nil || outcome != store.MaterializeApplied || updated == nil { + t.Fatalf("MaterializeFlush = %q, %v, %v; want applied", outcome, updated, err) + } + + got, _ := s.GetItem(item.ID) + if got.Content != "recovered body" { + t.Fatalf("content %q, want the recovered body", got.Content) + } + if got.ContentState != "" { + t.Fatalf("content_state %q after recovery, want clean", got.ContentState) + } + if got.Seq <= before.Seq { + t.Fatalf("seq %d not bumped past %d", got.Seq, before.Seq) + } + if got.LastModifiedBy != before.LastModifiedBy { + t.Fatalf("last_modified_by %q changed from %q; recovery attributes only the version row", got.LastModifiedBy, before.LastModifiedBy) + } + wm, ok, err := s.GetItemContentFlushedOpLogID(item.ID) + if err != nil || !ok || wm != cursor { + t.Fatalf("watermark = %d (ok %v, err %v), want the cursor %d", wm, ok, err, cursor) + } + // Exactly one version row, attributed to the system. + vs, err := s.ListItemVersions(item.ID) + if err != nil { + t.Fatal(err) + } + if len(vs) != versionsBefore+1 { + t.Fatalf("versions %d, want %d", len(vs), versionsBefore+1) + } + v := vs[0] + if v.CreatedBy != models.VersionCreatedBySystem || v.Source != models.VersionSourceRecovery || v.ChangeSummary != models.RecoveryChangeSummary { + t.Fatalf("version attribution = (%q, %q, %q), want (%q, %q, %q)", v.CreatedBy, v.Source, v.ChangeSummary, + models.VersionCreatedBySystem, models.VersionSourceRecovery, models.RecoveryChangeSummary) + } + // PLAN-2348 U2 (#1691): a recovery row is a system row, so no user_id; + // it is an update row, not a create; and it records its change's line + // counts like any other update, since it goes through the same insert. + wantAdded, wantRemoved := diff.LineCounts(before.Content, "recovered body") + if v.UserID != "" || v.IsCreate || v.LinesAdded == nil || v.LinesRemoved == nil || + *v.LinesAdded != wantAdded || *v.LinesRemoved != wantRemoved { + t.Fatalf("recovery version history data: user %q create %v lines %v/%v, want \"\" false %d/%d", + v.UserID, v.IsCreate, v.LinesAdded, v.LinesRemoved, wantAdded, wantRemoved) + } + // The op-log is NOT pruned: the next tab replays it. + if ops, _ := s.LoadYjsUpdatesSince(item.ID, 0); len(ops) != 2 { + t.Fatalf("op-log rows after recovery = %d, want 2 (never pruned)", len(ops)) + } + // A second recovery moments later still writes its own version row: + // the per-(actor, source) throttle must not let a recovery move the + // body with no version bracketing it. + cursor2 := appendFrames(t, s, item.ID, recoveryFrame(3)) + if outcome, _, err := s.MaterializeFlush(item.ID, cursor2, "recovered again"); err != nil || outcome != store.MaterializeApplied { + t.Fatalf("second MaterializeFlush = %q, %v", outcome, err) + } + if n := versionCount(t, s, item.ID); n != versionsBefore+2 { + t.Fatalf("versions after two recoveries %d, want %d", n, versionsBefore+2) + } + before, _ = s.GetItem(item.ID) + before.Seq = got.Seq // the token a caller read after the FIRST recovery + + // A writer holding the pre-recovery seq is refused, not allowed to + // overwrite the recovered text (BUG-3037 / BUG-3133). + stale := before.Seq + body := "caller's stale body" + _, err = s.UpdateItem(item.ID, models.ItemUpdate{Content: &body, ExpectedSeq: &stale}) + var conflict *store.UpdateConflictError + if !errors.As(err, &conflict) { + t.Fatalf("stale expected_seq update err = %v, want UpdateConflictError", err) + } + }) +} + +func TestMaterializeFlushAbortsWhenTheCursorMoved(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + _, _, item := seedStaleItem(t, s) + appendFrames(t, s, item.ID, recoveryFrame(1)) + in, err := s.LoadMaterializeInput(item.ID) + if err != nil || in == nil { + t.Fatalf("LoadMaterializeInput: %v, %v", in, err) + } + // A row lands between job build and write. + appendFrames(t, s, item.ID, recoveryFrame(2)) + before, _ := s.GetItem(item.ID) + nv := versionCount(t, s, item.ID) + + outcome, updated, err := s.MaterializeFlush(item.ID, in.Cursor, "recovered body") + if err != nil || outcome != store.MaterializeCursorMoved || updated != nil { + t.Fatalf("MaterializeFlush = %q, %v, %v; want cursor_moved", outcome, updated, err) + } + got, _ := s.GetItem(item.ID) + if got.Content != "stored body" || got.Seq != before.Seq || versionCount(t, s, item.ID) != nv { + t.Fatalf("an aborted recovery wrote: content %q seq %d->%d", got.Content, before.Seq, got.Seq) + } + if got.ContentState != models.ContentStatePendingFlush { + t.Fatalf("content_state %q, want still pending", got.ContentState) + } + }) +} + +func TestMaterializeFlushAbortsWhenNothingIsPending(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + _, _, item := seedStaleItem(t, s) + cursor := appendFrames(t, s, item.ID, recoveryFrame(1)) + if err := s.SetItemContentFlushedOpLogIDForTesting(item.ID, cursor); err != nil { + t.Fatal(err) + } + before, _ := s.GetItem(item.ID) + outcome, _, err := s.MaterializeFlush(item.ID, cursor, "recovered body") + if err != nil || outcome != store.MaterializeNothingPending { + t.Fatalf("MaterializeFlush = %q, %v; want nothing_pending", outcome, err) + } + got, _ := s.GetItem(item.ID) + if got.Content != "stored body" || got.Seq != before.Seq { + t.Fatalf("wrote with nothing pending: content %q seq %d->%d", got.Content, before.Seq, got.Seq) + } + }) +} + +func TestMaterializeFlushLeavesSetAsideRowsAlone(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + _, _, item := seedStaleItem(t, s) + setAsideOne(t, s, item.ID) + cursor := appendFrames(t, s, item.ID, recoveryFrame(9)) + + outcome, _, err := s.MaterializeFlush(item.ID, cursor, "recovered body") + if err != nil || outcome != store.MaterializeSetAside { + t.Fatalf("MaterializeFlush = %q, %v; want set_aside", outcome, err) + } + // Same answer when the body already matches (the stamp-only path). + outcome, _, err = s.MaterializeFlush(item.ID, cursor, "stored body") + if err != nil || outcome != store.MaterializeSetAside { + t.Fatalf("MaterializeFlush (equal body) = %q, %v; want set_aside", outcome, err) + } + rows, err := s.ListYjsSetAside(item.ID) + if err != nil || len(rows) != 1 { + t.Fatalf("set-aside rows = %d (%v), want 1, untouched", len(rows), err) + } + got, _ := s.GetItem(item.ID) + if got.Content != "stored body" || got.ContentState != models.ContentStateSetAside { + t.Fatalf("content %q state %q", got.Content, got.ContentState) + } + }) +} + +// A document that serializes to the stored body moves the watermark only: no +// seq bump, no version row. +func TestMaterializeFlushStampsWhenTheBodyAlreadyMatches(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + _, _, item := seedStaleItem(t, s) + cursor := appendFrames(t, s, item.ID, recoveryFrame(1)) + before, _ := s.GetItem(item.ID) + nv := versionCount(t, s, item.ID) + outcome, _, err := s.MaterializeFlush(item.ID, cursor, "stored body") + if err != nil || outcome != store.MaterializeStamped { + t.Fatalf("MaterializeFlush = %q, %v; want stamped", outcome, err) + } + got, _ := s.GetItem(item.ID) + if got.Seq != before.Seq || versionCount(t, s, item.ID) != nv || got.ContentState != "" { + t.Fatalf("stamp: seq %d->%d versions %d->%d state %q", before.Seq, got.Seq, nv, versionCount(t, s, item.ID), got.ContentState) + } + }) +} + +// The job covers the WHOLE op-log's content-bearing rows (not only those above +// the watermark), and the cursor is MAX over every row. +func TestLoadMaterializeInputReadsTheWholeOpLog(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + _, _, item := seedStaleItem(t, s) + first := appendFrames(t, s, item.ID, recoveryFrame(1)) + if err := s.SetItemContentFlushedOpLogIDForTesting(item.ID, first); err != nil { + t.Fatal(err) + } + appendFrames(t, s, item.ID, recoveryFrame(2)) + last := appendFrames(t, s, item.ID, syncStep1Frame) + + in, err := s.LoadMaterializeInput(item.ID) + if err != nil || in == nil { + t.Fatalf("LoadMaterializeInput: %v, %v", in, err) + } + if in.Cursor != last { + t.Fatalf("cursor %d, want MAX over all rows %d", in.Cursor, last) + } + if len(in.Rows) != 2 || string(in.Rows[0]) != string(recoveryFrame(1)) || string(in.Rows[1]) != string(recoveryFrame(2)) { + t.Fatalf("rows = %d, want both content-bearing rows in id order", len(in.Rows)) + } + if in.Pending != 1 || in.SetAside != 0 { + t.Fatalf("pending %d set-aside %d, want 1, 0", in.Pending, in.SetAside) + } + if len(in.SchemaVersions) != 1 || in.SchemaVersions[0] != "1" { + t.Fatalf("schema versions %v", in.SchemaVersions) + } + }) +} + +func TestListMaterializeCandidates(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + wsID, collID, pending := seedStaleItem(t, s) + appendFrames(t, s, pending.ID, recoveryFrame(1)) + + flushed, err := s.CreateItem(wsID, collID, models.ItemCreate{Title: "Flushed", Content: "x"}) + if err != nil { + t.Fatal(err) + } + c := appendFrames(t, s, flushed.ID, recoveryFrame(2)) + if err := s.SetItemContentFlushedOpLogIDForTesting(flushed.ID, c); err != nil { + t.Fatal(err) + } + aside, err := s.CreateItem(wsID, collID, models.ItemCreate{Title: "Aside", Content: "y"}) + if err != nil { + t.Fatal(err) + } + setAsideOne(t, s, aside.ID) + appendFrames(t, s, aside.ID, recoveryFrame(3)) + + future := time.Now().Add(time.Hour) + ids, err := s.ListMaterializeCandidates(future, store.MaterializeCursor{}, 10) + if err != nil { + t.Fatal(err) + } + if len(ids) != 1 || ids[0].ItemID != pending.ID { + t.Fatalf("candidates = %v, want only the pending item %s", ids, pending.ID) + } + // Not dormant yet: the newest row is younger than the cutoff. + ids, err = s.ListMaterializeCandidates(time.Now().Add(-time.Hour), store.MaterializeCursor{}, 10) + if err != nil || len(ids) != 0 { + t.Fatalf("candidates before the dormancy cutoff = %v (%v), want none", ids, err) + } + has, err := s.ItemHasPendingContent(pending.ID) + if err != nil || !has { + t.Fatalf("ItemHasPendingContent(pending) = %v, %v", has, err) + } + has, err = s.ItemHasPendingContent(flushed.ID) + if err != nil || has { + t.Fatalf("ItemHasPendingContent(flushed) = %v, %v", has, err) + } + }) +} + +// The sweep's read is keyset-paged: walking it one row at a time visits every +// candidate exactly once, in order, and then comes back empty. +func TestListMaterializeCandidatesPagesByKeyset(t *testing.T) { + eachBackend(t, func(t *testing.T, s *store.Store) { + wsID, collID, first := seedStaleItem(t, s) + want := map[string]bool{first.ID: true} + appendFrames(t, s, first.ID, recoveryFrame(1)) + for i := byte(2); i <= 5; i++ { + it, err := s.CreateItem(wsID, collID, models.ItemCreate{Title: fmt.Sprintf("P%d", i), Content: "x"}) + if err != nil { + t.Fatal(err) + } + appendFrames(t, s, it.ID, recoveryFrame(i)) + want[it.ID] = true + } + future := time.Now().Add(time.Hour) + all, err := s.ListMaterializeCandidates(future, store.MaterializeCursor{}, 100) + if err != nil || len(all) != 5 { + t.Fatalf("full read = %d (%v), want 5", len(all), err) + } + var cur store.MaterializeCursor + var walked []string + for i := 0; i < 10; i++ { + page, err := s.ListMaterializeCandidates(future, cur, 1) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + walked = append(walked, page[0].ItemID) + cur = store.MaterializeCursor{LastAt: page[0].LastAt, ItemID: page[0].ItemID} + } + if len(walked) != 5 { + t.Fatalf("walked %d candidates, want 5", len(walked)) + } + for i, c := range all { + if walked[i] != c.ItemID || !want[c.ItemID] { + t.Fatalf("page walk order %v differs from the full read", walked) + } + } + }) +} diff --git a/web/src/lib/collab/materializer/materializerLinkFixture.test.ts b/web/src/lib/collab/materializer/materializerLinkFixture.test.ts new file mode 100644 index 000000000..6207bda73 --- /dev/null +++ b/web/src/lib/collab/materializer/materializerLinkFixture.test.ts @@ -0,0 +1,91 @@ +// TASK-2198 U4, leg (i) / (ii): Y.Docs holding internal links, for the Go +// tests that run the op-log recovery with the server's WHOLE-workspace link +// index (internal/server/materialize_recovery_links_test.go). +// +// Each case is a markdown body a tab would hold in its editor AFTER loading: +// the stored [[REF|text]] form has already been rendered by wikiLinksToMarkdown +// into a [text](/owner/ws/collection/REF) link, which is what the Y.Doc +// stores. The case's document is built with the headless editor (the live +// editor's schema, materializerSchema.svelte.test.ts) and encoded as ONE +// y-protocols Update frame, with a fixed client id so regeneration is +// byte-stable. +// +// `raw` is the serializer's output BEFORE the flush pipeline (no link index): +// the Go test uses it as "what the document itself contains", so any title in +// the recovered body that is not in `raw` came from the index. +// +// Regenerate with PAD_GEN_MATERIALIZE_LINK_FIXTURE=1. Without it the test +// checks the committed fixture still decodes to the same `raw`. +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import * as Y from 'yjs'; +import { prosemirrorToYXmlFragment } from '@tiptap/y-tiptap'; +import { createHeadlessEditor, materialize } from './entry'; +import { updateFrame } from './replay'; + +const FIXTURE = path.resolve(__dirname, '../../../../../internal/server/testdata/materialize_links.json'); + +// X is SECR-1, "Secret Launch Codename", slug secret-launch-codename, in a +// collection the Go test restricts. Its title appears in a document ONLY in +// case `text-equals-current-title`. +const CASES: Array<{ name: string; markdown: string }> = [ + { name: 'ref-link-other-text', markdown: 'See [shown text](/alice/ws/secrets/SECR-1) here.' }, + { name: 'ref-link-old-title', markdown: 'See [Old Codename](/alice/ws/secrets/SECR-1) here.' }, + { name: 'slug-link-slug-text', markdown: 'See [the plan](/alice/ws/secrets/secret-launch-codename) here.' }, + { name: 'text-equals-current-title', markdown: 'See [Secret Launch Codename](/alice/ws/secrets/SECR-1) here.' }, + { name: 'broken-link', markdown: 'See [gone text](broken) here.' }, + { name: 'cross-workspace', markdown: 'See [xw text](/-/r/other/SECR-1) here.' }, + { name: 'typed-wiki-literal', markdown: 'Typed \\[\\[SECR-1\\]\\] literal.' }, + { name: 'public-ref-link', markdown: 'See [the public plan](/alice/ws/public/PUB-2) here.' }, + // BUG-3315 U2: a tab loads a link that follows its target's title with the + // follows-title marker in the link's title attribute, and the marker lives + // in the Y.Doc. Recovery must strip it exactly as a tab's save does. + { name: 'marker-follows-title', markdown: 'See [Secret Launch Codename](/alice/ws/secrets/SECR-1 "pad-follows-title:Secret Launch Codename") here.' }, + { name: 'marker-renamed', markdown: 'See [Old Codename](/alice/ws/secrets/SECR-1 "pad-follows-title:Old Codename") here.' }, + { name: 'marker-external-href', markdown: 'See [Old Codename](https://example.com/x "pad-follows-title:Old Codename") here.' }, +]; + +interface FixtureCase { + name: string; + markdown: string; + rows: string[]; + raw: string; +} + +function build(): FixtureCase[] { + const ed = createHeadlessEditor(); + try { + return CASES.map((c) => { + ed.commands.setContent(c.markdown); + // A marker case is only a marker case if the document holds it. + if (c.name.startsWith('marker-')) { + expect((ed.storage as unknown as { markdown: { getMarkdown(): string } }).markdown.getMarkdown()).toContain('pad-follows-title'); + } + const doc = new Y.Doc(); + doc.clientID = 2198; + prosemirrorToYXmlFragment(ed.state.doc, doc.getXmlFragment('default')); + const frame = updateFrame(Y.encodeStateAsUpdate(doc)); + doc.destroy(); + const rows = [Buffer.from(frame).toString('base64')]; + const raw = materialize(JSON.stringify({ rows, link_index: [], workspace_slug: 'ws' })); + return { name: c.name, markdown: c.markdown, rows, raw }; + }); + } finally { + ed.destroy(); + } +} + +describe('materializer link fixture (TASK-2198 U4)', () => { + it('is current', () => { + const cases = build(); + if (process.env.PAD_GEN_MATERIALIZE_LINK_FIXTURE === '1') { + fs.writeFileSync(FIXTURE, JSON.stringify(cases, null, '\t') + '\n'); + } + const committed = JSON.parse(fs.readFileSync(FIXTURE, 'utf8')) as FixtureCase[]; + expect(committed).toEqual(cases); + // Each document holds a link (or the literal), so the fixture is about + // what it claims to be about. + for (const c of cases) expect(c.raw).toMatch(/SECR-1|secret-launch-codename|gone text|PUB-2|example\.com/); + }); +}); diff --git a/web/src/lib/components/timeline/TimelineVersionCard.svelte b/web/src/lib/components/timeline/TimelineVersionCard.svelte index 4d26dd55b..1ac765fd1 100644 --- a/web/src/lib/components/timeline/TimelineVersionCard.svelte +++ b/web/src/lib/components/timeline/TimelineVersionCard.svelte @@ -188,8 +188,19 @@ } } + // 'system' is the op-log recovery's version row (TASK-2198 U4): the server + // rebuilt the body from an editor session that closed without saving, and + // the op-log records no author, so it must not read as a user's edit. function actorLabel(actor: string): string { - return actor === 'agent' ? 'Agent' : 'User'; + if (actor === 'agent') return 'Agent'; + if (actor === 'system') return 'System'; + return 'User'; + } + + function actorColor(actor: string): string { + if (actor === 'agent') return 'var(--accent-purple)'; + if (actor === 'system') return 'var(--text-secondary)'; + return 'var(--status-blue)'; } function sourceLabel(source: string): string { @@ -197,7 +208,8 @@ cli: 'CLI', web: 'Web', skill: 'Skill', - 'collab-snapshot': 'Autosave' + 'collab-snapshot': 'Autosave', + recovery: 'Recovered' }; return labels[source] ?? source; } @@ -215,7 +227,7 @@
{actorLabel(version.created_by)} diff --git a/web/src/lib/components/timeline/TimelineVersionCard.svelte.test.ts b/web/src/lib/components/timeline/TimelineVersionCard.svelte.test.ts index 873591c69..b7035a2e5 100644 --- a/web/src/lib/components/timeline/TimelineVersionCard.svelte.test.ts +++ b/web/src/lib/components/timeline/TimelineVersionCard.svelte.test.ts @@ -150,3 +150,30 @@ describe('TimelineVersionCard flush-before-restore (BUG-2271)', () => { }); }); }); + +// TASK-2198 U4: the op-log recovery's version row is the system's, and must +// not read as a user's edit. +describe('TimelineVersionCard recovery attribution (TASK-2198 U4)', () => { + it('labels a recovery version System / Recovered, and a user version User / Web', () => { + const labels = (v: Version) => { + const root = target(); + const inst = mount(TimelineVersionCard, { + target: root, + props: { version: v, wsSlug: 'ws', itemSlug: 'ITEM-1', currentContent: 'now', onRestore: () => {}, frozen: false }, + }); + flushSync(); + const text = root.querySelector('.badges')?.textContent?.replace(/\s+/g, ' ').trim(); + unmount(inst); + root.remove(); + return text; + }; + const recovered = { + ...version, + created_by: 'system', + source: 'recovery', + change_summary: 'recovered from an unsaved editor session', + } as unknown as Version; + expect(labels(recovered)).toBe('System Recovered'); + expect(labels(version)).toBe('User Web'); + }); +});