pad/internal/store/store.go

package store

import (
	"context"
	"database/sql"
	"embed"
	"fmt"
	"io/fs"
	"log/slog"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/PerpetualSoftware/pad/internal/collections"
	"github.com/PerpetualSoftware/pad/internal/links"
	"github.com/PerpetualSoftware/pad/internal/models"
	"github.com/google/uuid"
	_ "github.com/jackc/pgx/v5/stdlib" // PostgreSQL driver
	_ "modernc.org/sqlite"
)

//go:embed migrations/*.sql
var migrationsFS embed.FS

//go:embed pgmigrations/*.sql
var pgMigrationsFS embed.FS

// BinaryVersion labels the pre-migration snapshot file
// (<db>.pre-<VERSION>). The `pad` binary sets it to its build version at
// startup; it defaults to "dev" so tests and direct library callers still
// produce a sensibly-named snapshot. See snapshotBeforeMigrate.
var BinaryVersion = "dev"

// AllowSchemaAhead downgrades the schema-ahead migration guard (see
// guardSchemaAhead) from a fatal error to a logged warning. It is the
// escape hatch for the rare operator who has knowingly downgraded the
// binary and accepts running old code against a newer schema. The `pad`
// binary wires it to `start --force` / PAD_ALLOW_SCHEMA_AHEAD=1; tests
// toggle it directly.
var AllowSchemaAhead bool

type Store struct {
	db      *sql.DB
	dialect Dialect
	// deadlockRetryObserver is told each deadlock retry's site
	// (retryOnDeadlock); deadlockInjector is a TEST-ONLY hook, nil in
	// production, that fails an attempt just before its commit.
	deadlockRetryObserver func(site string)
	deadlockInjector      func(site string, attempt int) bool
	// usernameValidator, when set, is the rule a generated username must
	// pass (the server's ValidateUsername: format, length, reserved names).
	// An account claim regenerates one (TASK-3351).
	usernameValidator func(string) error
	// appAPIAudience is the app API resource, the only audience an install
	// client holds (SPEC-6 U5a); set by the server at startup.
	appAPIAudience string
	// dbPath is the on-disk SQLite file path (empty for Postgres and for
	// in-memory SQLite). Retained so the migration path can write a
	// pre-migration snapshot next to it. See snapshotBeforeMigrate.
	dbPath        string
	encryptionKey []byte // 32-byte AES-256 key for encrypting sensitive fields (e.g., TOTP secrets)

	// afterDebounceRead is a TEST-ONLY seam, nil in production. When set,
	// CreateActivityDebounced calls it between reading its candidate row
	// and issuing the merge UPDATE — the window in which a concurrent
	// writer can move the row out from under the read (BUG-2770). A hook
	// makes that interleaving deterministic; racing two real goroutines
	// reproduces it only sometimes, and a test that reproduces a race
	// sometimes is a detector with an unknown rate, not a regression test.
	// It is a Store field rather than a package var so parallel tests
	// cannot write each other's seam. Set it only while no request is in
	// flight against this Store: it is READ on a write path with no
	// synchronisation, so mutating it under live traffic is a -race
	// finding waiting to happen (codex round 2 — latent, not current:
	// every test that sets it does so synchronously and none is parallel).
	afterDebounceRead func()

	// afterDebounceMiss is a TEST-ONLY seam, nil in production. When set,
	// CreateActivityDebounced calls it after its candidate read found NO row
	// to coalesce with and before it inserts a fresh one: the window in which
	// a concurrent writer that also found nothing inserts too (BUG-2777).
	// Same usage constraints as afterDebounceRead.
	afterDebounceMiss func()

	// afterDebounceLockedMiss is a TEST-ONLY seam, nil in production. When
	// set, insertDebouncedOnMiss calls it while holding its per-document lock,
	// after the re-read found no candidate and before the insert (BUG-2777):
	// a concurrent miss must be seen WAITING here, not inserting. Same usage
	// constraints as afterDebounceRead.
	afterDebounceLockedMiss func()

	// decisionSets names the typed-decision question sets that apply to an
	// item (TASK-3117). Empty until the server installs a resolver, which it
	// does only when a decision provider is configured; see decisions.go.
	decisionSets decisionSetsPtr
	// decisionDebounce is the per-set re-ask window (TASK-3916), installed
	// by the decision runner beside decisionSets. Nil means no set waits.
	decisionDebounce decisionDebouncePtr

	// outboxRowCapOverride and outboxClaimBudgetOverride are TEST-ONLY seams,
	// zero in production. They lower MaxOutboxPayloadBytes and
	// maxOutboxClaimBytes so a test can cross those bounds with kilobytes
	// instead of hundreds of megabytes.
	//
	// ZERO MEANS THE PRODUCTION CONSTANT, never "no limit" — read through
	// outboxRowCap() / outboxClaimBudget(), never directly. A Store built
	// without the constructor therefore enforces the real bounds rather than
	// silently enforcing none, which is the failure mode a plain int field
	// would have had.
	//
	// Store fields rather than package vars so parallel tests cannot write
	// each other's seam, same reasoning as afterDebounceRead above. Set them
	// only while no request is in flight against this Store.
	outboxRowCapOverride      int
	outboxClaimBudgetOverride int
	// outboxScrubRowsOverride lowers maxOutboxScrubRows the same way. The
	// scrub's BYTE budget has no seam of its own because maxOutboxScrubBytes
	// is defined as maxOutboxClaimBytes — one number, so one seam.
	outboxScrubRowsOverride int
	// outboxClaimRowsOverride lowers maxOutboxClaimRows the same way.
	outboxClaimRowsOverride int

	// afterOutboxScrubBatch is a TEST-ONLY seam, nil in production. When set,
	// ScrubOutboxUserRefsTx calls it once per candidate batch with the number
	// of rows that batch carried.
	//
	// It exists because the scrub's BYTE budget is otherwise unobservable: it
	// changes peak memory and nothing else, so removing it leaves every
	// outcome assertion green (codex round 2). Counting batches is the one
	// visible consequence of the budget doing its job.
	afterOutboxScrubBatch func(rows int)

	// afterItemPreLockRead is a TEST-ONLY seam, nil in production. When set,
	// updateItemWithParentLinkOnce calls it after its pre-lock GetItem and
	// before it opens the transaction — the window in which a concurrent
	// writer can commit a change that this call must NOT attribute to its own
	// caller (BUG-2776). It exists because that window is what makes the
	// locked re-read load-bearing: without it a test cannot tell a pre-image
	// taken from the pre-lock read apart from one taken under the lock, and a
	// mutation swapping the two survives.
	//
	// Same usage constraint as afterDebounceRead: set it only while no other
	// request is in flight against this Store. It fires once per ATTEMPT, so
	// a hook that issues its own update must nil the field for the duration
	// of that nested call — and note that updateItemWithParentLinkOnce runs
	// under retryOnParentSetChanged, so a retried update fires it again
	// (codex round 3).
	afterItemPreLockRead func(itemID string)

	// beforeItemVersionsRead is a TEST-ONLY seam, nil in production. When set,
	// listItemVersionsPageQ calls it before its query: after the caller read
	// the item's body and before the version list is read, the window in which
	// a content write makes the two disagree (BUG-3814). Inside
	// ListItemVersionsResolvedSnapshot it fires after the snapshot's body read,
	// so a hook that writes commits outside the snapshot. A hook that writes
	// must guard against its own nested version reads.
	beforeItemVersionsRead func(itemID string)

	// beforeVersionsRead is beforeItemVersionsRead for the legacy documents'
	// versions table (BUG-3833): TEST-ONLY, nil in production. listVersionsQ
	// calls it before its query; inside ListVersionsResolvedSnapshot that is
	// after the snapshot's body read. A hook that writes must guard against
	// its own nested version reads.
	beforeVersionsRead func(documentID string)

	// beforeRotateRefresh is a TEST-ONLY seam, nil in production. When set,
	// RotateRefreshToken calls it first, outside any transaction: the window
	// between fosite validating a refresh token and rotating it, where a
	// revoke or a second refresh can land (BUG-3678).
	beforeRotateRefresh func(requestID string)

	// rotateFault is a TEST-ONLY seam, nil in production. When set,
	// RotateUserSessions calls it after its writes and before its COMMIT; a
	// non-nil return fails the transaction (TASK-3938).
	rotateFault func() error

	// afterItemRestoreOrMoveCommit is a TEST-ONLY seam, nil in production.
	// restoreItemOnce and MoveItemWithPreCheck call it right after their
	// COMMIT and before they return. It exists for BUG-2717: both used to
	// re-read the item after commit, so a concurrent write landing in that
	// window was handed back as though this mutation had produced it. A test
	// writes to the item from the hook and asserts the returned row is still
	// the one this mutation committed. Same usage constraint as the hooks
	// above: set it only while no other request is in flight on this Store.
	afterItemRestoreOrMoveCommit func(itemID string)

	// commitItemUpdate is a TEST-ONLY seam, nil in production. When set,
	// updateItemWithParentLinkOnce routes its final COMMIT through it instead
	// of calling tx.Commit directly.
	//
	// It exists for BUG-2994, whose whole subject is the one outcome no other
	// seam can produce: a transaction that DURABLY COMMITTED and whose
	// tx.Commit() nevertheless reported an error, which is what Postgres does
	// when the acknowledgement is lost at the connection boundary. A hook that
	// calls tx.Commit itself and then returns a non-nil error reproduces that
	// exactly — the effects are on disk, the caller is told they are not — and
	// it does so without faking a commit the store never ran, which is the
	// distinction that keeps the resulting test from being vacuous.
	//
	// Same usage constraint as the seams above: set it only while no other
	// request is in flight against this Store. It fires once per ATTEMPT, so
	// note that updateItemWithParentLinkOnce runs under retryOnParentSetChanged.
	commitItemUpdate func(tx *sql.Tx) error

	// commitAddWorkspaceMember is a TEST-ONLY seam, nil in production. Same shape
	// and same reason as commitItemUpdate above, for BUG-3026: AddWorkspaceMember
	// is a plain INSERT against PRIMARY KEY (workspace_id, user_id) returning the
	// raw tx.Commit() error, so a commit that landed and whose acknowledgement was
	// lost leaves the row on disk while the caller is told it is not — and the
	// cloud auto-create path then deleted the workspace over it.
	//
	// Nothing else reproduces that state. Every other injection makes the INSERT
	// genuinely fail, which is the case the code already handled (and which the
	// existing ghost-user FK test covers).
	//
	// AcceptWorkspaceInvitation commits through the same seam
	// (commitWorkspaceMemberTx, BUG-3281), so one hook drives both paths.
	commitAddWorkspaceMember func(tx *sql.Tx) error

	// failSeedCollections is a TEST-ONLY seam, nil in production. Same shape and
	// same reason as the two commit hooks above, for BUG-3087: the cloud
	// auto-create door must REMOVE a workspace whose template seed failed, and
	// nothing in the tree can make SeedCollectionsFromTemplate fail on a fresh
	// workspace without also breaking the store around it — the collection
	// prefix is not unique, a held trait declaration is SKIPPED rather than
	// refused, and a closed database fails the removal too.
	//
	// Consulted once, after the template's collections exist and before its
	// items are seeded, so the failure lands on the partial-init shape the
	// seeder's own comment describes ("DB error after some items were seeded").
	failSeedCollections func(workspaceID, templateName string) error

	// failCommentInsert is a TEST-ONLY seam, nil in production, for BUG-2716:
	// makes the comment INSERT fail inside createCommentTx, so a test can show
	// that the activity row written in the same transaction rolls back with
	// it. Nothing else fails a comment write on a fresh item without breaking
	// the store around it.
	failCommentInsert func(input models.CommentCreate) error

	// failStripeEventRead is a TEST-ONLY seam, nil in production: when it
	// returns an error, ClaimStripeEvent's read of the existing row fails with
	// it (retro CONVE-735 finding on BUG-3561). A transient read failure after
	// the insert conflict cannot be produced otherwise without breaking the
	// INSERT and UPDATE that run just before it.
	failStripeEventRead func(eventID string) error

	// afterDocumentPreLockRead is a TEST-ONLY seam, nil in production. When
	// set, UpdateDocument calls it after its pre-transaction read and before
	// it opens the transaction that takes the rename lock (BUG-2778) — the
	// window in which another rename of the SAME document can commit, making
	// the pre-read's title the WRONG old title for the backlink cascade.
	//
	// Same usage constraints as the seams above: set it only while no other
	// request is in flight against this Store, and a hook that issues its own
	// update must nil the field for the duration or it recurses.
	afterDocumentPreLockRead func(documentID string)

	// afterDocumentPreWrite is a TEST-ONLY seam, nil in production. When set,
	// UpdateDocument calls it immediately before its final UPDATE — the last
	// moment at which a concurrent soft-delete can still commit ahead of that
	// write on a path holding no row lock (BUG-2778). Same usage constraints
	// as the seams above.
	afterDocumentPreWrite func(documentID string)

	// afterLinkCascadeRead is a TEST-ONLY seam, nil in production. When set,
	// updateLinksInTx calls it after reading every linking document's content
	// and before writing any of them back — the window in which a concurrent
	// content edit to a linker is lost without a compare-and-set (BUG-2785).
	//
	// It is its own seam because no existing one reaches this gap:
	// afterDocumentPreLockRead fires before the transaction opens, and
	// afterDocumentPreWrite fires before the RENAMED document's own update,
	// which is a different window on a different row.
	//
	// Same usage constraints as the seams above. Note that a test driving it
	// only observes the defect on Postgres — SQLite's BEGIN IMMEDIATE holds
	// the write lock across this window, so a concurrent write cannot commit
	// inside it there.
	afterLinkCascadeRead func(workspaceID string)

	// onItemCascadeBodyBuilt is a TEST-ONLY seam, nil in production. When set,
	// cascadeTitleRename calls it once per rewritten body it BUILDS, with that
	// body's byte length.
	//
	// It exists because the property "the cap refuses BEFORE building the body
	// it refuses" cannot be pinned by an allocation ceiling on this path. After
	// BUG-2804's single-pass rewrite, building one extra body costs ~1x the body
	// — a signal small enough that a ceiling wide enough to be portable is also
	// wide enough to miss it. Worse, the ceiling is DIALECT-DEPENDENT: the same
	// refusal allocated ~69 MB on SQLite and ~242 MB on Postgres, whose driver
	// copies each row's bytes differently, so any absolute number is green on
	// one backend and red on the other for reasons that have nothing to do with
	// the guard.
	//
	// A count of bodies built is exact, dialect-independent, and discriminates
	// by one whole unit: refusing before the build yields N, refusing after
	// yields N+1.
	onItemCascadeBodyBuilt func(bytes int)

	// afterDebounceRefusal is a TEST-ONLY seam, nil in production. When set,
	// CreateActivityDebounced calls it after its merge UPDATE has affected
	// ZERO rows and before the probe that decides which of the UPDATE's two
	// arms refused (BUG-2779). It exists because that probe's error return is
	// otherwise unreachable: the failure has to land between two statements,
	// which no other seam can schedule. Same usage constraints as the seams
	// above.
	afterDebounceRefusal func()

	// afterRemapScan is a TEST-ONLY seam, nil in production. When set,
	// RemapAttachmentReferencesInWorkspace calls it after scanning every item's
	// content and fields and before writing any rewrite back: the window in
	// which a concurrent item write would be overwritten by the remap's stale
	// snapshot, were it not for the workspace seq lock the remap holds
	// (BUG-2797, closed by #1328). Same usage constraints as the seams above;
	// a hook that writes to the same workspace must do so from ANOTHER
	// goroutine, since the calling transaction holds the lock that write needs.
	afterRemapScan func(workspaceID string)

	// beforeMoveLock is a TEST-ONLY seam, nil in production. A move calls it
	// after its pre-lock read of the item and before it takes the workspace
	// lock: the window in which another writer can move the item first
	// (BUG-3665). No lock is held, so the hook may write directly.
	beforeMoveLock func(itemID string)

	// stopMaint signals the background WAL checkpointer to exit; maintDone
	// is closed once it has. Both are nil on the Postgres path (no WAL
	// file to checkpoint) and Close() guards on nil accordingly.
	stopMaint chan struct{}
	maintDone chan struct{}
}

// sqliteMaxOpenConns bounds the SQLite connection pool. SQLite is a
// single-writer database, so an UNBOUNDED pool (the prior default) lets a
// burst of concurrent writers each open a connection and block up to
// busy_timeout (30s) on BEGIN IMMEDIATE — unbounded goroutine/connection
// growth under load. WAL still serves many concurrent readers from the
// snapshot, so a modest cap preserves read concurrency while bounding
// the writer pile-up. (The Postgres path sets its own cap in
// openPostgresDB.)
const sqliteMaxOpenConns = 16

// walCheckpointInterval is how often the background checkpointer runs
// PRAGMA wal_checkpoint(TRUNCATE) to keep the WAL file from growing
// without bound. See startWALCheckpointer.
const walCheckpointInterval = 60 * time.Second

// D returns the store's dialect for building backend-specific SQL.
func (s *Store) D() Dialect { return s.dialect }

// DB returns the underlying *sql.DB (for use in migrations/testing).
func (s *Store) DB() *sql.DB { return s.db }

// BeginSnapshot opens a transaction that is both the write lock and a single
// read snapshot for everything run on it (BUG-3072).
//
// The DSN carries `_txlock=immediate`, so this is a BEGIN IMMEDIATE: it takes
// the write lock on Begin rather than on first write. MEASURED, because the
// whole design leans on it: with this transaction held, another handle's
// CreateItem blocked for the full 1.5s it was observed and completed only
// after the commit. SQLite has ONE write lock, so while this is open no other
// connection can COMMIT anything.
//
// THAT EXCLUSION IS WHAT MAKES A BUNDLE CONSISTENT — not the executor the
// reads use. Reads issued on the POOL while this is held do not deadlock (the
// pool is 16 deep) and, measured against the same probe, see exactly what the
// transaction sees, because nothing can change underneath either of them.
//
// So passing the returned *sql.Tx to the *Q read variants is not what buys
// correctness today; it is what stops correctness from depending on an
// argument about lock exclusion that holds only while ONE transaction spans
// the whole run. Split it per workspace, or commit in the middle, and pooled
// reads start diverging silently while transaction-scoped reads stay right.
//
// SQLite-specific in all of the above. On Postgres this is a plain Begin and
// the pool has no single-writer property at all, so there the executor is the
// only thing that makes a read part of the snapshot.
func (s *Store) BeginSnapshot() (*sql.Tx, error) { return s.db.Begin() }

// New creates a Store backed by SQLite at the given path.
//
// The DSN is configured for safe concurrent use under Go's connection pool:
//
//   - `_pragma=busy_timeout(30000)`: when a connection finds the database
//     locked, retry for up to 30 seconds before returning SQLITE_BUSY.
//     The pragma is applied per-connection by the driver, so every pool
//     member inherits it.
//
//     30s is much larger than the millisecond-scale p95 we observe under
//     normal load — the slack is there to absorb shared-runner jitter
//     under heavy CI contention. With the previous 5s value,
//     TestSQLiteConcurrentWritersNoBusy (25 writers × 5 ops, all
//     contending for BEGIN IMMEDIATE) intermittently failed on the
//     GitHub-hosted SQLite job: the cumulative wall-time of 125
//     serialized inserts on a slow runner can exceed 5s, leaving the
//     unluckiest writer to time out. See BUG-853. Genuine deadlocks
//     don't happen with WAL + BEGIN IMMEDIATE, so the only thing the
//     larger value costs is "how long we wait before declaring lock
//     contention pathological"; for Pad's workload, 30s is fine.
//
//   - `_pragma=foreign_keys(on)`: foreign-key enforcement is per-connection
//     in SQLite. Setting it via the DSN guarantees ALL pool members enforce
//     them, not just the one that received a `db.Exec("PRAGMA ...")` call.
//
//   - `_txlock=immediate`: makes every `db.Begin()` issue `BEGIN IMMEDIATE`
//     instead of the default `BEGIN DEFERRED`. With deferred mode, a
//     transaction starts holding only a SHARED lock and tries to upgrade
//     to a write lock on the first INSERT/UPDATE — and lock upgrades are
//     refused with SQLITE_BUSY *immediately* if another connection holds
//     the write lock, regardless of busy_timeout (SQLite refuses to wait
//     because that would risk deadlock between two connections both holding
//     SHARED locks). With IMMEDIATE, the write lock is acquired at BEGIN
//     time, and busy_timeout's wait-and-retry behaviour does apply, so
//     concurrent writers serialize cleanly instead of failing with
//     "database is locked". Reads are unaffected: single-statement SELECTs
//     don't open a transaction at the SQL layer, so they continue to run
//     concurrently against the WAL snapshot.
//
//     Tradeoff: IMMEDIATE widens the writer critical section. Some update
//     flows (e.g. items.UpdateItem, documents.UpdateDocument) now hold the
//     write lock during diff/version-throttle reads and slug-collision
//     checks, not just during the final INSERT/UPDATE. In practice these
//     reads are sub-millisecond and dominated by network/HTTP latency,
//     so concurrent writers wait briefly for cleanly serialized work
//     rather than failing fast. The pre-fix behaviour was "fail fast
//     with BUSY"; this is strictly better. If a future hot path produces
//     pathologically long write transactions (>100ms holding the lock),
//     the right move is to narrow that specific transaction — not to
//     revert this fix.
//
//     Rollout note for `_pragma=foreign_keys(on)`: FK enforcement was
//     previously per-connection, applied to only one pool member. If a
//     database was historically written through a different pool member
//     with FKs disabled, latent integrity violations may exist; the
//     `sql.Open` itself will not fail, but the next write touching a
//     stale relationship can now return an FK error. For Pad's data
//     model the realistic fallout is small (link tables already cascade),
//     but the integrity check `PRAGMA foreign_key_check` can surface
//     any pre-existing offenders.
//
//   - `journal_mode=WAL` is set via Exec below; WAL is a database-level
//     setting (stored in the file header), so it persists across
//     connections after the first one applies it.
func New(dbPath string) (*Store, error) {
	dsn := dbPath + "?_pragma=busy_timeout(30000)&_pragma=foreign_keys(on)&_txlock=immediate"
	// The NUL invariant's Layer A rides the DRIVER, not the call sites — see
	// nulguard.go for why that is the only shape whose completeness nobody has
	// to maintain.
	if err := registerGuardedDrivers(); err != nil {
		return nil, err
	}
	db, err := sql.Open(guardedSQLiteDriver, dsn)
	if err != nil {
		return nil, fmt.Errorf("open database: %w", err)
	}

	// REFUSE A DATABASE THAT HAS BEEN MIGRATED TO POSTGRES (BUG-3072).
	//
	// THIS IS FIRST, and the ordering is load-bearing rather than tidy.
	// Everything below it WRITES: the WAL pragma touches the header, migrate()
	// runs DDL and rebuild migrations that INSERT into protected tables, and
	// the three backfills after it write collections.prefix,
	// workspaces.owner_id and users.username. Run any of them against a marked
	// file and the open either trips the refusal triggers — surfacing as a
	// migration failure rather than the remedy — or quietly mutates a file this
	// function is about to declare abandoned.
	//
	// This covers every opener by construction: the three non-test callers of
	// New (the migration's own source, `pad db nul`, and `pad server start`)
	// all arrive here. It deliberately does NOT cover `pad db backup`, which
	// opens its own connection and runs VACUUM INTO — a marked file stays
	// backup-able, copyable and readable by sqlite3, which is the point of
	// refusing WRITES rather than access.
	if remedy, merr := migratedRemedyIfMarked(db); merr != nil {
		_ = db.Close()
		return nil, fmt.Errorf("check whether %s was migrated: %w", dbPath, merr)
	} else if remedy != "" {
		_ = db.Close()
		return nil, fmt.Errorf("refusing to open %s: %s", dbPath, remedy)
	}

	// Enable WAL mode (database-level: persists across connections).
	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
		return nil, fmt.Errorf("enable WAL: %w", err)
	}

	// Bound the pool (see sqliteMaxOpenConns). Keep idle == max so a read
	// burst doesn't churn connections open/closed (each open re-applies
	// the DSN pragmas), and recycle hourly as a safety valve.
	db.SetMaxOpenConns(sqliteMaxOpenConns)
	db.SetMaxIdleConns(sqliteMaxOpenConns)
	db.SetConnMaxLifetime(time.Hour)

	s := &Store{db: db, dialect: &sqliteDialect{}, dbPath: dbPath}
	if err := s.migrate(); err != nil {
		return nil, fmt.Errorf("migrate: %w", err)
	}

	if err := s.backfillItemNumbers(); err != nil {
		return nil, fmt.Errorf("backfill item numbers: %w", err)
	}

	if err := s.backfillWorkspaceOwners(); err != nil {
		return nil, fmt.Errorf("backfill workspace owners: %w", err)
	}

	if err := s.backfillUsernames(); err != nil {
		return nil, fmt.Errorf("backfill usernames: %w", err)
	}

	s.startWALCheckpointer()

	return s, nil
}

// startWALCheckpointer runs PRAGMA wal_checkpoint(TRUNCATE) on a timer so
// the WAL file can't grow without bound. SQLite's automatic
// wal_autocheckpoint is PASSIVE: it can only checkpoint frames no reader
// still needs and never truncates the file. Pad's web UI keeps a steady
// stream of pollers (dashboard/items/changes) and SSE readers active, so
// under a write burst (e.g. collab op-log appends) the WAL would
// otherwise keep growing and slow every large read. A periodic TRUNCATE
// checkpoint reclaims it during quiet windows. Best-effort: a busy
// checkpoint (readers active) just retries next tick. SQLite-only —
// Close() stops it; the Postgres constructor never calls this.
func (s *Store) startWALCheckpointer() {
	s.stopMaint = make(chan struct{})
	s.maintDone = make(chan struct{})
	go func() {
		defer close(s.maintDone)
		ticker := time.NewTicker(walCheckpointInterval)
		defer ticker.Stop()
		for {
			select {
			case <-s.stopMaint:
				return
			case <-ticker.C:
				if _, err := s.db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
					slog.Warn("wal checkpoint failed", "error", err)
				}
			}
		}
	}()
}

// openPostgresDB opens and configures a PostgreSQL connection pool.
func openPostgresDB(connStr string) (*sql.DB, error) {
	// Layer A on Postgres too, deliberately. Postgres refuses a NUL natively,
	// so the wrapper is not what makes the write safe there — it is what makes
	// the two dialects give the SAME answer, with the same typed error, which
	// is what the four-way differential test asserts and what BUG-2831's
	// dialect split cost us the last time two backends disagreed.
	if err := registerGuardedDrivers(); err != nil {
		return nil, err
	}
	db, err := sql.Open(guardedPostgresDriver, connStr)
	if err != nil {
		return nil, fmt.Errorf("open postgres: %w", err)
	}

	if err := db.Ping(); err != nil {
		return nil, fmt.Errorf("ping postgres: %w", err)
	}

	db.SetMaxOpenConns(25)
	db.SetMaxIdleConns(5)
	db.SetConnMaxLifetime(5 * time.Minute)

	return db, nil
}

// NewPostgres creates a Store backed by PostgreSQL.
// The connStr should be a PostgreSQL connection string (e.g. "postgres://user:pass@host/db").
func NewPostgres(connStr string) (*Store, error) {
	db, err := openPostgresDB(connStr)
	if err != nil {
		return nil, err
	}

	s := &Store{db: db, dialect: &postgresDialect{}}
	if err := s.migratePostgres(); err != nil {
		return nil, fmt.Errorf("migrate postgres: %w", err)
	}

	if err := s.backfillItemNumbers(); err != nil {
		return nil, fmt.Errorf("backfill item numbers: %w", err)
	}

	if err := s.backfillWorkspaceOwners(); err != nil {
		return nil, fmt.Errorf("backfill workspace owners: %w", err)
	}

	if err := s.backfillUsernames(); err != nil {
		return nil, fmt.Errorf("backfill usernames: %w", err)
	}

	return s, nil
}

func (s *Store) Close() error {
	// Stop the WAL checkpointer (SQLite only) and wait for it to exit so
	// it can't run a PRAGMA against a closing handle. nil on Postgres.
	if s.stopMaint != nil {
		close(s.stopMaint)
		<-s.maintDone
		s.stopMaint = nil
	}
	return s.db.Close()
}

// Ping verifies the database connection is alive.
func (s *Store) Ping() error {
	return s.db.Ping()
}

// readMigrationNames returns sorted .sql filenames from an embedded FS directory.
func readMigrationNames(fsys embed.FS, dir string) ([]string, error) {
	entries, err := fs.ReadDir(fsys, dir)
	if err != nil {
		return nil, fmt.Errorf("read migration dir %s: %w", dir, err)
	}
	var names []string
	for _, e := range entries {
		if !e.IsDir() && strings.HasSuffix(e.Name(), ".sql") {
			names = append(names, e.Name())
		}
	}
	sort.Strings(names)
	return names, nil
}

// slowSQLiteMigrations names the SQLite migrations whose cost grows with the
// database, so the server logs a line before starting one and a slow first
// boot after an upgrade has an explanation in the log. The runner applies
// migrations under the database's write lock with no other output.
var slowSQLiteMigrations = map[string]string{
	// BUG-2758. Measured with the sqlite3 CLI: 1.0 s for 9,766 items (a copy of
	// a 321 MB working database); 10.5 s for a synthetic 97,660 items
	// (235 MB of content). Roughly linear in item content.
	"095_rebuild_items_fts.sql": "rebuilding the item search index; this takes about 1 s per 10,000 items",
}

func (s *Store) migrate() error {
	// Create migrations tracking table
	_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations (
		version TEXT PRIMARY KEY,
		applied_at TEXT NOT NULL
	)`)
	if err != nil {
		return fmt.Errorf("create migrations table: %w", err)
	}

	migrations, err := readMigrationNames(migrationsFS, "migrations")
	if err != nil {
		return err
	}

	applied, err := appliedMigrations(s.db)
	if err != nil {
		return err
	}

	// Refuse to run against a DB that is AHEAD of this binary — i.e. a
	// downgrade, where old code would silently operate on a newer schema.
	if err := guardSchemaAhead(applied, migrations); err != nil {
		return err
	}

	// Snapshot the DB file before applying any pending migrations, so a
	// bad upgrade can be rolled back by copying the snapshot over the DB.
	// Only when this is an UPGRADE of an existing DB (pending migrations
	// AND at least one already applied) — a fresh install has nothing
	// worth snapshotting. Best-effort; a failed snapshot warns, not fatal.
	if len(applied) > 0 && hasPending(applied, migrations) {
		s.snapshotBeforeMigrate()
	}

	// AFTER the snapshot and BEFORE the migrations, both deliberately
	// (TASK-2710, codex round 5).
	//
	// Before the migrations, because 087 / 064 add partial unique indexes over
	// the collection-trait declarations and those CREATE statements fail on
	// exactly the databases that hold duplicates — the repair has to precede
	// them. No-ops on a fresh database, where the collections table does not
	// exist yet.
	//
	// After the snapshot, because the repair CHANGES DATA — it strips a
	// declaration, moving which collection owns a kernel behavior — and the
	// snapshot is the operator's rollback for a bad upgrade. Running it in the
	// constructor, ahead of migrate(), put the altered ownership INSIDE the
	// snapshot: restoring after a failed migration would have given back the
	// old schema with the repair already applied and unrecorded, so the one
	// thing the rollback could not undo was the only thing that had silently
	// changed routing.
	if err := s.dedupeTraitDeclarations(); err != nil {
		return fmt.Errorf("de-duplicate collection trait declarations: %w", err)
	}

	for _, name := range migrations {
		if applied[name] {
			continue
		}

		data, err := migrationsFS.ReadFile("migrations/" + name)
		if err != nil {
			return fmt.Errorf("read migration %s: %w", name, err)
		}

		if notice, ok := slowSQLiteMigrations[name]; ok {
			slog.Info("applying a slow migration", "migration", name, "note", notice)
		}
		if err := applySQLiteMigration(s.db, name, string(data)); err != nil {
			return err
		}

		// AFTER EACH MIGRATION, not once at the end (codex round 4).
		//
		// A table-rebuild migration drops that table's triggers, and until they
		// are back the invariant is unenforced for it. Restoring once after the
		// whole chain left that gap open across every remaining migration and
		// the FTS validation; restoring here narrows it to the moment between
		// one migration's commit and the next statement.
		//
		// It cannot be closed completely from here — the DROP and the recreate
		// are in different transactions, so a concurrent raw writer can still
		// commit between them — and that residual is stated rather than papered
		// over: see ensureNULTriggers, and S3's repair sweep, which is the only
		// thing that makes an EXISTING violating row go away regardless of how
		// it got there.
		if err := s.ensureNULTriggers(); err != nil {
			return err
		}
	}

	// Defensive: log a warning for any FTS trigger that should exist but
	// doesn't. Catches the BUG-822 class of regression — a table-rebuild
	// migration that recorded as applied but silently failed to recreate
	// its triggers, leaving search broken until someone notices.
	s.validateFTSInvariants()

	// A final re-assertion, covering a database that arrived with triggers
	// already missing (a manual drop, a restored backup taken mid-rebuild) and
	// therefore had no migration to trigger the per-migration check above.
	//
	// The NUL triggers are RE-ASSERTED, not merely checked (DOC-2823 S2, codex
	// round 1).
	//
	// SQLite drops a table's triggers when the table is dropped, and this
	// codebase rebuilds tables to change constraints — migrations 025, 055,
	// 056, 057, 068 and 072 all do it. A future rebuild would silently take
	// the NUL triggers with it, and the trigger files (084 and the later ones
	// in nulTriggerMigrations) would never run again because they are already
	// recorded as applied.
	//
	// The FTS check above warns in that situation and leaves search broken
	// until someone notices, which was judged the right cost for a derived
	// index. It is the wrong cost for a DATA INVARIANT: the whole point of
	// Layer B is that the file enforces the rule for writers we do not control,
	// so a silently absent trigger is silently no protection. Re-asserting is
	// cheap (CREATE TRIGGER IF NOT EXISTS over a fixed set) and turns a rebuild
	// from a silent loss into a no-op.
	if err := s.ensureNULTriggers(); err != nil {
		return err
	}

	return nil
}

// expectedFTSTriggers is the canonical list of FTS5 triggers that ship with
// the SQLite migrations. Each entry is (trigger name, table the trigger is
// attached to). The migrations file at the right of each row defines them:
//
//   - items_fts_*    — internal/store/migrations/001_initial.sql
//   - comments_fts_* — internal/store/migrations/007_comments.sql
//   - documents_*    — internal/store/migrations/001_initial.sql (restored by 046 after BUG-822 drift)
//
// If a future migration intentionally renames or removes any of these,
// update this list in the same commit.
var expectedFTSTriggers = []struct {
	name  string
	table string
}{
	{"items_fts_insert", "items"},
	{"items_fts_update", "items"},
	{"items_fts_delete", "items"},
	{"comments_fts_insert", "comments"},
	{"comments_fts_update", "comments"},
	{"comments_fts_delete", "comments"},
	{"documents_ai", "documents"},
	{"documents_au", "documents"},
	{"documents_ad", "documents"},
}

// validateFTSInvariants checks that every trigger in expectedFTSTriggers
// exists. Missing triggers are logged as structured warnings; this function
// is intentionally non-fatal and non-repairing.
//
// Non-fatal: a missing trigger doesn't prevent server startup — the
// operator may have legitimately removed one and just not updated this
// list yet, and we'd rather warn loudly than refuse to boot.
//
// Non-repairing: auto-creating triggers here would mask future legitimate
// removals and obscure the source of truth (the migrations directory).
// The recovery path is a targeted migration like 046_restore_documents_fts_triggers.sql.
//
// SQLite-only: Postgres uses a different trigger model and has its own
// search_vector update functions in pgmigrations.
func (s *Store) validateFTSInvariants() {
	if s.dialect.Driver() != DriverSQLite {
		return
	}
	for _, t := range expectedFTSTriggers {
		var name string
		err := s.db.QueryRow(
			`SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=? AND name=?`,
			t.table, t.name,
		).Scan(&name)
		if err == sql.ErrNoRows {
			slog.Warn(
				"FTS trigger missing — search may silently miss new rows; consider a restoration migration like 046_restore_documents_fts_triggers.sql",
				"trigger", t.name,
				"table", t.table,
			)
			continue
		}
		if err != nil {
			slog.Warn(
				"FTS trigger check errored",
				"trigger", t.name,
				"table", t.table,
				"err", err,
			)
		}
	}
}

// migratePostgres applies PostgreSQL migrations.
// PostgreSQL supports multi-statement execution natively, so we don't need execMulti.
func (s *Store) migratePostgres() error {
	// Create migrations tracking table
	_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations (
		version TEXT PRIMARY KEY,
		applied_at TEXT NOT NULL
	)`)
	if err != nil {
		return fmt.Errorf("create migrations table: %w", err)
	}

	migrations, err := readMigrationNames(pgMigrationsFS, "pgmigrations")
	if err != nil {
		return err
	}

	applied, err := appliedMigrations(s.db)
	if err != nil {
		return err
	}

	// Same schema-ahead guard as SQLite (a Postgres binary downgrade is
	// equally unsafe). No pre-migration snapshot on Postgres — backups
	// there are the operator's DBA responsibility (pg_dump / PITR), not
	// something we can safely fake with a file copy.
	if err := guardSchemaAhead(applied, migrations); err != nil {
		return err
	}

	// The trait repair, in the same position as SQLite's: before the
	// migrations, because 064's partial unique indexes fail on databases
	// holding duplicates. There is no snapshot here to sit after — see the
	// note above — so the ordering argument is one-sided on this dialect, but
	// the call lives in the same place so the two paths cannot drift.
	if err := s.dedupeTraitDeclarations(); err != nil {
		return fmt.Errorf("de-duplicate collection trait declarations: %w", err)
	}

	for _, name := range migrations {
		if applied[name] {
			continue
		}

		data, err := pgMigrationsFS.ReadFile("pgmigrations/" + name)
		if err != nil {
			return fmt.Errorf("read migration %s: %w", name, err)
		}

		if err := applyPostgresMigration(s.db, name, string(data)); err != nil {
			return err
		}
	}

	return nil
}

// sqlExecer is the subset of sql.DB / sql.Tx that execMulti needs. It lets
// the same statement-iteration loop run either against the raw pool or against
// a wrapping transaction (used by applySQLiteMigration for atomicity).
type sqlExecer interface {
	Exec(query string, args ...any) (sql.Result, error)
}

// Queryer is the read-only subset of *sql.DB / *sql.Tx a store read needs.
// It exists so a read can run either against the pool or on an in-flight
// transaction's own connection. The distinction matters most inside
// transactions that hold locks: a read routed through the POOL while the
// transaction's connection is held can wait on a free connection, and if
// every pooled connection is itself occupied by a transaction waiting on
// those locks, the system deadlocks by starvation (BUG-2409 — the
// cross-workspace copy held both workspace advisory locks while the
// attachment planner and its authorization callback read through the pool).
// A read routed through the transaction's Queryer cannot wait on the pool,
// because the transaction already owns its connection.
//
// Exported because the callback boundary crosses packages: the copy
// planner hands ITS executor to the server-side AttachmentAuthorizer, so
// the server's authorization reads run wherever the planner runs.
type Queryer interface {
	Query(query string, args ...any) (*sql.Rows, error)
	QueryRow(query string, args ...any) *sql.Row
}

// Q returns the store's pool as a Queryer — the executor for reads that run
// outside any transaction. Server code uses it to call the *Q read variants
// through the same signature the in-transaction path uses.
func (s *Store) Q() Queryer { return s.db }

// ContextQueryer is an executor that can run a read under a context: both
// *sql.DB and *sql.Tx are one.
type ContextQueryer interface {
	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}

// WithContext returns q as a Queryer whose every read runs under ctx, so the
// *Q readers can be bound to a deadline without a Context variant of each
// (BUG-3632). Query and QueryRow on a *sql.Tx run the driver call under
// context.Background(), even when the transaction was begun with a context:
// a stalled read was never cancelled, and the transaction's own
// rollback-on-cancel waited for it. Through this wrapper the driver is
// handed ctx, so the read stops at the deadline and its connection goes back.
// How far that reaches is the driver's: pgx cancels the statement wherever it
// is, while modernc SQLite interrupts only the query's first step, and a
// later rows.Next runs to its end before database/sql closes the rows. A
// SQLite read is not stalled by a lock in WAL mode, so what remains there is
// a slow step, not a wait.
func WithContext(ctx context.Context, q ContextQueryer) Queryer {
	return ctxQueryer{ctx: ctx, q: q}
}

type ctxQueryer struct {
	ctx context.Context
	q   ContextQueryer
}

func (c ctxQueryer) Query(query string, args ...any) (*sql.Rows, error) {
	return c.q.QueryContext(c.ctx, query, args...)
}

func (c ctxQueryer) QueryRow(query string, args ...any) *sql.Row {
	return c.q.QueryRowContext(c.ctx, query, args...)
}

// QContext is Q with every read bound to ctx (BUG-3632).
func (s *Store) QContext(ctx context.Context) Queryer { return WithContext(ctx, s.db) }

// execMulti executes multiple SQL statements by iteratively using
// database/sql's Exec which processes one statement at a time,
// then advancing past it using the driver's awareness of statement boundaries.
// This handles triggers, FTS5, and other complex SQL correctly.
func execMulti(db sqlExecer, sqlText string) error {
	for {
		sqlText = strings.TrimSpace(sqlText)
		if sqlText == "" {
			return nil
		}

		// Skip comment-only lines at the start
		if strings.HasPrefix(sqlText, "--") {
			idx := strings.Index(sqlText, "\n")
			if idx < 0 {
				return nil
			}
			sqlText = sqlText[idx+1:]
			continue
		}

		// Find the next complete statement by tracking BEGIN/END blocks
		end := findStatementEnd(sqlText)
		if end < 0 {
			// No more complete statements
			return nil
		}

		stmt := strings.TrimSpace(sqlText[:end+1])
		if stmt != "" && stmt != ";" {
			if _, err := db.Exec(stmt); err != nil {
				// Tolerate "duplicate column name" errors from ALTER TABLE ADD COLUMN.
				// This makes migrations idempotent when partially applied (e.g. server
				// crashed after adding a column but before recording the migration).
				upper := strings.ToUpper(strings.TrimSpace(stmt))
				isDupCol := strings.Contains(err.Error(), "duplicate column name")
				isAddCol := strings.HasPrefix(upper, "ALTER TABLE") && strings.Contains(upper, "ADD COLUMN")
				if !(isDupCol && isAddCol) {
					return fmt.Errorf("exec migration statement: %w\nStatement: %.200s", err, stmt)
				}
			}
		}
		sqlText = sqlText[end+1:]
	}
}

// extractPragmas scans the migration text line-by-line and lifts any
// top-level `PRAGMA ...;` statement out of the body. It returns the lifted
// pragma statements (in original order) and the remaining SQL text with
// those lines replaced by blank lines so line numbers in any error message
// still align with the source file.
//
// IDEA-1485: PRAGMA statements like `foreign_keys = OFF/ON` and
// `journal_mode = WAL` are no-ops (or errors) inside a SQLite transaction.
// To wrap the rest of the migration in a single BEGIN/COMMIT for atomicity,
// the runner emits any PRAGMA outside the wrapping transaction.
//
// Detection is intentionally conservative: only lines whose first
// non-whitespace token is `PRAGMA` (case-insensitive) and that contain a
// trailing `;` on the same line are lifted. PRAGMAs split across multiple
// lines or embedded inside triggers/CTEs are not matched — none of the
// existing migrations use that shape.
func extractPragmas(sqlText string) (pragmas []string, rest string) {
	var out strings.Builder
	out.Grow(len(sqlText))
	for _, line := range strings.SplitAfter(sqlText, "\n") {
		trimmed := strings.TrimSpace(line)
		if strings.HasPrefix(strings.ToUpper(trimmed), "PRAGMA ") && strings.HasSuffix(trimmed, ";") {
			pragmas = append(pragmas, trimmed)
			// Preserve the trailing newline (if present) so line numbers
			// in downstream error messages still match the source.
			if strings.HasSuffix(line, "\n") {
				out.WriteByte('\n')
			}
			continue
		}
		out.WriteString(line)
	}
	return pragmas, out.String()
}

// fkCheckResult is one PRAGMA foreign_key_check pass, run per table so a
// table whose check cannot run ("foreign key mismatch": an FK whose parent
// key has no unique index) does not blind the check for every other table
// (codex on #2120).
type fkCheckResult struct {
	// counts holds the violations per foreign key, keyed
	// "table|parent|fkid". Counts, not rowids (codex r1): a table rebuild
	// renumbers rowids, so a legacy violation would look new under a rowid
	// key, and a WITHOUT ROWID table reports no rowid at all.
	counts map[string]int
	// checked and unavailable partition the tables that existed: a table in
	// neither did not exist when the pass ran.
	checked     map[string]bool
	unavailable map[string]error
}

// fkCheckTablesQuery lists the tables the check runs on; a var so a test can
// make the listing itself fail.
var fkCheckTablesQuery = `SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`

// foreignKeyViolations runs PRAGMA foreign_key_check on tx for each table.
// Only listing the tables is fatal; a table whose check fails is recorded as
// unavailable.
func foreignKeyViolations(ctx context.Context, tx *sql.Tx) (fkCheckResult, error) {
	res := fkCheckResult{counts: map[string]int{}, checked: map[string]bool{}, unavailable: map[string]error{}}
	var tables []string
	rows, err := tx.QueryContext(ctx, fkCheckTablesQuery)
	if err != nil {
		return res, err
	}
	for rows.Next() {
		var t string
		if err := rows.Scan(&t); err != nil {
			rows.Close()
			return res, err
		}
		tables = append(tables, t)
	}
	if err := rows.Err(); err != nil {
		rows.Close()
		return res, err
	}
	if err := rows.Close(); err != nil {
		return res, err
	}
	for _, t := range tables {
		counts, err := foreignKeyViolationsOf(ctx, tx, t)
		if err != nil {
			res.unavailable[t] = err
			continue
		}
		res.checked[t] = true
		for fk, n := range counts {
			res.counts[fk] += n
		}
	}
	return res, nil
}

func foreignKeyViolationsOf(ctx context.Context, tx *sql.Tx, table string) (map[string]int, error) {
	rows, err := tx.QueryContext(ctx, `PRAGMA foreign_key_check("`+strings.ReplaceAll(table, `"`, `""`)+`")`)
	if err != nil {
		return nil, err
	}
	out := map[string]int{}
	for rows.Next() {
		var child, parent string
		var rowid sql.NullInt64
		var fkid int64
		if err := rows.Scan(&child, &rowid, &parent, &fkid); err != nil {
			rows.Close()
			return nil, err
		}
		out[child+"|"+parent+"|"+strconv.FormatInt(fkid, 10)]++
	}
	if err := rows.Err(); err != nil {
		rows.Close()
		return nil, err
	}
	// A close error is a failed check too (codex on #2120), never counts
	// reported as complete.
	if err := rows.Close(); err != nil {
		return nil, err
	}
	return out, nil
}

// applySQLiteMigration applies a single migration file atomically.
//
// Behavior (IDEA-1485):
//
//  1. PRAGMA statements are lifted out of the migration body (see
//     extractPragmas), since PRAGMA settings like `foreign_keys` and
//     `journal_mode` are no-ops (or errors) inside a SQLite transaction.
//
//  2. Lifted PRAGMAs are classified into two groups by their semantics:
//
//     - "before-tx" PRAGMAs (e.g. `foreign_keys = OFF`) are exec'd on
//     the pinned connection BEFORE BEGIN, so their effect carries
//     into the wrapping transaction.
//     - "after-tx" PRAGMAs (e.g. `foreign_keys = ON` and everything
//     else — `journal_mode=WAL`, etc.) are exec'd on the pinned
//     connection AFTER COMMIT, so they don't undo a preceding
//     `OFF` before the body runs.
//
//     Whenever a before-tx `foreign_keys=OFF` is exec'd, a deferred
//     `PRAGMA foreign_keys = ON` is registered against the pinned
//     connection. The defer fires on EVERY return — success, BeginTx
//     failure, execMulti failure, INSERT failure, Commit failure, or
//     post-tx pragma failure — so the connection never goes back to
//     the pool with FKs disabled. The restore is best-effort and does
//     not override the migration's primary error; if the restore
//     itself fails it's logged loudly.
//
//  3. The remaining body SQL — including the `INSERT INTO schema_migrations`
//     bookkeeping row — runs inside a single BEGIN/COMMIT on the SAME
//     dedicated connection. If anything fails (including a process
//     crash before COMMIT), the database rolls back to its pre-migration
//     state AND the schema_migrations row is absent, so the next
//     startup re-attempts the migration cleanly. This eliminates the
//     data-loss window described in the IDEA's "Problem" section for
//     the table-rebuild migrations (022 / 055).
//
// CRITICAL: SQLite's `PRAGMA foreign_keys` is per-connection. We MUST
// pin the migration to a single connection via db.Conn() so the PRAGMA
// emitted before BEGIN is visible to the transaction body. Using the
// raw *sql.DB would race against the pool: the PRAGMA might land on
// connection A and `db.Begin()` might grab connection B (where FKs are
// still on, from the per-connection DSN), causing the rebuild to fail.
func applySQLiteMigration(db *sql.DB, name, sqlText string) error {
	pragmas, body := extractPragmas(sqlText)

	ctx := context.Background()
	conn, err := db.Conn(ctx)
	if err != nil {
		return fmt.Errorf("apply migration %s: acquire conn: %w", name, err)
	}
	defer conn.Close()

	// Classify pragmas. "before" pragmas must take effect for the body
	// to behave correctly (e.g. foreign_keys=OFF for table-rebuilds);
	// "after" pragmas are restorations that must not undo a before-pragma
	// prematurely.
	var beforeTx, afterTx []string
	for _, p := range pragmas {
		switch {
		case isForeignKeysOff(p):
			beforeTx = append(beforeTx, p)
		default:
			// Everything else (foreign_keys=ON, journal_mode=WAL, etc.)
			// runs AFTER the body. foreign_keys=ON specifically must NOT
			// run before BEGIN because it would cancel out a paired OFF
			// in the same migration.
			afterTx = append(afterTx, p)
		}
	}

	fkOff := false
	for _, p := range beforeTx {
		if _, err := conn.ExecContext(ctx, p); err != nil {
			return fmt.Errorf("apply migration %s: pre-tx pragma %q: %w", name, p, err)
		}
		if isForeignKeysOff(p) {
			fkOff = true
			// FK-restore MUST be a defer, not a success-path block:
			// once foreign_keys=OFF lands on the pinned conn, ANY later
			// failure (BeginTx, execMulti, INSERT, Commit, post-tx
			// pragma) returns early and the conn would otherwise go
			// back to the pool with FKs disabled, silently bypassing
			// enforcement for whichever caller next checks it out.
			//
			// The restore is best-effort: it does NOT override the
			// migration's primary error. If the restore itself fails,
			// log loudly — the connection is about to be released to
			// the pool with FKs still off and a subsequent migration
			// (or worse, a request handler) would inherit that state.
			defer func() {
				if _, restoreErr := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON"); restoreErr != nil {
					slog.Error(
						"failed to restore PRAGMA foreign_keys=ON after migration; pool connection may be released with FKs disabled",
						"migration", name,
						"err", restoreErr,
					)
				}
			}()
		}
	}

	// Wrap the body + bookkeeping in a single transaction so they
	// commit (or roll back) atomically.
	tx, err := conn.BeginTx(ctx, nil)
	if err != nil {
		return fmt.Errorf("apply migration %s: begin tx: %w", name, err)
	}
	committed := false
	defer func() {
		if !committed {
			_ = tx.Rollback()
		}
	}()

	// With foreign keys off the body can leave a dangling reference that
	// nothing reports (BUG-3663). The violations it leaves are checked against
	// the ones the database already had: a legacy database may carry some
	// from before FKs were enforced on every connection (see New), and
	// refusing those here would turn this check into an upgrade blocker.
	//
	// The check itself can fail rather than report: foreign_key_check errors
	// with "foreign key mismatch" when an FK's parent key has no unique
	// index, which a rebuild can leave in a schema the chain is not yet done
	// with (CI on #2120: re-applying 056 over a database that has
	// item_decisions). It is a safety net, never a new way to refuse an
	// upgrade, so it runs per table: a table whose check fails on either
	// side is logged and skipped, and every other table is still compared.
	// A table the body creates has a baseline of zero. A table that was
	// unavailable before has no baseline, so its violations after are
	// logged, not refused: they may be legacy ones the check could not see.
	// A failure to list the tables at all skips the check, on either side.
	var fkBefore fkCheckResult
	fkCheck := fkOff
	if fkCheck {
		if fkBefore, err = foreignKeyViolations(ctx, tx); err != nil {
			slog.Warn("migration foreign key check unavailable; skipped", "migration", name, "stage", "before", "error", err)
			fkCheck = false
		}
		for t, cerr := range fkBefore.unavailable {
			slog.Warn("migration foreign key check unavailable for table; skipped", "migration", name, "stage", "before", "table", t, "error", cerr)
		}
	}
	if err := execMulti(tx, body); err != nil {
		return fmt.Errorf("apply migration %s: %w", name, err)
	}
	if fkCheck {
		after, err := foreignKeyViolations(ctx, tx)
		if err != nil {
			slog.Warn("migration foreign key check unavailable; skipped", "migration", name, "stage", "after", "error", err)
		}
		for t, cerr := range after.unavailable {
			slog.Warn("migration foreign key check unavailable for table; skipped", "migration", name, "stage", "after", "table", t, "error", cerr)
		}
		for fk, n := range after.counts {
			table := fk[:strings.Index(fk, "|")]
			if _, blind := fkBefore.unavailable[table]; blind {
				slog.Warn("migration foreign key violations on a table with no baseline; not refused", "migration", name, "table", table, "fk", fk, "count", n)
				continue
			}
			if n > fkBefore.counts[fk] {
				return fmt.Errorf("apply migration %s: leaves %d foreign key violation(s) on %s (table|parent|fkid) where it found %d", name, n, fk, fkBefore.counts[fk])
			}
		}
	}
	if _, err := tx.ExecContext(ctx, "INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)", name, now()); err != nil {
		return fmt.Errorf("record migration %s: %w", name, err)
	}
	if err := tx.Commit(); err != nil {
		return fmt.Errorf("apply migration %s: commit: %w", name, err)
	}
	committed = true

	for _, p := range afterTx {
		if _, err := conn.ExecContext(ctx, p); err != nil {
			return fmt.Errorf("apply migration %s: post-tx pragma %q: %w", name, p, err)
		}
	}

	// FK restore on the success path is handled by the deferred block
	// registered when the before-tx OFF was executed. The migration's
	// own `PRAGMA foreign_keys = ON` (if present) has already run via
	// afterTx; the deferred re-set is idempotent.
	return nil
}

// isForeignKeysOff reports whether a PRAGMA statement disables foreign keys.
// Matches the canonical forms used in migrations 022 and 055 plus the common
// variants (`OFF`, `0`, `false`), case-insensitive, ignoring whitespace.
func isForeignKeysOff(pragma string) bool {
	u := normalizePragma(pragma)
	return strings.HasPrefix(u, "PRAGMAFOREIGN_KEYS=OFF") ||
		strings.HasPrefix(u, "PRAGMAFOREIGN_KEYS=0") ||
		strings.HasPrefix(u, "PRAGMAFOREIGN_KEYS=FALSE")
}

func normalizePragma(p string) string {
	u := strings.ToUpper(p)
	u = strings.ReplaceAll(u, " ", "")
	u = strings.ReplaceAll(u, "\t", "")
	return u
}

// applyPostgresMigration applies a single Postgres migration file atomically.
//
// IDEA-1485: even though Postgres wraps each DDL statement in an implicit
// transaction, the migration body plus the `INSERT INTO schema_migrations`
// bookkeeping row need to commit together — otherwise a crash between
// `data Exec` and the bookkeeping INSERT leaves the migration applied but
// not recorded, and the next startup re-runs it against an already-mutated
// schema. Wrapping both in a single explicit transaction closes that window
// for multi-statement migration files.
func applyPostgresMigration(db *sql.DB, name, sqlText string) error {
	tx, err := db.Begin()
	if err != nil {
		return fmt.Errorf("apply migration %s: begin tx: %w", name, err)
	}
	committed := false
	defer func() {
		if !committed {
			_ = tx.Rollback()
		}
	}()

	if _, err := tx.Exec(sqlText); err != nil {
		return fmt.Errorf("apply migration %s: %w", name, err)
	}
	if _, err := tx.Exec("INSERT INTO schema_migrations (version, applied_at) VALUES ($1, $2)", name, now()); err != nil {
		return fmt.Errorf("record migration %s: %w", name, err)
	}
	if err := tx.Commit(); err != nil {
		return fmt.Errorf("apply migration %s: commit: %w", name, err)
	}
	committed = true
	return nil
}

// findStatementEnd finds the index of the semicolon that ends the next
// complete SQL statement, correctly handling BEGIN...END blocks.
func findStatementEnd(sql string) int {
	depth := 0
	i := 0
	for i < len(sql) {
		// Skip string literals
		if sql[i] == '\'' {
			i++
			for i < len(sql) {
				if sql[i] == '\'' {
					if i+1 < len(sql) && sql[i+1] == '\'' {
						i += 2 // escaped quote
						continue
					}
					break
				}
				i++
			}
			i++
			continue
		}

		// Skip line comments
		if i+1 < len(sql) && sql[i] == '-' && sql[i+1] == '-' {
			for i < len(sql) && sql[i] != '\n' {
				i++
			}
			continue
		}

		// Check for BEGIN (case-insensitive)
		if i+5 <= len(sql) && strings.EqualFold(sql[i:i+5], "BEGIN") {
			// Make sure it's a word boundary
			if (i == 0 || !isAlpha(sql[i-1])) && (i+5 >= len(sql) || !isAlpha(sql[i+5])) {
				depth++
				i += 5
				continue
			}
		}

		// Check for END (case-insensitive)
		if depth > 0 && i+3 <= len(sql) && strings.EqualFold(sql[i:i+3], "END") {
			if (i == 0 || !isAlpha(sql[i-1])) && (i+3 >= len(sql) || !isAlpha(sql[i+3])) {
				depth--
				i += 3
				continue
			}
		}

		// Semicolon outside of any BEGIN...END block = statement end
		if sql[i] == ';' && depth == 0 {
			return i
		}

		i++
	}
	return -1
}

// uniqueSlugExcluding generates a unique slug, excluding a specific document ID
// from the collision check. Used during title renames.
// It takes the executor rather than reaching for s.db, because all three of
// its callers run inside a transaction that already holds locks — the item
// update (items.go, under the workspace seq and parent-children locks), the
// document rename (documents.go, under the rename lock this bug added), and
// since IDEA-2874 the collection rename (collections.go, under the workspace
// seq lock and the row's FOR UPDATE). A read issued against the POOL from
// inside such a transaction needs a SECOND connection while the first is
// held, and if the pool is saturated by callers waiting on the very lock this
// transaction holds, that second connection never arrives: the deadlock is
// then in the application rather than in the database, where no SQLSTATE
// names it and no lock timeout breaks it. (An earlier version of this comment
// claimed all three were in-transaction when UpdateCollection still ran its
// scan BEFORE opening one; it is true now because the scan moved, not
// because the comment was re-read.)
func (s *Store) uniqueSlugExcluding(q rowQueryer, table, scopeCol, scopeVal, baseSlug, excludeID string) (string, error) {
	slug := baseSlug
	for i := 2; ; i++ {
		var count int
		query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s = ? AND slug = ? AND id != ?", table, scopeCol)
		err := q.QueryRow(s.q(query), scopeVal, slug, excludeID).Scan(&count)
		if err != nil {
			return "", err
		}
		if count == 0 {
			return slug, nil
		}
		slug = fmt.Sprintf("%s-%d", baseSlug, i)
	}
}

func isAlpha(b byte) bool {
	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_'
}

// IsValidFieldKey reports whether key is a field name the store will filter
// on; a list filter key that is not one is refused (BUG-3480) rather than
// dropped, which used to answer the unfiltered list.
func IsValidFieldKey(key string) bool { return isValidFieldKey(key) }

// isValidFieldKey checks that a field name contains only safe characters
// (alphanumeric, underscore, hyphen). This prevents SQL injection when
// field keys from user input are interpolated into JSON path expressions.
func isValidFieldKey(key string) bool {
	if key == "" {
		return false
	}
	for _, c := range key {
		if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') {
			return false
		}
	}
	return true
}

// q rebinds a query to the store's dialect (converts "?" to "$1", "$2", etc. for PostgreSQL).
func (s *Store) q(query string) string {
	return s.dialect.Rebind(query)
}

func newID() string {
	return uuid.New().String()
}

func now() string {
	return time.Now().UTC().Format(time.RFC3339)
}

// nowNano is a sub-second (nanosecond) UTC timestamp. Used where a stored
// timestamp doubles as an optimistic-concurrency token and must distinguish
// writes that land within the same wall-clock second (BUG-2265 collections):
// second-precision now() would collide, forcing a whole-second "advance into
// the future" to keep the token monotonic. Sub-second resolution makes
// collisions near-impossible, so the token advances naturally and never drifts
// meaningfully ahead of wall-clock. The value is a valid RFC3339 string
// (time.Parse(time.RFC3339, …) accepts the fractional seconds), so it
// round-trips through parseTime and the JSON API unchanged.
func nowNano() string {
	return time.Now().UTC().Format(time.RFC3339Nano)
}

func parseTime(s string) time.Time {
	t, _ := time.Parse(time.RFC3339, s)
	return t
}

func parseTimePtr(s *string) *time.Time {
	if s == nil {
		return nil
	}
	t := parseTime(*s)
	return &t
}

// slugify converts a string to a URL-friendly slug.
func slugify(s string) string {
	s = strings.ToLower(s)
	var result []byte
	prevHyphen := false
	for _, c := range s {
		if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') {
			result = append(result, byte(c))
			prevHyphen = false
		} else if c == '\'' {
			// Strip apostrophes without inserting a hyphen
			continue
		} else if !prevHyphen && len(result) > 0 {
			result = append(result, '-')
			prevHyphen = true
		}
	}
	// Trim trailing hyphen
	if len(result) > 0 && result[len(result)-1] == '-' {
		result = result[:len(result)-1]
	}
	return string(result)
}

// backfillItemNumbers assigns prefixes to collections that lack them and
// sequential item_number values to items that don't have one yet.
//
// On first run after the workspace-global numbering migration, this function
// detects the old per-collection unique index and performs a one-time
// renumbering of all items so that item_number is unique per workspace
// (not per collection). This allows items to keep their number when moved
// between collections (e.g. IDEA-42 → BUG-42).
func (s *Store) backfillItemNumbers() error {
	// 1. Backfill collection prefixes
	rows, err := s.db.Query(s.q("SELECT id, name FROM collections WHERE prefix = ''"))
	if err != nil {
		return fmt.Errorf("query collections for prefix backfill: %w", err)
	}
	defer rows.Close()

	type collInfo struct {
		id, name string
	}
	var colls []collInfo
	for rows.Next() {
		var c collInfo
		if err := rows.Scan(&c.id, &c.name); err != nil {
			return err
		}
		colls = append(colls, c)
	}
	if err := rows.Err(); err != nil {
		return err
	}

	for _, c := range colls {
		prefix := collections.DerivePrefix(c.name)
		if prefix == "" {
			prefix = "ITEM"
		}
		if _, err := s.db.Exec(s.q("UPDATE collections SET prefix = ? WHERE id = ?"), prefix, c.id); err != nil {
			return fmt.Errorf("update prefix for collection %s: %w", c.id, err)
		}
	}

	// 2. Migrate from per-collection to per-workspace numbering (one-time)
	if err := s.migrateToWorkspaceNumbering(); err != nil {
		return fmt.Errorf("migrate to workspace numbering: %w", err)
	}

	// 3. Backfill NULL item_numbers for any new items (per workspace)
	if err := s.backfillNullItemNumbers(); err != nil {
		return fmt.Errorf("backfill null item numbers: %w", err)
	}

	// 4. Ensure the workspace-level unique index exists
	_, err = s.db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_items_workspace_number ON items(workspace_id, item_number)")
	if err != nil {
		return fmt.Errorf("create workspace number index: %w", err)
	}

	return nil
}

// indexExists checks whether the named index exists in the database.
func (s *Store) indexExists(name string) bool {
	var query string
	switch s.dialect.Driver() {
	case DriverPostgres:
		query = "SELECT 1 FROM pg_indexes WHERE indexname = $1"
	default: // SQLite
		query = "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?"
	}
	var one int
	err := s.db.QueryRow(query, name).Scan(&one)
	return err == nil
}

// migrateToWorkspaceNumbering performs a one-time migration from per-collection
// item numbering to per-workspace item numbering. It detects the old index
// (collection_id, item_number) and, if present, renumbers all items within each
// workspace using a single sequential counter ordered by created_at.
//
// The entire operation runs in a single transaction — if anything fails the
// database is left unchanged and the migration will be retried on next startup.
func (s *Store) migrateToWorkspaceNumbering() error {
	if !s.indexExists("idx_items_collection_number") {
		// The old index was the only marker, and SQLite migration 056 drops
		// it: its items rebuild recreates only the indexes migrations made,
		// and this one was made in code (BUG-3664). A legacy database that
		// upgrades through 056 in one go arrives here without the marker,
		// still numbered per collection. So, with no workspace index yet,
		// numbers repeated within a workspace are the marker instead. This
		// also recovers a database an earlier release left in that state,
		// whose every startup failed creating the workspace index.
		if s.indexExists("idx_items_workspace_number") {
			return nil
		}
		dup, err := s.hasRepeatedWorkspaceItemNumbers()
		if err != nil {
			return err
		}
		if !dup {
			return nil
		}
	}

	tx, err := s.db.Begin()
	if err != nil {
		return fmt.Errorf("begin transaction: %w", err)
	}
	defer tx.Rollback()

	// Drop the old per-collection unique index
	if _, err := tx.Exec("DROP INDEX IF EXISTS idx_items_collection_number"); err != nil {
		return fmt.Errorf("drop old index: %w", err)
	}

	// Get all workspace IDs
	wsRows, err := tx.Query(s.q("SELECT id FROM workspaces"))
	if err != nil {
		return fmt.Errorf("query workspaces: %w", err)
	}
	var wsIDs []string
	for wsRows.Next() {
		var id string
		if err := wsRows.Scan(&id); err != nil {
			wsRows.Close()
			return err
		}
		wsIDs = append(wsIDs, id)
	}
	wsRows.Close()
	if err := wsRows.Err(); err != nil {
		return err
	}

	// Renumber items per workspace: assign 1, 2, 3… ordered by created_at, id
	for _, wsID := range wsIDs {
		itemRows, err := tx.Query(s.q(`
			SELECT i.id, i.item_number, COALESCE(c.prefix, ''), i.deleted_at IS NULL
			FROM items i LEFT JOIN collections c ON c.id = i.collection_id
			WHERE i.workspace_id = ? ORDER BY i.created_at ASC, i.id ASC`),
			wsID,
		)
		if err != nil {
			return fmt.Errorf("query items for workspace %s: %w", wsID, err)
		}

		type numbered struct {
			id     string
			old    sql.NullInt64
			prefix string
			live   bool
		}
		var items []numbered
		for itemRows.Next() {
			var it numbered
			if err := itemRows.Scan(&it.id, &it.old, &it.prefix, &it.live); err != nil {
				itemRows.Close()
				return err
			}
			items = append(items, it)
		}
		itemRows.Close()
		if err := itemRows.Err(); err != nil {
			return err
		}

		// Assign sequential numbers across the entire workspace, and keep
		// what each stored reference resolved to before (BUG-3750): a stored
		// [[IDEA-1]] whose item becomes IDEA-2 would otherwise resolve to
		// another item. Before the conversion, GetItemByRef took the exact
		// PREFIX-N match, else the live item holding number N; both are
		// recorded here with the new reference, and a key that does not name
		// one item (two collections with one prefix, or a number several live
		// items held) is dropped rather than guessed.
		exact := map[string]string{}
		ambiguous := map[string]bool{}
		byNumber := map[int64]string{}
		sharedNumber := map[int64]bool{}
		for i, it := range items {
			num := i + 1
			if _, err := tx.Exec(s.q("UPDATE items SET item_number = ? WHERE id = ?"), num, it.id); err != nil {
				return fmt.Errorf("renumber item %s to %d: %w", it.id, num, err)
			}
			if !it.old.Valid || !it.live {
				continue
			}
			to := ""
			if it.prefix != "" {
				to = formatRef(strings.ToUpper(it.prefix), num)
				from := strings.ToUpper(formatRef(it.prefix, int(it.old.Int64)))
				if _, seen := exact[from]; seen || ambiguous[from] {
					delete(exact, from)
					ambiguous[from] = true
				} else {
					exact[from] = to
				}
			}
			if _, seen := byNumber[it.old.Int64]; seen || sharedNumber[it.old.Int64] {
				delete(byNumber, it.old.Int64)
				sharedNumber[it.old.Int64] = true
			} else {
				byNumber[it.old.Int64] = to
			}
		}
		renamed := func(ref string) (string, bool) {
			prefix, number, ok := splitRef(ref)
			if !ok {
				return "", false
			}
			key := strings.ToUpper(prefix) + "-" + strconv.Itoa(number)
			to, found := exact[key]
			if !found && !ambiguous[key] {
				to, found = byNumber[int64(number)]
			}
			if !found || to == "" || strings.EqualFold(to, ref) {
				return "", false
			}
			return to, true
		}
		if err := s.rewriteRenumberedRefsTx(tx, wsID, renamed); err != nil {
			return err
		}
	}

	// Create the new workspace-level unique index inside the transaction
	if _, err := tx.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_items_workspace_number ON items(workspace_id, item_number)"); err != nil {
		return fmt.Errorf("create workspace number index: %w", err)
	}

	if err := tx.Commit(); err != nil {
		return fmt.Errorf("commit migration: %w", err)
	}

	return nil
}

// rewriteRenumberedRefsTx rewrites the stored reference links of one
// workspace, item bodies and comment bodies, by renamed (old ref -> new ref),
// as part of converting legacy numbering (BUG-3750).
func (s *Store) rewriteRenumberedRefsTx(tx *sql.Tx, workspaceID string, renamed func(ref string) (string, bool)) error {
	for _, target := range []struct{ table, column string }{{"items", "content"}, {"comments", "body"}} {
		rows, err := tx.Query(s.q(`SELECT id, `+target.column+` FROM `+target.table+` WHERE workspace_id = ? AND `+target.column+` LIKE '%[[%'`), workspaceID)
		if err != nil {
			return fmt.Errorf("renumber: read %s: %w", target.table, err)
		}
		type change struct{ id, text string }
		var changes []change
		for rows.Next() {
			var id, text string
			if err := rows.Scan(&id, &text); err != nil {
				rows.Close()
				return err
			}
			if out, n := links.RewriteRefs(text, renamed); n > 0 {
				changes = append(changes, change{id, out})
			}
		}
		if err := rows.Err(); err != nil {
			rows.Close()
			return err
		}
		rows.Close()
		for _, c := range changes {
			if _, err := tx.Exec(s.q(`UPDATE `+target.table+` SET `+target.column+` = ? WHERE id = ?`), c.text, c.id); err != nil {
				return fmt.Errorf("renumber: rewrite %s %s: %w", target.table, c.id, err)
			}
		}
	}
	return nil
}

// hasRepeatedWorkspaceItemNumbers reports whether two live or archived items
// in one workspace share an item_number: the shape of per-collection
// numbering, which the workspace index cannot be created over.
func (s *Store) hasRepeatedWorkspaceItemNumbers() (bool, error) {
	var one int
	err := s.db.QueryRow(`SELECT 1 FROM items WHERE item_number IS NOT NULL
		GROUP BY workspace_id, item_number HAVING COUNT(*) > 1 LIMIT 1`).Scan(&one)
	if err == sql.ErrNoRows {
		return false, nil
	}
	if err != nil {
		return false, fmt.Errorf("check repeated item numbers: %w", err)
	}
	return true, nil
}

// backfillNullItemNumbers assigns the next workspace-global item_number to any
// items that have a NULL item_number (e.g. from interrupted inserts or pre-
// migration data).
func (s *Store) backfillNullItemNumbers() error {
	// Get workspaces that have items with NULL item_number
	wsRows, err := s.db.Query(s.q(
		"SELECT DISTINCT workspace_id FROM items WHERE item_number IS NULL"))
	if err != nil {
		return fmt.Errorf("query workspaces with null item numbers: %w", err)
	}
	var wsIDs []string
	for wsRows.Next() {
		var id string
		if err := wsRows.Scan(&id); err != nil {
			wsRows.Close()
			return err
		}
		wsIDs = append(wsIDs, id)
	}
	wsRows.Close()
	if err := wsRows.Err(); err != nil {
		return err
	}

	for _, wsID := range wsIDs {
		itemRows, err := s.db.Query(
			s.q("SELECT id FROM items WHERE workspace_id = ? AND item_number IS NULL ORDER BY created_at ASC, id ASC"),
			wsID,
		)
		if err != nil {
			return fmt.Errorf("query null-numbered items for workspace %s: %w", wsID, err)
		}

		var itemIDs []string
		for itemRows.Next() {
			var id string
			if err := itemRows.Scan(&id); err != nil {
				itemRows.Close()
				return err
			}
			itemIDs = append(itemIDs, id)
		}
		itemRows.Close()
		if len(itemIDs) == 0 {
			continue
		}

		// Get current max in this workspace
		var maxNum int
		if err := s.db.QueryRow(s.q("SELECT COALESCE(MAX(item_number), 0) FROM items WHERE workspace_id = ?"), wsID).Scan(&maxNum); err != nil {
			return fmt.Errorf("get max item_number for workspace %s: %w", wsID, err)
		}

		for _, itemID := range itemIDs {
			maxNum++
			if _, err := s.db.Exec(s.q("UPDATE items SET item_number = ? WHERE id = ?"), maxNum, itemID); err != nil {
				return fmt.Errorf("update item_number for item %s: %w", itemID, err)
			}
		}
	}

	return nil
}

// uniqueSlug generates a unique slug within a scope by appending -2, -3, etc.
func (s *Store) uniqueSlug(table, scopeCol, scopeVal, baseSlug string) (string, error) {
	return s.uniqueSlugQ(s.db, table, scopeCol, scopeVal, baseSlug)
}

// uniqueSlugQ is uniqueSlug parameterized over the query surface (rowQueryer)
// so the collision scan can run either unlocked against *sql.DB or inside an
// in-flight *sql.Tx. The transactional form is what createItemTx uses: it
// allocates the slug under the same transaction — and the same workspace
// advisory lock — that performs the INSERT, so the scan is read-your-writes
// consistent and serialized against concurrent creators in that workspace.
func (s *Store) uniqueSlugQ(q rowQueryer, table, scopeCol, scopeVal, baseSlug string) (string, error) {
	slug := baseSlug
	for i := 2; ; i++ {
		var count int
		// Check all rows including soft-deleted to respect the DB UNIQUE constraint
		query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s = ? AND slug = ?", table, scopeCol)
		err := q.QueryRow(s.q(query), scopeVal, slug).Scan(&count)
		if err != nil {
			return "", err
		}
		if count == 0 {
			return slug, nil
		}
		slug = fmt.Sprintf("%s-%d", baseSlug, i)
	}
}