package store import ( "encoding/json" "fmt" "log/slog" "strings" "time" "github.com/PerpetualSoftware/pad/internal/models" ) // coerceJSONForImport normalizes an imported JSON column value to a // well-formed default of the expected shape. Used at the import boundary // (ImportWorkspace) where legacy / external workspace bundles may carry // empty-string or malformed JSON that bypasses the column's NOT NULL // DEFAULT (IDEA-1486) and that the handler-layer shape validators // (IDEA-1488) never see. The lenient policy (coerce + log) matches the // IDEA-1484 import-side precedent at export.go:206-209: don't break a // workspace import because one row carries malformed data. // // - raw == "" → return defaultJSON (empty-string sentinel). // - raw is well-formed JSON of the expected shape (non-nil object/array) // → return raw verbatim. // - raw is JSON `null`, unparseable, or wrong-shape → return defaultJSON // AND log a structured warning naming the field + row. The raw // value's LENGTH is logged (`raw_len`) but never its content, so // user data does not leak into logs. // // expectObject==true requires the parsed value to be a JSON object // (map[string]any); expectObject==false requires a JSON array // ([]any) — used for items.tags. // // IDEA-1486 R1 codex P2: `json.Unmarshal("null", &m)` returns nil error // and leaves the destination nil. Imported `fields: null` (or the // string literal `"null"` at the JSON-encoded-string layer) would // otherwise satisfy this validator and land as a JSONB null on // Postgres (which technically satisfies NOT NULL — SQL NULL ≠ JSONB // null) or the text "null" on SQLite. Downstream readers that expect // an object would choke. The non-nil check below forces JSON null to // the log-and-coerce path with the rest of the malformed shapes. func coerceJSONForImport(raw, defaultJSON, field, rowID, workspaceID string, expectObject bool) string { if raw == "" { return defaultJSON } if expectObject { var v map[string]any if err := json.Unmarshal([]byte(raw), &v); err == nil && v != nil { return raw } } else { var v []any if err := json.Unmarshal([]byte(raw), &v); err == nil && v != nil { return raw } } slog.Warn("import_workspace coerced malformed json", "field", field, "row_id", rowID, "workspace_id", workspaceID, "raw_len", len(raw)) return defaultJSON } // ExportWorkspace exports all data for a workspace into a portable format. func (s *Store) ExportWorkspace(slug string) (*models.WorkspaceExport, error) { ws, err := s.GetWorkspaceBySlug(slug) if err != nil { return nil, fmt.Errorf("workspace lookup: %w", err) } if ws == nil { return nil, fmt.Errorf("workspace not found: %s", slug) } export := &models.WorkspaceExport{ Version: 1, ExportedAt: time.Now().UTC().Format(time.RFC3339), Workspace: models.WorkspaceExportMeta{ Name: ws.Name, Slug: ws.Slug, Description: ws.Description, Settings: ws.Settings, }, } // Collections rows, err := s.db.Query(s.q(` SELECT id, name, slug, icon, description, schema, settings, prefix, sort_order, is_default, is_system, created_at, updated_at FROM collections WHERE workspace_id = ? AND deleted_at IS NULL ORDER BY sort_order, name`), ws.ID) if err != nil { return nil, fmt.Errorf("export collections: %w", err) } defer rows.Close() for rows.Next() { var c models.CollectionExport var isDefault, isSystem bool if err := rows.Scan(&c.ID, &c.Name, &c.Slug, &c.Icon, &c.Description, &c.Schema, &c.Settings, &c.Prefix, &c.SortOrder, &isDefault, &isSystem, &c.CreatedAt, &c.UpdatedAt); err != nil { return nil, fmt.Errorf("scan collection: %w", err) } c.IsDefault = isDefault c.IsSystem = isSystem export.Collections = append(export.Collections, c) } if err := rows.Err(); err != nil { return nil, err } // Items itemRows, err := s.db.Query(s.q(` SELECT id, collection_id, title, slug, content, fields, tags, pinned, sort_order, COALESCE(parent_id, ''), created_by, last_modified_by, source, COALESCE(item_number, 0), created_at, updated_at FROM items WHERE workspace_id = ? AND deleted_at IS NULL ORDER BY created_at, id`), ws.ID) if err != nil { return nil, fmt.Errorf("export items: %w", err) } defer itemRows.Close() for itemRows.Next() { var it models.ItemExport var pinned bool if err := itemRows.Scan(&it.ID, &it.CollectionID, &it.Title, &it.Slug, &it.Content, &it.Fields, &it.Tags, &pinned, &it.SortOrder, &it.ParentID, &it.CreatedBy, &it.LastModifiedBy, &it.Source, &it.ItemNumber, &it.CreatedAt, &it.UpdatedAt); err != nil { return nil, fmt.Errorf("scan item: %w", err) } it.Pinned = pinned export.Items = append(export.Items, it) } if err := itemRows.Err(); err != nil { return nil, err } // Comments commentRows, err := s.db.Query(s.q(` SELECT c.id, c.item_id, c.author, c.body, c.created_by, c.source, c.created_at, c.updated_at FROM comments c JOIN items i ON c.item_id = i.id WHERE c.workspace_id = ? AND i.deleted_at IS NULL ORDER BY c.created_at`), ws.ID) if err != nil { return nil, fmt.Errorf("export comments: %w", err) } defer commentRows.Close() for commentRows.Next() { var cm models.CommentExport if err := commentRows.Scan(&cm.ID, &cm.ItemID, &cm.Author, &cm.Body, &cm.CreatedBy, &cm.Source, &cm.CreatedAt, &cm.UpdatedAt); err != nil { return nil, fmt.Errorf("scan comment: %w", err) } export.Comments = append(export.Comments, cm) } if err := commentRows.Err(); err != nil { return nil, err } // Item links — exported in full, including links whose source or target item // is soft-deleted. This is intentional and differs from user-facing reads // (GetItemLinks/GetParentForItem/GetParentMap, which all filter on // items.deleted_at IS NULL — see BUG-734). Backups need to round-trip the // raw graph so that re-importing into a workspace where the deleted items // are restored preserves the original relationships. The import path // already silently skips links whose endpoints are missing entirely. linkRows, err := s.db.Query(s.q(` SELECT id, source_id, target_id, link_type, created_by, created_at FROM item_links WHERE workspace_id = ? ORDER BY created_at`), ws.ID) if err != nil { return nil, fmt.Errorf("export item links: %w", err) } defer linkRows.Close() for linkRows.Next() { var lk models.ItemLinkExport if err := linkRows.Scan(&lk.ID, &lk.SourceID, &lk.TargetID, &lk.LinkType, &lk.CreatedBy, &lk.CreatedAt); err != nil { return nil, fmt.Errorf("scan item link: %w", err) } export.ItemLinks = append(export.ItemLinks, lk) } if err := linkRows.Err(); err != nil { return nil, err } // Item versions versionRows, err := s.db.Query(s.q(` SELECT v.id, v.item_id, v.content, v.change_summary, v.created_by, v.source, v.is_diff, v.created_at FROM item_versions v JOIN items i ON v.item_id = i.id WHERE i.workspace_id = ? AND i.deleted_at IS NULL ORDER BY v.created_at, v.version_seq`), ws.ID) if err != nil { return nil, fmt.Errorf("export item versions: %w", err) } defer versionRows.Close() for versionRows.Next() { var ver models.ItemVersionExport var isDiff bool if err := versionRows.Scan(&ver.ID, &ver.ItemID, &ver.Content, &ver.ChangeSummary, &ver.CreatedBy, &ver.Source, &isDiff, &ver.CreatedAt); err != nil { return nil, fmt.Errorf("scan item version: %w", err) } ver.IsDiff = isDiff export.ItemVersions = append(export.ItemVersions, ver) } if err := versionRows.Err(); err != nil { return nil, err } return export, nil } // ImportWorkspace imports a workspace from an exported data structure. // It creates a new workspace with regenerated IDs, remapping all references. // If newName is non-empty, it overrides the workspace name and slug. func (s *Store) ImportWorkspace(data *models.WorkspaceExport, newName string, ownerID string) (*models.Workspace, error) { if data.Version != 1 { return nil, fmt.Errorf("unsupported export version: %d", data.Version) } // Determine workspace name/slug wsName := data.Workspace.Name wsSlug := data.Workspace.Slug if newName != "" { wsName = newName wsSlug = newName } ws, err := s.CreateWorkspace(models.WorkspaceCreate{ Name: wsName, Slug: wsSlug, Description: data.Workspace.Description, Settings: data.Workspace.Settings, OwnerID: ownerID, }) if err != nil { return nil, fmt.Errorf("create workspace: %w", err) } // Run all data inserts in a single transaction for atomicity tx, err := s.db.Begin() if err != nil { return nil, fmt.Errorf("begin transaction: %w", err) } defer tx.Rollback() // ID mapping: old ID -> new ID collMap := make(map[string]string) itemMap := make(map[string]string) // Import collections for _, c := range data.Collections { newCollID := newID() collMap[c.ID] = newCollID // Coerce empty-string / malformed settings to a valid JSON object // before insert. IDEA-1484 (PR #562) hardened collections.settings // to NOT NULL DEFAULT '{}', but this INSERT explicitly supplies // the settings column — so the DEFAULT clause does NOT fire when // c.Settings is "". Without this coercion, Postgres rejects `""` // at JSONB type-validation and SQLite silently stores invalid // JSON. Legacy bundles and plain-JSON workspace imports // (handlers_workspaces.go, handlers_import_bundle.go, // cmd/pad/main.go's migrate command) can still carry "" or // malformed settings, so normalization belongs at the import // boundary rather than at the schema level. IDEA-1488 extends // this to log-and-coerce on non-empty malformed JSON. settings := coerceJSONForImport(c.Settings, "{}", "collections.settings", c.ID, ws.ID, true) _, err := tx.Exec(s.q(` INSERT INTO collections (id, workspace_id, name, slug, icon, description, schema, settings, prefix, sort_order, is_default, is_system, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`), newCollID, ws.ID, c.Name, c.Slug, c.Icon, c.Description, c.Schema, settings, c.Prefix, c.SortOrder, s.dialect.BoolToInt(c.IsDefault), s.dialect.BoolToInt(c.IsSystem), c.CreatedAt, c.UpdatedAt) if err != nil { return nil, fmt.Errorf("import collection %s: %w", c.Name, err) } } // Import items (first pass: create items, remap collection_id) // Item numbers are assigned sequentially in created_at order to produce // workspace-global numbering. Exported item_number values are ignored // because old exports used per-collection numbering which can have // duplicates within a workspace. // // IDEA-1486 + IDEA-1488: precompute each item's coerced fields/tags so // the second-pass UPDATE (which re-applies fields after the ID remap) // uses the SAME normalized value as the first-pass INSERT. Without // this, a malformed it.Fields gets coerced to "{}" on INSERT but the // second pass would re-write the original malformed value verbatim, // undoing the coercion. The map is keyed by the original (exported) // item ID so the second-pass loop can look up the normalized value. coercedFields := make(map[string]string, len(data.Items)) coercedTags := make(map[string]string, len(data.Items)) var nextItemNumber int for _, it := range data.Items { newItemID := newID() itemMap[it.ID] = newItemID newCollID := collMap[it.CollectionID] if newCollID == "" { continue // skip orphaned items } // On first pass, parent_id may refer to an item not yet created, so use empty parentID := "" if it.ParentID != "" { if mapped, ok := itemMap[it.ParentID]; ok { parentID = mapped } } nextItemNumber++ // IDEA-1486 + IDEA-1488: coerce empty-string / malformed // fields/tags at the import boundary. After migration 056 / // pgmigrations 035 hardened items.fields and items.tags to // NOT NULL DEFAULT, an imported item with fields="" 500s on // Postgres at JSONB type-validation and silently stores // invalid JSON on SQLite. Mirror collections.settings' // coercion above. The IDEA-1488 leg: log-and-coerce (not // fail-stop) so a legacy bundle with one malformed item // still imports. fieldsJSON := coerceJSONForImport(it.Fields, "{}", "items.fields", it.ID, ws.ID, true) tagsJSON := coerceJSONForImport(it.Tags, "[]", "items.tags", it.ID, ws.ID, false) coercedFields[it.ID] = fieldsJSON coercedTags[it.ID] = tagsJSON // Stamp `seq` so workspace import populates the delta-sync cursor // column (PLAN-1343 / TASK-1352). Each INSERT reads MAX(seq)+1 // within this transaction, so imported rows get sequential // per-workspace seqs — clients post-import see them on the next // /items-index fetch, and any subsequent mutation keeps bumping // from a sensible floor instead of a flat MAX(seq)=0. _, err := tx.Exec(s.q(` INSERT INTO items (id, workspace_id, collection_id, title, slug, content, fields, tags, pinned, sort_order, parent_id, created_by, last_modified_by, source, item_number, created_at, updated_at, seq) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULLIF(?, ''), ?, ?, ?, ?, ?, ?, `+nextWorkspaceSeqSubquery+`)`), newItemID, ws.ID, newCollID, it.Title, it.Slug, it.Content, fieldsJSON, tagsJSON, s.dialect.BoolToInt(it.Pinned), it.SortOrder, parentID, it.CreatedBy, it.LastModifiedBy, it.Source, nextItemNumber, it.CreatedAt, it.UpdatedAt, ws.ID) if err != nil { return nil, fmt.Errorf("import item %s: %w", it.Title, err) } } // Second pass: remap parent_id and relation fields (now all items exist). // Use the coerced first-pass fields value as the remap input so a // malformed-and-coerced row doesn't get its coercion clobbered by the // raw export value (IDEA-1486 + IDEA-1488). _ = coercedTags // tags don't carry ID relations; second-pass only touches fields for _, it := range data.Items { newItemID := itemMap[it.ID] if newItemID == "" { continue } fieldsInput, ok := coercedFields[it.ID] if !ok { fieldsInput = it.Fields // defensive — should always be populated by first pass } // Remap relation fields now that ALL items are mapped fields := remapFieldIDs(fieldsInput, itemMap, collMap) parentID := "" if it.ParentID != "" { if mapped, ok := itemMap[it.ParentID]; ok { parentID = mapped } } _, err := tx.Exec(s.q(`UPDATE items SET fields = ?, parent_id = NULLIF(?, '') WHERE id = ?`), fields, parentID, newItemID) if err != nil { return nil, fmt.Errorf("remap item %s: %w", it.Title, err) } } // Import comments for _, cm := range data.Comments { newItemID := itemMap[cm.ItemID] if newItemID == "" { continue } _, err := tx.Exec(s.q(` INSERT INTO comments (id, item_id, workspace_id, author, body, created_by, source, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`), newID(), newItemID, ws.ID, cm.Author, cm.Body, cm.CreatedBy, cm.Source, cm.CreatedAt, cm.UpdatedAt) if err != nil { return nil, fmt.Errorf("import comment: %w", err) } } // Import item links for _, lk := range data.ItemLinks { newSourceID := itemMap[lk.SourceID] newTargetID := itemMap[lk.TargetID] if newSourceID == "" || newTargetID == "" { continue } _, err := tx.Exec(s.q(` INSERT INTO item_links (id, workspace_id, source_id, target_id, link_type, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)`), newID(), ws.ID, newSourceID, newTargetID, lk.LinkType, lk.CreatedBy, lk.CreatedAt) if err != nil { // Ignore duplicate links continue } } // Import item versions for _, ver := range data.ItemVersions { newItemID := itemMap[ver.ItemID] if newItemID == "" { continue } // version_seq (BUG-2270): re-derive a per-item monotonic seq at // import time. data.ItemVersions is exported ORDER BY created_at, // version_seq, so COALESCE(MAX,0)+1 reassigns 1,2,3… in that same // deterministic order and imported same-second versions keep a // stable tie-break instead of all defaulting to 0. _, err := tx.Exec(s.q(` INSERT INTO item_versions (id, item_id, content, change_summary, created_by, source, is_diff, created_at, version_seq) VALUES (?, ?, ?, ?, ?, ?, ?, ?, (SELECT COALESCE(MAX(version_seq), 0) + 1 FROM item_versions WHERE item_id = ?))`), newID(), newItemID, ver.Content, ver.ChangeSummary, ver.CreatedBy, ver.Source, s.dialect.BoolToInt(ver.IsDiff), ver.CreatedAt, newItemID) if err != nil { // Log detail but skip — version history is non-critical. // Migrated from fmt.Printf to slog.Warn alongside the // IDEA-1488 log-and-coerce additions; same severity, same // triage signal, but threads through the canonical logger. slog.Warn("import_workspace skipped item version", "item_id", ver.ItemID, "workspace_id", ws.ID, "err", err) continue } } if err := tx.Commit(); err != nil { return nil, fmt.Errorf("commit import: %w", err) } // Rebuild FTS indexes for the new workspace (outside transaction) s.rebuildFTSForWorkspace(ws.ID) return ws, nil } // rebuildFTSForWorkspace rebuilds the FTS index for all items in a workspace. // This is needed after import because direct INSERTs bypass the FTS triggers. // Only applicable to SQLite (PostgreSQL uses trigger-maintained tsvector columns). func (s *Store) rebuildFTSForWorkspace(wsID string) { if s.dialect.Driver() != DriverSQLite { return } rows, err := s.db.Query(s.q(`SELECT rowid, title, content, tags FROM items WHERE workspace_id = ? AND deleted_at IS NULL`), wsID) if err != nil { return } defer rows.Close() for rows.Next() { var rowid int64 var title, content, tags string if err := rows.Scan(&rowid, &title, &content, &tags); err != nil { continue } s.db.Exec(s.q(`INSERT INTO items_fts(rowid, title, content, tags) VALUES (?, ?, ?, ?)`), rowid, title, content, tags) } } // remapFieldIDs replaces old UUIDs in a JSON fields string with their new IDs. // This handles relation fields (e.g. parent: "uuid") without needing to parse the schema. // // IDEA-1486: empty-string input is normalized to "{}". Centralizing the // contract here keeps future callers safe by default — any UPDATE that // writes the result back into items.fields is guaranteed to satisfy the // post-migration NOT NULL DEFAULT '{}' invariant. Without this guard, // the second-pass UPDATE at ImportWorkspace would write "" verbatim on // items whose original fields were already empty, which silently stores // invalid JSON on SQLite and (post-migration) would have already 500'd // on the first-pass INSERT on Postgres if not for the import-boundary // coercion above. func remapFieldIDs(fieldsJSON string, itemMap, collMap map[string]string) string { if fieldsJSON == "" { return "{}" } result := fieldsJSON for oldID, newID := range itemMap { if oldID != "" && newID != "" { result = strings.ReplaceAll(result, oldID, newID) } } return result }