From ca158c22d8dfe942e327dc2accc163cbf6dcc7fb Mon Sep 17 00:00:00 2001 From: xarmian Date: Sun, 4 Oct 2026 07:19:07 -0400 Subject: [PATCH] feat(apps): app install staging: apps/1 manifest, fetch policy, pending reservations, preview (TASK-3397, U8a) (#1770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC-6 U8a (DOC-3371 §1, §2 steps 1-5, §9). Strict apps/1 parsing; an https/no-redirect fetch policy screened at dial time with an admin private-origin list; pending-install reservations under the owner lock and the instance advisory lock, holding the full charge until the fetch ends; the owner preview with every artifact normalized by the importer's own code (normalizeArtifact, prepareCreateFields, both extracted behaviour-preserving and pinned by a parity golden); apps_enabled and the admin apps settings; a background sweep of expired pending installs; migration 118/092. The preview is advisory: it may refuse what the import would accept, never the reverse; U8b re-checks atomically. Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC --- cmd/pad/cmd_server.go | 3 + internal/appfetch/appfetch.go | 214 +++++ internal/appfetch/appfetch_test.go | 145 ++++ internal/appmanifest/manifest.go | 433 ++++++++++ internal/appmanifest/manifest_test.go | 172 ++++ .../reserved_field_decode_guard_test.go | 3 +- internal/server/app_pending_sweep.go | 78 ++ internal/server/artifact_import.go | 7 + .../server/artifact_import_parity_test.go | 168 ++++ internal/server/artifact_normalize.go | 72 ++ internal/server/handlers_apps_install.go | 752 ++++++++++++++++++ internal/server/handlers_apps_install_test.go | 715 +++++++++++++++++ internal/server/handlers_artifact_import.go | 57 +- internal/server/handlers_items.go | 121 +-- internal/server/server.go | 19 + .../testdata/artifact_import_parity.json | 129 +++ internal/store/app_pending.go | 326 ++++++++ internal/store/app_pending_test.go | 218 +++++ internal/store/dialect_fieldvalue.go | 18 + .../migrations/118_app_install_lifecycle.sql | 65 ++ internal/store/nul_unprotected_baseline.txt | 27 + internal/store/nulguard_test.go | 6 +- .../092_app_install_lifecycle.sql | 42 + 23 files changed, 3696 insertions(+), 94 deletions(-) create mode 100644 internal/appfetch/appfetch.go create mode 100644 internal/appfetch/appfetch_test.go create mode 100644 internal/appmanifest/manifest.go create mode 100644 internal/appmanifest/manifest_test.go create mode 100644 internal/server/app_pending_sweep.go create mode 100644 internal/server/artifact_import_parity_test.go create mode 100644 internal/server/artifact_normalize.go create mode 100644 internal/server/handlers_apps_install.go create mode 100644 internal/server/handlers_apps_install_test.go create mode 100644 internal/server/testdata/artifact_import_parity.json create mode 100644 internal/store/app_pending.go create mode 100644 internal/store/app_pending_test.go create mode 100644 internal/store/migrations/118_app_install_lifecycle.sql create mode 100644 internal/store/pgmigrations/092_app_install_lifecycle.sql diff --git a/cmd/pad/cmd_server.go b/cmd/pad/cmd_server.go index fb39db9b4..bbb5731a1 100644 --- a/cmd/pad/cmd_server.go +++ b/cmd/pad/cmd_server.go @@ -742,6 +742,9 @@ func serveCmd() *cobra.Command { } srv.StartReminderTick() + // Expired pending app installs and their staged bytes (TASK-3397). + srv.StartAppPendingSweep() + // Typed decisions (PLAN-3114 / TASK-3117). With no provider // configured the runner is nil: no write enqueues a job and no // tick starts. A provider that is NAMED but cannot be built (an diff --git a/internal/appfetch/appfetch.go b/internal/appfetch/appfetch.go new file mode 100644 index 000000000..1c7bb63c5 --- /dev/null +++ b/internal/appfetch/appfetch.go @@ -0,0 +1,214 @@ +// Package appfetch is the fetch policy for app manifests and companion-pack +// artifacts (SPEC-6, DOC-3371 §2 step 1 and §9): +// - https only, and never a redirect: a 3xx is a failure, not followed; +// - no proxy from the environment; +// - a size cap per response, enforced while reading; +// - every connection screened at DIAL time against the address actually +// dialed: a non-public address (netpolicy.Blocked) is refused unless the +// URL's exact origin is on the instance admin's private-origin list with +// the fetch flag, and then only an address inside that entry's pinned +// IPs or CIDRs is dialed. Never a hostname alone, never a skip switch. +// +// Content hashes are the caller's check (the manifest declares them). +package appfetch + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/PerpetualSoftware/pad/internal/appmanifest" + "github.com/PerpetualSoftware/pad/internal/netpolicy" +) + +// PrivateOrigin is one entry of the admin's private-destination list (§9): +// an exact origin, the addresses it may resolve to, and what it may be used +// for. Only Fetch matters here; Webhook is A5's. +type PrivateOrigin struct { + Origin string `json:"origin"` + Allowed []string `json:"allowed"` // IPs or CIDRs + Fetch bool `json:"fetch"` + Webhook bool `json:"webhook"` + Path string `json:"path,omitempty"` // webhook only +} + +// ErrRefused is any fetch the policy refuses: scheme, redirect, status, +// address or size. The wrapped message says which. +var ErrRefused = errors.New("app fetch refused") + +func refused(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrRefused, fmt.Sprintf(format, args...)) +} + +// Fetcher fetches under the policy. Build one per install attempt with New. +type Fetcher struct { + client *http.Client + private map[string][]*net.IPNet // origin -> pinned nets, fetch-flagged only +} + +// lookupIP is the resolver; a variable so tests can resolve without DNS. +var lookupIP = net.DefaultResolver.LookupIP + +// New builds a Fetcher. private is the admin list (self-host only; Cloud +// passes nil, so no private destination is ever reachable). tlsConfig is nil +// in production; tests pass one that trusts their server. +func New(private []PrivateOrigin, timeout time.Duration, tlsConfig *tls.Config) (*Fetcher, error) { + f := &Fetcher{private: map[string][]*net.IPNet{}} + for i, p := range private { + if !p.Fetch { + continue + } + origin, err := appmanifest.NormalizeOrigin(p.Origin) + if err != nil { + return nil, fmt.Errorf("private origin %d: %v", i, err) + } + if len(p.Allowed) == 0 { + return nil, fmt.Errorf("private origin %s: no pinned addresses", origin) + } + for _, a := range p.Allowed { + n, err := parseNet(a) + if err != nil { + return nil, fmt.Errorf("private origin %s: %v", origin, err) + } + f.private[origin] = append(f.private[origin], n) + } + } + dialer := &net.Dialer{Timeout: 10 * time.Second} + transport := &http.Transport{ + Proxy: nil, + TLSClientConfig: tlsConfig, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 15 * time.Second, + MaxIdleConns: 4, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return f.dial(ctx, dialer, network, addr) + }, + } + f.client = &http.Client{ + Transport: transport, + Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } + return f, nil +} + +func parseNet(s string) (*net.IPNet, error) { + if strings.Contains(s, "/") { + _, n, err := net.ParseCIDR(s) + if err != nil { + return nil, fmt.Errorf("bad CIDR %q", s) + } + return n, nil + } + ip := net.ParseIP(s) + if ip == nil { + return nil, fmt.Errorf("bad IP %q", s) + } + bits := 128 + if ip.To4() != nil { + ip, bits = ip.To4(), 32 + } + return &net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)}, nil +} + +type originKey struct{} + +// dial screens the address actually dialed. The request's origin rides in the +// context, because the transport sees only host:port. +func (f *Fetcher) dial(ctx context.Context, d *net.Dialer, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, refused("bad address %q", addr) + } + origin, _ := ctx.Value(originKey{}).(string) + pinned := f.private[origin] + + var ips []net.IP + if ip := net.ParseIP(host); ip != nil { + ips = []net.IP{ip} + } else { + ips, err = lookupIP(ctx, "ip", host) + if err != nil { + return nil, refused("resolve %s: %v", host, err) + } + } + var lastErr error = refused("%s resolves to no permitted address", host) + for _, ip := range ips { + if !permitted(ip, pinned) { + continue + } + conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) + if err == nil { + return conn, nil + } + lastErr = err + } + return nil, lastErr +} + +// permitted: a pinned origin may dial only its pinned addresses (public or +// not); any other origin may dial only public addresses. +func permitted(ip net.IP, pinned []*net.IPNet) bool { + if len(pinned) > 0 { + for _, n := range pinned { + if n.Contains(ip) { + return true + } + } + return false + } + return !netpolicy.Blocked(ip) +} + +// Get fetches rawURL and returns at most maxBytes of body. Anything else +// (not https, a redirect, a non-200 status, a longer body, a refused +// address) is an error wrapping ErrRefused or a transport error. +func (f *Fetcher) Get(ctx context.Context, rawURL string, maxBytes int64) ([]byte, error) { + u, err := url.Parse(rawURL) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil { + return nil, refused("%q is not an https URL", rawURL) + } + origin, err := appmanifest.NormalizeOrigin("https://" + u.Host) + if err != nil { + return nil, refused("%q: %v", rawURL, err) + } + req, err := http.NewRequestWithContext(context.WithValue(ctx, originKey{}, origin), http.MethodGet, u.String(), nil) + if err != nil { + return nil, refused("%v", err) + } + req.Header.Set("Accept", "application/json, text/markdown, text/plain, */*") + req.Header.Set("User-Agent", "Pad-App-Installer/1") + resp, err := f.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + return nil, refused("%s redirected (%d); redirects are never followed", rawURL, resp.StatusCode) + } + if resp.StatusCode != http.StatusOK { + return nil, refused("%s answered %d", rawURL, resp.StatusCode) + } + if resp.ContentLength > maxBytes { + return nil, refused("%s is larger than %d bytes", rawURL, maxBytes) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > maxBytes { + return nil, refused("%s is larger than %d bytes", rawURL, maxBytes) + } + return body, nil +} diff --git a/internal/appfetch/appfetch_test.go b/internal/appfetch/appfetch_test.go new file mode 100644 index 000000000..1ded25a3a --- /dev/null +++ b/internal/appfetch/appfetch_test.go @@ -0,0 +1,145 @@ +package appfetch + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// tlsServer serves h over TLS on 127.0.0.1 and returns its origin plus a +// TLS config trusting it. +func tlsServer(t *testing.T, h http.Handler) (*httptest.Server, *tls.Config) { + t.Helper() + srv := httptest.NewTLSServer(h) + t.Cleanup(srv.Close) + pool := x509.NewCertPool() + pool.AddCert(srv.Certificate()) + return srv, &tls.Config{RootCAs: pool} +} + +func pinnedTo(srv *httptest.Server) []PrivateOrigin { + return []PrivateOrigin{{Origin: srv.URL, Allowed: []string{"127.0.0.1"}, Fetch: true}} +} + +func TestGet_PinnedPrivateOriginFetches(t *testing.T) { + srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("manifest")) })) + f, err := New(pinnedTo(srv), 5*time.Second, cfg) + if err != nil { + t.Fatal(err) + } + got, err := f.Get(context.Background(), srv.URL+"/.well-known/pad-app.json", 100) + if err != nil || string(got) != "manifest" { + t.Fatalf("got %q, %v", got, err) + } +} + +// Without the admin entry, the same loopback server is refused at dial time. +func TestGet_PrivateAddressRefusedWithoutAnEntry(t *testing.T) { + srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("x")) })) + for name, list := range map[string][]PrivateOrigin{ + "no list": nil, + "entry without fetch": {{Origin: srv.URL, Allowed: []string{"127.0.0.1"}, Fetch: false, Webhook: true}}, + "another origin": {{Origin: "https://other.example", Allowed: []string{"127.0.0.1"}, Fetch: true}}, + } { + t.Run(name, func(t *testing.T) { + f, err := New(list, 5*time.Second, cfg) + if err != nil { + t.Fatal(err) + } + if _, err := f.Get(context.Background(), srv.URL+"/m", 100); !errors.Is(err, ErrRefused) { + t.Fatalf("got %v, want ErrRefused", err) + } + }) + } +} + +// A pinned origin dials ONLY its pinned addresses, even public ones it +// resolves to elsewhere. +func TestGet_PinnedOriginOutsideItsPinsIsRefused(t *testing.T) { + srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("x")) })) + f, err := New([]PrivateOrigin{{Origin: srv.URL, Allowed: []string{"10.9.9.9/32"}, Fetch: true}}, 5*time.Second, cfg) + if err != nil { + t.Fatal(err) + } + if _, err := f.Get(context.Background(), srv.URL+"/m", 100); !errors.Is(err, ErrRefused) { + t.Fatalf("got %v", err) + } +} + +// A hostname resolving to a private address is refused at dial: DNS cannot +// launder an address the literal check would refuse. +func TestGet_HostnameResolvingPrivateIsRefused(t *testing.T) { + srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("x")) })) + _, port, _ := net.SplitHostPort(strings.TrimPrefix(srv.URL, "https://")) + orig := lookupIP + lookupIP = func(ctx context.Context, network, host string) ([]net.IP, error) { + return []net.IP{net.ParseIP("127.0.0.1")}, nil + } + t.Cleanup(func() { lookupIP = orig }) + f, err := New(nil, 5*time.Second, cfg) + if err != nil { + t.Fatal(err) + } + if _, err := f.Get(context.Background(), "https://app.example:"+port+"/m", 100); !errors.Is(err, ErrRefused) { + t.Fatalf("got %v", err) + } +} + +func TestGet_RedirectStatusAndSizeAreRefused(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/redirect", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ok", http.StatusFound) }) + mux.HandleFunc("/ok", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) }) + mux.HandleFunc("/missing", func(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) }) + mux.HandleFunc("/big", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(strings.Repeat("x", 101))) }) + mux.HandleFunc("/big-chunked", func(w http.ResponseWriter, r *http.Request) { + fl := w.(http.Flusher) + for i := 0; i < 11; i++ { + _, _ = w.Write([]byte(strings.Repeat("y", 10))) + fl.Flush() + } + }) + srv, cfg := tlsServer(t, mux) + f, err := New(pinnedTo(srv), 5*time.Second, cfg) + if err != nil { + t.Fatal(err) + } + for _, p := range []string{"/redirect", "/missing", "/big", "/big-chunked"} { + if _, err := f.Get(context.Background(), srv.URL+p, 100); !errors.Is(err, ErrRefused) { + t.Errorf("%s: got %v, want ErrRefused", p, err) + } + } + if got, err := f.Get(context.Background(), srv.URL+"/ok", 100); err != nil || string(got) != "ok" { + t.Errorf("/ok: %q %v", got, err) + } +} + +func TestGet_OnlyHTTPS(t *testing.T) { + f, err := New(nil, time.Second, nil) + if err != nil { + t.Fatal(err) + } + for _, u := range []string{"http://example.com/m", "ftp://example.com/m", "https://user:pw@example.com/m", "/relative"} { + if _, err := f.Get(context.Background(), u, 10); !errors.Is(err, ErrRefused) { + t.Errorf("%s: got %v", u, err) + } + } +} + +func TestNew_RejectsBadEntries(t *testing.T) { + for _, p := range []PrivateOrigin{ + {Origin: "http://x.example", Allowed: []string{"10.0.0.1"}, Fetch: true}, + {Origin: "https://x.example", Fetch: true}, + {Origin: "https://x.example", Allowed: []string{"not-an-ip"}, Fetch: true}, + } { + if _, err := New([]PrivateOrigin{p}, time.Second, nil); err == nil { + t.Errorf("accepted %+v", p) + } + } +} diff --git a/internal/appmanifest/manifest.go b/internal/appmanifest/manifest.go new file mode 100644 index 000000000..e9fea85a6 --- /dev/null +++ b/internal/appmanifest/manifest.go @@ -0,0 +1,433 @@ +// Package appmanifest parses and validates an app's apps/1 manifest (SPEC-6, +// DOC-3371 §1). It is pure: no network, no database. The fetch policy lives +// in internal/appfetch and the staging and provisioning in internal/store. +// +// The manifest is published at {base_url}/.well-known/pad-app.json and is a +// contract surface. Validation is strict: an unknown key is refused rather +// than ignored, so a manifest written for a later contract cannot install +// here with part of it silently dropped. +package appmanifest + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "regexp" + "strings" + + "github.com/PerpetualSoftware/pad/internal/models" +) + +// Contract versions this server speaks. +const ( + AppsContract = 1 + EventsContract = 1 +) + +// Caps (DOC-3371 §2 "Staging caps"). +const ( + MaxManifestBytes = 256 << 10 + MaxArtifacts = 32 + MaxCollections = 32 + MaxEvents = 64 + MaxItemActions = 32 + MaxRedirectURIs = 16 + MaxConfigSchemaBytes = 64 << 10 + maxTextLen = 2000 +) + +// WellKnownPath is where the manifest is published under base_url. +const WellKnownPath = "/.well-known/pad-app.json" + +// SubscribableEvents are the v1 events an app may subscribe to: those whose +// subject resolves to exactly one companion item or comment (§5). +var SubscribableEvents = map[string]bool{ + "item.created": true, "item.updated": true, "item.status_changed": true, + "item.deleted": true, "item.restored": true, + "comment.created": true, "comment.updated": true, "comment.deleted": true, +} + +// Manifest is a parsed, validated apps/1 manifest. +type Manifest struct { + ID string `json:"id"` + Version string `json:"version"` + MinContract MinContract `json:"min_contract"` + Title string `json:"title"` + Description string `json:"description"` + Publisher string `json:"publisher"` + Homepage string `json:"homepage"` + BaseURL string `json:"base_url"` + RedirectURIs []string `json:"redirect_uris"` + Scopes Scopes `json:"scopes"` + Events []Event `json:"events"` + WebhookURL string `json:"webhook_url"` + CompanionPack CompanionPack `json:"companion_pack"` + ItemActions []ItemAction `json:"item_actions"` + ConfigSchema json.RawMessage `json:"config_schema"` + Docs string `json:"docs"` + + // Origin is the normalized origin of BaseURL: the app's identity. + Origin string `json:"-"` +} + +type MinContract struct { + Apps int `json:"apps"` + Events int `json:"events"` +} + +type Scopes struct { + Service Access `json:"service"` + Delegated Access `json:"delegated"` +} + +type Access struct { + Access string `json:"access"` +} + +type Event struct { + Name string `json:"name"` + Collections []string `json:"collections"` +} + +type CompanionPack struct { + Collections []Collection `json:"collections"` + Artifacts []Artifact `json:"artifacts"` +} + +type Collection struct { + Key string `json:"key"` + Slug string `json:"slug"` + Name string `json:"name"` + Schema json.RawMessage `json:"schema"` + + // Parsed is Schema, parsed. + Parsed models.CollectionSchema `json:"-"` +} + +type Artifact struct { + Key string `json:"key"` + URL string `json:"url"` + SHA256 string `json:"sha256"` +} + +type ItemAction struct { + Key string `json:"key"` + Label string `json:"label"` + Collections []string `json:"collections"` + Path string `json:"path"` +} + +// Error is one validation failure, at a JSON path in the manifest. +type Error struct { + Path string `json:"path"` + Reason string `json:"reason"` +} + +func (e *Error) Error() string { return "app manifest: " + e.Path + ": " + e.Reason } + +func fail(path, format string, args ...any) error { + return &Error{Path: path, Reason: fmt.Sprintf(format, args...)} +} + +// IsError reports whether err is a manifest validation failure. +func IsError(err error) bool { + var e *Error + return errors.As(err, &e) +} + +var ( + keyRE = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) + slugRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$`) + idRE = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,63}/[a-z0-9][a-z0-9._-]{0,63}$`) + semverRE = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`) + sha256RE = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +// Parse decodes and validates manifest bytes. The first failure is returned +// as an *Error naming its path. +func Parse(data []byte) (*Manifest, error) { + if len(data) > MaxManifestBytes { + return nil, fail("$", "manifest exceeds %d KiB", MaxManifestBytes>>10) + } + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + var m Manifest + if err := dec.Decode(&m); err != nil { + return nil, fail("$", "not a valid apps/1 manifest: %v", err) + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return nil, fail("$", "trailing data after the manifest") + } + if err := m.validate(); err != nil { + return nil, err + } + return &m, nil +} + +func (m *Manifest) validate() error { + if !idRE.MatchString(m.ID) { + return fail("id", "must be publisher/name in lower case") + } + if !semverRE.MatchString(m.Version) { + return fail("version", "must be a semantic version") + } + if m.MinContract.Apps < 1 || m.MinContract.Apps > AppsContract { + return fail("min_contract.apps", "this server speaks apps/%d", AppsContract) + } + if m.MinContract.Events < 1 || m.MinContract.Events > EventsContract { + return fail("min_contract.events", "this server speaks events/%d", EventsContract) + } + for _, f := range []struct{ path, v string }{{"title", m.Title}, {"publisher", m.Publisher}} { + if strings.TrimSpace(f.v) == "" { + return fail(f.path, "is required") + } + } + for _, f := range []struct{ path, v string }{{"title", m.Title}, {"description", m.Description}, {"publisher", m.Publisher}} { + if len(f.v) > maxTextLen { + return fail(f.path, "is longer than %d characters", maxTextLen) + } + } + + origin, err := NormalizeOrigin(m.BaseURL) + if err != nil { + return fail("base_url", "%v", err) + } + m.Origin = origin + if m.Homepage != "" { + if err := m.under(m.Homepage); err != nil { + return fail("homepage", "%v", err) + } + } + if len(m.RedirectURIs) == 0 || len(m.RedirectURIs) > MaxRedirectURIs { + return fail("redirect_uris", "between 1 and %d are required", MaxRedirectURIs) + } + for i, u := range m.RedirectURIs { + if err := m.under(u); err != nil { + return fail(fmt.Sprintf("redirect_uris[%d]", i), "%v", err) + } + } + for _, f := range []struct{ path, v string }{{"scopes.service.access", m.Scopes.Service.Access}, {"scopes.delegated.access", m.Scopes.Delegated.Access}} { + if f.v != "read" && f.v != "write" { + return fail(f.path, "must be read or write") + } + } + if m.WebhookURL != "" { + if err := m.under(m.WebhookURL); err != nil { + return fail("webhook_url", "%v", err) + } + } + if m.Docs != "" { + if err := m.under(m.Docs); err != nil { + return fail("docs", "%v", err) + } + } + + // Companion pack. + pack := m.CompanionPack + if len(pack.Collections) == 0 || len(pack.Collections) > MaxCollections { + return fail("companion_pack.collections", "between 1 and %d are required", MaxCollections) + } + keys := map[string]bool{} + slugs := map[string]bool{} + for i := range pack.Collections { + c := &m.CompanionPack.Collections[i] + p := fmt.Sprintf("companion_pack.collections[%d]", i) + if !keyRE.MatchString(c.Key) { + return fail(p+".key", "must be a lower-case key") + } + if keys[c.Key] { + return fail(p+".key", "duplicate key %q", c.Key) + } + keys[c.Key] = true + if !slugRE.MatchString(c.Slug) { + return fail(p+".slug", "must be a lower-case slug") + } + if slugs[c.Slug] { + return fail(p+".slug", "duplicate slug %q", c.Slug) + } + slugs[c.Slug] = true + if len(c.Name) > 200 { + return fail(p+".name", "is longer than 200 characters") + } + if len(c.Schema) == 0 { + return fail(p+".schema", "is required") + } + // Strict, like the manifest itself: a misspelled key ("requried") + // would otherwise be dropped silently, losing what it meant (codex + // round 2). + sdec := json.NewDecoder(bytes.NewReader(c.Schema)) + sdec.DisallowUnknownFields() + if err := sdec.Decode(&c.Parsed); err != nil { + return fail(p+".schema", "is not a collection schema: %v", err) + } + } + if len(pack.Artifacts) > MaxArtifacts { + return fail("companion_pack.artifacts", "at most %d", MaxArtifacts) + } + artKeys := map[string]bool{} + for i, a := range pack.Artifacts { + p := fmt.Sprintf("companion_pack.artifacts[%d]", i) + if !keyRE.MatchString(a.Key) { + return fail(p+".key", "must be a lower-case key") + } + if artKeys[a.Key] { + return fail(p+".key", "duplicate key %q", a.Key) + } + artKeys[a.Key] = true + if err := m.under(a.URL); err != nil { + return fail(p+".url", "%v", err) + } + if !sha256RE.MatchString(a.SHA256) { + return fail(p+".sha256", "must be 64 lower-case hex characters") + } + } + + // Events: v1-subscribable names, companion keys only. + if len(m.Events) > MaxEvents { + return fail("events", "at most %d", MaxEvents) + } + if len(m.Events) > 0 && m.WebhookURL == "" { + return fail("webhook_url", "is required when events are declared") + } + for i, e := range m.Events { + p := fmt.Sprintf("events[%d]", i) + if !SubscribableEvents[e.Name] { + return fail(p+".name", "%q is not subscribable in apps/1", e.Name) + } + if len(e.Collections) == 0 { + return fail(p+".collections", "at least one companion collection key is required") + } + for j, k := range e.Collections { + if !keys[k] { + return fail(fmt.Sprintf("%s.collections[%d]", p, j), "%q is not a companion collection key", k) + } + } + } + + // Item actions. + if len(m.ItemActions) > MaxItemActions { + return fail("item_actions", "at most %d", MaxItemActions) + } + actKeys := map[string]bool{} + for i, a := range m.ItemActions { + p := fmt.Sprintf("item_actions[%d]", i) + if !keyRE.MatchString(a.Key) { + return fail(p+".key", "must be a lower-case key") + } + if actKeys[a.Key] { + return fail(p+".key", "duplicate key %q", a.Key) + } + actKeys[a.Key] = true + if strings.TrimSpace(a.Label) == "" || len(a.Label) > 80 { + return fail(p+".label", "is required, at most 80 characters") + } + if len(a.Collections) == 0 { + return fail(p+".collections", "at least one companion collection key is required") + } + for j, k := range a.Collections { + if !keys[k] { + return fail(fmt.Sprintf("%s.collections[%d]", p, j), "%q is not a companion collection key", k) + } + } + if !absolutePath(a.Path) { + return fail(p+".path", "must be an absolute path under base_url") + } + } + + // Config schema: a JSON object, size-capped; values are owner-edited and + // never secrets (§1). It is not evaluated here. + if len(m.ConfigSchema) > 0 { + if len(m.ConfigSchema) > MaxConfigSchemaBytes { + return fail("config_schema", "exceeds %d KiB", MaxConfigSchemaBytes>>10) + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(m.ConfigSchema, &obj); err != nil { + return fail("config_schema", "must be a JSON Schema object") + } + } + return nil +} + +// NormalizeOrigin returns the canonical origin of an https base URL: +// lower-case host, no default port, and no path, query, fragment or +// userinfo. It is the app's identity (DOC-3371 Definitions). +func NormalizeOrigin(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return "", fmt.Errorf("is not a URL") + } + if u.Scheme != "https" { + return "", fmt.Errorf("must be https") + } + if u.User != nil { + return "", fmt.Errorf("must not carry credentials") + } + if u.Host == "" || u.Hostname() == "" { + return "", fmt.Errorf("must have a host") + } + if (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" { + return "", fmt.Errorf("must be an origin, with no path, query or fragment") + } + host := strings.ToLower(u.Hostname()) + port := u.Port() + if port == "443" { + port = "" + } + if port != "" { + return "https://" + joinHostPort(host, port), nil + } + if strings.Contains(host, ":") { + return "https://[" + host + "]", nil + } + return "https://" + host, nil +} + +func joinHostPort(host, port string) string { + if strings.Contains(host, ":") { + return "[" + host + "]:" + port + } + return host + ":" + port +} + +// under requires raw to be an https URL on exactly the manifest's origin. +func (m *Manifest) under(raw string) error { + u, err := url.Parse(raw) + if err != nil || u.Scheme != "https" || u.Host == "" { + return fmt.Errorf("must be an https URL under base_url") + } + if u.User != nil { + return fmt.Errorf("must not carry credentials") + } + o, err := NormalizeOrigin("https://" + u.Host) + if err != nil || o != m.Origin { + return fmt.Errorf("must be under base_url (%s)", m.Origin) + } + return nil +} + +// absolutePath accepts a path to join onto the origin: it starts with one +// "/", and has no scheme, authority, backslash or control character. A +// "//host/x" would keep the origin when concatenated but name another host +// when resolved as a reference, so it is refused outright. +func absolutePath(p string) bool { + if !strings.HasPrefix(p, "/") || strings.HasPrefix(p, "//") || len(p) > 1024 { + return false + } + for _, r := range p { + if r == '\\' || r < 0x20 || r == 0x7f { + return false + } + } + u, err := url.Parse(p) + return err == nil && u.Scheme == "" && u.Host == "" && u.User == nil && u.Opaque == "" +} + +// ManifestURL is where a base URL's manifest is published. +func ManifestURL(origin string) string { return origin + WellKnownPath } + +// SHA256Hex is the lower-case hex sha256 the manifest and digests use. +func SHA256Hex(sum [32]byte) string { return hex.EncodeToString(sum[:]) } diff --git a/internal/appmanifest/manifest_test.go b/internal/appmanifest/manifest_test.go new file mode 100644 index 000000000..fd3d3db8c --- /dev/null +++ b/internal/appmanifest/manifest_test.go @@ -0,0 +1,172 @@ +package appmanifest + +import ( + "encoding/json" + "strings" + "testing" +) + +func validManifest() map[string]any { + return map[string]any{ + "id": "acme/support-portal", + "version": "1.2.0", + "min_contract": map[string]any{"apps": 1, "events": 1}, + "title": "Support Portal", + "description": "Requester portal", + "publisher": "Acme", + "homepage": "https://portal.acme.example/about", + "base_url": "https://Portal.Acme.Example:443/", + "redirect_uris": []any{"https://portal.acme.example/oauth/callback"}, + "scopes": map[string]any{ + "service": map[string]any{"access": "write"}, + "delegated": map[string]any{"access": "read"}, + }, + "events": []any{map[string]any{"name": "item.created", "collections": []any{"tickets"}}}, + "webhook_url": "https://portal.acme.example/hooks/pad", + "companion_pack": map[string]any{ + "collections": []any{map[string]any{"key": "tickets", "slug": "tickets", "name": "Tickets", + "schema": map[string]any{"fields": []any{map[string]any{"key": "status", "label": "Status", "type": "select", "options": []any{"open", "solved"}}}}}}, + "artifacts": []any{map[string]any{"key": "triage", "url": "https://portal.acme.example/pack/triage.md", "sha256": strings.Repeat("ab", 32)}}, + }, + "item_actions": []any{map[string]any{"key": "open", "label": "Open in portal", "collections": []any{"tickets"}, "path": "/tickets"}}, + "config_schema": map[string]any{"type": "object"}, + "docs": "https://portal.acme.example/llms.txt", + } +} + +func encode(t *testing.T, m map[string]any) []byte { + t.Helper() + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestParse_Valid(t *testing.T) { + m, err := Parse(encode(t, validManifest())) + if err != nil { + t.Fatal(err) + } + if m.Origin != "https://portal.acme.example" { + t.Errorf("origin %q", m.Origin) + } + if len(m.CompanionPack.Collections[0].Parsed.Fields) != 1 { + t.Errorf("schema not parsed") + } + if ManifestURL(m.Origin) != "https://portal.acme.example/.well-known/pad-app.json" { + t.Errorf("manifest url %q", ManifestURL(m.Origin)) + } +} + +// Each mutation breaks one rule; Parse must refuse it at the named path. +func TestParse_Refusals(t *testing.T) { + type mut func(m map[string]any) + pack := func(m map[string]any) map[string]any { return m["companion_pack"].(map[string]any) } + coll := func(m map[string]any) map[string]any { return pack(m)["collections"].([]any)[0].(map[string]any) } + art := func(m map[string]any) map[string]any { return pack(m)["artifacts"].([]any)[0].(map[string]any) } + cases := map[string]struct { + mut mut + path string + }{ + "unknown top-level key": {func(m map[string]any) { m["extra"] = 1 }, "$"}, + "bad id": {func(m map[string]any) { m["id"] = "Acme Portal" }, "id"}, + "bad version": {func(m map[string]any) { m["version"] = "1.2" }, "version"}, + "future apps contract": {func(m map[string]any) { m["min_contract"] = map[string]any{"apps": 2, "events": 1} }, "min_contract.apps"}, + "future events contract": {func(m map[string]any) { m["min_contract"] = map[string]any{"apps": 1, "events": 9} }, "min_contract.events"}, + "missing title": {func(m map[string]any) { m["title"] = " " }, "title"}, + "http base_url": {func(m map[string]any) { m["base_url"] = "http://portal.acme.example" }, "base_url"}, + "base_url with a path": {func(m map[string]any) { m["base_url"] = "https://portal.acme.example/app" }, "base_url"}, + "base_url with userinfo": {func(m map[string]any) { m["base_url"] = "https://u:p@portal.acme.example" }, "base_url"}, + "no redirect uris": {func(m map[string]any) { m["redirect_uris"] = []any{} }, "redirect_uris"}, + "redirect off origin": {func(m map[string]any) { m["redirect_uris"] = []any{"https://evil.example/cb"} }, "redirect_uris[0]"}, + "redirect on other port": {func(m map[string]any) { m["redirect_uris"] = []any{"https://portal.acme.example:8443/cb"} }, "redirect_uris[0]"}, + "redirect over http": {func(m map[string]any) { m["redirect_uris"] = []any{"http://portal.acme.example/cb"} }, "redirect_uris[0]"}, + "bad service access": {func(m map[string]any) { m["scopes"].(map[string]any)["service"] = map[string]any{"access": "admin"} }, "scopes.service.access"}, + "webhook off origin": {func(m map[string]any) { m["webhook_url"] = "https://evil.example/h" }, "webhook_url"}, + "events without webhook": {func(m map[string]any) { delete(m, "webhook_url") }, "webhook_url"}, + "unsubscribable event": {func(m map[string]any) { + m["events"] = []any{map[string]any{"name": "item.bulk_updated", "collections": []any{"tickets"}}} + }, "events[0].name"}, + "attachment event": {func(m map[string]any) { + m["events"] = []any{map[string]any{"name": "attachment.added", "collections": []any{"tickets"}}} + }, "events[0].name"}, + "event on unknown key": {func(m map[string]any) { + m["events"] = []any{map[string]any{"name": "item.created", "collections": []any{"other"}}} + }, "events[0].collections[0]"}, + "no collections": {func(m map[string]any) { pack(m)["collections"] = []any{} }, "companion_pack.collections"}, + "bad collection slug": {func(m map[string]any) { coll(m)["slug"] = "Tickets!" }, "companion_pack.collections[0].slug"}, + "bad collection schema": {func(m map[string]any) { coll(m)["schema"] = "not an object" }, "companion_pack.collections[0].schema"}, + "artifact off origin": {func(m map[string]any) { art(m)["url"] = "https://cdn.example/x.md" }, "companion_pack.artifacts[0].url"}, + "artifact bad sha256": {func(m map[string]any) { art(m)["sha256"] = "ABC" }, "companion_pack.artifacts[0].sha256"}, + "too many artifacts": {func(m map[string]any) { + var arts []any + for i := 0; i < MaxArtifacts+1; i++ { + arts = append(arts, map[string]any{"key": "a" + strings.Repeat("x", i%3) + string(rune('a'+i%26)) + string(rune('a'+i/26)), "url": "https://portal.acme.example/a.md", "sha256": strings.Repeat("ab", 32)}) + } + pack(m)["artifacts"] = arts + }, "companion_pack.artifacts"}, + "action on unknown key": {func(m map[string]any) { + m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"nope"}, "path": "/x"}} + }, "item_actions[0].collections[0]"}, + "action relative path": {func(m map[string]any) { + m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"tickets"}, "path": "x"}} + }, "item_actions[0].path"}, + "action protocol-relative": {func(m map[string]any) { + m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"tickets"}, "path": "//evil.example/x"}} + }, "item_actions[0].path"}, + "action backslash path": {func(m map[string]any) { + m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"tickets"}, "path": "/\\evil.example/x"}} + }, "item_actions[0].path"}, + "config schema not object": {func(m map[string]any) { m["config_schema"] = []any{1} }, "config_schema"}, + "schema with a misspelled key": {func(m map[string]any) { + coll(m)["schema"] = map[string]any{"fields": []any{map[string]any{"key": "status", "label": "Status", "type": "text", "requried": true}}} + }, "companion_pack.collections[0].schema"}, + "homepage off origin": {func(m map[string]any) { m["homepage"] = "https://other.example/" }, "homepage"}, + "docs off origin": {func(m map[string]any) { m["docs"] = "https://evil.example/llms.txt" }, "docs"}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + m := validManifest() + c.mut(m) + _, err := Parse(encode(t, m)) + var e *Error + if err == nil || !IsError(err) { + t.Fatalf("got %v, want a manifest error at %s", err, c.path) + } + e = err.(*Error) + if e.Path != c.path { + t.Fatalf("refused at %s (%s), want %s", e.Path, e.Reason, c.path) + } + }) + } +} + +func TestParse_SizeAndTrailingData(t *testing.T) { + if _, err := Parse(make([]byte, MaxManifestBytes+1)); !IsError(err) { + t.Fatalf("oversize manifest: %v", err) + } + b := append(encode(t, validManifest()), []byte(` {}`)...) + if _, err := Parse(b); !IsError(err) { + t.Fatalf("trailing data: %v", err) + } +} + +func TestNormalizeOrigin(t *testing.T) { + for in, want := range map[string]string{ + "https://EXAMPLE.com": "https://example.com", + "https://example.com:443/": "https://example.com", + "https://example.com:8443": "https://example.com:8443", + "https://[::1]:8443": "https://[::1]:8443", + } { + got, err := NormalizeOrigin(in) + if err != nil || got != want { + t.Errorf("%s: %q %v, want %q", in, got, err, want) + } + } + for _, bad := range []string{"http://example.com", "https://example.com/x", "https://example.com?q", "https://u@example.com", "example.com", "https://"} { + if _, err := NormalizeOrigin(bad); err == nil { + t.Errorf("%s accepted", bad) + } + } +} diff --git a/internal/models/reserved_field_decode_guard_test.go b/internal/models/reserved_field_decode_guard_test.go index 6ecb2b3b8..1b76c6d00 100644 --- a/internal/models/reserved_field_decode_guard_test.go +++ b/internal/models/reserved_field_decode_guard_test.go @@ -72,6 +72,7 @@ var rawDecodeAllowed = map[string]string{ "internal/server/handlers_collections.go::prevSchema": "the grandfather test's own baseline: a stripped prevSchema would reclassify every EXISTING declaration as newly introduced and refuse every update to such a collection", "cmd/pad/cmd_collection.go::schema": "CLI --schema parse and the collection list's own status-option rendering — the collection's definition, not any item's fields", "internal/mcp/dispatch_http_routes.go::schema": "MCP collection schema INPUT, the remote twin of handlers_collections.go", + "internal/appmanifest/manifest.go::Parsed": "an app manifest's companion collection DEFINITION (TASK-3397), decoded strictly so a misspelled key is refused; the install preview then runs validateNoReservedFieldKeys on it, as collection create does", // The move / copy family. These decode raw and strip at the point of use // via items.SchemaForMigratedFields (BUG-2674), which is the same @@ -556,7 +557,7 @@ func TestGuardAllowListHasNoDeadEntries(t *testing.T) { // // 22 since BUG-3202 added models/fields_json.go's generic number-preserving // decoder, a fourth INTERFACE destination. -const rawDecodeSiteCount = 22 +const rawDecodeSiteCount = 23 func TestGuardSeesEveryKnownDecodeSite(t *testing.T) { sites := collectSchemaDecodes(t, repoRoot(t)) diff --git a/internal/server/app_pending_sweep.go b/internal/server/app_pending_sweep.go new file mode 100644 index 000000000..d4489e67d --- /dev/null +++ b/internal/server/app_pending_sweep.go @@ -0,0 +1,78 @@ +package server + +import ( + "log/slog" + "sync" + "time" +) + +// The expired pending-install sweep (SPEC-6 U8a, DOC-3371 §2: "expiry (1 +// hour, swept)"; TASK-3397). Expired reservations already stop counting +// against the caps, because every count reads expires_at; this sweep is what +// deletes their staged bytes, so an abandoned preview does not keep them +// until some later install happens to sweep (codex round 1). Same lifecycle +// shape as the reminder tick: its own mutex and stop channel, tracked by +// Server.bg so Stop() drains it, started only from the real bootstrap path. + +const defaultAppPendingSweepInterval = 5 * time.Minute + +type appPendingSweepConfig struct { + mu sync.Mutex + interval time.Duration + stop chan struct{} + running bool +} + +// StartAppPendingSweep starts the sweep. Idempotent. +func (s *Server) StartAppPendingSweep() { + s.appPendingSweep.mu.Lock() + if s.appPendingSweep.running { + s.appPendingSweep.mu.Unlock() + return + } + if s.appPendingSweep.interval == 0 { + s.appPendingSweep.interval = defaultAppPendingSweepInterval + } + s.appPendingSweep.stop = make(chan struct{}) + s.appPendingSweep.running = true + interval, stop := s.appPendingSweep.interval, s.appPendingSweep.stop + s.appPendingSweep.mu.Unlock() + + s.bg.Add(1) + go func() { + defer s.bg.Done() + defer s.recoverSweeper("app-pending-sweep") + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + s.runAppPendingSweep() + } + } + }() +} + +func (s *Server) stopAppPendingSweep() { + s.appPendingSweep.mu.Lock() + defer s.appPendingSweep.mu.Unlock() + if !s.appPendingSweep.running { + return + } + close(s.appPendingSweep.stop) + s.appPendingSweep.running = false +} + +// runAppPendingSweep is one pass. +func (s *Server) runAppPendingSweep() { + n, err := s.store.SweepExpiredPendingInstalls() + if err != nil { + slog.Warn("apps: pending install sweep failed", "error", err) + return + } + if n > 0 { + slog.Info("apps: swept expired pending installs", "count", n) + } +} diff --git a/internal/server/artifact_import.go b/internal/server/artifact_import.go index fca3b06c6..16f381b7f 100644 --- a/internal/server/artifact_import.go +++ b/internal/server/artifact_import.go @@ -107,6 +107,13 @@ func parseArtifactRequest(w http.ResponseWriter, r *http.Request, maxBytes int64 return artifact.Artifact{}, fmt.Errorf("artifact import: read body: %w", err) } + return decodeArtifactBytes(data) +} + +// decodeArtifactBytes is parseArtifactRequest's steps (2) to (5) over bytes +// already read and size-capped: the import door and the app installer's +// staged artifacts (TASK-3397) run the same guards and decode. +func decodeArtifactBytes(data []byte) (artifact.Artifact, error) { // (2) The artifact body is TEXT bound for text columns, and this // handler reads it directly rather than through decodeJSON, so it // inherits neither BUG-2803's refusal nor the path/query rule (a body is diff --git a/internal/server/artifact_import_parity_test.go b/internal/server/artifact_import_parity_test.go new file mode 100644 index 000000000..59e05fb14 --- /dev/null +++ b/internal/server/artifact_import_parity_test.go @@ -0,0 +1,168 @@ +package server + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/PerpetualSoftware/pad/internal/artifact" +) + +// TASK-3397 (U8a) refactors the artifact importer's preprocess into one +// function the app installer's preview and provisioning share. This test pins +// that the HUMAN import path's output is unchanged by it: every fixture is +// imported through POST /import-artifact and the stored item (title, content, +// fields) plus the response's warnings are compared against a golden captured +// from the importer BEFORE the refactor. +// +// Regenerate (only for an intended behaviour change): +// +// PAD_UPDATE_ARTIFACT_PARITY=1 go test ./internal/server/ -run TestArtifactImportParity + +const artifactParityGolden = "testdata/artifact_import_parity.json" + +type parityCase struct { + name string + setup func(t *testing.T, srv *Server, ws string) + body func(t *testing.T) []byte +} + +type parityResult struct { + Status int `json:"status"` + Title string `json:"title"` + Content string `json:"content"` + Fields map[string]any `json:"fields"` + Warnings []string `json:"warnings"` +} + +func encodeArtifact(t *testing.T, a artifact.Artifact) []byte { + t.Helper() + a.FormatVersion = artifact.FormatVersion + data, err := artifact.Encode(a) + if err != nil { + t.Fatal(err) + } + return data +} + +func fixtureFile(name string) func(t *testing.T) []byte { + return func(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", name)) + if err != nil { + t.Fatal(err) + } + return data + } +} + +func artifactParityCases() []parityCase { + return []parityCase{ + {name: "golden playbook", body: fixtureFile("playbook.golden.md")}, + {name: "golden convention", body: fixtureFile("convention.golden.md")}, + {name: "convention, active, valid selects", body: func(t *testing.T) []byte { + return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: "Clean Convention", + Fields: map[string]any{"status": "active", "trigger": "on-commit", "scope": "all", "priority": "must"}, Body: "A clean convention.\n"}) + }}, + {name: "convention, foreign select value", body: func(t *testing.T) []byte { + return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: "Foreign", + Fields: map[string]any{"status": "active", "trigger": "on-candidate-advance", "scope": "all"}, Body: "x\n"}) + }}, + {name: "playbook with arguments", body: func(t *testing.T) []byte { + return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindPlaybook, Title: "Playbook With Args", + Fields: map[string]any{"status": "active", "trigger": "manual", "scope": "all", "invocation_slug": "with-args", + "arguments": []map[string]any{{"name": "target", "type": "ref", "required": true, "description": "the thing"}}}, + Body: "Do the thing.\n"}) + }}, + {name: "playbook, slug collision", setup: func(t *testing.T, srv *Server, ws string) { + t.Helper() + data := encodeArtifact(t, artifact.Artifact{Kind: artifact.KindPlaybook, Title: "First", + Fields: map[string]any{"status": "draft", "invocation_slug": "ship"}, Body: "first\n"}) + if rr := doArtifactRequest(srv, "POST", "/api/v1/workspaces/"+ws+"/import-artifact", data); rr.Code != http.StatusCreated { + t.Fatalf("setup import: %d %s", rr.Code, rr.Body.String()) + } + }, body: func(t *testing.T) []byte { + return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindPlaybook, Title: "Second", + Fields: map[string]any{"status": "active", "invocation_slug": "ship"}, Body: "second\n"}) + }}, + {name: "with provenance", body: func(t *testing.T) []byte { + return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: "Provenanced", + Fields: map[string]any{"status": "draft", "trigger": "on-commit"}, + Body: "Body.\n", + Provenance: artifact.Provenance{Workspace: "elsewhere", Author: "someone", ExportedAt: "2026-01-02T03:04:05Z"}}) + }}, + {name: "blank title", body: func(t *testing.T) []byte { + return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: " ", + Fields: map[string]any{"status": "draft"}, Body: "x\n"}) + }}, + } +} + +func runParityCase(t *testing.T, c parityCase) parityResult { + t.Helper() + srv := testServer(t) + ws := createWSForTest(t, srv) + if c.setup != nil { + c.setup(t, srv, ws) + } + rr := doArtifactRequest(srv, "POST", "/api/v1/workspaces/"+ws+"/import-artifact", c.body(t)) + res := parityResult{Status: rr.Code} + if rr.Code != http.StatusCreated { + return res + } + var resp artifactImportResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + res.Warnings = resp.Warnings + sort.Strings(res.Warnings) + item := getItem(t, srv, ws, resp.Slug) + res.Title, res.Content = item.Title, item.Content + if err := json.Unmarshal([]byte(item.Fields), &res.Fields); err != nil { + t.Fatal(err) + } + return res +} + +func TestArtifactImportParity(t *testing.T) { + got := map[string]parityResult{} + for _, c := range artifactParityCases() { + got[c.name] = runParityCase(t, c) + } + if os.Getenv("PAD_UPDATE_ARTIFACT_PARITY") == "1" { + b, err := json.MarshalIndent(got, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(artifactParityGolden, append(b, '\n'), 0o644); err != nil { + t.Fatal(err) + } + return + } + raw, err := os.ReadFile(artifactParityGolden) + if err != nil { + t.Fatal(err) + } + var want map[string]parityResult + if err := json.Unmarshal(raw, &want); err != nil { + t.Fatal(err) + } + if len(want) != len(got) { + t.Fatalf("golden has %d cases, test has %d", len(want), len(got)) + } + for name, w := range want { + g, ok := got[name] + if !ok { + t.Errorf("case %q missing", name) + continue + } + wb, _ := json.Marshal(w) + gb, _ := json.Marshal(g) + if string(wb) != string(gb) { + t.Errorf("%s changed:\nwant %s\ngot %s", name, wb, gb) + } + } +} diff --git a/internal/server/artifact_normalize.go b/internal/server/artifact_normalize.go new file mode 100644 index 000000000..09cb327cf --- /dev/null +++ b/internal/server/artifact_normalize.go @@ -0,0 +1,72 @@ +package server + +import ( + "encoding/json" + "fmt" + + "github.com/PerpetualSoftware/pad/internal/artifact" + "github.com/PerpetualSoftware/pad/internal/models" +) + +// normalizedArtifact is what the importer stores for an artifact: the item's +// title, content and fields after the forgiving preprocess, and the warnings +// that preprocess produced. +type normalizedArtifact struct { + Title string + Content string + // FieldsJSON is the stored fields blob. + FieldsJSON string + // Fields is FieldsJSON decoded back into canonical JSON types ([]any, + // map[string]any), the shape the create path validates. + Fields map[string]any + Warnings []string +} + +// normalizeArtifact is the artifact importer's preprocess, shared by the human +// import (handleImportArtifact) and the app installer's preview and +// provisioning (SPEC-6 U8, DOC-3371 §2 step 3), so the item an owner reviews +// is the item that is stored. It never mutates art. +// +// Its output depends on the destination: select values the schema does not +// offer are cleared, and a playbook's invocation_slug is de-collided against +// slugTaken. That is why provisioning re-runs it on its own transaction and +// compares the digest with the one reviewed. +// +// The caller validates the title (models.ValidateItemTitle) first; a title +// that fails there must not reach this function. +func normalizeArtifact(art artifact.Artifact, coll *models.Collection, schema models.CollectionSchema, slugTaken func(string) (bool, error)) (*normalizedArtifact, error) { + // Copy the artifact fields so the preprocess never mutates the decoded + // value (keeps the parse layer's output immutable from the caller's POV). + fields := make(map[string]any, len(art.Fields)) + for k, v := range art.Fields { + fields[k] = v + } + warnings := preprocessArtifactFields(art.Kind, schema, fields, slugTaken) + + fieldsJSON, err := json.Marshal(fields) + if err != nil { + return nil, fmt.Errorf("marshal fields: %w", err) + } + // Normalize the field map through JSON so its nested types match what the + // normal create path validates. handleCreateItem builds its fieldMap by + // unmarshalling the JSON request body, so structured values are canonical + // JSON types ([]any, map[string]any). The artifact decode produces Go-native + // types (e.g. arguments as []map[string]any), which ValidateFields' json case + // rejects — round-tripping fixes that without special-casing any field. + normalized := make(map[string]any) + if err := json.Unmarshal(fieldsJSON, &normalized); err != nil { + return nil, fmt.Errorf("normalize fields: %w", err) + } + + body := art.Body + if footer := artifactProvenanceFooter(art.Provenance); footer != "" { + body = body + footer + } + return &normalizedArtifact{ + Title: models.NormalizeItemTitle(art.Title), + Content: body, + FieldsJSON: string(fieldsJSON), + Fields: normalized, + Warnings: warnings, + }, nil +} diff --git a/internal/server/handlers_apps_install.go b/internal/server/handlers_apps_install.go new file mode 100644 index 000000000..99ee7560c --- /dev/null +++ b/internal/server/handlers_apps_install.go @@ -0,0 +1,752 @@ +package server + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "sort" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/PerpetualSoftware/pad/internal/appfetch" + "github.com/PerpetualSoftware/pad/internal/appmanifest" + "github.com/PerpetualSoftware/pad/internal/links" + "github.com/PerpetualSoftware/pad/internal/models" + "github.com/PerpetualSoftware/pad/internal/store" +) + +// App install staging and preview (SPEC-6 U8a, DOC-3371 §2 steps 1-5, §9; +// TASK-3397). Provisioning, the install code and redeem are U8b. +// +// POST /workspaces/{ws}/apps/install/preview {base_url} +// reserve (owner + instance caps) -> fetch the manifest and every artifact +// exactly once under the fetch policy -> stage the raw bytes -> validate +// apps/1 -> normalize each artifact with the importer's own preprocess -> +// conflict check -> a preview the owner reviews before consenting. +// GET /workspaces/{ws}/apps/install/pending/{pendingID} the staged preview +// DELETE /workspaces/{ws}/apps/install/pending/{pendingID} discard it +// +// Owner-only, and only while appsAvailable(). Nothing is provisioned here. + +// settingAppsPrivateOrigins is the admin's private-destination list (§9), +// JSON-encoded []appfetch.PrivateOrigin. Self-host only. +const settingAppsPrivateOrigins = "apps_private_origins" + +// appFetchDeadline bounds one install's whole fetch (§2 caps). +const appFetchDeadline = 30 * time.Second + +// appArtifactMaxBytes is the per-artifact cap: the importer's own default +// (§2: defaultImportArtifactMaxBytes). +const appArtifactMaxBytes = defaultImportArtifactMaxBytes + +// appNotReviewedNotice is shown with every app on every surface (Dave Q3). +const appNotReviewedNotice = "Not reviewed by Pad. You are installing code published at this origin." + +// appDeferredNotice explains declared webhook events and item actions that +// this server does not provision yet (A5/A6). They are stored with the +// install and activated when supported, without a reinstall. +const appDeferredNotice = "Webhook events and item actions will activate when this server supports them; they are recorded with the install now." + +// appPreview is what the owner reviews before consenting (§2 step 4). +type appPreview struct { + PendingID string `json:"pending_id"` + ExpiresAt time.Time `json:"expires_at"` + Origin string `json:"origin"` + ManifestURL string `json:"manifest_url"` + ManifestSHA256 string `json:"manifest_sha256"` + AppID string `json:"app_id"` + Version string `json:"version"` + Title string `json:"title"` + Description string `json:"description,omitempty"` + Publisher string `json:"publisher"` + Homepage string `json:"homepage,omitempty"` + ReviewedByPad bool `json:"reviewed_by_pad"` + Notice string `json:"notice"` + ServiceAccess string `json:"service_access"` + DelegatedAccess string `json:"delegated_access"` + ReadsSystem string `json:"reads_system_collections"` + Collections []appPreviewCollection `json:"collections"` + Events []appmanifest.Event `json:"events"` + WebhookURL string `json:"webhook_url,omitempty"` + ItemActions []appmanifest.ItemAction `json:"item_actions"` + DeferredNotice string `json:"deferred_notice,omitempty"` + Artifacts []appPreviewArtifact `json:"artifacts"` + ConfigSchema json.RawMessage `json:"config_schema,omitempty"` + RedirectURIs []string `json:"redirect_uris"` + Docs string `json:"docs,omitempty"` +} + +type appPreviewCollection struct { + Key string `json:"key"` + Slug string `json:"slug"` + Name string `json:"name"` + Schema json.RawMessage `json:"schema"` + // Adopt: a collection with this slug exists and was created by an install + // of this same origin; it is adopted rather than created. + Adopt bool `json:"adopt"` +} + +type appPreviewArtifact struct { + Key string `json:"key"` + URL string `json:"url"` + Kind string `json:"kind"` + DestinationCollection string `json:"destination_collection"` + RawSHA256 string `json:"raw_sha256"` + Raw string `json:"raw"` + Normalized normalizedItem `json:"normalized"` + NormalizedSHA256 string `json:"normalized_sha256"` + // Changes are the importer's warnings: every field it dropped or changed. + Changes []string `json:"changes"` +} + +// normalizedItem is the item an artifact will be stored as. Its canonical +// JSON (encoding/json sorts map keys) is what normalized_sha256 hashes. +type normalizedItem struct { + Title string `json:"title"` + Content string `json:"content"` + Fields map[string]any `json:"fields"` +} + +func (n normalizedItem) digest() (string, error) { + b, err := json.Marshal(n) + if err != nil { + return "", err + } + return appmanifest.SHA256Hex(sha256.Sum256(b)), nil +} + +// appInstallError is a refusal the preview reports to the owner. +type appInstallError struct { + status int + code string + msg string + path string +} + +func (e *appInstallError) Error() string { return e.msg } + +func installErr(status int, code, format string, args ...any) *appInstallError { + return &appInstallError{status: status, code: code, msg: fmt.Sprintf(format, args...)} +} + +func (e *appInstallError) write(w http.ResponseWriter) { + details := map[string]interface{}{} + if e.path != "" { + details["path"] = e.path + } + writeError2(w, e.status, e.code, e.msg, details) +} + +// requireAppsOwner gates every install route: apps available here, and the +// caller an owner of the workspace. +func (s *Server) requireAppsOwner(w http.ResponseWriter, r *http.Request) (workspaceID string, owner *models.User, ok bool) { + if !s.appsAvailable() { + writeError(w, http.StatusNotFound, "not_found", "Apps are not enabled on this server") + return "", nil, false + } + if !requireMinRole(w, r, "owner") { + return "", nil, false + } + workspaceID, ok = s.getWorkspaceID(w, r) + if !ok { + return "", nil, false + } + owner = currentUser(r) + if owner == nil { + writeError(w, http.StatusUnauthorized, "unauthorized", "Sign in to install apps") + return "", nil, false + } + return workspaceID, owner, true +} + +func (s *Server) handleAppInstallPreview(w http.ResponseWriter, r *http.Request) { + workspaceID, owner, ok := s.requireAppsOwner(w, r) + if !ok { + return + } + var in struct { + BaseURL string `json:"base_url"` + } + if err := decodeJSON(r, &in); err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Body must be {\"base_url\": \"https://...\"}") + return + } + origin, err := appmanifest.NormalizeOrigin(in.BaseURL) + if err != nil { + writeError2(w, http.StatusBadRequest, "invalid_base_url", "base_url "+err.Error(), map[string]interface{}{"path": "base_url"}) + return + } + if _, err := s.store.SweepExpiredPendingInstalls(); err != nil { + slog.Warn("apps: sweeping expired pending installs failed", "error", err) + } + + pending, err := s.store.ReservePendingInstall(workspaceID, owner.ID, origin) + if errors.Is(err, store.ErrPendingLimit) { + writeError(w, http.StatusTooManyRequests, "pending_install_limit", "Too many app installs are pending; finish or discard one and try again") + return + } + if err != nil { + writeInternalError(w, err) + return + } + preview, ierr := s.stageAppInstall(r, workspaceID, pending, origin) + if ierr != nil { + if err := s.store.DeletePendingInstall(pending.ID); err != nil { + slog.Warn("apps: deleting a failed pending install", "pending", pending.ID, "error", err) + } + var ae *appInstallError + if errors.As(ierr, &ae) { + ae.write(w) + return + } + writeInternalError(w, ierr) + return + } + writeJSON(w, http.StatusOK, preview) +} + +// stageAppInstall fetches, stages, validates and previews under a reservation +// already taken. Any error leaves the caller to delete the reservation. +func (s *Server) stageAppInstall(r *http.Request, workspaceID string, pending *store.PendingInstall, origin string) (*appPreview, error) { + ctx, cancel := context.WithTimeout(r.Context(), appFetchDeadline) + defer cancel() + fetcher, err := appfetch.New(s.appsPrivateOrigins(), appFetchDeadline, s.appFetchTLS) + if err != nil { + return nil, fmt.Errorf("apps: private origin list: %w", err) + } + + manifestURL := appmanifest.ManifestURL(origin) + raw, err := fetcher.Get(ctx, manifestURL, appmanifest.MaxManifestBytes) + if err != nil { + return nil, installErr(http.StatusBadGateway, "manifest_fetch_failed", "Could not fetch %s: %v", manifestURL, err) + } + manifestSHA := appmanifest.SHA256Hex(sha256.Sum256(raw)) + if err := s.store.StagePendingBlob(pending.ID, store.PendingBlob{Key: store.PendingManifestKey, URL: manifestURL, SHA256: manifestSHA, Data: raw}); err != nil { + return nil, s.stageErr(err) + } + m, err := appmanifest.Parse(raw) + if err != nil { + var me *appmanifest.Error + if errors.As(err, &me) { + e := installErr(http.StatusUnprocessableEntity, "invalid_manifest", "%s", me.Error()) + e.path = me.Path + return nil, e + } + return nil, err + } + if m.Origin != origin { + e := installErr(http.StatusUnprocessableEntity, "invalid_manifest", "the manifest's base_url (%s) is not the origin it was fetched from (%s)", m.Origin, origin) + e.path = "base_url" + return nil, e + } + + // Fetch every artifact exactly once, checking the declared sha256. + rawArtifacts := make([][]byte, len(m.CompanionPack.Artifacts)) + for i, a := range m.CompanionPack.Artifacts { + body, err := fetcher.Get(ctx, a.URL, appArtifactMaxBytes) + if err != nil { + e := installErr(http.StatusBadGateway, "artifact_fetch_failed", "Could not fetch artifact %q: %v", a.Key, err) + e.path = fmt.Sprintf("companion_pack.artifacts[%d]", i) + return nil, e + } + if got := appmanifest.SHA256Hex(sha256.Sum256(body)); got != a.SHA256 { + e := installErr(http.StatusUnprocessableEntity, "artifact_hash_mismatch", "Artifact %q does not match its declared sha256", a.Key) + e.path = fmt.Sprintf("companion_pack.artifacts[%d].sha256", i) + return nil, e + } + if err := s.store.StagePendingBlob(pending.ID, store.PendingBlob{Key: a.Key, URL: a.URL, SHA256: a.SHA256, Data: body}); err != nil { + return nil, s.stageErr(err) + } + rawArtifacts[i] = body + } + // No further bytes can arrive: everything is staged. + + preview, err := s.buildAppPreview(r, workspaceID, m, manifestSHA, rawArtifacts) + if err != nil { + return nil, err + } + preview.PendingID = pending.ID + // The stored preview omits each artifact's raw text: the staged blobs + // already hold those bytes, and GET fills them back in from there. + stored := *preview + stored.Artifacts = make([]appPreviewArtifact, len(preview.Artifacts)) + for i, a := range preview.Artifacts { + a.Raw = "" + stored.Artifacts[i] = a + } + previewJSON, err := json.Marshal(stored) + if err != nil { + return nil, err + } + if err := s.store.FinishPendingInstall(pending.ID, manifestSHA, string(previewJSON)); err != nil { + return nil, s.stageErr(err) + } + got, err := s.store.GetPendingInstall(pending.ID, workspaceID, pending.OwnerID) + if err != nil { + return nil, err + } + preview.ExpiresAt = got.ExpiresAt + return preview, nil +} + +func (s *Server) stageErr(err error) error { + switch { + case errors.Is(err, store.ErrPendingOverCap): + return installErr(http.StatusRequestEntityTooLarge, "pack_too_large", "The app's manifest and artifacts exceed the %d MiB install cap", store.PendingInstallBytes>>20) + case errors.Is(err, store.ErrPendingNotFound): + return installErr(http.StatusGatewayTimeout, "install_fetch_expired", "The install took too long to fetch; try again") + } + return err +} + +// buildAppPreview validates the pack against this workspace and normalizes +// every artifact exactly as the importer would store it. +func (s *Server) buildAppPreview(r *http.Request, workspaceID string, m *appmanifest.Manifest, manifestSHA string, rawArtifacts [][]byte) (*appPreview, error) { + p := &appPreview{ + Origin: m.Origin, ManifestURL: appmanifest.ManifestURL(m.Origin), ManifestSHA256: manifestSHA, + AppID: m.ID, Version: m.Version, Title: m.Title, Description: m.Description, Publisher: m.Publisher, Homepage: m.Homepage, + ReviewedByPad: false, Notice: appNotReviewedNotice, + ServiceAccess: m.Scopes.Service.Access, DelegatedAccess: m.Scopes.Delegated.Access, + ReadsSystem: "The app can read this workspace's Conventions and Playbooks.", + Events: m.Events, WebhookURL: m.WebhookURL, ItemActions: m.ItemActions, + ConfigSchema: m.ConfigSchema, RedirectURIs: m.RedirectURIs, Docs: m.Docs, + } + if len(m.Events) > 0 || len(m.ItemActions) > 0 { + p.DeferredNotice = appDeferredNotice + } + + // Conflict check (§2 step 5, L6). + slugs := make([]string, len(m.CompanionPack.Collections)) + for i, c := range m.CompanionPack.Collections { + slugs[i] = c.Slug + } + owners, err := s.store.CollectionSlugOwners(workspaceID, slugs) + if err != nil { + return nil, err + } + for i, c := range m.CompanionPack.Collections { + pc := appPreviewCollection{Key: c.Key, Slug: c.Slug, Name: c.Name, Schema: c.Schema} + if owner, exists := owners[c.Slug]; exists { + if owner != m.Origin { + e := installErr(http.StatusConflict, "collection_conflict", "This workspace already has a collection %q that this app did not create", c.Slug) + e.path = fmt.Sprintf("companion_pack.collections[%d].slug", i) + return nil, e + } + pc.Adopt = true + } + if err := validateNoReservedFieldKeys(c.Parsed, nil); err != nil { + e := installErr(http.StatusUnprocessableEntity, "invalid_manifest", "collection %q: %v", c.Key, err) + e.path = fmt.Sprintf("companion_pack.collections[%d].schema", i) + return nil, e + } + p.Collections = append(p.Collections, pc) + } + + // Unique values claimed by earlier artifacts of this same pack, per + // destination collection and field: the import creates them in order, so + // a later artifact de-collides its invocation_slug against them, and any + // other unique-scoped value it repeats is the conflict the import's unique + // check would refuse (codex rounds 2-4). Provisioning inserts in the same + // order on one transaction and so reaches the same outcome. + claimed := map[string]map[string]map[string]bool{} + for i, a := range m.CompanionPack.Artifacts { + pa, err := s.previewArtifact(r, workspaceID, a, rawArtifacts[i], claimed) + if err != nil { + var ae *appInstallError + if errors.As(err, &ae) && ae.path == "" { + ae.path = fmt.Sprintf("companion_pack.artifacts[%d]", i) + } + return nil, err + } + p.Artifacts = append(p.Artifacts, *pa) + } + return p, nil +} + +// previewArtifact runs the importer's own guards, decode and preprocess over +// the STAGED bytes (DOC-3371 §2 step 3). +func (s *Server) previewArtifact(r *http.Request, workspaceID string, a appmanifest.Artifact, raw []byte, claimed map[string]map[string]map[string]bool) (*appPreviewArtifact, error) { + art, err := decodeArtifactBytes(raw) + if err != nil { + return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q is not a valid Pad artifact: %v", a.Key, err) + } + if msg := models.ValidateItemTitle(models.NormalizeItemTitle(art.Title)); msg != "" { + return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q: %s", a.Key, msg) + } + collID, err := s.collectionIDForKind(workspaceID, art.Kind, nil) + if err != nil { + return nil, err + } + if collID == "" { + return nil, installErr(http.StatusUnprocessableEntity, "artifact_destination_missing", "This workspace has no collection that accepts %q artifacts (artifact %q)", art.Kind, a.Key) + } + coll, err := s.store.GetCollection(collID) + if err != nil { + return nil, err + } + var schema models.CollectionSchema + if err := models.UnmarshalItemFieldSchema([]byte(coll.Schema), &schema); err != nil { + return nil, fmt.Errorf("apps: destination schema: %w", err) + } + if claimed[coll.ID] == nil { + claimed[coll.ID] = map[string]map[string]bool{} + } + packClaims := claimed[coll.ID] + norm, err := normalizeArtifact(art, coll, schema, func(slug string) (bool, error) { + if packClaims["invocation_slug"]["s:"+slug] { + return true, nil + } + return s.invocationSlugTaken(workspaceID, coll.ID, slug) + }) + if err != nil { + return nil, err + } + + // The create's own field pipeline (coercion, defaults, validation, + // relation resolution in the import's carry posture, the unique check), + // so the digest covers exactly what the import stores (codex round 1). + fields, dropped, unresolved, undeclared, cerr := s.prepareCreateFields(r, workspaceID, coll, schema, norm.Fields, relationsCarry) + if cerr != nil { + return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q cannot be stored in %q: %s", a.Key, coll.Slug, cerr.message) + } + for _, k := range undeclared { + norm.Warnings = append(norm.Warnings, fmt.Sprintf("field %q is not declared by the destination collection's schema; stored as-is", k)) + } + for _, k := range unresolved { + norm.Warnings = append(norm.Warnings, fmt.Sprintf("field %q was imported as-is: it does not name an item in this workspace", k)) + } + for _, k := range dropped { + norm.Warnings = append(norm.Warnings, fmt.Sprintf("field %q was discarded: the destination collection's schema declares a default for it that is not a valid reference", k)) + } + // Claims are recorded from the FINAL fields, after defaults, which is + // what the import stores (codex round 3), for invocation_slug and every + // unique-scoped field (codex round 4). + // Which fields: invocation_slug always (a database unique index enforces + // it whatever the schema says), plus exactly the fields the import's + // checkUniqueFields enforces, through the same predicate (codex round 6). + // Deduplicated: a schema may declare a key twice, and one artifact must + // never conflict with itself (codex round 5). + uniqueKeys := []string{"invocation_slug"} + seenKey := map[string]bool{"invocation_slug": true} + for _, def := range schema.Fields { + if uniqueEnforced(def) && !seenKey[def.Key] { + seenKey[def.Key] = true + uniqueKeys = append(uniqueKeys, def.Key) + } + } + // invocation_slug is also covered by a database unique index per + // collection (056 / pg 033), whatever the schema says, so its FINAL value + // (a default included) is checked against stored items too, through the + // index's own expression (codex round 8), and must be + // text: SQLite keys the index on the JSON-typed value and Postgres on its + // text, so a non-text slug is refused rather than guessed at (codex round + // 7). Provisioning re-checks every constraint inside one transaction, so + // a divergence left here costs a failed install, never a partial one. + if raw, ok := fields["invocation_slug"]; ok && raw != nil { + slug, isText := raw.(string) + if !isText { + return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q would store a non-text invocation_slug", a.Key) + } + if slug != "" { + taken, err := s.store.InvocationSlugIndexTaken(coll.ID, slug) + if err != nil { + return nil, err + } + if taken { + return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q would store invocation_slug %q, which an item in %q already holds", a.Key, slug, coll.Slug) + } + } + } + for _, key := range uniqueKeys { + // INVARIANT (lead ruling): the preview may refuse what the import + // would accept, never the reverse; U8b's provisioning re-checks every + // constraint inside one transaction, so that re-check is the + // guarantee and this preview is advisory. + // + // Each value claims every form the import's lookup (JSONFieldEquals) + // could match it by: text, boolean, and number as its float64 (equal + // doubles is a superset of SQLite's double comparison and Postgres's + // exact numeric equality). Two values collide when they share any + // form (codex rounds 4-12). + forms := claimForms(fields[key]) + for _, f := range forms { + if packClaims[key][f] { + return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q would store a %s value that an earlier artifact of this pack already holds", a.Key, key) + } + } + if packClaims[key] == nil { + packClaims[key] = map[string]bool{} + } + for _, f := range forms { + packClaims[key][f] = true + } + } + // App-authored content: the owner only REVIEWED it, so it may not carry + // a pad-attachment: token or a [[workspace::REF]] link, and the human + // path's attachment stamp and cross-workspace resolution never run on it + // (lead ruling, U8b Q3). Ordinary [[Title]] links stay: they resolve + // workspace-wide as on any owner write, and the preview shows them. + if err := refuseAppAuthoredReferences(norm.Content, fields); err != nil { + return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q: %v", a.Key, err) + } + item := normalizedItem{Title: norm.Title, Content: norm.Content, Fields: fields} + digest, err := item.digest() + if err != nil { + return nil, err + } + changes := append([]string{}, norm.Warnings...) + sort.Strings(changes) + return &appPreviewArtifact{ + Key: a.Key, URL: a.URL, Kind: string(art.Kind), DestinationCollection: coll.Slug, + RawSHA256: a.SHA256, Raw: string(raw), Normalized: item, NormalizedSHA256: digest, Changes: changes, + }, nil +} + +func (s *Server) handleGetAppInstallPending(w http.ResponseWriter, r *http.Request) { + workspaceID, owner, ok := s.requireAppsOwner(w, r) + if !ok { + return + } + p, err := s.store.GetPendingInstall(chi.URLParam(r, "pendingID"), workspaceID, owner.ID) + if errors.Is(err, store.ErrPendingNotFound) || (err == nil && p.State != "staged") { + writeError(w, http.StatusNotFound, "not_found", "Pending install not found") + return + } + if err != nil { + writeInternalError(w, err) + return + } + var preview appPreview + if err := json.Unmarshal([]byte(p.Preview), &preview); err != nil { + writeInternalError(w, err) + return + } + blobs, err := s.store.PendingInstallBlobs(p.ID) + if err != nil { + writeInternalError(w, err) + return + } + for i := range preview.Artifacts { + preview.Artifacts[i].Raw = string(blobs[preview.Artifacts[i].Key].Data) + } + preview.ExpiresAt = p.ExpiresAt + writeJSON(w, http.StatusOK, preview) +} + +func (s *Server) handleDeleteAppInstallPending(w http.ResponseWriter, r *http.Request) { + workspaceID, owner, ok := s.requireAppsOwner(w, r) + if !ok { + return + } + p, err := s.store.GetPendingInstall(chi.URLParam(r, "pendingID"), workspaceID, owner.ID) + if errors.Is(err, store.ErrPendingNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Pending install not found") + return + } + if err != nil { + writeInternalError(w, err) + return + } + if err := s.store.DeletePendingInstall(p.ID); err != nil { + writeInternalError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// appsPrivateOrigins is the admin list, or nil on Cloud (no private +// destination is ever reachable there) or when unset or unreadable. +func (s *Server) appsPrivateOrigins() []appfetch.PrivateOrigin { + if s.cloudMode { + return nil + } + v, err := s.store.GetPlatformSetting(settingAppsPrivateOrigins) + if err != nil || v == "" { + return nil + } + var list []appfetch.PrivateOrigin + if err := json.Unmarshal([]byte(v), &list); err != nil { + slog.Warn("apps: the private-origin list is not valid JSON; ignoring it", "error", err) + return nil + } + return list +} + +// SetAppFetchTLS overrides the TLS config app fetches use. Tests only. +func (s *Server) SetAppFetchTLS(c *tls.Config) { s.appFetchTLS = c } + +// --- Admin: GET/PUT /api/v1/admin/apps ---------------------------------- + +type appsSettingsResponse struct { + Enabled bool `json:"enabled"` + Available bool `json:"available"` + HTTPSIssuer bool `json:"https_issuer"` + Cloud bool `json:"cloud"` + PrivateOrigins []appfetch.PrivateOrigin `json:"private_origins"` +} + +func (s *Server) appsSettings() appsSettingsResponse { + v, _ := s.store.GetPlatformSetting(settingAppsEnabled) + list := s.appsPrivateOrigins() + if list == nil { + list = []appfetch.PrivateOrigin{} + } + return appsSettingsResponse{ + Enabled: s.cloudMode || v == "true", Available: s.appsAvailable(), + HTTPSIssuer: s.oauthServer != nil && (s.cloudMode || s.mcpEndpoints.HTTPS()), + Cloud: s.cloudMode, PrivateOrigins: list, + } +} + +// handleGetAppsSettings answers on Cloud too, as GET /admin/mcp does, with +// cloud: true: reading the state is harmless, and only the PUT is +// operator-managed there (managed_by_operator). +func (s *Server) handleGetAppsSettings(w http.ResponseWriter, r *http.Request) { + if user := currentUser(r); user == nil || user.Role != "admin" { + writeError(w, http.StatusForbidden, "forbidden", "Admin access required") + return + } + writeJSON(w, http.StatusOK, s.appsSettings()) +} + +func (s *Server) handleUpdateAppsSettings(w http.ResponseWriter, r *http.Request) { + if user := currentUser(r); user == nil || user.Role != "admin" { + writeError(w, http.StatusForbidden, "forbidden", "Admin access required") + return + } + if s.cloudMode { + writeError(w, http.StatusForbidden, "managed_by_operator", "Apps on this instance are managed by the operator.") + return + } + var in struct { + Enabled *bool `json:"enabled"` + PrivateOrigins *[]appfetch.PrivateOrigin `json:"private_origins"` + } + if err := decodeJSON(r, &in); err != nil || (in.Enabled == nil && in.PrivateOrigins == nil) { + writeError(w, http.StatusBadRequest, "bad_request", "Body must set enabled and/or private_origins") + return + } + var changed []string + if in.PrivateOrigins != nil { + list := *in.PrivateOrigins + if list == nil { + list = []appfetch.PrivateOrigin{} + } + // Validate by building a fetcher from it: the same parser the fetch + // path uses, so a list that saves is a list that works. + if _, err := appfetch.New(list, time.Second, nil); err != nil { + writeError(w, http.StatusBadRequest, "invalid_private_origins", err.Error()) + return + } + b, err := json.Marshal(list) + if err != nil { + writeInternalError(w, err) + return + } + if err := s.store.SetPlatformSetting(settingAppsPrivateOrigins, string(b)); err != nil { + writeInternalError(w, err) + return + } + changed = append(changed, settingAppsPrivateOrigins) + } + if in.Enabled != nil { + value := "false" + if *in.Enabled { + value = "true" + } + if err := s.store.SetPlatformSetting(settingAppsEnabled, value); err != nil { + writeInternalError(w, err) + return + } + changed = append(changed, settingAppsEnabled) + } + s.logAuditEvent(models.ActionSettingsChanged, r, settingsChangedMeta(changed)) + writeJSON(w, http.StatusOK, s.appsSettings()) +} + +// claimForms are the forms a field value can be matched by in the import's +// uniqueness lookup (JSONFieldEquals): a string as text, and also as a +// boolean when it reads true/false and as a number when it is a JSON number; +// a boolean as a boolean; a number as its canonical number. Absent, null and +// empty values claim nothing, as they occupy no unique slot. Composite values +// claim their JSON text. +func claimForms(v any) []string { + switch t := v.(type) { + case nil: + return nil + case string: + if t == "" { + return nil + } + forms := []string{"s:" + t} + if t == "true" || t == "false" { + forms = append(forms, "b:"+t) + } + for _, n := range store.NumericClaimForms(t) { + forms = append(forms, "n:"+n) + } + return forms + case bool: + return []string{fmt.Sprintf("b:%v", t)} + case float64: + // Canonicalized from its JSON serialization, which is the text the + // stored value has and the import compares against, through the same + // numericArg path a later string takes (codex round 11: formatting + // the float directly diverged near 2^63 and past 2^53). + if b, err := json.Marshal(t); err == nil { + if forms := numberForms(string(b)); len(forms) > 0 { + return forms + } + } + case json.Number: + if forms := numberForms(t.String()); len(forms) > 0 { + return forms + } + } + b, err := json.Marshal(v) + if err != nil || len(b) == 0 { + return nil + } + return []string{"j:" + string(b)} +} + +func numberForms(text string) []string { + var forms []string + for _, n := range store.NumericClaimForms(text) { + forms = append(forms, "n:"+n) + } + return forms +} + +// refuseAppAuthoredReferences refuses any pad-attachment: token, in any +// spelling, in an app artifact's content or field values, and any +// workspace-qualified wiki link in its content. +func refuseAppAuthoredReferences(content string, fields map[string]any) error { + if strings.Contains(strings.ToLower(content), "pad-attachment:") { + return errors.New("pad-attachment: references are not accepted in app content") + } + // Fail closed: a field set that cannot be serialized cannot be checked, + // so it is refused, never waved through. Unreachable from a preview today + // (fields come from decoded JSON, which always re-encodes); the direct + // test reaches it. + b, err := json.Marshal(fields) + if err != nil { + return fmt.Errorf("app content fields cannot be checked: %w", err) + } + if strings.Contains(strings.ToLower(string(b)), "pad-attachment:") { + return errors.New("pad-attachment: references are not accepted in app content") + } + for _, l := range links.ExtractWikiLinks(content) { + if l.Kind == links.WikiLinkKindWorkspaceRef { + return errors.New("workspace-qualified links are not accepted in app content") + } + } + return nil +} diff --git a/internal/server/handlers_apps_install_test.go b/internal/server/handlers_apps_install_test.go new file mode 100644 index 000000000..05480e4ba --- /dev/null +++ b/internal/server/handlers_apps_install_test.go @@ -0,0 +1,715 @@ +package server + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/PerpetualSoftware/pad/internal/config" + "github.com/PerpetualSoftware/pad/internal/models" + "github.com/PerpetualSoftware/pad/internal/oauth" +) + +// TASK-3397 (U8a): app install staging and preview. + +type appsEnv struct { + srv *Server + token string + ws string + wsID string + app *httptest.Server + files map[string][]byte // path -> body served by the app + status map[string]int // path -> status override +} + +func appSHA(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } + +func newAppsEnv(t *testing.T) *appsEnv { + t.Helper() + srv := testServer(t) + token := bootstrapFirstUser(t, srv, "owner@test.com", "Owner") + rr := doRequestWithCookie(srv, "POST", "/api/v1/workspaces", map[string]string{"name": "Apps"}, token) + if rr.Code != http.StatusCreated { + t.Fatalf("create workspace: %d %s", rr.Code, rr.Body.String()) + } + var ws models.Workspace + parseJSON(t, rr, &ws) + + srv.SetMCPConfig(config.MCPEndpoints{Origin: "https://pad.test.example", OriginVar: "PAD_URL", + ResourceURL: "https://pad.test.example/mcp", AuthServerURL: "https://pad.test.example"}, nil) + o, err := oauth.NewServer(oauth.Config{Store: srv.store, HMACSecret: bytes32ForTest(), AllowedAudience: "https://pad.test.example/mcp"}) + if err != nil { + t.Fatal(err) + } + srv.SetOAuthServer(o) + + e := &appsEnv{srv: srv, token: token, ws: ws.Slug, wsID: ws.ID, files: map[string][]byte{}, status: map[string]int{}} + e.app = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if code, ok := e.status[r.URL.Path]; ok { + if code >= 300 && code < 400 { + http.Redirect(w, r, "/elsewhere", code) + return + } + w.WriteHeader(code) + return + } + body, ok := e.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(e.app.Close) + pool := x509.NewCertPool() + pool.AddCert(e.app.Certificate()) + srv.SetAppFetchTLS(&tls.Config{RootCAs: pool}) + + // Admin: enable apps and pin the test app's loopback origin. + rr = doRequestWithCookie(srv, "PUT", "/api/v1/admin/apps", map[string]any{ + "enabled": true, + "private_origins": []map[string]any{{"origin": e.app.URL, "allowed": []string{"127.0.0.1"}, "fetch": true}}, + }, token) + if rr.Code != http.StatusOK { + t.Fatalf("enable apps: %d %s", rr.Code, rr.Body.String()) + } + e.publish(t, e.manifest(t)) + return e +} + +func (e *appsEnv) origin() string { return e.app.URL } + +// manifest returns a valid manifest for the test app with one companion +// collection and the golden playbook as its artifact. +func (e *appsEnv) manifest(t *testing.T) map[string]any { + t.Helper() + pb, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", "playbook.golden.md")) + if err != nil { + t.Fatal(err) + } + e.files["/pack/ship.md"] = pb + return map[string]any{ + "id": "acme/portal", "version": "1.0.0", "min_contract": map[string]any{"apps": 1, "events": 1}, + "title": "Portal", "publisher": "Acme", "base_url": e.origin(), + "redirect_uris": []any{e.origin() + "/oauth/callback"}, + "scopes": map[string]any{"service": map[string]any{"access": "write"}, "delegated": map[string]any{"access": "read"}}, + "events": []any{map[string]any{"name": "item.created", "collections": []any{"tickets"}}}, + "webhook_url": e.origin() + "/hooks", + "companion_pack": map[string]any{ + "collections": []any{map[string]any{"key": "tickets", "slug": "portal-tickets", "name": "Portal tickets", + "schema": map[string]any{"fields": []any{map[string]any{"key": "status", "label": "Status", "type": "select", "options": []any{"open", "solved"}}}}}}, + "artifacts": []any{map[string]any{"key": "ship", "url": e.origin() + "/pack/ship.md", "sha256": appSHA(pb)}}, + }, + "item_actions": []any{map[string]any{"key": "open", "label": "Open", "collections": []any{"tickets"}, "path": "/t"}}, + } +} + +func (e *appsEnv) publish(t *testing.T, m map[string]any) { + t.Helper() + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + e.files["/.well-known/pad-app.json"] = b +} + +func (e *appsEnv) preview(t *testing.T) *httptest.ResponseRecorder { + t.Helper() + return doRequestWithCookie(e.srv, "POST", "/api/v1/workspaces/"+e.ws+"/apps/install/preview", map[string]any{"base_url": e.origin()}, e.token) +} + +func (e *appsEnv) pendingRows(t *testing.T) int { + t.Helper() + var n int + if err := e.srv.store.DB().QueryRow(`SELECT COUNT(*) FROM app_install_pending`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestAppInstallPreview_StagesAndPreviews(t *testing.T) { + e := newAppsEnv(t) + rr := e.preview(t) + if rr.Code != http.StatusOK { + t.Fatalf("preview: %d %s", rr.Code, rr.Body.String()) + } + var p appPreview + parseJSON(t, rr, &p) + if p.ReviewedByPad || !strings.Contains(p.Notice, "Not reviewed by Pad") { + t.Errorf("not-reviewed notice missing: %+v", p.Notice) + } + if p.Origin != e.origin() || p.ServiceAccess != "write" || p.DelegatedAccess != "read" || p.DeferredNotice == "" { + t.Errorf("preview header: %+v", p) + } + if len(p.Collections) != 1 || p.Collections[0].Adopt { + t.Errorf("collections: %+v", p.Collections) + } + if len(p.Artifacts) != 1 { + t.Fatalf("artifacts: %+v", p.Artifacts) + } + a := p.Artifacts[0] + if a.Raw == "" || a.RawSHA256 != appSHA(e.files["/pack/ship.md"]) || a.Normalized.Fields["status"] != "draft" { + t.Errorf("artifact preview: raw=%d sha=%s status=%v", len(a.Raw), a.RawSHA256, a.Normalized.Fields["status"]) + } + // Every field the importer changed is shown: the status reset, and the + // select values this workspace does not offer. + joined := strings.Join(a.Changes, "\n") + for _, want := range []string{`reset to "draft"`, `"trigger" value "on-intent"`} { + if !strings.Contains(joined, want) { + t.Errorf("change %q not shown: %v", want, a.Changes) + } + } + if want, _ := a.Normalized.digest(); a.NormalizedSHA256 != want { + t.Errorf("normalized digest %s, want %s", a.NormalizedSHA256, want) + } + + // The staged record serves the same preview, raw bytes included. + rr = doRequestWithCookie(e.srv, "GET", "/api/v1/workspaces/"+e.ws+"/apps/install/pending/"+p.PendingID, nil, e.token) + if rr.Code != http.StatusOK { + t.Fatalf("get pending: %d %s", rr.Code, rr.Body.String()) + } + var again appPreview + parseJSON(t, rr, &again) + if again.Artifacts[0].Raw != a.Raw || again.Artifacts[0].NormalizedSHA256 != a.NormalizedSHA256 { + t.Error("the stored preview differs from the one returned") + } + var staged int64 + if err := e.srv.store.DB().QueryRow(`SELECT reserved_bytes FROM app_install_pending WHERE id = ?`, p.PendingID).Scan(&staged); err != nil { + t.Fatal(err) + } + if want := int64(len(e.files["/.well-known/pad-app.json"]) + len(e.files["/pack/ship.md"])); staged != want { + t.Errorf("charge after staging %d, want the %d staged bytes", staged, want) + } + + rr = doRequestWithCookie(e.srv, "DELETE", "/api/v1/workspaces/"+e.ws+"/apps/install/pending/"+p.PendingID, nil, e.token) + if rr.Code != http.StatusNoContent || e.pendingRows(t) != 0 { + t.Fatalf("discard: %d, %d rows left", rr.Code, e.pendingRows(t)) + } +} + +// Every refusal releases its reservation and stages nothing. +func TestAppInstallPreview_Refusals(t *testing.T) { + for name, c := range map[string]struct { + mutate func(t *testing.T, e *appsEnv) + status int + code string + }{ + "artifact hash mismatch": {func(t *testing.T, e *appsEnv) { e.files["/pack/ship.md"] = []byte("tampered") }, 422, "artifact_hash_mismatch"}, + "unsubscribable event": {func(t *testing.T, e *appsEnv) { + m := e.manifest(t) + m["events"] = []any{map[string]any{"name": "item.bulk_updated", "collections": []any{"tickets"}}} + e.publish(t, m) + }, 422, "invalid_manifest"}, + "manifest names another origin": {func(t *testing.T, e *appsEnv) { + m := e.manifest(t) + m["base_url"] = "https://other.example" + m["redirect_uris"] = []any{"https://other.example/cb"} + m["webhook_url"] = "https://other.example/h" + pack := m["companion_pack"].(map[string]any) + pack["artifacts"] = []any{} + e.publish(t, m) + }, 422, "invalid_manifest"}, + "manifest redirects": {func(t *testing.T, e *appsEnv) { e.status["/.well-known/pad-app.json"] = http.StatusFound }, 502, "manifest_fetch_failed"}, + "artifact missing": {func(t *testing.T, e *appsEnv) { delete(e.files, "/pack/ship.md") }, 502, "artifact_fetch_failed"}, + "slug taken by a human collection": {func(t *testing.T, e *appsEnv) { + if _, err := e.srv.store.CreateCollection(e.wsID, models.CollectionCreate{Name: "Portal tickets", Slug: "portal-tickets", Schema: `{"fields":[]}`}); err != nil { + t.Fatal(err) + } + }, 409, "collection_conflict"}, + } { + t.Run(name, func(t *testing.T) { + e := newAppsEnv(t) + c.mutate(t, e) + rr := e.preview(t) + if rr.Code != c.status || !strings.Contains(rr.Body.String(), c.code) { + t.Fatalf("got %d %s, want %d %s", rr.Code, rr.Body.String(), c.status, c.code) + } + if n := e.pendingRows(t); n != 0 { + t.Fatalf("a refused preview left %d pending record(s)", n) + } + }) + } +} + +func TestAppInstallPreview_Gates(t *testing.T) { + e := newAppsEnv(t) + // Apps off: the routes answer 404. + if rr := doRequestWithCookie(e.srv, "PUT", "/api/v1/admin/apps", map[string]any{"enabled": false}, e.token); rr.Code != http.StatusOK { + t.Fatal(rr.Body.String()) + } + if rr := e.preview(t); rr.Code != http.StatusNotFound { + t.Fatalf("apps off: %d", rr.Code) + } + if rr := doRequestWithCookie(e.srv, "PUT", "/api/v1/admin/apps", map[string]any{"enabled": true}, e.token); rr.Code != http.StatusOK { + t.Fatal(rr.Body.String()) + } + // A bad base URL is refused before any reservation. + rr := doRequestWithCookie(e.srv, "POST", "/api/v1/workspaces/"+e.ws+"/apps/install/preview", map[string]any{"base_url": "http://x.example"}, e.token) + if rr.Code != http.StatusBadRequest || e.pendingRows(t) != 0 { + t.Fatalf("http base url: %d", rr.Code) + } + // The per-owner cap. + for i := 0; i < 3; i++ { + if rr := e.preview(t); rr.Code != http.StatusOK { + t.Fatalf("preview %d: %d %s", i, rr.Code, rr.Body.String()) + } + } + if rr := e.preview(t); rr.Code != http.StatusTooManyRequests { + t.Fatalf("fourth pending install: %d %s", rr.Code, rr.Body.String()) + } +} + +func TestAppsAdminSettings(t *testing.T) { + e := newAppsEnv(t) + rr := doRequestWithCookie(e.srv, "PUT", "/api/v1/admin/apps", map[string]any{ + "private_origins": []map[string]any{{"origin": "https://x.example", "fetch": true}}, + }, e.token) + if rr.Code != http.StatusBadRequest { + t.Fatalf("an entry without pinned addresses was accepted: %d", rr.Code) + } + rr = doRequestWithCookie(e.srv, "GET", "/api/v1/admin/apps", nil, e.token) + var s appsSettingsResponse + parseJSON(t, rr, &s) + if !s.Enabled || !s.Available || len(s.PrivateOrigins) != 1 { + t.Fatalf("settings: %+v", s) + } + // Without an https issuer apps are unavailable even when enabled. + e.srv.SetMCPConfig(config.MCPEndpoints{Origin: "http://pad.test.example", OriginVar: "PAD_URL", + ResourceURL: "http://pad.test.example/mcp", AuthServerURL: "http://pad.test.example"}, nil) + if e.srv.appsAvailable() { + t.Fatal("apps available over an http issuer") + } +} + +// The preview's normalized item is what the import would STORE: the create's +// own defaults and validation run on it (codex round 1). A destination field +// the artifact cannot satisfy refuses the preview; a schema default appears +// in the previewed fields and so in the digest. +func TestAppInstallPreview_RunsTheCreateFieldPipeline(t *testing.T) { + addField := func(t *testing.T, e *appsEnv, field map[string]any) { + t.Helper() + coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks") + if err != nil || coll == nil { + t.Fatalf("playbooks collection: %v", err) + } + var schema map[string]any + if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil { + t.Fatal(err) + } + schema["fields"] = append(schema["fields"].([]any), field) + b, _ := json.Marshal(schema) + sch := string(b) + if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil { + t.Fatal(err) + } + } + + t.Run("required field refuses", func(t *testing.T) { + e := newAppsEnv(t) + addField(t, e, map[string]any{"key": "owner_team", "label": "Owner team", "type": "text", "required": true}) + rr := e.preview(t) + if rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "invalid_artifact") { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } + if n := e.pendingRows(t); n != 0 { + t.Fatalf("%d pending left", n) + } + }) + t.Run("default is previewed", func(t *testing.T) { + e := newAppsEnv(t) + addField(t, e, map[string]any{"key": "team", "label": "Team", "type": "text", "default": "core"}) + rr := e.preview(t) + if rr.Code != http.StatusOK { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } + var p appPreview + parseJSON(t, rr, &p) + if got := p.Artifacts[0].Normalized.Fields["team"]; got != "core" { + t.Fatalf("the schema default is not in the previewed item: %v", got) + } + }) +} + +// An abandoned preview's staged bytes are deleted by the background sweep, +// not only when a later install happens to sweep (codex round 1). +func TestAppPendingSweep_DeletesExpiredStagedBytes(t *testing.T) { + e := newAppsEnv(t) + rr := e.preview(t) + if rr.Code != http.StatusOK { + t.Fatalf("preview: %d %s", rr.Code, rr.Body.String()) + } + if _, err := e.srv.store.DB().Exec(`UPDATE app_install_pending SET expires_at = '2000-01-01T00:00:00Z'`); err != nil { + t.Fatal(err) + } + e.srv.runAppPendingSweep() + var records, blobs int + _ = e.srv.store.DB().QueryRow(`SELECT COUNT(*) FROM app_install_pending`).Scan(&records) + _ = e.srv.store.DB().QueryRow(`SELECT COUNT(*) FROM app_install_pending_blobs`).Scan(&blobs) + if records != 0 || blobs != 0 { + t.Fatalf("after the sweep: %d records, %d blobs", records, blobs) + } + // The loop starts and stops cleanly. + e.srv.StartAppPendingSweep() + e.srv.StartAppPendingSweep() + e.srv.stopAppPendingSweep() + e.srv.stopAppPendingSweep() +} + +// Two artifacts of one pack asking for the same invocation_slug are +// allocated distinct slugs in the preview, as their sequential import would +// be (codex round 2). +func TestAppInstallPreview_PackArtifactsDeCollideWithEachOther(t *testing.T) { + e := newAppsEnv(t) + m := e.manifest(t) + second := []byte(strings.Replace(string(e.files["/pack/ship.md"]), "title: Ship a change", "title: Ship again", 1)) + e.files["/pack/ship2.md"] = second + pack := m["companion_pack"].(map[string]any) + pack["artifacts"] = append(pack["artifacts"].([]any), map[string]any{"key": "ship2", "url": e.origin() + "/pack/ship2.md", "sha256": appSHA(second)}) + e.publish(t, m) + rr := e.preview(t) + if rr.Code != http.StatusOK { + t.Fatalf("preview: %d %s", rr.Code, rr.Body.String()) + } + var p appPreview + parseJSON(t, rr, &p) + a, b := p.Artifacts[0].Normalized.Fields["invocation_slug"], p.Artifacts[1].Normalized.Fields["invocation_slug"] + if a == b { + t.Fatalf("both artifacts previewed invocation_slug %v", a) + } + if a != "ship" || b != "ship-2" { + t.Fatalf("slugs %v, %v; want ship, ship-2 (the sequential import's allocation)", a, b) + } +} + +// A schema DEFAULT counts as a claim: an artifact without a slug, stored with +// the default, makes a later artifact asking for that slug de-collide, as +// the sequential import does (codex round 3). +func TestAppInstallPreview_DefaultSlugIsClaimed(t *testing.T) { + e := newAppsEnv(t) + coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks") + if err != nil || coll == nil { + t.Fatalf("playbooks collection: %v", err) + } + var schema map[string]any + if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil { + t.Fatal(err) + } + for _, f := range schema["fields"].([]any) { + if fm := f.(map[string]any); fm["key"] == "invocation_slug" { + fm["default"] = "ship" + } + } + b, _ := json.Marshal(schema) + sch := string(b) + if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil { + t.Fatal(err) + } + m := e.manifest(t) + orig := string(e.files["/pack/ship.md"]) + noSlug := []byte(strings.Replace(strings.Replace(orig, "invocation_slug: ship\n", "", 1), "title: Ship a change", "title: Defaulted", 1)) + e.files["/pack/a.md"] = noSlug + e.files["/pack/b.md"] = []byte(strings.Replace(orig, "title: Ship a change", "title: Explicit", 1)) + pack := m["companion_pack"].(map[string]any) + pack["artifacts"] = []any{ + map[string]any{"key": "a", "url": e.origin() + "/pack/a.md", "sha256": appSHA(e.files["/pack/a.md"])}, + map[string]any{"key": "b", "url": e.origin() + "/pack/b.md", "sha256": appSHA(e.files["/pack/b.md"])}, + } + e.publish(t, m) + rr := e.preview(t) + if rr.Code != http.StatusOK { + t.Fatalf("preview: %d %s", rr.Code, rr.Body.String()) + } + var p appPreview + parseJSON(t, rr, &p) + if a, b := p.Artifacts[0].Normalized.Fields["invocation_slug"], p.Artifacts[1].Normalized.Fields["invocation_slug"]; a != "ship" || b != "ship-2" { + t.Fatalf("slugs %v, %v; want ship (the default), ship-2", a, b) + } +} + +// Any unique-scoped destination field, not only invocation_slug: two pack +// artifacts storing the same unique value (here through a default) refuse +// the preview, as the second sequential import would 409 (codex round 4). +func TestAppInstallPreview_PackUniqueValuesConflict(t *testing.T) { + e := newAppsEnv(t) + coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks") + if err != nil || coll == nil { + t.Fatalf("playbooks collection: %v", err) + } + var schema map[string]any + if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil { + t.Fatal(err) + } + codeField := map[string]any{"key": "code", "label": "Code", "type": "text", "unique_scope": "workspace_collection", "default": "shared"} + // Declared twice: a single artifact must still not conflict with itself + // (codex round 5). + schema["fields"] = append(schema["fields"].([]any), codeField, codeField) + b, _ := json.Marshal(schema) + sch := string(b) + if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil { + t.Fatal(err) + } + if rr := e.preview(t); rr.Code != http.StatusOK { + t.Fatalf("a single artifact conflicted with itself: %d %s", rr.Code, rr.Body.String()) + } + if rr := doRequestWithCookie(e.srv, "DELETE", "/api/v1/workspaces/"+e.ws+"/apps/install/pending/"+pendingIDOf(t, e), nil, e.token); rr.Code != http.StatusNoContent { + t.Fatalf("discard: %d", rr.Code) + } + m := e.manifest(t) + second := []byte(strings.Replace(strings.Replace(string(e.files["/pack/ship.md"]), "title: Ship a change", "title: Other", 1), "invocation_slug: ship", "invocation_slug: other", 1)) + e.files["/pack/other.md"] = second + pack := m["companion_pack"].(map[string]any) + pack["artifacts"] = append(pack["artifacts"].([]any), map[string]any{"key": "other", "url": e.origin() + "/pack/other.md", "sha256": appSHA(second)}) + e.publish(t, m) + rr := e.preview(t) + if rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "code") { + t.Fatalf("got %d %s; want 422 naming the unique field", rr.Code, rr.Body.String()) + } + if n := e.pendingRows(t); n != 0 { + t.Fatalf("%d pending left", n) + } +} + +func pendingIDOf(t *testing.T, e *appsEnv) string { + t.Helper() + var id string + if err := e.srv.store.DB().QueryRow(`SELECT id FROM app_install_pending ORDER BY created_at DESC LIMIT 1`).Scan(&id); err != nil { + t.Fatal(err) + } + return id +} + +// A unique_scope the import does NOT enforce is not a pack conflict either +// (codex round 6): two artifacts sharing that default both import, so both +// preview. +func TestAppInstallPreview_UnenforcedUniqueScopeIsNotAConflict(t *testing.T) { + e := newAppsEnv(t) + coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks") + if err != nil || coll == nil { + t.Fatalf("playbooks collection: %v", err) + } + var schema map[string]any + if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil { + t.Fatal(err) + } + schema["fields"] = append(schema["fields"].([]any), map[string]any{"key": "code", "label": "Code", "type": "text", "unique_scope": "collection", "default": "shared"}) + b, _ := json.Marshal(schema) + sch := string(b) + if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil { + t.Fatal(err) + } + m := e.manifest(t) + second := []byte(strings.Replace(strings.Replace(string(e.files["/pack/ship.md"]), "title: Ship a change", "title: Other", 1), "invocation_slug: ship", "invocation_slug: other", 1)) + e.files["/pack/other.md"] = second + pack := m["companion_pack"].(map[string]any) + pack["artifacts"] = append(pack["artifacts"].([]any), map[string]any{"key": "other", "url": e.origin() + "/pack/other.md", "sha256": appSHA(second)}) + e.publish(t, m) + if rr := e.preview(t); rr.Code != http.StatusOK { + t.Fatalf("an unenforced scope refused the preview: %d %s", rr.Code, rr.Body.String()) + } +} + +// invocation_slug is unique per collection at the DATABASE, whatever the +// schema declares (codex round 7): a defaulted slug that a stored item +// already holds, and a non-text slug, both refuse the preview. +func TestAppInstallPreview_InvocationSlugIndexIsMirrored(t *testing.T) { + setDefault := func(t *testing.T, e *appsEnv, def any) string { + t.Helper() + coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "conventions") + if err != nil || coll == nil { + t.Fatalf("conventions collection: %v", err) + } + var schema map[string]any + if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil { + t.Fatal(err) + } + typ := "text" + if _, isNum := def.(int); isNum { + typ = "number" + } + schema["fields"] = append(schema["fields"].([]any), map[string]any{"key": "invocation_slug", "label": "Slug", "type": typ, "default": def}) + b, _ := json.Marshal(schema) + sch := string(b) + if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil { + t.Fatal(err) + } + return coll.ID + } + useConvention := func(t *testing.T, e *appsEnv) { + t.Helper() + cv, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", "convention.golden.md")) + if err != nil { + t.Fatal(err) + } + e.files["/pack/conv.md"] = cv + m := e.manifest(t) + m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "conv", "url": e.origin() + "/pack/conv.md", "sha256": appSHA(cv)}} + e.publish(t, m) + } + + t.Run("a stored item holds the defaulted slug", func(t *testing.T) { + e := newAppsEnv(t) + collID := setDefault(t, e, "shared") + if _, err := e.srv.store.CreateItem(e.wsID, collID, models.ItemCreate{Title: "Holder", Fields: `{"invocation_slug":"shared","status":"draft"}`}); err != nil { + t.Fatal(err) + } + useConvention(t, e) + if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "shared") { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } + }) + t.Run("a non-text slug", func(t *testing.T) { + e := newAppsEnv(t) + setDefault(t, e, 42) + useConvention(t, e) + if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "non-text") { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } + }) +} + +// The stored-slug check uses the index's own expression: a stored ARRAY +// "[]" collides with a defaulted text "[]" there, so the preview refuses +// (codex round 8). +func TestAppInstallPreview_InvocationSlugIndexExpression(t *testing.T) { + e := newAppsEnv(t) + coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "conventions") + if err != nil || coll == nil { + t.Fatalf("conventions collection: %v", err) + } + var schema map[string]any + if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil { + t.Fatal(err) + } + schema["fields"] = append(schema["fields"].([]any), map[string]any{"key": "invocation_slug", "label": "Slug", "type": "text", "default": "[]"}) + b, _ := json.Marshal(schema) + sch := string(b) + if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil { + t.Fatal(err) + } + if _, err := e.srv.store.DB().Exec(e.srv.store.D().Rebind(`INSERT INTO items (id, workspace_id, collection_id, title, slug, content, fields, tags, created_at, updated_at, item_number) + VALUES (?, ?, ?, 'Array holder', 'array-holder', '', '{"invocation_slug":[],"status":"draft"}', '[]', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 99999)`), + "arr-"+e.wsID[:8], e.wsID, coll.ID); err != nil { + t.Fatal(err) + } + cv, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", "convention.golden.md")) + if err != nil { + t.Fatal(err) + } + e.files["/pack/conv.md"] = cv + m := e.manifest(t) + m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "conv", "url": e.origin() + "/pack/conv.md", "sha256": appSHA(cv)}} + e.publish(t, m) + if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } +} + +// In-pack claims collide wherever the import's lookup (JSONFieldEquals) +// would match: boolean true vs "true", number 1000 vs "1e3" (codex rounds +// 9-10). +func TestClaimForms(t *testing.T) { + collide := func(a, b any) bool { + seen := map[string]bool{} + for _, f := range claimForms(a) { + seen[f] = true + } + for _, f := range claimForms(b) { + if seen[f] { + return true + } + } + return false + } + for _, c := range []struct { + a, b any + want bool + }{ + {true, "true", true}, + {float64(1000), "1e3", true}, + {float64(1000), "1000", true}, + {float64(9210000000000000000), "9210000000000000000", true}, + {float64(1000000000000000100), "1000000000000000100", true}, + // codex round 12: exponent spelling past numericArg's integer bound, + // its negative, and a decimal spelling that SQLite rounds to the same + // double. + {float64(9210000000000000000), "9.21e18", true}, + {float64(-9210000000000000000), "-9.21e18", true}, + {float64(1000000000000000100), "1000000000000000100.0", true}, + {float64(0.5), "0.50", true}, + {float64(2), "3", false}, + {"ship", "ship", true}, + {"ship", "other", false}, + {float64(1), true, false}, + {nil, "", false}, + {"", "", false}, + } { + if got := collide(c.a, c.b); got != c.want { + t.Errorf("collide(%#v, %#v) = %v, want %v", c.a, c.b, got, c.want) + } + } +} + +// App-authored artifact content may not carry pad-attachment: tokens or +// [[workspace::REF]] links; ordinary [[Title]] links are fine (lead ruling, +// U8b Q3). +func TestAppInstallPreview_AppContentReferenceRules(t *testing.T) { + for name, c := range map[string]struct { + edit string + ok bool + }{ + "attachment token in the body": {"![x](PAD-Attachment:abc)", false}, + "workspace-qualified link": {"see [[other-ws::TASK-1]]", false}, + "ordinary title link": {"see [[Some Title]]", true}, + } { + t.Run(name, func(t *testing.T) { + e := newAppsEnv(t) + m := e.manifest(t) + body := []byte(string(e.files["/pack/ship.md"]) + "\n" + c.edit + "\n") + e.files["/pack/ship.md"] = body + m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "ship", "url": e.origin() + "/pack/ship.md", "sha256": appSHA(body)}} + e.publish(t, m) + rr := e.preview(t) + if c.ok && rr.Code != http.StatusOK { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } + if !c.ok && (rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "not accepted in app content")) { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } + }) + } + t.Run("attachment token in a field", func(t *testing.T) { + e := newAppsEnv(t) + m := e.manifest(t) + orig := string(e.files["/pack/ship.md"]) + if !strings.Contains(orig, "description: the item to ship") { + t.Fatal("fixture changed: no arguments description to carry the token") + } + body := []byte(strings.Replace(orig, "description: the item to ship", "description: \"see pad-attachment:abc\"", 1)) + e.files["/pack/ship.md"] = body + m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "ship", "url": e.origin() + "/pack/ship.md", "sha256": appSHA(body)}} + e.publish(t, m) + if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "not accepted in app content") { + t.Fatalf("got %d %s", rr.Code, rr.Body.String()) + } + }) +} + +// Lead review of U8a: the fields check fails closed. A field set that cannot +// be serialized is refused, never passed unchecked. +func TestRefuseAppAuthoredReferences_FailsClosedOnUnserializableFields(t *testing.T) { + err := refuseAppAuthoredReferences("plain content", map[string]any{"bad": make(chan int)}) + if err == nil { + t.Fatal("an unserializable field set passed the reference check") + } + if err := refuseAppAuthoredReferences("plain content", map[string]any{"ok": "value"}); err != nil { + t.Fatalf("control: a clean field set was refused: %v", err) + } +} diff --git a/internal/server/handlers_artifact_import.go b/internal/server/handlers_artifact_import.go index 5150e5f9e..8d45f7a14 100644 --- a/internal/server/handlers_artifact_import.go +++ b/internal/server/handlers_artifact_import.go @@ -1,7 +1,6 @@ package server import ( - "encoding/json" "errors" "fmt" "net/http" @@ -91,8 +90,7 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) { // anyone noticing. Both now call the same models pair, so the claim is // enforced rather than asserted; create adopted this door's trim, which is // the stricter and correct reading. - art.Title = models.NormalizeItemTitle(art.Title) - if msg := models.ValidateItemTitle(art.Title); msg != "" { + if msg := models.ValidateItemTitle(models.NormalizeItemTitle(art.Title)); msg != "" { writeError(w, http.StatusBadRequest, "bad_request", msg) return } @@ -143,42 +141,19 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) { return } - // Copy the artifact fields so the preprocess never mutates the decoded - // value (keeps the parse layer's output immutable from the handler's POV). - fields := make(map[string]any, len(art.Fields)) - for k, v := range art.Fields { - fields[k] = v - } - - warnings := s.preprocessArtifactFields(workspaceID, coll, art.Kind, schema, fields) - - fieldsJSON, err := json.Marshal(fields) + norm, err := normalizeArtifact(art, coll, schema, func(slug string) (bool, error) { + return s.invocationSlugTaken(workspaceID, coll.ID, slug) + }) if err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to marshal fields") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to normalize the artifact") return } - - // Normalize the field map through JSON so its nested types match what the - // normal create path validates. handleCreateItem builds its fieldMap by - // unmarshalling the JSON request body, so structured values are canonical - // JSON types ([]any, map[string]any). The artifact decode produces Go-native - // types (e.g. arguments as []map[string]any), which ValidateFields' json case - // rejects — round-tripping fixes that without special-casing any field. - normalizedFields := make(map[string]any) - if err := json.Unmarshal(fieldsJSON, &normalizedFields); err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to normalize fields") - return - } - - body := art.Body - if footer := artifactProvenanceFooter(art.Provenance); footer != "" { - body = body + footer - } + warnings := norm.Warnings input := models.ItemCreate{ - Title: art.Title, - Content: body, - Fields: string(fieldsJSON), + Title: norm.Title, + Content: norm.Content, + Fields: norm.FieldsJSON, } // Attribution: a normal agent/api create. CreatedBy and Source are both // left blank so createItemChecked stamps them from the request auth @@ -203,7 +178,7 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) { // ruling, day 57. An artifact was written elsewhere and the importer did // not choose its field values; refusing would break the import of exactly // the artifacts most likely to hold pre-validation junk. - item, cerr := s.createItemChecked(r, workspaceID, coll, schema, input, normalizedFields, "", relationsCarry) + item, cerr := s.createItemChecked(r, workspaceID, coll, schema, input, norm.Fields, "", relationsCarry) if cerr != nil { cerr.write(w, r) return @@ -265,7 +240,7 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) { // - status is forced to "draft" regardless of the artifact's value. // - For playbooks, a non-empty invocation_slug that's already taken in the // destination collection is suffixed (-2, -3, …) until free. -func (s *Server) preprocessArtifactFields(workspaceID string, coll *models.Collection, kind artifact.Kind, schema models.CollectionSchema, fields map[string]any) []string { +func preprocessArtifactFields(kind artifact.Kind, schema models.CollectionSchema, fields map[string]any, slugTaken func(string) (bool, error)) []string { var warnings []string // Blank foreign select values (trigger/scope/priority and any other @@ -302,7 +277,7 @@ func (s *Server) preprocessArtifactFields(workspaceID string, coll *models.Colle // De-collide invocation_slug for playbooks. if kind == artifact.KindPlaybook { if slug, _ := fields["invocation_slug"].(string); slug != "" { - free, changed := s.freeInvocationSlug(workspaceID, coll.ID, slug) + free, changed := freeInvocationSlug(slug, slugTaken) if changed { fields["invocation_slug"] = free warnings = append(warnings, @@ -315,12 +290,12 @@ func (s *Server) preprocessArtifactFields(workspaceID string, coll *models.Colle } // freeInvocationSlug returns an invocation_slug that's free in the destination -// collection. If the requested slug is already taken it appends -2, -3, … -// until an unused value is found. Returns (slug, changed). -func (s *Server) freeInvocationSlug(workspaceID, collectionID, requested string) (string, bool) { +// collection, per taken. If the requested slug is already taken it appends +// -2, -3, … until an unused value is found. Returns (slug, changed). +func freeInvocationSlug(requested string, isTaken func(string) (bool, error)) (string, bool) { candidate := requested for n := 2; ; n++ { - taken, err := s.invocationSlugTaken(workspaceID, collectionID, candidate) + taken, err := isTaken(candidate) if err != nil { // On a query error, fall back to the requested slug and let the // create-time uniqueness precheck/constraint surface a conflict. diff --git a/internal/server/handlers_items.go b/internal/server/handlers_items.go index 3957b8d0e..686baebf7 100644 --- a/internal/server/handlers_items.go +++ b/internal/server/handlers_items.go @@ -861,56 +861,9 @@ const ( ) func (s *Server) createItemChecked(r *http.Request, workspaceID string, coll *models.Collection, schema models.CollectionSchema, input models.ItemCreate, fieldMap map[string]any, parentValue string, posture relationPosture) (*models.Item, *itemCreateError) { - // Coerce strings to their declared types before validating (BUG-2850). - fieldMap = items.CoerceFields(fieldMap, schema) - // BUG-3028: on an ordinary create every value is SUPPLIED, so a blank - // relation is removed before validation and a required one is refused as - // required. An artifact import (relationsCarry) carries values nobody typed - // in this request, so its blanks are removed after validation instead — - // the same carry-not-refuse posture it already has for unresolvable ones. - if posture == relationsRefuse { - items.DropBlankRelations(fieldMap, schema, nil) - } - // Snapshot before validation, which INJECTS schema defaults without - // type-checking them (codex round 7). The resolver below skips a - // non-string, so an injected `default: 42` on a relation field reached the - // blob unchallenged — the same hole the migrate doors had, by the same - // route, and closed with the same pass. - relBefore := store.RelationKeysPresent(schema, fieldMap) - // BUG-3079: a default that fails its own type check is DISCARDED and - // reported, not stored. The list joins the relation drops below in the one - // `warnings.dropped_fields` channel — they are the same event to a caller. - defaultDrops, err := items.ValidateFieldsWithDrops(fieldMap, schema) - if err != nil { - return nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: err.Error()} - } - items.DropBlankRelations(fieldMap, schema, nil) - // Referent validation for relation values (TASK-2878). AFTER the shape - // check, so "must be a string" and "names nothing" are never both reported - // for one value, and after coercion so the value is in its final form. - // The four steps live in one place — see resolveRelationsForWrite. - relRefusals, droppedDefaults, relErr := s.resolveRelationsForWrite( - r, workspaceID, workspaceRole(r), schema, fieldMap, relBefore, posture) - if relErr != nil { - return nil, &itemCreateError{status: http.StatusInternalServerError, code: "internal_error", message: "Failed to resolve relation references"} - } - droppedDefaults = append(defaultDrops, droppedDefaults...) - var unresolved []string - if len(relRefusals) > 0 { - if posture == relationsRefuse { - return nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: relationIssuesMessage(relRefusals)} - } - // Carry: the values stay in fieldMap exactly as supplied — the - // resolver leaves what it cannot resolve untouched — and the write - // says which ones, so an import is never silently lossy. - for _, ri := range relRefusals { - unresolved = append(unresolved, ri.Key) - } - } - undeclared := items.UndeclaredFieldKeys(fieldMap, schema) - - if err := s.checkUniqueFields(workspaceID, coll.ID, "", schema, fieldMap); err != nil { - return nil, &itemCreateError{status: http.StatusConflict, code: "conflict", message: err.Error()} + fieldMap, droppedDefaults, unresolved, undeclared, cerr := s.prepareCreateFields(r, workspaceID, coll, schema, fieldMap, posture) + if cerr != nil { + return nil, cerr } // Marshal validated/defaulted fields back @@ -1034,6 +987,67 @@ func (s *Server) createItemChecked(r *http.Request, workspaceID string, coll *mo return item, nil } +// prepareCreateFields is createItemChecked's field pipeline, in its order: +// coerce to declared types, drop blank relations (refuse posture), apply and +// validate schema defaults, resolve relation referents, name undeclared keys, +// and the unique-field check. It returns the fields as a create would store +// them, plus the three warning lists. Extracted so the app installer's +// preview (TASK-3397) computes its digest over exactly what the import +// stores; createItemChecked is unchanged in behaviour. +func (s *Server) prepareCreateFields(r *http.Request, workspaceID string, coll *models.Collection, schema models.CollectionSchema, fieldMap map[string]any, posture relationPosture) (fields map[string]any, droppedDefaults, unresolved, undeclared []string, cerr *itemCreateError) { + // Coerce strings to their declared types before validating (BUG-2850). + fieldMap = items.CoerceFields(fieldMap, schema) + // BUG-3028: on an ordinary create every value is SUPPLIED, so a blank + // relation is removed before validation and a required one is refused as + // required. An artifact import (relationsCarry) carries values nobody typed + // in this request, so its blanks are removed after validation instead — + // the same carry-not-refuse posture it already has for unresolvable ones. + if posture == relationsRefuse { + items.DropBlankRelations(fieldMap, schema, nil) + } + // Snapshot before validation, which INJECTS schema defaults without + // type-checking them (codex round 7). The resolver below skips a + // non-string, so an injected `default: 42` on a relation field reached the + // blob unchallenged — the same hole the migrate doors had, by the same + // route, and closed with the same pass. + relBefore := store.RelationKeysPresent(schema, fieldMap) + // BUG-3079: a default that fails its own type check is DISCARDED and + // reported, not stored. The list joins the relation drops below in the one + // `warnings.dropped_fields` channel — they are the same event to a caller. + defaultDrops, err := items.ValidateFieldsWithDrops(fieldMap, schema) + if err != nil { + return nil, nil, nil, nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: err.Error()} + } + items.DropBlankRelations(fieldMap, schema, nil) + // Referent validation for relation values (TASK-2878). AFTER the shape + // check, so "must be a string" and "names nothing" are never both reported + // for one value, and after coercion so the value is in its final form. + // The four steps live in one place — see resolveRelationsForWrite. + relRefusals, droppedDefaults, relErr := s.resolveRelationsForWrite( + r, workspaceID, workspaceRole(r), schema, fieldMap, relBefore, posture) + if relErr != nil { + return nil, nil, nil, nil, &itemCreateError{status: http.StatusInternalServerError, code: "internal_error", message: "Failed to resolve relation references"} + } + droppedDefaults = append(defaultDrops, droppedDefaults...) + if len(relRefusals) > 0 { + if posture == relationsRefuse { + return nil, nil, nil, nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: relationIssuesMessage(relRefusals)} + } + // Carry: the values stay in fieldMap exactly as supplied — the + // resolver leaves what it cannot resolve untouched — and the write + // says which ones, so an import is never silently lossy. + for _, ri := range relRefusals { + unresolved = append(unresolved, ri.Key) + } + } + undeclared = items.UndeclaredFieldKeys(fieldMap, schema) + + if err := s.checkUniqueFields(workspaceID, coll.ID, "", schema, fieldMap); err != nil { + return nil, nil, nil, nil, &itemCreateError{status: http.StatusConflict, code: "conflict", message: err.Error()} + } + return fieldMap, droppedDefaults, unresolved, undeclared, nil +} + // writeItemResolveError distinguishes a soft-deleted (archived) item from a // genuinely-missing one for mutation handlers. An archived item still appears // in include-archived list results and can be fetched read-only via GET, so a @@ -3880,9 +3894,14 @@ func (s *Server) extractParentLink( return parentValue, parentProvided, true } +// uniqueEnforced reports whether checkUniqueFields enforces def. The app +// installer's preview uses the same predicate (TASK-3397), so the two cannot +// disagree about which fields are unique. +func uniqueEnforced(def models.FieldDef) bool { return def.UniqueScope == "workspace_collection" } + func (s *Server) checkUniqueFields(workspaceID, collectionID, excludeItemID string, schema models.CollectionSchema, fieldMap map[string]any) error { for _, def := range schema.Fields { - if def.UniqueScope != "workspace_collection" { + if !uniqueEnforced(def) { continue } raw, ok := fieldMap[def.Key] diff --git a/internal/server/server.go b/internal/server/server.go index 512fc15b1..8974b6036 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/subtle" + "crypto/tls" "encoding/base64" "encoding/json" "errors" @@ -169,6 +170,9 @@ type Server struct { // build that doesn't want the dependency — still serves every // other endpoint and stores originals untouched. imageProcessor attachments.Processor + // appFetchTLS is the TLS config app manifest/artifact fetches use; nil + // (the system roots) in production, set by tests (SetAppFetchTLS). + appFetchTLS *tls.Config // MCP Streamable HTTP transport (PLAN-943 TASK-950). Wired via // SetMCPTransport at startup on every install (PLAN-2310 DR-4); nil @@ -339,6 +343,10 @@ type Server struct { // the loop via stopReminderTick. reminderTick reminderTickConfig + // appPendingSweep deletes expired pending app installs and their staged + // bytes (TASK-3397). Started via StartAppPendingSweep. + appPendingSweep appPendingSweepConfig + // decisionTick holds the typed-decision runner and its evaluation loop // (TASK-3117). Same shape as reminderTick; started via StartDecisionTick // and stopped by Stop() via stopDecisionTick. @@ -617,6 +625,7 @@ func (s *Server) Stop() { // Item reminder tick (IDEA-2641). Same lifecycle pattern; an in-flight // pass is tracked on s.bg and awaited below. s.stopReminderTick() + s.stopAppPendingSweep() s.stopDecisionTick() // MCP audit writer / sweeper run on s.bg too. Signal first so // the workers see the close BEFORE Wait() blocks; without the @@ -1722,6 +1731,8 @@ func (s *Server) setupRouter() { r.Put("/decision-provider", s.handleUpdateDecisionSettings) r.Get("/mcp", s.handleGetMCPSettings) r.Put("/mcp", s.handleUpdateMCPSettings) + r.Get("/apps", s.handleGetAppsSettings) + r.Put("/apps", s.handleUpdateAppsSettings) r.Post("/test-email", s.handleTestEmail) // Cloud sidecar endpoints — only exist in cloud mode. requireCloudMode @@ -2180,6 +2191,14 @@ func (s *Server) setupRouter() { // short TTL — see handleGetWorkspaceStorageUsage. r.Get("/storage/usage", s.handleGetWorkspaceStorageUsage) + // App install staging (SPEC-6 U8a, TASK-3397). Owner-only and + // gated on appsAvailable inside each handler. + r.Route("/apps/install", func(r chi.Router) { + r.Post("/preview", s.handleAppInstallPreview) + r.Get("/pending/{pendingID}", s.handleGetAppInstallPending) + r.Delete("/pending/{pendingID}", s.handleDeleteAppInstallPending) + }) + // Webhooks r.Route("/webhooks", func(r chi.Router) { r.Get("/", s.handleListWebhooks) diff --git a/internal/server/testdata/artifact_import_parity.json b/internal/server/testdata/artifact_import_parity.json new file mode 100644 index 000000000..2d09258d7 --- /dev/null +++ b/internal/server/testdata/artifact_import_parity.json @@ -0,0 +1,129 @@ +{ + "blank title": { + "status": 400, + "title": "", + "content": "", + "fields": null, + "warnings": null + }, + "convention, active, valid selects": { + "status": 201, + "title": "Clean Convention", + "content": "A clean convention.\n", + "fields": { + "priority": "must", + "scope": "all", + "status": "draft", + "trigger": "on-commit" + }, + "warnings": [ + "status \"active\" reset to \"draft\" on import" + ] + }, + "convention, foreign select value": { + "status": 201, + "title": "Foreign", + "content": "x\n", + "fields": { + "scope": "all", + "status": "draft", + "trigger": "" + }, + "warnings": [ + "field \"trigger\" value \"on-candidate-advance\" is not a valid option in this workspace; cleared on import", + "status \"active\" reset to \"draft\" on import" + ] + }, + "golden convention": { + "status": 201, + "title": "Run the test suite before pushing", + "content": "Always run `make test` before pushing to main.\n\n\n---\n\n_Imported artifact from workspace `demo`, exported by xarmian at 2026-06-22T00:00:00Z._\n", + "fields": { + "priority": "", + "role": "engineer", + "scope": "", + "status": "draft", + "trigger": "on-commit" + }, + "warnings": [ + "field \"priority\" value \"high\" is not a valid option in this workspace; cleared on import", + "field \"scope\" value \"repo\" is not a valid option in this workspace; cleared on import", + "status \"active\" reset to \"draft\" on import" + ] + }, + "golden playbook": { + "status": 201, + "title": "Ship a change", + "content": "## Steps\n\n1. Run the tests\n2. Open a PR\n\n\n---\n\n_Imported artifact from workspace `demo`, exported by xarmian at 2026-06-22T00:00:00Z._\n", + "fields": { + "arguments": [ + { + "description": "the item to ship", + "name": "ref", + "required": true, + "type": "ref" + }, + { + "default": "squash", + "name": "merge-strategy", + "required": false, + "type": "enum" + } + ], + "invocation_slug": "ship", + "scope": "", + "status": "draft", + "trigger": "" + }, + "warnings": [ + "field \"scope\" value \"workspace\" is not a valid option in this workspace; cleared on import", + "field \"trigger\" value \"on-intent\" is not a valid option in this workspace; cleared on import", + "status \"active\" reset to \"draft\" on import" + ] + }, + "playbook with arguments": { + "status": 201, + "title": "Playbook With Args", + "content": "Do the thing.\n", + "fields": { + "arguments": [ + { + "description": "the thing", + "name": "target", + "required": true, + "type": "ref" + } + ], + "invocation_slug": "with-args", + "scope": "all", + "status": "draft", + "trigger": "manual" + }, + "warnings": [ + "status \"active\" reset to \"draft\" on import" + ] + }, + "playbook, slug collision": { + "status": 201, + "title": "Second", + "content": "second\n", + "fields": { + "invocation_slug": "ship-2", + "status": "draft" + }, + "warnings": [ + "invocation_slug \"ship\" was already in use; imported as \"ship-2\"", + "status \"active\" reset to \"draft\" on import" + ] + }, + "with provenance": { + "status": 201, + "title": "Provenanced", + "content": "Body.\n\n\n---\n\n_Imported artifact from workspace `elsewhere`, exported by someone at 2026-01-02T03:04:05Z._\n", + "fields": { + "status": "draft", + "trigger": "on-commit" + }, + "warnings": null + } +} diff --git a/internal/store/app_pending.go b/internal/store/app_pending.go new file mode 100644 index 000000000..1125652bc --- /dev/null +++ b/internal/store/app_pending.go @@ -0,0 +1,326 @@ +package store + +import ( + "database/sql" + "errors" + "fmt" + "strings" + "time" +) + +// Pending app installs (SPEC-6 U8a, DOC-3371 §2 "Staging caps"; TASK-3397). +// +// A reservation is taken BEFORE any network fetch, in one transaction that +// locks the owner's users row (FOR NO KEY UPDATE, the enforceUserLimitTx +// shape) and then the instance-wide pending-bytes advisory lock, and counts: +// at most PendingPerOwner live reservations per owner, and at most +// PendingInstanceBytes of live reserved bytes on the instance. It charges the +// FULL PendingInstallBytes. The charge is kept, whatever has been staged so +// far, until no further bytes can arrive (FinishPendingInstall); only then is +// it reduced to the bytes actually staged, under the instance lock. Releasing +// capacity mid-fetch would let the remaining artifacts of concurrent installs +// together exceed the instance cap. +// +// "Live" is expires_at in the future. A fetching reservation expires shortly +// after the fetch deadline, a staged one after an hour; the sweep deletes +// expired ones, and the blobs go with them (ON DELETE CASCADE). On SQLite +// BEGIN IMMEDIATE serializes all of this, and the lock clauses are skipped. +// +// LOCK ORDER: owner row, then the instance lock. The other transactions here +// take the instance lock alone, so no cycle forms. + +// Staging caps (DOC-3371 §2). +const ( + PendingInstallBytes int64 = 4 << 20 + PendingInstanceBytes int64 = 64 << 20 + PendingPerOwner = 3 + PendingFetchTTL = 2 * time.Minute + PendingStagedTTL = time.Hour +) + +// PendingManifestKey is the blob key the manifest is staged under. +const PendingManifestKey = "$manifest" + +var ( + // ErrPendingLimit: the owner already has PendingPerOwner live pending + // installs, or the instance's pending bytes are at their cap. The + // refusal carries no counts. + ErrPendingLimit = errors.New("app install: too many pending installs") + // ErrPendingOverCap: staging a blob would exceed the install's byte cap. + ErrPendingOverCap = errors.New("app install: the pack exceeds its size cap") + // ErrPendingNotFound: no live pending install with that id for that + // workspace and owner. + ErrPendingNotFound = errors.New("app install: pending install not found") +) + +// PendingInstall is a pending-install record. +type PendingInstall struct { + ID string + WorkspaceID string + OwnerID string + Origin string + State string + ReservedBytes int64 + ManifestSHA256 string + Preview string + ExpiresAt time.Time +} + +// PendingBlob is one staged blob: the manifest or an artifact. +type PendingBlob struct { + Key string + URL string + SHA256 string + Data []byte +} + +func (s *Store) lockPendingBytes(tx *sql.Tx) error { + if s.dialect.Driver() != DriverPostgres { + return nil + } + if _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtext('pad:app-pending-bytes'))`); err != nil { + return fmt.Errorf("pending install: instance lock: %w", err) + } + return nil +} + +func timeText(t time.Time) string { return t.UTC().Format(time.RFC3339) } + +// ReservePendingInstall takes a reservation for an install the owner is about +// to fetch, or refuses with ErrPendingLimit. No fetch may begin without one. +func (s *Store) ReservePendingInstall(workspaceID, ownerID, origin string) (*PendingInstall, error) { + tx, err := s.db.Begin() + if err != nil { + return nil, fmt.Errorf("pending install: begin: %w", err) + } + defer tx.Rollback() + + ownerQ := `SELECT id FROM users WHERE id = ?` + if s.dialect.Driver() == DriverPostgres { + ownerQ += ` FOR NO KEY UPDATE` + } + var got string + if err := tx.QueryRow(s.q(ownerQ), ownerID).Scan(&got); err != nil { + return nil, fmt.Errorf("pending install: lock owner: %w", err) + } + if err := s.lockPendingBytes(tx); err != nil { + return nil, err + } + nowT := time.Now() + nowS := timeText(nowT) + var mine int + if err := tx.QueryRow(s.q(`SELECT COUNT(*) FROM app_install_pending WHERE owner_id = ? AND expires_at > ?`), ownerID, nowS).Scan(&mine); err != nil { + return nil, fmt.Errorf("pending install: count owner: %w", err) + } + var instance int64 + if err := tx.QueryRow(s.q(`SELECT COALESCE(SUM(reserved_bytes), 0) FROM app_install_pending WHERE expires_at > ?`), nowS).Scan(&instance); err != nil { + return nil, fmt.Errorf("pending install: sum instance: %w", err) + } + if mine >= PendingPerOwner || instance+PendingInstallBytes > PendingInstanceBytes { + return nil, ErrPendingLimit + } + p := &PendingInstall{ + ID: newID(), WorkspaceID: workspaceID, OwnerID: ownerID, Origin: origin, State: "fetching", + ReservedBytes: PendingInstallBytes, ExpiresAt: nowT.Add(PendingFetchTTL).UTC().Truncate(time.Second), + } + if _, err := tx.Exec(s.q(`INSERT INTO app_install_pending (id, workspace_id, owner_id, origin, state, reserved_bytes, expires_at, created_at, updated_at) + VALUES (?, ?, ?, ?, 'fetching', ?, ?, ?, ?)`), + p.ID, workspaceID, ownerID, origin, PendingInstallBytes, timeText(p.ExpiresAt), nowS, nowS); err != nil { + return nil, fmt.Errorf("pending install: insert: %w", err) + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("pending install: commit: %w", err) + } + return p, nil +} + +// StagePendingBlob writes one fetched blob into a fetching reservation. The +// staged total may not exceed the reservation's charge (the per-install +// cap), which is never reduced while fetching. +func (s *Store) StagePendingBlob(pendingID string, b PendingBlob) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("pending blob: begin: %w", err) + } + defer tx.Rollback() + q := `SELECT reserved_bytes FROM app_install_pending WHERE id = ? AND state = 'fetching' AND expires_at > ?` + if s.dialect.Driver() == DriverPostgres { + q += ` FOR UPDATE` + } + var reserved int64 + switch err := tx.QueryRow(s.q(q), pendingID, timeText(time.Now())).Scan(&reserved); { + case errors.Is(err, sql.ErrNoRows): + return ErrPendingNotFound + case err != nil: + return fmt.Errorf("pending blob: lock: %w", err) + } + var staged int64 + if err := tx.QueryRow(s.q(`SELECT COALESCE(SUM(size_bytes), 0) FROM app_install_pending_blobs WHERE pending_id = ?`), pendingID).Scan(&staged); err != nil { + return fmt.Errorf("pending blob: sum: %w", err) + } + if staged+int64(len(b.Data)) > reserved { + return ErrPendingOverCap + } + if _, err := tx.Exec(s.q(`INSERT INTO app_install_pending_blobs (pending_id, key, url, sha256, size_bytes, data) VALUES (?, ?, ?, ?, ?, ?)`), + pendingID, b.Key, b.URL, b.SHA256, len(b.Data), b.Data); err != nil { + return fmt.Errorf("pending blob: insert: %w", err) + } + return tx.Commit() +} + +// FinishPendingInstall ends the fetch: under the instance lock, the charge is +// reduced to the bytes actually staged, the record is marked staged with its +// manifest hash and preview, and it is kept for PendingStagedTTL. +func (s *Store) FinishPendingInstall(pendingID, manifestSHA256, preview string) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("pending install: begin: %w", err) + } + defer tx.Rollback() + if err := s.lockPendingBytes(tx); err != nil { + return err + } + var staged int64 + if err := tx.QueryRow(s.q(`SELECT COALESCE(SUM(size_bytes), 0) FROM app_install_pending_blobs WHERE pending_id = ?`), pendingID).Scan(&staged); err != nil { + return fmt.Errorf("pending install: sum: %w", err) + } + nowT := time.Now() + res, err := tx.Exec(s.q(`UPDATE app_install_pending SET state = 'staged', reserved_bytes = ?, manifest_sha256 = ?, preview = ?, expires_at = ?, updated_at = ? + WHERE id = ? AND state = 'fetching' AND expires_at > ?`), + staged, manifestSHA256, preview, timeText(nowT.Add(PendingStagedTTL)), timeText(nowT), pendingID, timeText(nowT)) + if err != nil { + return fmt.Errorf("pending install: finish: %w", err) + } + if n, _ := res.RowsAffected(); n != 1 { + return ErrPendingNotFound + } + return tx.Commit() +} + +// DeletePendingInstall removes a reservation and every staged blob: a failed +// or cancelled fetch, or an owner discarding a preview. A missing record is +// not an error. +func (s *Store) DeletePendingInstall(pendingID string) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("pending install: begin: %w", err) + } + defer tx.Rollback() + if err := s.lockPendingBytes(tx); err != nil { + return err + } + if _, err := tx.Exec(s.q(`DELETE FROM app_install_pending WHERE id = ?`), pendingID); err != nil { + return fmt.Errorf("pending install: delete: %w", err) + } + return tx.Commit() +} + +// SweepExpiredPendingInstalls deletes every expired reservation and its +// blobs, returning how many records it removed. +func (s *Store) SweepExpiredPendingInstalls() (int64, error) { + tx, err := s.db.Begin() + if err != nil { + return 0, fmt.Errorf("pending sweep: begin: %w", err) + } + defer tx.Rollback() + if err := s.lockPendingBytes(tx); err != nil { + return 0, err + } + res, err := tx.Exec(s.q(`DELETE FROM app_install_pending WHERE expires_at <= ?`), timeText(time.Now())) + if err != nil { + return 0, fmt.Errorf("pending sweep: %w", err) + } + n, _ := res.RowsAffected() + return n, tx.Commit() +} + +// GetPendingInstall returns a LIVE pending install of this owner in this +// workspace, or ErrPendingNotFound (another owner's, another workspace's, an +// expired and a missing one all answer the same). +func (s *Store) GetPendingInstall(pendingID, workspaceID, ownerID string) (*PendingInstall, error) { + var p PendingInstall + var manifest, preview sql.NullString + var expires string + err := s.db.QueryRow(s.q(`SELECT id, workspace_id, owner_id, origin, state, reserved_bytes, manifest_sha256, preview, expires_at + FROM app_install_pending WHERE id = ? AND workspace_id = ? AND owner_id = ? AND expires_at > ?`), + pendingID, workspaceID, ownerID, timeText(time.Now())). + Scan(&p.ID, &p.WorkspaceID, &p.OwnerID, &p.Origin, &p.State, &p.ReservedBytes, &manifest, &preview, &expires) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrPendingNotFound + } + if err != nil { + return nil, fmt.Errorf("get pending install: %w", err) + } + p.ManifestSHA256, p.Preview, p.ExpiresAt = manifest.String, preview.String, parseTime(expires) + return &p, nil +} + +// PendingInstallBlobs returns a pending install's staged blobs, by key. +func (s *Store) PendingInstallBlobs(pendingID string) (map[string]PendingBlob, error) { + rows, err := s.db.Query(s.q(`SELECT key, url, sha256, data FROM app_install_pending_blobs WHERE pending_id = ?`), pendingID) + if err != nil { + return nil, fmt.Errorf("pending blobs: %w", err) + } + defer rows.Close() + out := map[string]PendingBlob{} + for rows.Next() { + var b PendingBlob + if err := rows.Scan(&b.Key, &b.URL, &b.SHA256, &b.Data); err != nil { + return nil, err + } + out[b.Key] = b + } + return out, rows.Err() +} + +// CollectionSlugOwners reports, for each of slugs that already names a +// collection in the workspace (live or soft-deleted, since the unique index +// covers both), the origin of the app install that created it, or "" when no +// install did. Slugs absent from the result are free. It backs the install +// conflict check (DOC-3371 §2 step 5, L6). +func (s *Store) CollectionSlugOwners(workspaceID string, slugs []string) (map[string]string, error) { + out := map[string]string{} + if len(slugs) == 0 { + return out, nil + } + in := strings.TrimSuffix(strings.Repeat("?,", len(slugs)), ",") + args := []any{workspaceID} + for _, sl := range slugs { + args = append(args, sl) + } + rows, err := s.db.Query(s.q(`SELECT c.slug, COALESCE(a.origin, '') FROM collections c + LEFT JOIN app_installs a ON a.id = c.via_app + WHERE c.workspace_id = ? AND c.slug IN (`+in+`)`), args...) + if err != nil { + return nil, fmt.Errorf("collection slug owners: %w", err) + } + defer rows.Close() + for rows.Next() { + var slug, origin string + if err := rows.Scan(&slug, &origin); err != nil { + return nil, err + } + out[slug] = origin + } + return out, rows.Err() +} + +// InvocationSlugIndexTaken reports whether a live item in collectionID would +// collide with slug in idx_items_invocation_slug_per_collection, using the +// index's OWN expression on each dialect (SQLite json_extract, Postgres +// ->>), so it agrees with the index on every stored shape, arrays and +// objects included (TASK-3397, codex round 8). It backs the app installer's +// preview of a defaulted slug. +func (s *Store) InvocationSlugIndexTaken(collectionID, slug string) (bool, error) { + expr := `json_extract(fields, '$.invocation_slug')` + if s.dialect.Driver() == DriverPostgres { + expr = `(fields->>'invocation_slug')` + } + var n int + err := s.db.QueryRow(s.q(`SELECT COUNT(*) FROM items WHERE collection_id = ? AND deleted_at IS NULL + AND `+expr+` IS NOT NULL AND `+expr+` != '' AND `+expr+` = ?`), collectionID, slug).Scan(&n) + if err != nil { + return false, fmt.Errorf("invocation slug index lookup: %w", err) + } + return n > 0, nil +} diff --git a/internal/store/app_pending_test.go b/internal/store/app_pending_test.go new file mode 100644 index 000000000..5656158a5 --- /dev/null +++ b/internal/store/app_pending_test.go @@ -0,0 +1,218 @@ +package store + +import ( + "bytes" + "errors" + "fmt" + "sync" + "testing" + "time" +) + +// TASK-3397 (U8a): pending app-install reservations and their caps. + +func pendingOwner(t *testing.T, s *Store, n int) string { + t.Helper() + return createTestUser(t, s, fmt.Sprintf("owner-%d-%d@example.com", n, time.Now().UnixNano()), "Owner", "password123").ID +} + +func TestPendingInstall_PerOwnerCap(t *testing.T) { + s := testStore(t) + ws := createTestWorkspace(t, s, "Apps") + owner := pendingOwner(t, s, 1) + for i := 0; i < PendingPerOwner; i++ { + if _, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example"); err != nil { + t.Fatalf("reservation %d: %v", i, err) + } + } + if _, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example"); !errors.Is(err, ErrPendingLimit) { + t.Fatalf("over the per-owner cap: %v", err) + } + // Another owner is unaffected. + if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, 2), "https://app.example"); err != nil { + t.Fatalf("another owner: %v", err) + } +} + +func TestPendingInstall_InstanceBytesCap(t *testing.T) { + s := testStore(t) + ws := createTestWorkspace(t, s, "Apps") + max := int(PendingInstanceBytes / PendingInstallBytes) + for i := 0; i < max; i++ { + if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, i), "https://app.example"); err != nil { + t.Fatalf("reservation %d of %d: %v", i, max, err) + } + } + if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, max), "https://app.example"); !errors.Is(err, ErrPendingLimit) { + t.Fatalf("over the instance cap: %v", err) + } +} + +// The full charge is held while fetching, however little has been staged, and +// reduced to the staged bytes only when the fetch ends. +func TestPendingInstall_ChargeIsHeldUntilTheFetchEnds(t *testing.T) { + s := testStore(t) + ws := createTestWorkspace(t, s, "Apps") + max := int(PendingInstanceBytes / PendingInstallBytes) + var first *PendingInstall + for i := 0; i < max; i++ { + p, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, i), "https://app.example") + if err != nil { + t.Fatal(err) + } + if first == nil { + first = p + } + } + if err := s.StagePendingBlob(first.ID, PendingBlob{Key: PendingManifestKey, URL: "https://app.example/m", SHA256: "x", Data: []byte("{}")}); err != nil { + t.Fatal(err) + } + // Staging 2 bytes did not release capacity. + if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, 99), "https://app.example"); !errors.Is(err, ErrPendingLimit) { + t.Fatalf("mid-fetch staging released capacity: %v", err) + } + reserved := func() int64 { + var sum int64 + if err := s.db.QueryRow(`SELECT COALESCE(SUM(reserved_bytes), 0) FROM app_install_pending`).Scan(&sum); err != nil { + t.Fatal(err) + } + return sum + } + if got := reserved(); got != PendingInstanceBytes { + t.Fatalf("reserved %d mid-fetch, want the full %d", got, PendingInstanceBytes) + } + if err := s.FinishPendingInstall(first.ID, "sha", `{"p":1}`); err != nil { + t.Fatal(err) + } + if got, want := reserved(), PendingInstanceBytes-PendingInstallBytes+2; got != want { + t.Fatalf("reserved %d after finish, want %d (the charge reduced to the 2 staged bytes)", got, want) + } + got, err := s.GetPendingInstall(first.ID, ws.ID, first.OwnerID) + if err != nil { + t.Fatal(err) + } + if got.State != "staged" || got.ReservedBytes != 2 || got.Preview != `{"p":1}` { + t.Fatalf("after finish: %+v", got) + } +} + +func TestPendingInstall_StagingCannotExceedTheInstallCap(t *testing.T) { + s := testStore(t) + ws := createTestWorkspace(t, s, "Apps") + p, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, 1), "https://app.example") + if err != nil { + t.Fatal(err) + } + half := bytes.Repeat([]byte("a"), int(PendingInstallBytes/2)) + for i, c := range []struct { + data []byte + want error + }{{half, nil}, {half, nil}, {[]byte("x"), ErrPendingOverCap}} { + err := s.StagePendingBlob(p.ID, PendingBlob{Key: fmt.Sprintf("a%d", i), URL: "https://app.example/a", SHA256: "x", Data: c.data}) + if !errors.Is(err, c.want) { + t.Fatalf("blob %d: %v, want %v", i, err, c.want) + } + } +} + +func TestPendingInstall_ScopingDeleteAndSweep(t *testing.T) { + s := testStore(t) + ws := createTestWorkspace(t, s, "Apps") + other := createTestWorkspace(t, s, "Other") + owner := pendingOwner(t, s, 1) + p, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example") + if err != nil { + t.Fatal(err) + } + data := []byte("manifest bytes\x00with a nul") + if err := s.StagePendingBlob(p.ID, PendingBlob{Key: PendingManifestKey, URL: "https://app.example/m", SHA256: "h", Data: data}); err != nil { + t.Fatal(err) + } + blobs, err := s.PendingInstallBlobs(p.ID) + if err != nil || !bytes.Equal(blobs[PendingManifestKey].Data, data) { + t.Fatalf("blob round trip: %v %q", err, blobs[PendingManifestKey].Data) + } + for name, args := range map[string][3]string{ + "another owner": {p.ID, ws.ID, pendingOwner(t, s, 2)}, + "another workspace": {p.ID, other.ID, owner}, + "unknown": {newID(), ws.ID, owner}, + } { + if _, err := s.GetPendingInstall(args[0], args[1], args[2]); !errors.Is(err, ErrPendingNotFound) { + t.Errorf("%s: %v", name, err) + } + } + + // Expire it, then sweep: the record and its blobs go together. + if _, err := s.db.Exec(s.q(`UPDATE app_install_pending SET expires_at = ? WHERE id = ?`), timeText(time.Now().Add(-time.Minute)), p.ID); err != nil { + t.Fatal(err) + } + if _, err := s.GetPendingInstall(p.ID, ws.ID, owner); !errors.Is(err, ErrPendingNotFound) { + t.Errorf("expired record still readable: %v", err) + } + // Expired but NOT yet swept, it already no longer counts against its + // owner: the caps read expires_at, not the sweep. + for i := 0; i < PendingPerOwner; i++ { + if _, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example"); err != nil { + t.Fatalf("with an expired, unswept reservation, reservation %d: %v", i, err) + } + } + n, err := s.SweepExpiredPendingInstalls() + if err != nil || n != 1 { + t.Fatalf("sweep: %d %v", n, err) + } + var left int + if err := s.db.QueryRow(s.q(`SELECT COUNT(*) FROM app_install_pending_blobs WHERE pending_id = ?`), p.ID).Scan(&left); err != nil || left != 0 { + t.Fatalf("blobs left after sweep: %d %v", left, err) + } +} + +// Concurrent reservations never exceed either cap. On SQLite BEGIN IMMEDIATE +// serializes them; on Postgres the owner row lock and the instance advisory +// lock do. +func TestPendingInstall_ConcurrentReservationsRespectBothCaps(t *testing.T) { + s := testStore(t) + ws := createTestWorkspace(t, s, "Apps") + shared := pendingOwner(t, s, 0) + owners := make([]string, 24) + for i := range owners { + owners[i] = pendingOwner(t, s, i+1) + } + var mu sync.Mutex + okShared, okTotal := 0, 0 + var wg sync.WaitGroup + run := func(owner string, isShared bool) { + defer wg.Done() + _, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example") + switch { + case err == nil: + mu.Lock() + okTotal++ + if isShared { + okShared++ + } + mu.Unlock() + case errors.Is(err, ErrPendingLimit): + default: + t.Errorf("unexpected: %v", err) + } + } + for i := 0; i < 8; i++ { + wg.Add(1) + go run(shared, true) + } + for _, o := range owners { + wg.Add(1) + go run(o, false) + } + wg.Wait() + if okShared > PendingPerOwner { + t.Errorf("one owner holds %d reservations, cap %d", okShared, PendingPerOwner) + } + if max := int(PendingInstanceBytes / PendingInstallBytes); okTotal > max { + t.Errorf("%d reservations, instance cap %d", okTotal, max) + } + var sum int64 + if err := s.db.QueryRow(`SELECT COALESCE(SUM(reserved_bytes), 0) FROM app_install_pending`).Scan(&sum); err != nil || sum > PendingInstanceBytes { + t.Errorf("reserved bytes %d > %d (%v)", sum, PendingInstanceBytes, err) + } +} diff --git a/internal/store/dialect_fieldvalue.go b/internal/store/dialect_fieldvalue.go index 43ea1dd2a..f2c64d7d2 100644 --- a/internal/store/dialect_fieldvalue.go +++ b/internal/store/dialect_fieldvalue.go @@ -160,3 +160,21 @@ func (d *postgresDialect) JSONFieldOrder(column, key, dir string) string { num := fmt.Sprintf("(CASE WHEN %s = 'number' THEN (%s->'%s')::numeric END)", t, column, key) return fmt.Sprintf("%s ASC, %s %s, %s %s", rank, num, dir, d.JSONFieldText(column, key), dir) } + +// NumericClaimForms is the form under which JSONFieldEquals could find two +// numeric values equal, on either dialect, for arg when numericArg accepts +// it: its float64 value. SQLite compares an INTEGER and a REAL as doubles, +// and Postgres compares numeric EXACTLY; two exactly equal values round to +// the same double, so equal doubles is a superset of both. The app +// installer's in-pack uniqueness claims key numbers on it (TASK-3397, codex +// rounds 10-12). +func NumericClaimForms(arg string) []string { + if _, ok := numericArg(arg); !ok { + return nil + } + f, err := strconv.ParseFloat(arg, 64) + if err != nil && !math.IsInf(f, 0) { + return nil + } + return []string{"f:" + strconv.FormatFloat(f, 'g', -1, 64)} +} diff --git a/internal/store/migrations/118_app_install_lifecycle.sql b/internal/store/migrations/118_app_install_lifecycle.sql new file mode 100644 index 000000000..d2a757e95 --- /dev/null +++ b/internal/store/migrations/118_app_install_lifecycle.sql @@ -0,0 +1,65 @@ +-- Migration 118: app install staging and provenance (SPEC-6 U8a, TASK-3397). +-- +-- app_install_pending is the pending-install record (DOC-3371 §2 step 1): a +-- reservation taken BEFORE any network fetch, under the owner's users row and +-- an instance-wide lock, that charges the full per-install byte cap until the +-- fetch has ended. The staged bytes live in app_install_pending_blobs (the +-- manifest under the key '$manifest', then one row per artifact), so a +-- failure, the fetch deadline or expiry deletes the reservation and every +-- staged byte in one statement (ON DELETE CASCADE). A fetching record expires +-- two minutes after it is taken (the 30 s fetch deadline plus margin), so a +-- process that dies mid-fetch holds its charge only that long; a staged one +-- is kept for an hour awaiting consent. +-- +-- The app_installs columns are written by provisioning (U8b): the install's +-- bot (the OAuth client is U5's, keyed by oauth_clients.app_install_id), the manifest's hash and version, the declared events +-- and item actions (stored now, provisioned when A5/A6 land), the access +-- levels, the owner-edited config and every raw and normalized digest. +-- +-- collections.via_app is the install that created a companion collection. The +-- conflict check (§2 step 5, L6) adopts an existing slug only when the install +-- that created it has the same origin; any other existing slug is an error. +-- +-- items.source_pack / source_artifact_sha256 / installed_sha256 stamp a +-- companion artifact with the pack it came from and both digests (§2 step 6), +-- so a later installer can adopt it. +CREATE TABLE IF NOT EXISTS app_install_pending ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + origin TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'fetching' CHECK (state IN ('fetching', 'staged')), + reserved_bytes INTEGER NOT NULL, + manifest_sha256 TEXT, + preview TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_app_install_pending_owner ON app_install_pending (owner_id, expires_at); +CREATE INDEX IF NOT EXISTS idx_app_install_pending_expires ON app_install_pending (expires_at); + +CREATE TABLE IF NOT EXISTS app_install_pending_blobs ( + pending_id TEXT NOT NULL REFERENCES app_install_pending(id) ON DELETE CASCADE, + key TEXT NOT NULL, + url TEXT NOT NULL, + sha256 TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (pending_id, key) +); + +ALTER TABLE app_installs ADD COLUMN bot_user_id TEXT REFERENCES users(id) ON DELETE SET NULL; +ALTER TABLE app_installs ADD COLUMN manifest_sha256 TEXT; +ALTER TABLE app_installs ADD COLUMN manifest_version TEXT; +ALTER TABLE app_installs ADD COLUMN manifest TEXT; +ALTER TABLE app_installs ADD COLUMN service_access TEXT; +ALTER TABLE app_installs ADD COLUMN delegated_access TEXT; +ALTER TABLE app_installs ADD COLUMN config TEXT NOT NULL DEFAULT '{}'; +ALTER TABLE app_installs ADD COLUMN digests TEXT NOT NULL DEFAULT '{}'; + +ALTER TABLE collections ADD COLUMN via_app TEXT REFERENCES app_installs(id) ON DELETE SET NULL; + +ALTER TABLE items ADD COLUMN source_pack TEXT; +ALTER TABLE items ADD COLUMN source_artifact_sha256 TEXT; +ALTER TABLE items ADD COLUMN installed_sha256 TEXT; diff --git a/internal/store/nul_unprotected_baseline.txt b/internal/store/nul_unprotected_baseline.txt index d20d1110e..a88b92bd6 100644 --- a/internal/store/nul_unprotected_baseline.txt +++ b/internal/store/nul_unprotected_baseline.txt @@ -16,9 +16,32 @@ api_tokens.prefix api_tokens.token_hash api_tokens.user_id api_tokens.workspace_id +app_install_pending.created_at +app_install_pending.expires_at +app_install_pending.id +app_install_pending.manifest_sha256 +app_install_pending.origin +app_install_pending.owner_id +app_install_pending.preview +app_install_pending.state +app_install_pending.updated_at +app_install_pending.workspace_id +app_install_pending_blobs.data +app_install_pending_blobs.key +app_install_pending_blobs.pending_id +app_install_pending_blobs.sha256 +app_install_pending_blobs.url +app_installs.bot_user_id +app_installs.config app_installs.created_at +app_installs.delegated_access +app_installs.digests app_installs.id +app_installs.manifest +app_installs.manifest_sha256 +app_installs.manifest_version app_installs.origin +app_installs.service_access app_installs.state app_installs.updated_at app_installs.workspace_id @@ -55,6 +78,7 @@ collections.created_at collections.deleted_at collections.id collections.updated_at +collections.via_app collections.workspace_id comment_reactions.comment_id comment_reactions.created_at @@ -177,10 +201,13 @@ items.created_by_user_id items.created_via_app items.deleted_at items.id +items.installed_sha256 items.last_modified_by_user_id items.lease_acquired_at items.lease_expires_at items.parent_id +items.source_artifact_sha256 +items.source_pack items.updated_at items.via_app items.workspace_id diff --git a/internal/store/nulguard_test.go b/internal/store/nulguard_test.go index a6cd07819..9732b4ddd 100644 --- a/internal/store/nulguard_test.go +++ b/internal/store/nulguard_test.go @@ -34,7 +34,11 @@ func TestBinaryColumnCensus(t *testing.T) { t.Parallel() // item_yjs_updates_set_aside.update_data holds op-log frames moved verbatim // by a schema rebuild (BUG-3244): the same raw Yjs bytes, NULs included. - want := map[string]bool{"item_yjs_updates.update_data": true, "item_yjs_updates_set_aside.update_data": true} + // app_install_pending_blobs.data holds an app's manifest and artifact + // bytes exactly as fetched (TASK-3397), before any decoding: arbitrary + // bytes, NULs included, bound only as []byte by StagePendingBlob. + want := map[string]bool{"item_yjs_updates.update_data": true, "item_yjs_updates_set_aside.update_data": true, + "app_install_pending_blobs.data": true} // One pattern per dialect spelling, applied to the raw migration text so // nothing depends on a Go-side model of the schema. diff --git a/internal/store/pgmigrations/092_app_install_lifecycle.sql b/internal/store/pgmigrations/092_app_install_lifecycle.sql new file mode 100644 index 000000000..b61ecda39 --- /dev/null +++ b/internal/store/pgmigrations/092_app_install_lifecycle.sql @@ -0,0 +1,42 @@ +-- Migration 092: app install staging and provenance (SPEC-6 U8a, TASK-3397). +-- Mirrors SQLite migration 118; see it for the rationale. +CREATE TABLE IF NOT EXISTS app_install_pending ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + origin TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'fetching' CHECK (state IN ('fetching', 'staged')), + reserved_bytes BIGINT NOT NULL, + manifest_sha256 TEXT, + preview TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_app_install_pending_owner ON app_install_pending (owner_id, expires_at); +CREATE INDEX IF NOT EXISTS idx_app_install_pending_expires ON app_install_pending (expires_at); + +CREATE TABLE IF NOT EXISTS app_install_pending_blobs ( + pending_id TEXT NOT NULL REFERENCES app_install_pending(id) ON DELETE CASCADE, + key TEXT NOT NULL, + url TEXT NOT NULL, + sha256 TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + data BYTEA NOT NULL, + PRIMARY KEY (pending_id, key) +); + +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS bot_user_id TEXT REFERENCES users(id) ON DELETE SET NULL; +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS manifest_sha256 TEXT; +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS manifest_version TEXT; +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS manifest TEXT; +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS service_access TEXT; +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS delegated_access TEXT; +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS config TEXT NOT NULL DEFAULT '{}'; +ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS digests TEXT NOT NULL DEFAULT '{}'; + +ALTER TABLE collections ADD COLUMN IF NOT EXISTS via_app TEXT REFERENCES app_installs(id) ON DELETE SET NULL; + +ALTER TABLE items ADD COLUMN IF NOT EXISTS source_pack TEXT; +ALTER TABLE items ADD COLUMN IF NOT EXISTS source_artifact_sha256 TEXT; +ALTER TABLE items ADD COLUMN IF NOT EXISTS installed_sha256 TEXT;