feat(apps): app install staging: apps/1 manifest, fetch policy, pending reservations, preview (TASK-3397, U8a) (#1770)

SPEC-6 U8a (DOC-3371 §1, §2 steps 1-5, §9). Strict apps/1 parsing;
an https/no-redirect fetch policy screened at dial time with an admin
private-origin list; pending-install reservations under the owner lock
and the instance advisory lock, holding the full charge until the fetch
ends; the owner preview with every artifact normalized by the importer's
own code (normalizeArtifact, prepareCreateFields, both extracted
behaviour-preserving and pinned by a parity golden); apps_enabled and
the admin apps settings; a background sweep of expired pending
installs; migration 118/092. The preview is advisory: it may refuse
what the import would accept, never the reverse; U8b re-checks
atomically.

Claude-Session: https://claude.ai/code/session_01H63zaWhkb5pAafx2WGjrVC
This commit is contained in:
xarmian
2026-10-04 07:19:07 -04:00
committed by GitHub
parent e5262160e1
commit ca158c22d8
23 changed files with 3696 additions and 94 deletions
+3
View File
@@ -742,6 +742,9 @@ func serveCmd() *cobra.Command {
}
srv.StartReminderTick()
// Expired pending app installs and their staged bytes (TASK-3397).
srv.StartAppPendingSweep()
// Typed decisions (PLAN-3114 / TASK-3117). With no provider
// configured the runner is nil: no write enqueues a job and no
// tick starts. A provider that is NAMED but cannot be built (an
+214
View File
@@ -0,0 +1,214 @@
// Package appfetch is the fetch policy for app manifests and companion-pack
// artifacts (SPEC-6, DOC-3371 §2 step 1 and §9):
// - https only, and never a redirect: a 3xx is a failure, not followed;
// - no proxy from the environment;
// - a size cap per response, enforced while reading;
// - every connection screened at DIAL time against the address actually
// dialed: a non-public address (netpolicy.Blocked) is refused unless the
// URL's exact origin is on the instance admin's private-origin list with
// the fetch flag, and then only an address inside that entry's pinned
// IPs or CIDRs is dialed. Never a hostname alone, never a skip switch.
//
// Content hashes are the caller's check (the manifest declares them).
package appfetch
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"time"
"github.com/PerpetualSoftware/pad/internal/appmanifest"
"github.com/PerpetualSoftware/pad/internal/netpolicy"
)
// PrivateOrigin is one entry of the admin's private-destination list (§9):
// an exact origin, the addresses it may resolve to, and what it may be used
// for. Only Fetch matters here; Webhook is A5's.
type PrivateOrigin struct {
Origin string `json:"origin"`
Allowed []string `json:"allowed"` // IPs or CIDRs
Fetch bool `json:"fetch"`
Webhook bool `json:"webhook"`
Path string `json:"path,omitempty"` // webhook only
}
// ErrRefused is any fetch the policy refuses: scheme, redirect, status,
// address or size. The wrapped message says which.
var ErrRefused = errors.New("app fetch refused")
func refused(format string, args ...any) error {
return fmt.Errorf("%w: %s", ErrRefused, fmt.Sprintf(format, args...))
}
// Fetcher fetches under the policy. Build one per install attempt with New.
type Fetcher struct {
client *http.Client
private map[string][]*net.IPNet // origin -> pinned nets, fetch-flagged only
}
// lookupIP is the resolver; a variable so tests can resolve without DNS.
var lookupIP = net.DefaultResolver.LookupIP
// New builds a Fetcher. private is the admin list (self-host only; Cloud
// passes nil, so no private destination is ever reachable). tlsConfig is nil
// in production; tests pass one that trusts their server.
func New(private []PrivateOrigin, timeout time.Duration, tlsConfig *tls.Config) (*Fetcher, error) {
f := &Fetcher{private: map[string][]*net.IPNet{}}
for i, p := range private {
if !p.Fetch {
continue
}
origin, err := appmanifest.NormalizeOrigin(p.Origin)
if err != nil {
return nil, fmt.Errorf("private origin %d: %v", i, err)
}
if len(p.Allowed) == 0 {
return nil, fmt.Errorf("private origin %s: no pinned addresses", origin)
}
for _, a := range p.Allowed {
n, err := parseNet(a)
if err != nil {
return nil, fmt.Errorf("private origin %s: %v", origin, err)
}
f.private[origin] = append(f.private[origin], n)
}
}
dialer := &net.Dialer{Timeout: 10 * time.Second}
transport := &http.Transport{
Proxy: nil,
TLSClientConfig: tlsConfig,
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 15 * time.Second,
MaxIdleConns: 4,
IdleConnTimeout: 30 * time.Second,
DisableCompression: true,
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return f.dial(ctx, dialer, network, addr)
},
}
f.client = &http.Client{
Transport: transport,
Timeout: timeout,
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
return f, nil
}
func parseNet(s string) (*net.IPNet, error) {
if strings.Contains(s, "/") {
_, n, err := net.ParseCIDR(s)
if err != nil {
return nil, fmt.Errorf("bad CIDR %q", s)
}
return n, nil
}
ip := net.ParseIP(s)
if ip == nil {
return nil, fmt.Errorf("bad IP %q", s)
}
bits := 128
if ip.To4() != nil {
ip, bits = ip.To4(), 32
}
return &net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)}, nil
}
type originKey struct{}
// dial screens the address actually dialed. The request's origin rides in the
// context, because the transport sees only host:port.
func (f *Fetcher) dial(ctx context.Context, d *net.Dialer, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, refused("bad address %q", addr)
}
origin, _ := ctx.Value(originKey{}).(string)
pinned := f.private[origin]
var ips []net.IP
if ip := net.ParseIP(host); ip != nil {
ips = []net.IP{ip}
} else {
ips, err = lookupIP(ctx, "ip", host)
if err != nil {
return nil, refused("resolve %s: %v", host, err)
}
}
var lastErr error = refused("%s resolves to no permitted address", host)
for _, ip := range ips {
if !permitted(ip, pinned) {
continue
}
conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
if err == nil {
return conn, nil
}
lastErr = err
}
return nil, lastErr
}
// permitted: a pinned origin may dial only its pinned addresses (public or
// not); any other origin may dial only public addresses.
func permitted(ip net.IP, pinned []*net.IPNet) bool {
if len(pinned) > 0 {
for _, n := range pinned {
if n.Contains(ip) {
return true
}
}
return false
}
return !netpolicy.Blocked(ip)
}
// Get fetches rawURL and returns at most maxBytes of body. Anything else
// (not https, a redirect, a non-200 status, a longer body, a refused
// address) is an error wrapping ErrRefused or a transport error.
func (f *Fetcher) Get(ctx context.Context, rawURL string, maxBytes int64) ([]byte, error) {
u, err := url.Parse(rawURL)
if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
return nil, refused("%q is not an https URL", rawURL)
}
origin, err := appmanifest.NormalizeOrigin("https://" + u.Host)
if err != nil {
return nil, refused("%q: %v", rawURL, err)
}
req, err := http.NewRequestWithContext(context.WithValue(ctx, originKey{}, origin), http.MethodGet, u.String(), nil)
if err != nil {
return nil, refused("%v", err)
}
req.Header.Set("Accept", "application/json, text/markdown, text/plain, */*")
req.Header.Set("User-Agent", "Pad-App-Installer/1")
resp, err := f.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode >= 300 && resp.StatusCode < 400 {
return nil, refused("%s redirected (%d); redirects are never followed", rawURL, resp.StatusCode)
}
if resp.StatusCode != http.StatusOK {
return nil, refused("%s answered %d", rawURL, resp.StatusCode)
}
if resp.ContentLength > maxBytes {
return nil, refused("%s is larger than %d bytes", rawURL, maxBytes)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
if err != nil {
return nil, err
}
if int64(len(body)) > maxBytes {
return nil, refused("%s is larger than %d bytes", rawURL, maxBytes)
}
return body, nil
}
+145
View File
@@ -0,0 +1,145 @@
package appfetch
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// tlsServer serves h over TLS on 127.0.0.1 and returns its origin plus a
// TLS config trusting it.
func tlsServer(t *testing.T, h http.Handler) (*httptest.Server, *tls.Config) {
t.Helper()
srv := httptest.NewTLSServer(h)
t.Cleanup(srv.Close)
pool := x509.NewCertPool()
pool.AddCert(srv.Certificate())
return srv, &tls.Config{RootCAs: pool}
}
func pinnedTo(srv *httptest.Server) []PrivateOrigin {
return []PrivateOrigin{{Origin: srv.URL, Allowed: []string{"127.0.0.1"}, Fetch: true}}
}
func TestGet_PinnedPrivateOriginFetches(t *testing.T) {
srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("manifest")) }))
f, err := New(pinnedTo(srv), 5*time.Second, cfg)
if err != nil {
t.Fatal(err)
}
got, err := f.Get(context.Background(), srv.URL+"/.well-known/pad-app.json", 100)
if err != nil || string(got) != "manifest" {
t.Fatalf("got %q, %v", got, err)
}
}
// Without the admin entry, the same loopback server is refused at dial time.
func TestGet_PrivateAddressRefusedWithoutAnEntry(t *testing.T) {
srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("x")) }))
for name, list := range map[string][]PrivateOrigin{
"no list": nil,
"entry without fetch": {{Origin: srv.URL, Allowed: []string{"127.0.0.1"}, Fetch: false, Webhook: true}},
"another origin": {{Origin: "https://other.example", Allowed: []string{"127.0.0.1"}, Fetch: true}},
} {
t.Run(name, func(t *testing.T) {
f, err := New(list, 5*time.Second, cfg)
if err != nil {
t.Fatal(err)
}
if _, err := f.Get(context.Background(), srv.URL+"/m", 100); !errors.Is(err, ErrRefused) {
t.Fatalf("got %v, want ErrRefused", err)
}
})
}
}
// A pinned origin dials ONLY its pinned addresses, even public ones it
// resolves to elsewhere.
func TestGet_PinnedOriginOutsideItsPinsIsRefused(t *testing.T) {
srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("x")) }))
f, err := New([]PrivateOrigin{{Origin: srv.URL, Allowed: []string{"10.9.9.9/32"}, Fetch: true}}, 5*time.Second, cfg)
if err != nil {
t.Fatal(err)
}
if _, err := f.Get(context.Background(), srv.URL+"/m", 100); !errors.Is(err, ErrRefused) {
t.Fatalf("got %v", err)
}
}
// A hostname resolving to a private address is refused at dial: DNS cannot
// launder an address the literal check would refuse.
func TestGet_HostnameResolvingPrivateIsRefused(t *testing.T) {
srv, cfg := tlsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("x")) }))
_, port, _ := net.SplitHostPort(strings.TrimPrefix(srv.URL, "https://"))
orig := lookupIP
lookupIP = func(ctx context.Context, network, host string) ([]net.IP, error) {
return []net.IP{net.ParseIP("127.0.0.1")}, nil
}
t.Cleanup(func() { lookupIP = orig })
f, err := New(nil, 5*time.Second, cfg)
if err != nil {
t.Fatal(err)
}
if _, err := f.Get(context.Background(), "https://app.example:"+port+"/m", 100); !errors.Is(err, ErrRefused) {
t.Fatalf("got %v", err)
}
}
func TestGet_RedirectStatusAndSizeAreRefused(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/redirect", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ok", http.StatusFound) })
mux.HandleFunc("/ok", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) })
mux.HandleFunc("/missing", func(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) })
mux.HandleFunc("/big", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(strings.Repeat("x", 101))) })
mux.HandleFunc("/big-chunked", func(w http.ResponseWriter, r *http.Request) {
fl := w.(http.Flusher)
for i := 0; i < 11; i++ {
_, _ = w.Write([]byte(strings.Repeat("y", 10)))
fl.Flush()
}
})
srv, cfg := tlsServer(t, mux)
f, err := New(pinnedTo(srv), 5*time.Second, cfg)
if err != nil {
t.Fatal(err)
}
for _, p := range []string{"/redirect", "/missing", "/big", "/big-chunked"} {
if _, err := f.Get(context.Background(), srv.URL+p, 100); !errors.Is(err, ErrRefused) {
t.Errorf("%s: got %v, want ErrRefused", p, err)
}
}
if got, err := f.Get(context.Background(), srv.URL+"/ok", 100); err != nil || string(got) != "ok" {
t.Errorf("/ok: %q %v", got, err)
}
}
func TestGet_OnlyHTTPS(t *testing.T) {
f, err := New(nil, time.Second, nil)
if err != nil {
t.Fatal(err)
}
for _, u := range []string{"http://example.com/m", "ftp://example.com/m", "https://user:pw@example.com/m", "/relative"} {
if _, err := f.Get(context.Background(), u, 10); !errors.Is(err, ErrRefused) {
t.Errorf("%s: got %v", u, err)
}
}
}
func TestNew_RejectsBadEntries(t *testing.T) {
for _, p := range []PrivateOrigin{
{Origin: "http://x.example", Allowed: []string{"10.0.0.1"}, Fetch: true},
{Origin: "https://x.example", Fetch: true},
{Origin: "https://x.example", Allowed: []string{"not-an-ip"}, Fetch: true},
} {
if _, err := New([]PrivateOrigin{p}, time.Second, nil); err == nil {
t.Errorf("accepted %+v", p)
}
}
}
+433
View File
@@ -0,0 +1,433 @@
// Package appmanifest parses and validates an app's apps/1 manifest (SPEC-6,
// DOC-3371 §1). It is pure: no network, no database. The fetch policy lives
// in internal/appfetch and the staging and provisioning in internal/store.
//
// The manifest is published at {base_url}/.well-known/pad-app.json and is a
// contract surface. Validation is strict: an unknown key is refused rather
// than ignored, so a manifest written for a later contract cannot install
// here with part of it silently dropped.
package appmanifest
import (
"bytes"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"regexp"
"strings"
"github.com/PerpetualSoftware/pad/internal/models"
)
// Contract versions this server speaks.
const (
AppsContract = 1
EventsContract = 1
)
// Caps (DOC-3371 §2 "Staging caps").
const (
MaxManifestBytes = 256 << 10
MaxArtifacts = 32
MaxCollections = 32
MaxEvents = 64
MaxItemActions = 32
MaxRedirectURIs = 16
MaxConfigSchemaBytes = 64 << 10
maxTextLen = 2000
)
// WellKnownPath is where the manifest is published under base_url.
const WellKnownPath = "/.well-known/pad-app.json"
// SubscribableEvents are the v1 events an app may subscribe to: those whose
// subject resolves to exactly one companion item or comment (§5).
var SubscribableEvents = map[string]bool{
"item.created": true, "item.updated": true, "item.status_changed": true,
"item.deleted": true, "item.restored": true,
"comment.created": true, "comment.updated": true, "comment.deleted": true,
}
// Manifest is a parsed, validated apps/1 manifest.
type Manifest struct {
ID string `json:"id"`
Version string `json:"version"`
MinContract MinContract `json:"min_contract"`
Title string `json:"title"`
Description string `json:"description"`
Publisher string `json:"publisher"`
Homepage string `json:"homepage"`
BaseURL string `json:"base_url"`
RedirectURIs []string `json:"redirect_uris"`
Scopes Scopes `json:"scopes"`
Events []Event `json:"events"`
WebhookURL string `json:"webhook_url"`
CompanionPack CompanionPack `json:"companion_pack"`
ItemActions []ItemAction `json:"item_actions"`
ConfigSchema json.RawMessage `json:"config_schema"`
Docs string `json:"docs"`
// Origin is the normalized origin of BaseURL: the app's identity.
Origin string `json:"-"`
}
type MinContract struct {
Apps int `json:"apps"`
Events int `json:"events"`
}
type Scopes struct {
Service Access `json:"service"`
Delegated Access `json:"delegated"`
}
type Access struct {
Access string `json:"access"`
}
type Event struct {
Name string `json:"name"`
Collections []string `json:"collections"`
}
type CompanionPack struct {
Collections []Collection `json:"collections"`
Artifacts []Artifact `json:"artifacts"`
}
type Collection struct {
Key string `json:"key"`
Slug string `json:"slug"`
Name string `json:"name"`
Schema json.RawMessage `json:"schema"`
// Parsed is Schema, parsed.
Parsed models.CollectionSchema `json:"-"`
}
type Artifact struct {
Key string `json:"key"`
URL string `json:"url"`
SHA256 string `json:"sha256"`
}
type ItemAction struct {
Key string `json:"key"`
Label string `json:"label"`
Collections []string `json:"collections"`
Path string `json:"path"`
}
// Error is one validation failure, at a JSON path in the manifest.
type Error struct {
Path string `json:"path"`
Reason string `json:"reason"`
}
func (e *Error) Error() string { return "app manifest: " + e.Path + ": " + e.Reason }
func fail(path, format string, args ...any) error {
return &Error{Path: path, Reason: fmt.Sprintf(format, args...)}
}
// IsError reports whether err is a manifest validation failure.
func IsError(err error) bool {
var e *Error
return errors.As(err, &e)
}
var (
keyRE = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
slugRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$`)
idRE = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,63}/[a-z0-9][a-z0-9._-]{0,63}$`)
semverRE = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`)
sha256RE = regexp.MustCompile(`^[0-9a-f]{64}$`)
)
// Parse decodes and validates manifest bytes. The first failure is returned
// as an *Error naming its path.
func Parse(data []byte) (*Manifest, error) {
if len(data) > MaxManifestBytes {
return nil, fail("$", "manifest exceeds %d KiB", MaxManifestBytes>>10)
}
dec := json.NewDecoder(bytes.NewReader(data))
dec.DisallowUnknownFields()
var m Manifest
if err := dec.Decode(&m); err != nil {
return nil, fail("$", "not a valid apps/1 manifest: %v", err)
}
if _, err := dec.Token(); !errors.Is(err, io.EOF) {
return nil, fail("$", "trailing data after the manifest")
}
if err := m.validate(); err != nil {
return nil, err
}
return &m, nil
}
func (m *Manifest) validate() error {
if !idRE.MatchString(m.ID) {
return fail("id", "must be publisher/name in lower case")
}
if !semverRE.MatchString(m.Version) {
return fail("version", "must be a semantic version")
}
if m.MinContract.Apps < 1 || m.MinContract.Apps > AppsContract {
return fail("min_contract.apps", "this server speaks apps/%d", AppsContract)
}
if m.MinContract.Events < 1 || m.MinContract.Events > EventsContract {
return fail("min_contract.events", "this server speaks events/%d", EventsContract)
}
for _, f := range []struct{ path, v string }{{"title", m.Title}, {"publisher", m.Publisher}} {
if strings.TrimSpace(f.v) == "" {
return fail(f.path, "is required")
}
}
for _, f := range []struct{ path, v string }{{"title", m.Title}, {"description", m.Description}, {"publisher", m.Publisher}} {
if len(f.v) > maxTextLen {
return fail(f.path, "is longer than %d characters", maxTextLen)
}
}
origin, err := NormalizeOrigin(m.BaseURL)
if err != nil {
return fail("base_url", "%v", err)
}
m.Origin = origin
if m.Homepage != "" {
if err := m.under(m.Homepage); err != nil {
return fail("homepage", "%v", err)
}
}
if len(m.RedirectURIs) == 0 || len(m.RedirectURIs) > MaxRedirectURIs {
return fail("redirect_uris", "between 1 and %d are required", MaxRedirectURIs)
}
for i, u := range m.RedirectURIs {
if err := m.under(u); err != nil {
return fail(fmt.Sprintf("redirect_uris[%d]", i), "%v", err)
}
}
for _, f := range []struct{ path, v string }{{"scopes.service.access", m.Scopes.Service.Access}, {"scopes.delegated.access", m.Scopes.Delegated.Access}} {
if f.v != "read" && f.v != "write" {
return fail(f.path, "must be read or write")
}
}
if m.WebhookURL != "" {
if err := m.under(m.WebhookURL); err != nil {
return fail("webhook_url", "%v", err)
}
}
if m.Docs != "" {
if err := m.under(m.Docs); err != nil {
return fail("docs", "%v", err)
}
}
// Companion pack.
pack := m.CompanionPack
if len(pack.Collections) == 0 || len(pack.Collections) > MaxCollections {
return fail("companion_pack.collections", "between 1 and %d are required", MaxCollections)
}
keys := map[string]bool{}
slugs := map[string]bool{}
for i := range pack.Collections {
c := &m.CompanionPack.Collections[i]
p := fmt.Sprintf("companion_pack.collections[%d]", i)
if !keyRE.MatchString(c.Key) {
return fail(p+".key", "must be a lower-case key")
}
if keys[c.Key] {
return fail(p+".key", "duplicate key %q", c.Key)
}
keys[c.Key] = true
if !slugRE.MatchString(c.Slug) {
return fail(p+".slug", "must be a lower-case slug")
}
if slugs[c.Slug] {
return fail(p+".slug", "duplicate slug %q", c.Slug)
}
slugs[c.Slug] = true
if len(c.Name) > 200 {
return fail(p+".name", "is longer than 200 characters")
}
if len(c.Schema) == 0 {
return fail(p+".schema", "is required")
}
// Strict, like the manifest itself: a misspelled key ("requried")
// would otherwise be dropped silently, losing what it meant (codex
// round 2).
sdec := json.NewDecoder(bytes.NewReader(c.Schema))
sdec.DisallowUnknownFields()
if err := sdec.Decode(&c.Parsed); err != nil {
return fail(p+".schema", "is not a collection schema: %v", err)
}
}
if len(pack.Artifacts) > MaxArtifacts {
return fail("companion_pack.artifacts", "at most %d", MaxArtifacts)
}
artKeys := map[string]bool{}
for i, a := range pack.Artifacts {
p := fmt.Sprintf("companion_pack.artifacts[%d]", i)
if !keyRE.MatchString(a.Key) {
return fail(p+".key", "must be a lower-case key")
}
if artKeys[a.Key] {
return fail(p+".key", "duplicate key %q", a.Key)
}
artKeys[a.Key] = true
if err := m.under(a.URL); err != nil {
return fail(p+".url", "%v", err)
}
if !sha256RE.MatchString(a.SHA256) {
return fail(p+".sha256", "must be 64 lower-case hex characters")
}
}
// Events: v1-subscribable names, companion keys only.
if len(m.Events) > MaxEvents {
return fail("events", "at most %d", MaxEvents)
}
if len(m.Events) > 0 && m.WebhookURL == "" {
return fail("webhook_url", "is required when events are declared")
}
for i, e := range m.Events {
p := fmt.Sprintf("events[%d]", i)
if !SubscribableEvents[e.Name] {
return fail(p+".name", "%q is not subscribable in apps/1", e.Name)
}
if len(e.Collections) == 0 {
return fail(p+".collections", "at least one companion collection key is required")
}
for j, k := range e.Collections {
if !keys[k] {
return fail(fmt.Sprintf("%s.collections[%d]", p, j), "%q is not a companion collection key", k)
}
}
}
// Item actions.
if len(m.ItemActions) > MaxItemActions {
return fail("item_actions", "at most %d", MaxItemActions)
}
actKeys := map[string]bool{}
for i, a := range m.ItemActions {
p := fmt.Sprintf("item_actions[%d]", i)
if !keyRE.MatchString(a.Key) {
return fail(p+".key", "must be a lower-case key")
}
if actKeys[a.Key] {
return fail(p+".key", "duplicate key %q", a.Key)
}
actKeys[a.Key] = true
if strings.TrimSpace(a.Label) == "" || len(a.Label) > 80 {
return fail(p+".label", "is required, at most 80 characters")
}
if len(a.Collections) == 0 {
return fail(p+".collections", "at least one companion collection key is required")
}
for j, k := range a.Collections {
if !keys[k] {
return fail(fmt.Sprintf("%s.collections[%d]", p, j), "%q is not a companion collection key", k)
}
}
if !absolutePath(a.Path) {
return fail(p+".path", "must be an absolute path under base_url")
}
}
// Config schema: a JSON object, size-capped; values are owner-edited and
// never secrets (§1). It is not evaluated here.
if len(m.ConfigSchema) > 0 {
if len(m.ConfigSchema) > MaxConfigSchemaBytes {
return fail("config_schema", "exceeds %d KiB", MaxConfigSchemaBytes>>10)
}
var obj map[string]json.RawMessage
if err := json.Unmarshal(m.ConfigSchema, &obj); err != nil {
return fail("config_schema", "must be a JSON Schema object")
}
}
return nil
}
// NormalizeOrigin returns the canonical origin of an https base URL:
// lower-case host, no default port, and no path, query, fragment or
// userinfo. It is the app's identity (DOC-3371 Definitions).
func NormalizeOrigin(raw string) (string, error) {
u, err := url.Parse(strings.TrimSpace(raw))
if err != nil {
return "", fmt.Errorf("is not a URL")
}
if u.Scheme != "https" {
return "", fmt.Errorf("must be https")
}
if u.User != nil {
return "", fmt.Errorf("must not carry credentials")
}
if u.Host == "" || u.Hostname() == "" {
return "", fmt.Errorf("must have a host")
}
if (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" {
return "", fmt.Errorf("must be an origin, with no path, query or fragment")
}
host := strings.ToLower(u.Hostname())
port := u.Port()
if port == "443" {
port = ""
}
if port != "" {
return "https://" + joinHostPort(host, port), nil
}
if strings.Contains(host, ":") {
return "https://[" + host + "]", nil
}
return "https://" + host, nil
}
func joinHostPort(host, port string) string {
if strings.Contains(host, ":") {
return "[" + host + "]:" + port
}
return host + ":" + port
}
// under requires raw to be an https URL on exactly the manifest's origin.
func (m *Manifest) under(raw string) error {
u, err := url.Parse(raw)
if err != nil || u.Scheme != "https" || u.Host == "" {
return fmt.Errorf("must be an https URL under base_url")
}
if u.User != nil {
return fmt.Errorf("must not carry credentials")
}
o, err := NormalizeOrigin("https://" + u.Host)
if err != nil || o != m.Origin {
return fmt.Errorf("must be under base_url (%s)", m.Origin)
}
return nil
}
// absolutePath accepts a path to join onto the origin: it starts with one
// "/", and has no scheme, authority, backslash or control character. A
// "//host/x" would keep the origin when concatenated but name another host
// when resolved as a reference, so it is refused outright.
func absolutePath(p string) bool {
if !strings.HasPrefix(p, "/") || strings.HasPrefix(p, "//") || len(p) > 1024 {
return false
}
for _, r := range p {
if r == '\\' || r < 0x20 || r == 0x7f {
return false
}
}
u, err := url.Parse(p)
return err == nil && u.Scheme == "" && u.Host == "" && u.User == nil && u.Opaque == ""
}
// ManifestURL is where a base URL's manifest is published.
func ManifestURL(origin string) string { return origin + WellKnownPath }
// SHA256Hex is the lower-case hex sha256 the manifest and digests use.
func SHA256Hex(sum [32]byte) string { return hex.EncodeToString(sum[:]) }
+172
View File
@@ -0,0 +1,172 @@
package appmanifest
import (
"encoding/json"
"strings"
"testing"
)
func validManifest() map[string]any {
return map[string]any{
"id": "acme/support-portal",
"version": "1.2.0",
"min_contract": map[string]any{"apps": 1, "events": 1},
"title": "Support Portal",
"description": "Requester portal",
"publisher": "Acme",
"homepage": "https://portal.acme.example/about",
"base_url": "https://Portal.Acme.Example:443/",
"redirect_uris": []any{"https://portal.acme.example/oauth/callback"},
"scopes": map[string]any{
"service": map[string]any{"access": "write"},
"delegated": map[string]any{"access": "read"},
},
"events": []any{map[string]any{"name": "item.created", "collections": []any{"tickets"}}},
"webhook_url": "https://portal.acme.example/hooks/pad",
"companion_pack": map[string]any{
"collections": []any{map[string]any{"key": "tickets", "slug": "tickets", "name": "Tickets",
"schema": map[string]any{"fields": []any{map[string]any{"key": "status", "label": "Status", "type": "select", "options": []any{"open", "solved"}}}}}},
"artifacts": []any{map[string]any{"key": "triage", "url": "https://portal.acme.example/pack/triage.md", "sha256": strings.Repeat("ab", 32)}},
},
"item_actions": []any{map[string]any{"key": "open", "label": "Open in portal", "collections": []any{"tickets"}, "path": "/tickets"}},
"config_schema": map[string]any{"type": "object"},
"docs": "https://portal.acme.example/llms.txt",
}
}
func encode(t *testing.T, m map[string]any) []byte {
t.Helper()
b, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return b
}
func TestParse_Valid(t *testing.T) {
m, err := Parse(encode(t, validManifest()))
if err != nil {
t.Fatal(err)
}
if m.Origin != "https://portal.acme.example" {
t.Errorf("origin %q", m.Origin)
}
if len(m.CompanionPack.Collections[0].Parsed.Fields) != 1 {
t.Errorf("schema not parsed")
}
if ManifestURL(m.Origin) != "https://portal.acme.example/.well-known/pad-app.json" {
t.Errorf("manifest url %q", ManifestURL(m.Origin))
}
}
// Each mutation breaks one rule; Parse must refuse it at the named path.
func TestParse_Refusals(t *testing.T) {
type mut func(m map[string]any)
pack := func(m map[string]any) map[string]any { return m["companion_pack"].(map[string]any) }
coll := func(m map[string]any) map[string]any { return pack(m)["collections"].([]any)[0].(map[string]any) }
art := func(m map[string]any) map[string]any { return pack(m)["artifacts"].([]any)[0].(map[string]any) }
cases := map[string]struct {
mut mut
path string
}{
"unknown top-level key": {func(m map[string]any) { m["extra"] = 1 }, "$"},
"bad id": {func(m map[string]any) { m["id"] = "Acme Portal" }, "id"},
"bad version": {func(m map[string]any) { m["version"] = "1.2" }, "version"},
"future apps contract": {func(m map[string]any) { m["min_contract"] = map[string]any{"apps": 2, "events": 1} }, "min_contract.apps"},
"future events contract": {func(m map[string]any) { m["min_contract"] = map[string]any{"apps": 1, "events": 9} }, "min_contract.events"},
"missing title": {func(m map[string]any) { m["title"] = " " }, "title"},
"http base_url": {func(m map[string]any) { m["base_url"] = "http://portal.acme.example" }, "base_url"},
"base_url with a path": {func(m map[string]any) { m["base_url"] = "https://portal.acme.example/app" }, "base_url"},
"base_url with userinfo": {func(m map[string]any) { m["base_url"] = "https://u:p@portal.acme.example" }, "base_url"},
"no redirect uris": {func(m map[string]any) { m["redirect_uris"] = []any{} }, "redirect_uris"},
"redirect off origin": {func(m map[string]any) { m["redirect_uris"] = []any{"https://evil.example/cb"} }, "redirect_uris[0]"},
"redirect on other port": {func(m map[string]any) { m["redirect_uris"] = []any{"https://portal.acme.example:8443/cb"} }, "redirect_uris[0]"},
"redirect over http": {func(m map[string]any) { m["redirect_uris"] = []any{"http://portal.acme.example/cb"} }, "redirect_uris[0]"},
"bad service access": {func(m map[string]any) { m["scopes"].(map[string]any)["service"] = map[string]any{"access": "admin"} }, "scopes.service.access"},
"webhook off origin": {func(m map[string]any) { m["webhook_url"] = "https://evil.example/h" }, "webhook_url"},
"events without webhook": {func(m map[string]any) { delete(m, "webhook_url") }, "webhook_url"},
"unsubscribable event": {func(m map[string]any) {
m["events"] = []any{map[string]any{"name": "item.bulk_updated", "collections": []any{"tickets"}}}
}, "events[0].name"},
"attachment event": {func(m map[string]any) {
m["events"] = []any{map[string]any{"name": "attachment.added", "collections": []any{"tickets"}}}
}, "events[0].name"},
"event on unknown key": {func(m map[string]any) {
m["events"] = []any{map[string]any{"name": "item.created", "collections": []any{"other"}}}
}, "events[0].collections[0]"},
"no collections": {func(m map[string]any) { pack(m)["collections"] = []any{} }, "companion_pack.collections"},
"bad collection slug": {func(m map[string]any) { coll(m)["slug"] = "Tickets!" }, "companion_pack.collections[0].slug"},
"bad collection schema": {func(m map[string]any) { coll(m)["schema"] = "not an object" }, "companion_pack.collections[0].schema"},
"artifact off origin": {func(m map[string]any) { art(m)["url"] = "https://cdn.example/x.md" }, "companion_pack.artifacts[0].url"},
"artifact bad sha256": {func(m map[string]any) { art(m)["sha256"] = "ABC" }, "companion_pack.artifacts[0].sha256"},
"too many artifacts": {func(m map[string]any) {
var arts []any
for i := 0; i < MaxArtifacts+1; i++ {
arts = append(arts, map[string]any{"key": "a" + strings.Repeat("x", i%3) + string(rune('a'+i%26)) + string(rune('a'+i/26)), "url": "https://portal.acme.example/a.md", "sha256": strings.Repeat("ab", 32)})
}
pack(m)["artifacts"] = arts
}, "companion_pack.artifacts"},
"action on unknown key": {func(m map[string]any) {
m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"nope"}, "path": "/x"}}
}, "item_actions[0].collections[0]"},
"action relative path": {func(m map[string]any) {
m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"tickets"}, "path": "x"}}
}, "item_actions[0].path"},
"action protocol-relative": {func(m map[string]any) {
m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"tickets"}, "path": "//evil.example/x"}}
}, "item_actions[0].path"},
"action backslash path": {func(m map[string]any) {
m["item_actions"] = []any{map[string]any{"key": "o", "label": "Open", "collections": []any{"tickets"}, "path": "/\\evil.example/x"}}
}, "item_actions[0].path"},
"config schema not object": {func(m map[string]any) { m["config_schema"] = []any{1} }, "config_schema"},
"schema with a misspelled key": {func(m map[string]any) {
coll(m)["schema"] = map[string]any{"fields": []any{map[string]any{"key": "status", "label": "Status", "type": "text", "requried": true}}}
}, "companion_pack.collections[0].schema"},
"homepage off origin": {func(m map[string]any) { m["homepage"] = "https://other.example/" }, "homepage"},
"docs off origin": {func(m map[string]any) { m["docs"] = "https://evil.example/llms.txt" }, "docs"},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
m := validManifest()
c.mut(m)
_, err := Parse(encode(t, m))
var e *Error
if err == nil || !IsError(err) {
t.Fatalf("got %v, want a manifest error at %s", err, c.path)
}
e = err.(*Error)
if e.Path != c.path {
t.Fatalf("refused at %s (%s), want %s", e.Path, e.Reason, c.path)
}
})
}
}
func TestParse_SizeAndTrailingData(t *testing.T) {
if _, err := Parse(make([]byte, MaxManifestBytes+1)); !IsError(err) {
t.Fatalf("oversize manifest: %v", err)
}
b := append(encode(t, validManifest()), []byte(` {}`)...)
if _, err := Parse(b); !IsError(err) {
t.Fatalf("trailing data: %v", err)
}
}
func TestNormalizeOrigin(t *testing.T) {
for in, want := range map[string]string{
"https://EXAMPLE.com": "https://example.com",
"https://example.com:443/": "https://example.com",
"https://example.com:8443": "https://example.com:8443",
"https://[::1]:8443": "https://[::1]:8443",
} {
got, err := NormalizeOrigin(in)
if err != nil || got != want {
t.Errorf("%s: %q %v, want %q", in, got, err, want)
}
}
for _, bad := range []string{"http://example.com", "https://example.com/x", "https://example.com?q", "https://u@example.com", "example.com", "https://"} {
if _, err := NormalizeOrigin(bad); err == nil {
t.Errorf("%s accepted", bad)
}
}
}
@@ -72,6 +72,7 @@ var rawDecodeAllowed = map[string]string{
"internal/server/handlers_collections.go::prevSchema": "the grandfather test's own baseline: a stripped prevSchema would reclassify every EXISTING declaration as newly introduced and refuse every update to such a collection",
"cmd/pad/cmd_collection.go::schema": "CLI --schema parse and the collection list's own status-option rendering — the collection's definition, not any item's fields",
"internal/mcp/dispatch_http_routes.go::schema": "MCP collection schema INPUT, the remote twin of handlers_collections.go",
"internal/appmanifest/manifest.go::Parsed": "an app manifest's companion collection DEFINITION (TASK-3397), decoded strictly so a misspelled key is refused; the install preview then runs validateNoReservedFieldKeys on it, as collection create does",
// The move / copy family. These decode raw and strip at the point of use
// via items.SchemaForMigratedFields (BUG-2674), which is the same
@@ -556,7 +557,7 @@ func TestGuardAllowListHasNoDeadEntries(t *testing.T) {
//
// 22 since BUG-3202 added models/fields_json.go's generic number-preserving
// decoder, a fourth INTERFACE destination.
const rawDecodeSiteCount = 22
const rawDecodeSiteCount = 23
func TestGuardSeesEveryKnownDecodeSite(t *testing.T) {
sites := collectSchemaDecodes(t, repoRoot(t))
+78
View File
@@ -0,0 +1,78 @@
package server
import (
"log/slog"
"sync"
"time"
)
// The expired pending-install sweep (SPEC-6 U8a, DOC-3371 §2: "expiry (1
// hour, swept)"; TASK-3397). Expired reservations already stop counting
// against the caps, because every count reads expires_at; this sweep is what
// deletes their staged bytes, so an abandoned preview does not keep them
// until some later install happens to sweep (codex round 1). Same lifecycle
// shape as the reminder tick: its own mutex and stop channel, tracked by
// Server.bg so Stop() drains it, started only from the real bootstrap path.
const defaultAppPendingSweepInterval = 5 * time.Minute
type appPendingSweepConfig struct {
mu sync.Mutex
interval time.Duration
stop chan struct{}
running bool
}
// StartAppPendingSweep starts the sweep. Idempotent.
func (s *Server) StartAppPendingSweep() {
s.appPendingSweep.mu.Lock()
if s.appPendingSweep.running {
s.appPendingSweep.mu.Unlock()
return
}
if s.appPendingSweep.interval == 0 {
s.appPendingSweep.interval = defaultAppPendingSweepInterval
}
s.appPendingSweep.stop = make(chan struct{})
s.appPendingSweep.running = true
interval, stop := s.appPendingSweep.interval, s.appPendingSweep.stop
s.appPendingSweep.mu.Unlock()
s.bg.Add(1)
go func() {
defer s.bg.Done()
defer s.recoverSweeper("app-pending-sweep")
t := time.NewTicker(interval)
defer t.Stop()
for {
select {
case <-stop:
return
case <-t.C:
s.runAppPendingSweep()
}
}
}()
}
func (s *Server) stopAppPendingSweep() {
s.appPendingSweep.mu.Lock()
defer s.appPendingSweep.mu.Unlock()
if !s.appPendingSweep.running {
return
}
close(s.appPendingSweep.stop)
s.appPendingSweep.running = false
}
// runAppPendingSweep is one pass.
func (s *Server) runAppPendingSweep() {
n, err := s.store.SweepExpiredPendingInstalls()
if err != nil {
slog.Warn("apps: pending install sweep failed", "error", err)
return
}
if n > 0 {
slog.Info("apps: swept expired pending installs", "count", n)
}
}
+7
View File
@@ -107,6 +107,13 @@ func parseArtifactRequest(w http.ResponseWriter, r *http.Request, maxBytes int64
return artifact.Artifact{}, fmt.Errorf("artifact import: read body: %w", err)
}
return decodeArtifactBytes(data)
}
// decodeArtifactBytes is parseArtifactRequest's steps (2) to (5) over bytes
// already read and size-capped: the import door and the app installer's
// staged artifacts (TASK-3397) run the same guards and decode.
func decodeArtifactBytes(data []byte) (artifact.Artifact, error) {
// (2) The artifact body is TEXT bound for text columns, and this
// handler reads it directly rather than through decodeJSON, so it
// inherits neither BUG-2803's refusal nor the path/query rule (a body is
@@ -0,0 +1,168 @@
package server
import (
"encoding/json"
"net/http"
"os"
"path/filepath"
"sort"
"testing"
"github.com/PerpetualSoftware/pad/internal/artifact"
)
// TASK-3397 (U8a) refactors the artifact importer's preprocess into one
// function the app installer's preview and provisioning share. This test pins
// that the HUMAN import path's output is unchanged by it: every fixture is
// imported through POST /import-artifact and the stored item (title, content,
// fields) plus the response's warnings are compared against a golden captured
// from the importer BEFORE the refactor.
//
// Regenerate (only for an intended behaviour change):
//
// PAD_UPDATE_ARTIFACT_PARITY=1 go test ./internal/server/ -run TestArtifactImportParity
const artifactParityGolden = "testdata/artifact_import_parity.json"
type parityCase struct {
name string
setup func(t *testing.T, srv *Server, ws string)
body func(t *testing.T) []byte
}
type parityResult struct {
Status int `json:"status"`
Title string `json:"title"`
Content string `json:"content"`
Fields map[string]any `json:"fields"`
Warnings []string `json:"warnings"`
}
func encodeArtifact(t *testing.T, a artifact.Artifact) []byte {
t.Helper()
a.FormatVersion = artifact.FormatVersion
data, err := artifact.Encode(a)
if err != nil {
t.Fatal(err)
}
return data
}
func fixtureFile(name string) func(t *testing.T) []byte {
return func(t *testing.T) []byte {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", name))
if err != nil {
t.Fatal(err)
}
return data
}
}
func artifactParityCases() []parityCase {
return []parityCase{
{name: "golden playbook", body: fixtureFile("playbook.golden.md")},
{name: "golden convention", body: fixtureFile("convention.golden.md")},
{name: "convention, active, valid selects", body: func(t *testing.T) []byte {
return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: "Clean Convention",
Fields: map[string]any{"status": "active", "trigger": "on-commit", "scope": "all", "priority": "must"}, Body: "A clean convention.\n"})
}},
{name: "convention, foreign select value", body: func(t *testing.T) []byte {
return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: "Foreign",
Fields: map[string]any{"status": "active", "trigger": "on-candidate-advance", "scope": "all"}, Body: "x\n"})
}},
{name: "playbook with arguments", body: func(t *testing.T) []byte {
return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindPlaybook, Title: "Playbook With Args",
Fields: map[string]any{"status": "active", "trigger": "manual", "scope": "all", "invocation_slug": "with-args",
"arguments": []map[string]any{{"name": "target", "type": "ref", "required": true, "description": "the thing"}}},
Body: "Do the thing.\n"})
}},
{name: "playbook, slug collision", setup: func(t *testing.T, srv *Server, ws string) {
t.Helper()
data := encodeArtifact(t, artifact.Artifact{Kind: artifact.KindPlaybook, Title: "First",
Fields: map[string]any{"status": "draft", "invocation_slug": "ship"}, Body: "first\n"})
if rr := doArtifactRequest(srv, "POST", "/api/v1/workspaces/"+ws+"/import-artifact", data); rr.Code != http.StatusCreated {
t.Fatalf("setup import: %d %s", rr.Code, rr.Body.String())
}
}, body: func(t *testing.T) []byte {
return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindPlaybook, Title: "Second",
Fields: map[string]any{"status": "active", "invocation_slug": "ship"}, Body: "second\n"})
}},
{name: "with provenance", body: func(t *testing.T) []byte {
return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: "Provenanced",
Fields: map[string]any{"status": "draft", "trigger": "on-commit"},
Body: "Body.\n",
Provenance: artifact.Provenance{Workspace: "elsewhere", Author: "someone", ExportedAt: "2026-01-02T03:04:05Z"}})
}},
{name: "blank title", body: func(t *testing.T) []byte {
return encodeArtifact(t, artifact.Artifact{Kind: artifact.KindConvention, Title: " ",
Fields: map[string]any{"status": "draft"}, Body: "x\n"})
}},
}
}
func runParityCase(t *testing.T, c parityCase) parityResult {
t.Helper()
srv := testServer(t)
ws := createWSForTest(t, srv)
if c.setup != nil {
c.setup(t, srv, ws)
}
rr := doArtifactRequest(srv, "POST", "/api/v1/workspaces/"+ws+"/import-artifact", c.body(t))
res := parityResult{Status: rr.Code}
if rr.Code != http.StatusCreated {
return res
}
var resp artifactImportResponse
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
res.Warnings = resp.Warnings
sort.Strings(res.Warnings)
item := getItem(t, srv, ws, resp.Slug)
res.Title, res.Content = item.Title, item.Content
if err := json.Unmarshal([]byte(item.Fields), &res.Fields); err != nil {
t.Fatal(err)
}
return res
}
func TestArtifactImportParity(t *testing.T) {
got := map[string]parityResult{}
for _, c := range artifactParityCases() {
got[c.name] = runParityCase(t, c)
}
if os.Getenv("PAD_UPDATE_ARTIFACT_PARITY") == "1" {
b, err := json.MarshalIndent(got, "", " ")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(artifactParityGolden, append(b, '\n'), 0o644); err != nil {
t.Fatal(err)
}
return
}
raw, err := os.ReadFile(artifactParityGolden)
if err != nil {
t.Fatal(err)
}
var want map[string]parityResult
if err := json.Unmarshal(raw, &want); err != nil {
t.Fatal(err)
}
if len(want) != len(got) {
t.Fatalf("golden has %d cases, test has %d", len(want), len(got))
}
for name, w := range want {
g, ok := got[name]
if !ok {
t.Errorf("case %q missing", name)
continue
}
wb, _ := json.Marshal(w)
gb, _ := json.Marshal(g)
if string(wb) != string(gb) {
t.Errorf("%s changed:\nwant %s\ngot %s", name, wb, gb)
}
}
}
+72
View File
@@ -0,0 +1,72 @@
package server
import (
"encoding/json"
"fmt"
"github.com/PerpetualSoftware/pad/internal/artifact"
"github.com/PerpetualSoftware/pad/internal/models"
)
// normalizedArtifact is what the importer stores for an artifact: the item's
// title, content and fields after the forgiving preprocess, and the warnings
// that preprocess produced.
type normalizedArtifact struct {
Title string
Content string
// FieldsJSON is the stored fields blob.
FieldsJSON string
// Fields is FieldsJSON decoded back into canonical JSON types ([]any,
// map[string]any), the shape the create path validates.
Fields map[string]any
Warnings []string
}
// normalizeArtifact is the artifact importer's preprocess, shared by the human
// import (handleImportArtifact) and the app installer's preview and
// provisioning (SPEC-6 U8, DOC-3371 §2 step 3), so the item an owner reviews
// is the item that is stored. It never mutates art.
//
// Its output depends on the destination: select values the schema does not
// offer are cleared, and a playbook's invocation_slug is de-collided against
// slugTaken. That is why provisioning re-runs it on its own transaction and
// compares the digest with the one reviewed.
//
// The caller validates the title (models.ValidateItemTitle) first; a title
// that fails there must not reach this function.
func normalizeArtifact(art artifact.Artifact, coll *models.Collection, schema models.CollectionSchema, slugTaken func(string) (bool, error)) (*normalizedArtifact, error) {
// Copy the artifact fields so the preprocess never mutates the decoded
// value (keeps the parse layer's output immutable from the caller's POV).
fields := make(map[string]any, len(art.Fields))
for k, v := range art.Fields {
fields[k] = v
}
warnings := preprocessArtifactFields(art.Kind, schema, fields, slugTaken)
fieldsJSON, err := json.Marshal(fields)
if err != nil {
return nil, fmt.Errorf("marshal fields: %w", err)
}
// Normalize the field map through JSON so its nested types match what the
// normal create path validates. handleCreateItem builds its fieldMap by
// unmarshalling the JSON request body, so structured values are canonical
// JSON types ([]any, map[string]any). The artifact decode produces Go-native
// types (e.g. arguments as []map[string]any), which ValidateFields' json case
// rejects — round-tripping fixes that without special-casing any field.
normalized := make(map[string]any)
if err := json.Unmarshal(fieldsJSON, &normalized); err != nil {
return nil, fmt.Errorf("normalize fields: %w", err)
}
body := art.Body
if footer := artifactProvenanceFooter(art.Provenance); footer != "" {
body = body + footer
}
return &normalizedArtifact{
Title: models.NormalizeItemTitle(art.Title),
Content: body,
FieldsJSON: string(fieldsJSON),
Fields: normalized,
Warnings: warnings,
}, nil
}
+752
View File
@@ -0,0 +1,752 @@
package server
import (
"context"
"crypto/sha256"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"sort"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/PerpetualSoftware/pad/internal/appfetch"
"github.com/PerpetualSoftware/pad/internal/appmanifest"
"github.com/PerpetualSoftware/pad/internal/links"
"github.com/PerpetualSoftware/pad/internal/models"
"github.com/PerpetualSoftware/pad/internal/store"
)
// App install staging and preview (SPEC-6 U8a, DOC-3371 §2 steps 1-5, §9;
// TASK-3397). Provisioning, the install code and redeem are U8b.
//
// POST /workspaces/{ws}/apps/install/preview {base_url}
// reserve (owner + instance caps) -> fetch the manifest and every artifact
// exactly once under the fetch policy -> stage the raw bytes -> validate
// apps/1 -> normalize each artifact with the importer's own preprocess ->
// conflict check -> a preview the owner reviews before consenting.
// GET /workspaces/{ws}/apps/install/pending/{pendingID} the staged preview
// DELETE /workspaces/{ws}/apps/install/pending/{pendingID} discard it
//
// Owner-only, and only while appsAvailable(). Nothing is provisioned here.
// settingAppsPrivateOrigins is the admin's private-destination list (§9),
// JSON-encoded []appfetch.PrivateOrigin. Self-host only.
const settingAppsPrivateOrigins = "apps_private_origins"
// appFetchDeadline bounds one install's whole fetch (§2 caps).
const appFetchDeadline = 30 * time.Second
// appArtifactMaxBytes is the per-artifact cap: the importer's own default
// (§2: defaultImportArtifactMaxBytes).
const appArtifactMaxBytes = defaultImportArtifactMaxBytes
// appNotReviewedNotice is shown with every app on every surface (Dave Q3).
const appNotReviewedNotice = "Not reviewed by Pad. You are installing code published at this origin."
// appDeferredNotice explains declared webhook events and item actions that
// this server does not provision yet (A5/A6). They are stored with the
// install and activated when supported, without a reinstall.
const appDeferredNotice = "Webhook events and item actions will activate when this server supports them; they are recorded with the install now."
// appPreview is what the owner reviews before consenting (§2 step 4).
type appPreview struct {
PendingID string `json:"pending_id"`
ExpiresAt time.Time `json:"expires_at"`
Origin string `json:"origin"`
ManifestURL string `json:"manifest_url"`
ManifestSHA256 string `json:"manifest_sha256"`
AppID string `json:"app_id"`
Version string `json:"version"`
Title string `json:"title"`
Description string `json:"description,omitempty"`
Publisher string `json:"publisher"`
Homepage string `json:"homepage,omitempty"`
ReviewedByPad bool `json:"reviewed_by_pad"`
Notice string `json:"notice"`
ServiceAccess string `json:"service_access"`
DelegatedAccess string `json:"delegated_access"`
ReadsSystem string `json:"reads_system_collections"`
Collections []appPreviewCollection `json:"collections"`
Events []appmanifest.Event `json:"events"`
WebhookURL string `json:"webhook_url,omitempty"`
ItemActions []appmanifest.ItemAction `json:"item_actions"`
DeferredNotice string `json:"deferred_notice,omitempty"`
Artifacts []appPreviewArtifact `json:"artifacts"`
ConfigSchema json.RawMessage `json:"config_schema,omitempty"`
RedirectURIs []string `json:"redirect_uris"`
Docs string `json:"docs,omitempty"`
}
type appPreviewCollection struct {
Key string `json:"key"`
Slug string `json:"slug"`
Name string `json:"name"`
Schema json.RawMessage `json:"schema"`
// Adopt: a collection with this slug exists and was created by an install
// of this same origin; it is adopted rather than created.
Adopt bool `json:"adopt"`
}
type appPreviewArtifact struct {
Key string `json:"key"`
URL string `json:"url"`
Kind string `json:"kind"`
DestinationCollection string `json:"destination_collection"`
RawSHA256 string `json:"raw_sha256"`
Raw string `json:"raw"`
Normalized normalizedItem `json:"normalized"`
NormalizedSHA256 string `json:"normalized_sha256"`
// Changes are the importer's warnings: every field it dropped or changed.
Changes []string `json:"changes"`
}
// normalizedItem is the item an artifact will be stored as. Its canonical
// JSON (encoding/json sorts map keys) is what normalized_sha256 hashes.
type normalizedItem struct {
Title string `json:"title"`
Content string `json:"content"`
Fields map[string]any `json:"fields"`
}
func (n normalizedItem) digest() (string, error) {
b, err := json.Marshal(n)
if err != nil {
return "", err
}
return appmanifest.SHA256Hex(sha256.Sum256(b)), nil
}
// appInstallError is a refusal the preview reports to the owner.
type appInstallError struct {
status int
code string
msg string
path string
}
func (e *appInstallError) Error() string { return e.msg }
func installErr(status int, code, format string, args ...any) *appInstallError {
return &appInstallError{status: status, code: code, msg: fmt.Sprintf(format, args...)}
}
func (e *appInstallError) write(w http.ResponseWriter) {
details := map[string]interface{}{}
if e.path != "" {
details["path"] = e.path
}
writeError2(w, e.status, e.code, e.msg, details)
}
// requireAppsOwner gates every install route: apps available here, and the
// caller an owner of the workspace.
func (s *Server) requireAppsOwner(w http.ResponseWriter, r *http.Request) (workspaceID string, owner *models.User, ok bool) {
if !s.appsAvailable() {
writeError(w, http.StatusNotFound, "not_found", "Apps are not enabled on this server")
return "", nil, false
}
if !requireMinRole(w, r, "owner") {
return "", nil, false
}
workspaceID, ok = s.getWorkspaceID(w, r)
if !ok {
return "", nil, false
}
owner = currentUser(r)
if owner == nil {
writeError(w, http.StatusUnauthorized, "unauthorized", "Sign in to install apps")
return "", nil, false
}
return workspaceID, owner, true
}
func (s *Server) handleAppInstallPreview(w http.ResponseWriter, r *http.Request) {
workspaceID, owner, ok := s.requireAppsOwner(w, r)
if !ok {
return
}
var in struct {
BaseURL string `json:"base_url"`
}
if err := decodeJSON(r, &in); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Body must be {\"base_url\": \"https://...\"}")
return
}
origin, err := appmanifest.NormalizeOrigin(in.BaseURL)
if err != nil {
writeError2(w, http.StatusBadRequest, "invalid_base_url", "base_url "+err.Error(), map[string]interface{}{"path": "base_url"})
return
}
if _, err := s.store.SweepExpiredPendingInstalls(); err != nil {
slog.Warn("apps: sweeping expired pending installs failed", "error", err)
}
pending, err := s.store.ReservePendingInstall(workspaceID, owner.ID, origin)
if errors.Is(err, store.ErrPendingLimit) {
writeError(w, http.StatusTooManyRequests, "pending_install_limit", "Too many app installs are pending; finish or discard one and try again")
return
}
if err != nil {
writeInternalError(w, err)
return
}
preview, ierr := s.stageAppInstall(r, workspaceID, pending, origin)
if ierr != nil {
if err := s.store.DeletePendingInstall(pending.ID); err != nil {
slog.Warn("apps: deleting a failed pending install", "pending", pending.ID, "error", err)
}
var ae *appInstallError
if errors.As(ierr, &ae) {
ae.write(w)
return
}
writeInternalError(w, ierr)
return
}
writeJSON(w, http.StatusOK, preview)
}
// stageAppInstall fetches, stages, validates and previews under a reservation
// already taken. Any error leaves the caller to delete the reservation.
func (s *Server) stageAppInstall(r *http.Request, workspaceID string, pending *store.PendingInstall, origin string) (*appPreview, error) {
ctx, cancel := context.WithTimeout(r.Context(), appFetchDeadline)
defer cancel()
fetcher, err := appfetch.New(s.appsPrivateOrigins(), appFetchDeadline, s.appFetchTLS)
if err != nil {
return nil, fmt.Errorf("apps: private origin list: %w", err)
}
manifestURL := appmanifest.ManifestURL(origin)
raw, err := fetcher.Get(ctx, manifestURL, appmanifest.MaxManifestBytes)
if err != nil {
return nil, installErr(http.StatusBadGateway, "manifest_fetch_failed", "Could not fetch %s: %v", manifestURL, err)
}
manifestSHA := appmanifest.SHA256Hex(sha256.Sum256(raw))
if err := s.store.StagePendingBlob(pending.ID, store.PendingBlob{Key: store.PendingManifestKey, URL: manifestURL, SHA256: manifestSHA, Data: raw}); err != nil {
return nil, s.stageErr(err)
}
m, err := appmanifest.Parse(raw)
if err != nil {
var me *appmanifest.Error
if errors.As(err, &me) {
e := installErr(http.StatusUnprocessableEntity, "invalid_manifest", "%s", me.Error())
e.path = me.Path
return nil, e
}
return nil, err
}
if m.Origin != origin {
e := installErr(http.StatusUnprocessableEntity, "invalid_manifest", "the manifest's base_url (%s) is not the origin it was fetched from (%s)", m.Origin, origin)
e.path = "base_url"
return nil, e
}
// Fetch every artifact exactly once, checking the declared sha256.
rawArtifacts := make([][]byte, len(m.CompanionPack.Artifacts))
for i, a := range m.CompanionPack.Artifacts {
body, err := fetcher.Get(ctx, a.URL, appArtifactMaxBytes)
if err != nil {
e := installErr(http.StatusBadGateway, "artifact_fetch_failed", "Could not fetch artifact %q: %v", a.Key, err)
e.path = fmt.Sprintf("companion_pack.artifacts[%d]", i)
return nil, e
}
if got := appmanifest.SHA256Hex(sha256.Sum256(body)); got != a.SHA256 {
e := installErr(http.StatusUnprocessableEntity, "artifact_hash_mismatch", "Artifact %q does not match its declared sha256", a.Key)
e.path = fmt.Sprintf("companion_pack.artifacts[%d].sha256", i)
return nil, e
}
if err := s.store.StagePendingBlob(pending.ID, store.PendingBlob{Key: a.Key, URL: a.URL, SHA256: a.SHA256, Data: body}); err != nil {
return nil, s.stageErr(err)
}
rawArtifacts[i] = body
}
// No further bytes can arrive: everything is staged.
preview, err := s.buildAppPreview(r, workspaceID, m, manifestSHA, rawArtifacts)
if err != nil {
return nil, err
}
preview.PendingID = pending.ID
// The stored preview omits each artifact's raw text: the staged blobs
// already hold those bytes, and GET fills them back in from there.
stored := *preview
stored.Artifacts = make([]appPreviewArtifact, len(preview.Artifacts))
for i, a := range preview.Artifacts {
a.Raw = ""
stored.Artifacts[i] = a
}
previewJSON, err := json.Marshal(stored)
if err != nil {
return nil, err
}
if err := s.store.FinishPendingInstall(pending.ID, manifestSHA, string(previewJSON)); err != nil {
return nil, s.stageErr(err)
}
got, err := s.store.GetPendingInstall(pending.ID, workspaceID, pending.OwnerID)
if err != nil {
return nil, err
}
preview.ExpiresAt = got.ExpiresAt
return preview, nil
}
func (s *Server) stageErr(err error) error {
switch {
case errors.Is(err, store.ErrPendingOverCap):
return installErr(http.StatusRequestEntityTooLarge, "pack_too_large", "The app's manifest and artifacts exceed the %d MiB install cap", store.PendingInstallBytes>>20)
case errors.Is(err, store.ErrPendingNotFound):
return installErr(http.StatusGatewayTimeout, "install_fetch_expired", "The install took too long to fetch; try again")
}
return err
}
// buildAppPreview validates the pack against this workspace and normalizes
// every artifact exactly as the importer would store it.
func (s *Server) buildAppPreview(r *http.Request, workspaceID string, m *appmanifest.Manifest, manifestSHA string, rawArtifacts [][]byte) (*appPreview, error) {
p := &appPreview{
Origin: m.Origin, ManifestURL: appmanifest.ManifestURL(m.Origin), ManifestSHA256: manifestSHA,
AppID: m.ID, Version: m.Version, Title: m.Title, Description: m.Description, Publisher: m.Publisher, Homepage: m.Homepage,
ReviewedByPad: false, Notice: appNotReviewedNotice,
ServiceAccess: m.Scopes.Service.Access, DelegatedAccess: m.Scopes.Delegated.Access,
ReadsSystem: "The app can read this workspace's Conventions and Playbooks.",
Events: m.Events, WebhookURL: m.WebhookURL, ItemActions: m.ItemActions,
ConfigSchema: m.ConfigSchema, RedirectURIs: m.RedirectURIs, Docs: m.Docs,
}
if len(m.Events) > 0 || len(m.ItemActions) > 0 {
p.DeferredNotice = appDeferredNotice
}
// Conflict check (§2 step 5, L6).
slugs := make([]string, len(m.CompanionPack.Collections))
for i, c := range m.CompanionPack.Collections {
slugs[i] = c.Slug
}
owners, err := s.store.CollectionSlugOwners(workspaceID, slugs)
if err != nil {
return nil, err
}
for i, c := range m.CompanionPack.Collections {
pc := appPreviewCollection{Key: c.Key, Slug: c.Slug, Name: c.Name, Schema: c.Schema}
if owner, exists := owners[c.Slug]; exists {
if owner != m.Origin {
e := installErr(http.StatusConflict, "collection_conflict", "This workspace already has a collection %q that this app did not create", c.Slug)
e.path = fmt.Sprintf("companion_pack.collections[%d].slug", i)
return nil, e
}
pc.Adopt = true
}
if err := validateNoReservedFieldKeys(c.Parsed, nil); err != nil {
e := installErr(http.StatusUnprocessableEntity, "invalid_manifest", "collection %q: %v", c.Key, err)
e.path = fmt.Sprintf("companion_pack.collections[%d].schema", i)
return nil, e
}
p.Collections = append(p.Collections, pc)
}
// Unique values claimed by earlier artifacts of this same pack, per
// destination collection and field: the import creates them in order, so
// a later artifact de-collides its invocation_slug against them, and any
// other unique-scoped value it repeats is the conflict the import's unique
// check would refuse (codex rounds 2-4). Provisioning inserts in the same
// order on one transaction and so reaches the same outcome.
claimed := map[string]map[string]map[string]bool{}
for i, a := range m.CompanionPack.Artifacts {
pa, err := s.previewArtifact(r, workspaceID, a, rawArtifacts[i], claimed)
if err != nil {
var ae *appInstallError
if errors.As(err, &ae) && ae.path == "" {
ae.path = fmt.Sprintf("companion_pack.artifacts[%d]", i)
}
return nil, err
}
p.Artifacts = append(p.Artifacts, *pa)
}
return p, nil
}
// previewArtifact runs the importer's own guards, decode and preprocess over
// the STAGED bytes (DOC-3371 §2 step 3).
func (s *Server) previewArtifact(r *http.Request, workspaceID string, a appmanifest.Artifact, raw []byte, claimed map[string]map[string]map[string]bool) (*appPreviewArtifact, error) {
art, err := decodeArtifactBytes(raw)
if err != nil {
return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q is not a valid Pad artifact: %v", a.Key, err)
}
if msg := models.ValidateItemTitle(models.NormalizeItemTitle(art.Title)); msg != "" {
return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q: %s", a.Key, msg)
}
collID, err := s.collectionIDForKind(workspaceID, art.Kind, nil)
if err != nil {
return nil, err
}
if collID == "" {
return nil, installErr(http.StatusUnprocessableEntity, "artifact_destination_missing", "This workspace has no collection that accepts %q artifacts (artifact %q)", art.Kind, a.Key)
}
coll, err := s.store.GetCollection(collID)
if err != nil {
return nil, err
}
var schema models.CollectionSchema
if err := models.UnmarshalItemFieldSchema([]byte(coll.Schema), &schema); err != nil {
return nil, fmt.Errorf("apps: destination schema: %w", err)
}
if claimed[coll.ID] == nil {
claimed[coll.ID] = map[string]map[string]bool{}
}
packClaims := claimed[coll.ID]
norm, err := normalizeArtifact(art, coll, schema, func(slug string) (bool, error) {
if packClaims["invocation_slug"]["s:"+slug] {
return true, nil
}
return s.invocationSlugTaken(workspaceID, coll.ID, slug)
})
if err != nil {
return nil, err
}
// The create's own field pipeline (coercion, defaults, validation,
// relation resolution in the import's carry posture, the unique check),
// so the digest covers exactly what the import stores (codex round 1).
fields, dropped, unresolved, undeclared, cerr := s.prepareCreateFields(r, workspaceID, coll, schema, norm.Fields, relationsCarry)
if cerr != nil {
return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q cannot be stored in %q: %s", a.Key, coll.Slug, cerr.message)
}
for _, k := range undeclared {
norm.Warnings = append(norm.Warnings, fmt.Sprintf("field %q is not declared by the destination collection's schema; stored as-is", k))
}
for _, k := range unresolved {
norm.Warnings = append(norm.Warnings, fmt.Sprintf("field %q was imported as-is: it does not name an item in this workspace", k))
}
for _, k := range dropped {
norm.Warnings = append(norm.Warnings, fmt.Sprintf("field %q was discarded: the destination collection's schema declares a default for it that is not a valid reference", k))
}
// Claims are recorded from the FINAL fields, after defaults, which is
// what the import stores (codex round 3), for invocation_slug and every
// unique-scoped field (codex round 4).
// Which fields: invocation_slug always (a database unique index enforces
// it whatever the schema says), plus exactly the fields the import's
// checkUniqueFields enforces, through the same predicate (codex round 6).
// Deduplicated: a schema may declare a key twice, and one artifact must
// never conflict with itself (codex round 5).
uniqueKeys := []string{"invocation_slug"}
seenKey := map[string]bool{"invocation_slug": true}
for _, def := range schema.Fields {
if uniqueEnforced(def) && !seenKey[def.Key] {
seenKey[def.Key] = true
uniqueKeys = append(uniqueKeys, def.Key)
}
}
// invocation_slug is also covered by a database unique index per
// collection (056 / pg 033), whatever the schema says, so its FINAL value
// (a default included) is checked against stored items too, through the
// index's own expression (codex round 8), and must be
// text: SQLite keys the index on the JSON-typed value and Postgres on its
// text, so a non-text slug is refused rather than guessed at (codex round
// 7). Provisioning re-checks every constraint inside one transaction, so
// a divergence left here costs a failed install, never a partial one.
if raw, ok := fields["invocation_slug"]; ok && raw != nil {
slug, isText := raw.(string)
if !isText {
return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q would store a non-text invocation_slug", a.Key)
}
if slug != "" {
taken, err := s.store.InvocationSlugIndexTaken(coll.ID, slug)
if err != nil {
return nil, err
}
if taken {
return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q would store invocation_slug %q, which an item in %q already holds", a.Key, slug, coll.Slug)
}
}
}
for _, key := range uniqueKeys {
// INVARIANT (lead ruling): the preview may refuse what the import
// would accept, never the reverse; U8b's provisioning re-checks every
// constraint inside one transaction, so that re-check is the
// guarantee and this preview is advisory.
//
// Each value claims every form the import's lookup (JSONFieldEquals)
// could match it by: text, boolean, and number as its float64 (equal
// doubles is a superset of SQLite's double comparison and Postgres's
// exact numeric equality). Two values collide when they share any
// form (codex rounds 4-12).
forms := claimForms(fields[key])
for _, f := range forms {
if packClaims[key][f] {
return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q would store a %s value that an earlier artifact of this pack already holds", a.Key, key)
}
}
if packClaims[key] == nil {
packClaims[key] = map[string]bool{}
}
for _, f := range forms {
packClaims[key][f] = true
}
}
// App-authored content: the owner only REVIEWED it, so it may not carry
// a pad-attachment: token or a [[workspace::REF]] link, and the human
// path's attachment stamp and cross-workspace resolution never run on it
// (lead ruling, U8b Q3). Ordinary [[Title]] links stay: they resolve
// workspace-wide as on any owner write, and the preview shows them.
if err := refuseAppAuthoredReferences(norm.Content, fields); err != nil {
return nil, installErr(http.StatusUnprocessableEntity, "invalid_artifact", "Artifact %q: %v", a.Key, err)
}
item := normalizedItem{Title: norm.Title, Content: norm.Content, Fields: fields}
digest, err := item.digest()
if err != nil {
return nil, err
}
changes := append([]string{}, norm.Warnings...)
sort.Strings(changes)
return &appPreviewArtifact{
Key: a.Key, URL: a.URL, Kind: string(art.Kind), DestinationCollection: coll.Slug,
RawSHA256: a.SHA256, Raw: string(raw), Normalized: item, NormalizedSHA256: digest, Changes: changes,
}, nil
}
func (s *Server) handleGetAppInstallPending(w http.ResponseWriter, r *http.Request) {
workspaceID, owner, ok := s.requireAppsOwner(w, r)
if !ok {
return
}
p, err := s.store.GetPendingInstall(chi.URLParam(r, "pendingID"), workspaceID, owner.ID)
if errors.Is(err, store.ErrPendingNotFound) || (err == nil && p.State != "staged") {
writeError(w, http.StatusNotFound, "not_found", "Pending install not found")
return
}
if err != nil {
writeInternalError(w, err)
return
}
var preview appPreview
if err := json.Unmarshal([]byte(p.Preview), &preview); err != nil {
writeInternalError(w, err)
return
}
blobs, err := s.store.PendingInstallBlobs(p.ID)
if err != nil {
writeInternalError(w, err)
return
}
for i := range preview.Artifacts {
preview.Artifacts[i].Raw = string(blobs[preview.Artifacts[i].Key].Data)
}
preview.ExpiresAt = p.ExpiresAt
writeJSON(w, http.StatusOK, preview)
}
func (s *Server) handleDeleteAppInstallPending(w http.ResponseWriter, r *http.Request) {
workspaceID, owner, ok := s.requireAppsOwner(w, r)
if !ok {
return
}
p, err := s.store.GetPendingInstall(chi.URLParam(r, "pendingID"), workspaceID, owner.ID)
if errors.Is(err, store.ErrPendingNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Pending install not found")
return
}
if err != nil {
writeInternalError(w, err)
return
}
if err := s.store.DeletePendingInstall(p.ID); err != nil {
writeInternalError(w, err)
return
}
w.WriteHeader(http.StatusNoContent)
}
// appsPrivateOrigins is the admin list, or nil on Cloud (no private
// destination is ever reachable there) or when unset or unreadable.
func (s *Server) appsPrivateOrigins() []appfetch.PrivateOrigin {
if s.cloudMode {
return nil
}
v, err := s.store.GetPlatformSetting(settingAppsPrivateOrigins)
if err != nil || v == "" {
return nil
}
var list []appfetch.PrivateOrigin
if err := json.Unmarshal([]byte(v), &list); err != nil {
slog.Warn("apps: the private-origin list is not valid JSON; ignoring it", "error", err)
return nil
}
return list
}
// SetAppFetchTLS overrides the TLS config app fetches use. Tests only.
func (s *Server) SetAppFetchTLS(c *tls.Config) { s.appFetchTLS = c }
// --- Admin: GET/PUT /api/v1/admin/apps ----------------------------------
type appsSettingsResponse struct {
Enabled bool `json:"enabled"`
Available bool `json:"available"`
HTTPSIssuer bool `json:"https_issuer"`
Cloud bool `json:"cloud"`
PrivateOrigins []appfetch.PrivateOrigin `json:"private_origins"`
}
func (s *Server) appsSettings() appsSettingsResponse {
v, _ := s.store.GetPlatformSetting(settingAppsEnabled)
list := s.appsPrivateOrigins()
if list == nil {
list = []appfetch.PrivateOrigin{}
}
return appsSettingsResponse{
Enabled: s.cloudMode || v == "true", Available: s.appsAvailable(),
HTTPSIssuer: s.oauthServer != nil && (s.cloudMode || s.mcpEndpoints.HTTPS()),
Cloud: s.cloudMode, PrivateOrigins: list,
}
}
// handleGetAppsSettings answers on Cloud too, as GET /admin/mcp does, with
// cloud: true: reading the state is harmless, and only the PUT is
// operator-managed there (managed_by_operator).
func (s *Server) handleGetAppsSettings(w http.ResponseWriter, r *http.Request) {
if user := currentUser(r); user == nil || user.Role != "admin" {
writeError(w, http.StatusForbidden, "forbidden", "Admin access required")
return
}
writeJSON(w, http.StatusOK, s.appsSettings())
}
func (s *Server) handleUpdateAppsSettings(w http.ResponseWriter, r *http.Request) {
if user := currentUser(r); user == nil || user.Role != "admin" {
writeError(w, http.StatusForbidden, "forbidden", "Admin access required")
return
}
if s.cloudMode {
writeError(w, http.StatusForbidden, "managed_by_operator", "Apps on this instance are managed by the operator.")
return
}
var in struct {
Enabled *bool `json:"enabled"`
PrivateOrigins *[]appfetch.PrivateOrigin `json:"private_origins"`
}
if err := decodeJSON(r, &in); err != nil || (in.Enabled == nil && in.PrivateOrigins == nil) {
writeError(w, http.StatusBadRequest, "bad_request", "Body must set enabled and/or private_origins")
return
}
var changed []string
if in.PrivateOrigins != nil {
list := *in.PrivateOrigins
if list == nil {
list = []appfetch.PrivateOrigin{}
}
// Validate by building a fetcher from it: the same parser the fetch
// path uses, so a list that saves is a list that works.
if _, err := appfetch.New(list, time.Second, nil); err != nil {
writeError(w, http.StatusBadRequest, "invalid_private_origins", err.Error())
return
}
b, err := json.Marshal(list)
if err != nil {
writeInternalError(w, err)
return
}
if err := s.store.SetPlatformSetting(settingAppsPrivateOrigins, string(b)); err != nil {
writeInternalError(w, err)
return
}
changed = append(changed, settingAppsPrivateOrigins)
}
if in.Enabled != nil {
value := "false"
if *in.Enabled {
value = "true"
}
if err := s.store.SetPlatformSetting(settingAppsEnabled, value); err != nil {
writeInternalError(w, err)
return
}
changed = append(changed, settingAppsEnabled)
}
s.logAuditEvent(models.ActionSettingsChanged, r, settingsChangedMeta(changed))
writeJSON(w, http.StatusOK, s.appsSettings())
}
// claimForms are the forms a field value can be matched by in the import's
// uniqueness lookup (JSONFieldEquals): a string as text, and also as a
// boolean when it reads true/false and as a number when it is a JSON number;
// a boolean as a boolean; a number as its canonical number. Absent, null and
// empty values claim nothing, as they occupy no unique slot. Composite values
// claim their JSON text.
func claimForms(v any) []string {
switch t := v.(type) {
case nil:
return nil
case string:
if t == "" {
return nil
}
forms := []string{"s:" + t}
if t == "true" || t == "false" {
forms = append(forms, "b:"+t)
}
for _, n := range store.NumericClaimForms(t) {
forms = append(forms, "n:"+n)
}
return forms
case bool:
return []string{fmt.Sprintf("b:%v", t)}
case float64:
// Canonicalized from its JSON serialization, which is the text the
// stored value has and the import compares against, through the same
// numericArg path a later string takes (codex round 11: formatting
// the float directly diverged near 2^63 and past 2^53).
if b, err := json.Marshal(t); err == nil {
if forms := numberForms(string(b)); len(forms) > 0 {
return forms
}
}
case json.Number:
if forms := numberForms(t.String()); len(forms) > 0 {
return forms
}
}
b, err := json.Marshal(v)
if err != nil || len(b) == 0 {
return nil
}
return []string{"j:" + string(b)}
}
func numberForms(text string) []string {
var forms []string
for _, n := range store.NumericClaimForms(text) {
forms = append(forms, "n:"+n)
}
return forms
}
// refuseAppAuthoredReferences refuses any pad-attachment: token, in any
// spelling, in an app artifact's content or field values, and any
// workspace-qualified wiki link in its content.
func refuseAppAuthoredReferences(content string, fields map[string]any) error {
if strings.Contains(strings.ToLower(content), "pad-attachment:") {
return errors.New("pad-attachment: references are not accepted in app content")
}
// Fail closed: a field set that cannot be serialized cannot be checked,
// so it is refused, never waved through. Unreachable from a preview today
// (fields come from decoded JSON, which always re-encodes); the direct
// test reaches it.
b, err := json.Marshal(fields)
if err != nil {
return fmt.Errorf("app content fields cannot be checked: %w", err)
}
if strings.Contains(strings.ToLower(string(b)), "pad-attachment:") {
return errors.New("pad-attachment: references are not accepted in app content")
}
for _, l := range links.ExtractWikiLinks(content) {
if l.Kind == links.WikiLinkKindWorkspaceRef {
return errors.New("workspace-qualified links are not accepted in app content")
}
}
return nil
}
@@ -0,0 +1,715 @@
package server
import (
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/PerpetualSoftware/pad/internal/config"
"github.com/PerpetualSoftware/pad/internal/models"
"github.com/PerpetualSoftware/pad/internal/oauth"
)
// TASK-3397 (U8a): app install staging and preview.
type appsEnv struct {
srv *Server
token string
ws string
wsID string
app *httptest.Server
files map[string][]byte // path -> body served by the app
status map[string]int // path -> status override
}
func appSHA(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
func newAppsEnv(t *testing.T) *appsEnv {
t.Helper()
srv := testServer(t)
token := bootstrapFirstUser(t, srv, "owner@test.com", "Owner")
rr := doRequestWithCookie(srv, "POST", "/api/v1/workspaces", map[string]string{"name": "Apps"}, token)
if rr.Code != http.StatusCreated {
t.Fatalf("create workspace: %d %s", rr.Code, rr.Body.String())
}
var ws models.Workspace
parseJSON(t, rr, &ws)
srv.SetMCPConfig(config.MCPEndpoints{Origin: "https://pad.test.example", OriginVar: "PAD_URL",
ResourceURL: "https://pad.test.example/mcp", AuthServerURL: "https://pad.test.example"}, nil)
o, err := oauth.NewServer(oauth.Config{Store: srv.store, HMACSecret: bytes32ForTest(), AllowedAudience: "https://pad.test.example/mcp"})
if err != nil {
t.Fatal(err)
}
srv.SetOAuthServer(o)
e := &appsEnv{srv: srv, token: token, ws: ws.Slug, wsID: ws.ID, files: map[string][]byte{}, status: map[string]int{}}
e.app = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if code, ok := e.status[r.URL.Path]; ok {
if code >= 300 && code < 400 {
http.Redirect(w, r, "/elsewhere", code)
return
}
w.WriteHeader(code)
return
}
body, ok := e.files[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
_, _ = w.Write(body)
}))
t.Cleanup(e.app.Close)
pool := x509.NewCertPool()
pool.AddCert(e.app.Certificate())
srv.SetAppFetchTLS(&tls.Config{RootCAs: pool})
// Admin: enable apps and pin the test app's loopback origin.
rr = doRequestWithCookie(srv, "PUT", "/api/v1/admin/apps", map[string]any{
"enabled": true,
"private_origins": []map[string]any{{"origin": e.app.URL, "allowed": []string{"127.0.0.1"}, "fetch": true}},
}, token)
if rr.Code != http.StatusOK {
t.Fatalf("enable apps: %d %s", rr.Code, rr.Body.String())
}
e.publish(t, e.manifest(t))
return e
}
func (e *appsEnv) origin() string { return e.app.URL }
// manifest returns a valid manifest for the test app with one companion
// collection and the golden playbook as its artifact.
func (e *appsEnv) manifest(t *testing.T) map[string]any {
t.Helper()
pb, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", "playbook.golden.md"))
if err != nil {
t.Fatal(err)
}
e.files["/pack/ship.md"] = pb
return map[string]any{
"id": "acme/portal", "version": "1.0.0", "min_contract": map[string]any{"apps": 1, "events": 1},
"title": "Portal", "publisher": "Acme", "base_url": e.origin(),
"redirect_uris": []any{e.origin() + "/oauth/callback"},
"scopes": map[string]any{"service": map[string]any{"access": "write"}, "delegated": map[string]any{"access": "read"}},
"events": []any{map[string]any{"name": "item.created", "collections": []any{"tickets"}}},
"webhook_url": e.origin() + "/hooks",
"companion_pack": map[string]any{
"collections": []any{map[string]any{"key": "tickets", "slug": "portal-tickets", "name": "Portal tickets",
"schema": map[string]any{"fields": []any{map[string]any{"key": "status", "label": "Status", "type": "select", "options": []any{"open", "solved"}}}}}},
"artifacts": []any{map[string]any{"key": "ship", "url": e.origin() + "/pack/ship.md", "sha256": appSHA(pb)}},
},
"item_actions": []any{map[string]any{"key": "open", "label": "Open", "collections": []any{"tickets"}, "path": "/t"}},
}
}
func (e *appsEnv) publish(t *testing.T, m map[string]any) {
t.Helper()
b, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
e.files["/.well-known/pad-app.json"] = b
}
func (e *appsEnv) preview(t *testing.T) *httptest.ResponseRecorder {
t.Helper()
return doRequestWithCookie(e.srv, "POST", "/api/v1/workspaces/"+e.ws+"/apps/install/preview", map[string]any{"base_url": e.origin()}, e.token)
}
func (e *appsEnv) pendingRows(t *testing.T) int {
t.Helper()
var n int
if err := e.srv.store.DB().QueryRow(`SELECT COUNT(*) FROM app_install_pending`).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
func TestAppInstallPreview_StagesAndPreviews(t *testing.T) {
e := newAppsEnv(t)
rr := e.preview(t)
if rr.Code != http.StatusOK {
t.Fatalf("preview: %d %s", rr.Code, rr.Body.String())
}
var p appPreview
parseJSON(t, rr, &p)
if p.ReviewedByPad || !strings.Contains(p.Notice, "Not reviewed by Pad") {
t.Errorf("not-reviewed notice missing: %+v", p.Notice)
}
if p.Origin != e.origin() || p.ServiceAccess != "write" || p.DelegatedAccess != "read" || p.DeferredNotice == "" {
t.Errorf("preview header: %+v", p)
}
if len(p.Collections) != 1 || p.Collections[0].Adopt {
t.Errorf("collections: %+v", p.Collections)
}
if len(p.Artifacts) != 1 {
t.Fatalf("artifacts: %+v", p.Artifacts)
}
a := p.Artifacts[0]
if a.Raw == "" || a.RawSHA256 != appSHA(e.files["/pack/ship.md"]) || a.Normalized.Fields["status"] != "draft" {
t.Errorf("artifact preview: raw=%d sha=%s status=%v", len(a.Raw), a.RawSHA256, a.Normalized.Fields["status"])
}
// Every field the importer changed is shown: the status reset, and the
// select values this workspace does not offer.
joined := strings.Join(a.Changes, "\n")
for _, want := range []string{`reset to "draft"`, `"trigger" value "on-intent"`} {
if !strings.Contains(joined, want) {
t.Errorf("change %q not shown: %v", want, a.Changes)
}
}
if want, _ := a.Normalized.digest(); a.NormalizedSHA256 != want {
t.Errorf("normalized digest %s, want %s", a.NormalizedSHA256, want)
}
// The staged record serves the same preview, raw bytes included.
rr = doRequestWithCookie(e.srv, "GET", "/api/v1/workspaces/"+e.ws+"/apps/install/pending/"+p.PendingID, nil, e.token)
if rr.Code != http.StatusOK {
t.Fatalf("get pending: %d %s", rr.Code, rr.Body.String())
}
var again appPreview
parseJSON(t, rr, &again)
if again.Artifacts[0].Raw != a.Raw || again.Artifacts[0].NormalizedSHA256 != a.NormalizedSHA256 {
t.Error("the stored preview differs from the one returned")
}
var staged int64
if err := e.srv.store.DB().QueryRow(`SELECT reserved_bytes FROM app_install_pending WHERE id = ?`, p.PendingID).Scan(&staged); err != nil {
t.Fatal(err)
}
if want := int64(len(e.files["/.well-known/pad-app.json"]) + len(e.files["/pack/ship.md"])); staged != want {
t.Errorf("charge after staging %d, want the %d staged bytes", staged, want)
}
rr = doRequestWithCookie(e.srv, "DELETE", "/api/v1/workspaces/"+e.ws+"/apps/install/pending/"+p.PendingID, nil, e.token)
if rr.Code != http.StatusNoContent || e.pendingRows(t) != 0 {
t.Fatalf("discard: %d, %d rows left", rr.Code, e.pendingRows(t))
}
}
// Every refusal releases its reservation and stages nothing.
func TestAppInstallPreview_Refusals(t *testing.T) {
for name, c := range map[string]struct {
mutate func(t *testing.T, e *appsEnv)
status int
code string
}{
"artifact hash mismatch": {func(t *testing.T, e *appsEnv) { e.files["/pack/ship.md"] = []byte("tampered") }, 422, "artifact_hash_mismatch"},
"unsubscribable event": {func(t *testing.T, e *appsEnv) {
m := e.manifest(t)
m["events"] = []any{map[string]any{"name": "item.bulk_updated", "collections": []any{"tickets"}}}
e.publish(t, m)
}, 422, "invalid_manifest"},
"manifest names another origin": {func(t *testing.T, e *appsEnv) {
m := e.manifest(t)
m["base_url"] = "https://other.example"
m["redirect_uris"] = []any{"https://other.example/cb"}
m["webhook_url"] = "https://other.example/h"
pack := m["companion_pack"].(map[string]any)
pack["artifacts"] = []any{}
e.publish(t, m)
}, 422, "invalid_manifest"},
"manifest redirects": {func(t *testing.T, e *appsEnv) { e.status["/.well-known/pad-app.json"] = http.StatusFound }, 502, "manifest_fetch_failed"},
"artifact missing": {func(t *testing.T, e *appsEnv) { delete(e.files, "/pack/ship.md") }, 502, "artifact_fetch_failed"},
"slug taken by a human collection": {func(t *testing.T, e *appsEnv) {
if _, err := e.srv.store.CreateCollection(e.wsID, models.CollectionCreate{Name: "Portal tickets", Slug: "portal-tickets", Schema: `{"fields":[]}`}); err != nil {
t.Fatal(err)
}
}, 409, "collection_conflict"},
} {
t.Run(name, func(t *testing.T) {
e := newAppsEnv(t)
c.mutate(t, e)
rr := e.preview(t)
if rr.Code != c.status || !strings.Contains(rr.Body.String(), c.code) {
t.Fatalf("got %d %s, want %d %s", rr.Code, rr.Body.String(), c.status, c.code)
}
if n := e.pendingRows(t); n != 0 {
t.Fatalf("a refused preview left %d pending record(s)", n)
}
})
}
}
func TestAppInstallPreview_Gates(t *testing.T) {
e := newAppsEnv(t)
// Apps off: the routes answer 404.
if rr := doRequestWithCookie(e.srv, "PUT", "/api/v1/admin/apps", map[string]any{"enabled": false}, e.token); rr.Code != http.StatusOK {
t.Fatal(rr.Body.String())
}
if rr := e.preview(t); rr.Code != http.StatusNotFound {
t.Fatalf("apps off: %d", rr.Code)
}
if rr := doRequestWithCookie(e.srv, "PUT", "/api/v1/admin/apps", map[string]any{"enabled": true}, e.token); rr.Code != http.StatusOK {
t.Fatal(rr.Body.String())
}
// A bad base URL is refused before any reservation.
rr := doRequestWithCookie(e.srv, "POST", "/api/v1/workspaces/"+e.ws+"/apps/install/preview", map[string]any{"base_url": "http://x.example"}, e.token)
if rr.Code != http.StatusBadRequest || e.pendingRows(t) != 0 {
t.Fatalf("http base url: %d", rr.Code)
}
// The per-owner cap.
for i := 0; i < 3; i++ {
if rr := e.preview(t); rr.Code != http.StatusOK {
t.Fatalf("preview %d: %d %s", i, rr.Code, rr.Body.String())
}
}
if rr := e.preview(t); rr.Code != http.StatusTooManyRequests {
t.Fatalf("fourth pending install: %d %s", rr.Code, rr.Body.String())
}
}
func TestAppsAdminSettings(t *testing.T) {
e := newAppsEnv(t)
rr := doRequestWithCookie(e.srv, "PUT", "/api/v1/admin/apps", map[string]any{
"private_origins": []map[string]any{{"origin": "https://x.example", "fetch": true}},
}, e.token)
if rr.Code != http.StatusBadRequest {
t.Fatalf("an entry without pinned addresses was accepted: %d", rr.Code)
}
rr = doRequestWithCookie(e.srv, "GET", "/api/v1/admin/apps", nil, e.token)
var s appsSettingsResponse
parseJSON(t, rr, &s)
if !s.Enabled || !s.Available || len(s.PrivateOrigins) != 1 {
t.Fatalf("settings: %+v", s)
}
// Without an https issuer apps are unavailable even when enabled.
e.srv.SetMCPConfig(config.MCPEndpoints{Origin: "http://pad.test.example", OriginVar: "PAD_URL",
ResourceURL: "http://pad.test.example/mcp", AuthServerURL: "http://pad.test.example"}, nil)
if e.srv.appsAvailable() {
t.Fatal("apps available over an http issuer")
}
}
// The preview's normalized item is what the import would STORE: the create's
// own defaults and validation run on it (codex round 1). A destination field
// the artifact cannot satisfy refuses the preview; a schema default appears
// in the previewed fields and so in the digest.
func TestAppInstallPreview_RunsTheCreateFieldPipeline(t *testing.T) {
addField := func(t *testing.T, e *appsEnv, field map[string]any) {
t.Helper()
coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks")
if err != nil || coll == nil {
t.Fatalf("playbooks collection: %v", err)
}
var schema map[string]any
if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil {
t.Fatal(err)
}
schema["fields"] = append(schema["fields"].([]any), field)
b, _ := json.Marshal(schema)
sch := string(b)
if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil {
t.Fatal(err)
}
}
t.Run("required field refuses", func(t *testing.T) {
e := newAppsEnv(t)
addField(t, e, map[string]any{"key": "owner_team", "label": "Owner team", "type": "text", "required": true})
rr := e.preview(t)
if rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "invalid_artifact") {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
if n := e.pendingRows(t); n != 0 {
t.Fatalf("%d pending left", n)
}
})
t.Run("default is previewed", func(t *testing.T) {
e := newAppsEnv(t)
addField(t, e, map[string]any{"key": "team", "label": "Team", "type": "text", "default": "core"})
rr := e.preview(t)
if rr.Code != http.StatusOK {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
var p appPreview
parseJSON(t, rr, &p)
if got := p.Artifacts[0].Normalized.Fields["team"]; got != "core" {
t.Fatalf("the schema default is not in the previewed item: %v", got)
}
})
}
// An abandoned preview's staged bytes are deleted by the background sweep,
// not only when a later install happens to sweep (codex round 1).
func TestAppPendingSweep_DeletesExpiredStagedBytes(t *testing.T) {
e := newAppsEnv(t)
rr := e.preview(t)
if rr.Code != http.StatusOK {
t.Fatalf("preview: %d %s", rr.Code, rr.Body.String())
}
if _, err := e.srv.store.DB().Exec(`UPDATE app_install_pending SET expires_at = '2000-01-01T00:00:00Z'`); err != nil {
t.Fatal(err)
}
e.srv.runAppPendingSweep()
var records, blobs int
_ = e.srv.store.DB().QueryRow(`SELECT COUNT(*) FROM app_install_pending`).Scan(&records)
_ = e.srv.store.DB().QueryRow(`SELECT COUNT(*) FROM app_install_pending_blobs`).Scan(&blobs)
if records != 0 || blobs != 0 {
t.Fatalf("after the sweep: %d records, %d blobs", records, blobs)
}
// The loop starts and stops cleanly.
e.srv.StartAppPendingSweep()
e.srv.StartAppPendingSweep()
e.srv.stopAppPendingSweep()
e.srv.stopAppPendingSweep()
}
// Two artifacts of one pack asking for the same invocation_slug are
// allocated distinct slugs in the preview, as their sequential import would
// be (codex round 2).
func TestAppInstallPreview_PackArtifactsDeCollideWithEachOther(t *testing.T) {
e := newAppsEnv(t)
m := e.manifest(t)
second := []byte(strings.Replace(string(e.files["/pack/ship.md"]), "title: Ship a change", "title: Ship again", 1))
e.files["/pack/ship2.md"] = second
pack := m["companion_pack"].(map[string]any)
pack["artifacts"] = append(pack["artifacts"].([]any), map[string]any{"key": "ship2", "url": e.origin() + "/pack/ship2.md", "sha256": appSHA(second)})
e.publish(t, m)
rr := e.preview(t)
if rr.Code != http.StatusOK {
t.Fatalf("preview: %d %s", rr.Code, rr.Body.String())
}
var p appPreview
parseJSON(t, rr, &p)
a, b := p.Artifacts[0].Normalized.Fields["invocation_slug"], p.Artifacts[1].Normalized.Fields["invocation_slug"]
if a == b {
t.Fatalf("both artifacts previewed invocation_slug %v", a)
}
if a != "ship" || b != "ship-2" {
t.Fatalf("slugs %v, %v; want ship, ship-2 (the sequential import's allocation)", a, b)
}
}
// A schema DEFAULT counts as a claim: an artifact without a slug, stored with
// the default, makes a later artifact asking for that slug de-collide, as
// the sequential import does (codex round 3).
func TestAppInstallPreview_DefaultSlugIsClaimed(t *testing.T) {
e := newAppsEnv(t)
coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks")
if err != nil || coll == nil {
t.Fatalf("playbooks collection: %v", err)
}
var schema map[string]any
if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil {
t.Fatal(err)
}
for _, f := range schema["fields"].([]any) {
if fm := f.(map[string]any); fm["key"] == "invocation_slug" {
fm["default"] = "ship"
}
}
b, _ := json.Marshal(schema)
sch := string(b)
if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil {
t.Fatal(err)
}
m := e.manifest(t)
orig := string(e.files["/pack/ship.md"])
noSlug := []byte(strings.Replace(strings.Replace(orig, "invocation_slug: ship\n", "", 1), "title: Ship a change", "title: Defaulted", 1))
e.files["/pack/a.md"] = noSlug
e.files["/pack/b.md"] = []byte(strings.Replace(orig, "title: Ship a change", "title: Explicit", 1))
pack := m["companion_pack"].(map[string]any)
pack["artifacts"] = []any{
map[string]any{"key": "a", "url": e.origin() + "/pack/a.md", "sha256": appSHA(e.files["/pack/a.md"])},
map[string]any{"key": "b", "url": e.origin() + "/pack/b.md", "sha256": appSHA(e.files["/pack/b.md"])},
}
e.publish(t, m)
rr := e.preview(t)
if rr.Code != http.StatusOK {
t.Fatalf("preview: %d %s", rr.Code, rr.Body.String())
}
var p appPreview
parseJSON(t, rr, &p)
if a, b := p.Artifacts[0].Normalized.Fields["invocation_slug"], p.Artifacts[1].Normalized.Fields["invocation_slug"]; a != "ship" || b != "ship-2" {
t.Fatalf("slugs %v, %v; want ship (the default), ship-2", a, b)
}
}
// Any unique-scoped destination field, not only invocation_slug: two pack
// artifacts storing the same unique value (here through a default) refuse
// the preview, as the second sequential import would 409 (codex round 4).
func TestAppInstallPreview_PackUniqueValuesConflict(t *testing.T) {
e := newAppsEnv(t)
coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks")
if err != nil || coll == nil {
t.Fatalf("playbooks collection: %v", err)
}
var schema map[string]any
if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil {
t.Fatal(err)
}
codeField := map[string]any{"key": "code", "label": "Code", "type": "text", "unique_scope": "workspace_collection", "default": "shared"}
// Declared twice: a single artifact must still not conflict with itself
// (codex round 5).
schema["fields"] = append(schema["fields"].([]any), codeField, codeField)
b, _ := json.Marshal(schema)
sch := string(b)
if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil {
t.Fatal(err)
}
if rr := e.preview(t); rr.Code != http.StatusOK {
t.Fatalf("a single artifact conflicted with itself: %d %s", rr.Code, rr.Body.String())
}
if rr := doRequestWithCookie(e.srv, "DELETE", "/api/v1/workspaces/"+e.ws+"/apps/install/pending/"+pendingIDOf(t, e), nil, e.token); rr.Code != http.StatusNoContent {
t.Fatalf("discard: %d", rr.Code)
}
m := e.manifest(t)
second := []byte(strings.Replace(strings.Replace(string(e.files["/pack/ship.md"]), "title: Ship a change", "title: Other", 1), "invocation_slug: ship", "invocation_slug: other", 1))
e.files["/pack/other.md"] = second
pack := m["companion_pack"].(map[string]any)
pack["artifacts"] = append(pack["artifacts"].([]any), map[string]any{"key": "other", "url": e.origin() + "/pack/other.md", "sha256": appSHA(second)})
e.publish(t, m)
rr := e.preview(t)
if rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "code") {
t.Fatalf("got %d %s; want 422 naming the unique field", rr.Code, rr.Body.String())
}
if n := e.pendingRows(t); n != 0 {
t.Fatalf("%d pending left", n)
}
}
func pendingIDOf(t *testing.T, e *appsEnv) string {
t.Helper()
var id string
if err := e.srv.store.DB().QueryRow(`SELECT id FROM app_install_pending ORDER BY created_at DESC LIMIT 1`).Scan(&id); err != nil {
t.Fatal(err)
}
return id
}
// A unique_scope the import does NOT enforce is not a pack conflict either
// (codex round 6): two artifacts sharing that default both import, so both
// preview.
func TestAppInstallPreview_UnenforcedUniqueScopeIsNotAConflict(t *testing.T) {
e := newAppsEnv(t)
coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "playbooks")
if err != nil || coll == nil {
t.Fatalf("playbooks collection: %v", err)
}
var schema map[string]any
if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil {
t.Fatal(err)
}
schema["fields"] = append(schema["fields"].([]any), map[string]any{"key": "code", "label": "Code", "type": "text", "unique_scope": "collection", "default": "shared"})
b, _ := json.Marshal(schema)
sch := string(b)
if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil {
t.Fatal(err)
}
m := e.manifest(t)
second := []byte(strings.Replace(strings.Replace(string(e.files["/pack/ship.md"]), "title: Ship a change", "title: Other", 1), "invocation_slug: ship", "invocation_slug: other", 1))
e.files["/pack/other.md"] = second
pack := m["companion_pack"].(map[string]any)
pack["artifacts"] = append(pack["artifacts"].([]any), map[string]any{"key": "other", "url": e.origin() + "/pack/other.md", "sha256": appSHA(second)})
e.publish(t, m)
if rr := e.preview(t); rr.Code != http.StatusOK {
t.Fatalf("an unenforced scope refused the preview: %d %s", rr.Code, rr.Body.String())
}
}
// invocation_slug is unique per collection at the DATABASE, whatever the
// schema declares (codex round 7): a defaulted slug that a stored item
// already holds, and a non-text slug, both refuse the preview.
func TestAppInstallPreview_InvocationSlugIndexIsMirrored(t *testing.T) {
setDefault := func(t *testing.T, e *appsEnv, def any) string {
t.Helper()
coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "conventions")
if err != nil || coll == nil {
t.Fatalf("conventions collection: %v", err)
}
var schema map[string]any
if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil {
t.Fatal(err)
}
typ := "text"
if _, isNum := def.(int); isNum {
typ = "number"
}
schema["fields"] = append(schema["fields"].([]any), map[string]any{"key": "invocation_slug", "label": "Slug", "type": typ, "default": def})
b, _ := json.Marshal(schema)
sch := string(b)
if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil {
t.Fatal(err)
}
return coll.ID
}
useConvention := func(t *testing.T, e *appsEnv) {
t.Helper()
cv, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", "convention.golden.md"))
if err != nil {
t.Fatal(err)
}
e.files["/pack/conv.md"] = cv
m := e.manifest(t)
m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "conv", "url": e.origin() + "/pack/conv.md", "sha256": appSHA(cv)}}
e.publish(t, m)
}
t.Run("a stored item holds the defaulted slug", func(t *testing.T) {
e := newAppsEnv(t)
collID := setDefault(t, e, "shared")
if _, err := e.srv.store.CreateItem(e.wsID, collID, models.ItemCreate{Title: "Holder", Fields: `{"invocation_slug":"shared","status":"draft"}`}); err != nil {
t.Fatal(err)
}
useConvention(t, e)
if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "shared") {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
})
t.Run("a non-text slug", func(t *testing.T) {
e := newAppsEnv(t)
setDefault(t, e, 42)
useConvention(t, e)
if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "non-text") {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
})
}
// The stored-slug check uses the index's own expression: a stored ARRAY
// "[]" collides with a defaulted text "[]" there, so the preview refuses
// (codex round 8).
func TestAppInstallPreview_InvocationSlugIndexExpression(t *testing.T) {
e := newAppsEnv(t)
coll, err := e.srv.store.GetCollectionBySlug(e.wsID, "conventions")
if err != nil || coll == nil {
t.Fatalf("conventions collection: %v", err)
}
var schema map[string]any
if err := json.Unmarshal([]byte(coll.Schema), &schema); err != nil {
t.Fatal(err)
}
schema["fields"] = append(schema["fields"].([]any), map[string]any{"key": "invocation_slug", "label": "Slug", "type": "text", "default": "[]"})
b, _ := json.Marshal(schema)
sch := string(b)
if _, err := e.srv.store.UpdateCollection(coll.ID, models.CollectionUpdate{Schema: &sch}); err != nil {
t.Fatal(err)
}
if _, err := e.srv.store.DB().Exec(e.srv.store.D().Rebind(`INSERT INTO items (id, workspace_id, collection_id, title, slug, content, fields, tags, created_at, updated_at, item_number)
VALUES (?, ?, ?, 'Array holder', 'array-holder', '', '{"invocation_slug":[],"status":"draft"}', '[]', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 99999)`),
"arr-"+e.wsID[:8], e.wsID, coll.ID); err != nil {
t.Fatal(err)
}
cv, err := os.ReadFile(filepath.Join("..", "artifact", "testdata", "convention.golden.md"))
if err != nil {
t.Fatal(err)
}
e.files["/pack/conv.md"] = cv
m := e.manifest(t)
m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "conv", "url": e.origin() + "/pack/conv.md", "sha256": appSHA(cv)}}
e.publish(t, m)
if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
}
// In-pack claims collide wherever the import's lookup (JSONFieldEquals)
// would match: boolean true vs "true", number 1000 vs "1e3" (codex rounds
// 9-10).
func TestClaimForms(t *testing.T) {
collide := func(a, b any) bool {
seen := map[string]bool{}
for _, f := range claimForms(a) {
seen[f] = true
}
for _, f := range claimForms(b) {
if seen[f] {
return true
}
}
return false
}
for _, c := range []struct {
a, b any
want bool
}{
{true, "true", true},
{float64(1000), "1e3", true},
{float64(1000), "1000", true},
{float64(9210000000000000000), "9210000000000000000", true},
{float64(1000000000000000100), "1000000000000000100", true},
// codex round 12: exponent spelling past numericArg's integer bound,
// its negative, and a decimal spelling that SQLite rounds to the same
// double.
{float64(9210000000000000000), "9.21e18", true},
{float64(-9210000000000000000), "-9.21e18", true},
{float64(1000000000000000100), "1000000000000000100.0", true},
{float64(0.5), "0.50", true},
{float64(2), "3", false},
{"ship", "ship", true},
{"ship", "other", false},
{float64(1), true, false},
{nil, "", false},
{"", "", false},
} {
if got := collide(c.a, c.b); got != c.want {
t.Errorf("collide(%#v, %#v) = %v, want %v", c.a, c.b, got, c.want)
}
}
}
// App-authored artifact content may not carry pad-attachment: tokens or
// [[workspace::REF]] links; ordinary [[Title]] links are fine (lead ruling,
// U8b Q3).
func TestAppInstallPreview_AppContentReferenceRules(t *testing.T) {
for name, c := range map[string]struct {
edit string
ok bool
}{
"attachment token in the body": {"![x](PAD-Attachment:abc)", false},
"workspace-qualified link": {"see [[other-ws::TASK-1]]", false},
"ordinary title link": {"see [[Some Title]]", true},
} {
t.Run(name, func(t *testing.T) {
e := newAppsEnv(t)
m := e.manifest(t)
body := []byte(string(e.files["/pack/ship.md"]) + "\n" + c.edit + "\n")
e.files["/pack/ship.md"] = body
m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "ship", "url": e.origin() + "/pack/ship.md", "sha256": appSHA(body)}}
e.publish(t, m)
rr := e.preview(t)
if c.ok && rr.Code != http.StatusOK {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
if !c.ok && (rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "not accepted in app content")) {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
})
}
t.Run("attachment token in a field", func(t *testing.T) {
e := newAppsEnv(t)
m := e.manifest(t)
orig := string(e.files["/pack/ship.md"])
if !strings.Contains(orig, "description: the item to ship") {
t.Fatal("fixture changed: no arguments description to carry the token")
}
body := []byte(strings.Replace(orig, "description: the item to ship", "description: \"see pad-attachment:abc\"", 1))
e.files["/pack/ship.md"] = body
m["companion_pack"].(map[string]any)["artifacts"] = []any{map[string]any{"key": "ship", "url": e.origin() + "/pack/ship.md", "sha256": appSHA(body)}}
e.publish(t, m)
if rr := e.preview(t); rr.Code != http.StatusUnprocessableEntity || !strings.Contains(rr.Body.String(), "not accepted in app content") {
t.Fatalf("got %d %s", rr.Code, rr.Body.String())
}
})
}
// Lead review of U8a: the fields check fails closed. A field set that cannot
// be serialized is refused, never passed unchecked.
func TestRefuseAppAuthoredReferences_FailsClosedOnUnserializableFields(t *testing.T) {
err := refuseAppAuthoredReferences("plain content", map[string]any{"bad": make(chan int)})
if err == nil {
t.Fatal("an unserializable field set passed the reference check")
}
if err := refuseAppAuthoredReferences("plain content", map[string]any{"ok": "value"}); err != nil {
t.Fatalf("control: a clean field set was refused: %v", err)
}
}
+16 -41
View File
@@ -1,7 +1,6 @@
package server
import (
"encoding/json"
"errors"
"fmt"
"net/http"
@@ -91,8 +90,7 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) {
// anyone noticing. Both now call the same models pair, so the claim is
// enforced rather than asserted; create adopted this door's trim, which is
// the stricter and correct reading.
art.Title = models.NormalizeItemTitle(art.Title)
if msg := models.ValidateItemTitle(art.Title); msg != "" {
if msg := models.ValidateItemTitle(models.NormalizeItemTitle(art.Title)); msg != "" {
writeError(w, http.StatusBadRequest, "bad_request", msg)
return
}
@@ -143,42 +141,19 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) {
return
}
// Copy the artifact fields so the preprocess never mutates the decoded
// value (keeps the parse layer's output immutable from the handler's POV).
fields := make(map[string]any, len(art.Fields))
for k, v := range art.Fields {
fields[k] = v
}
warnings := s.preprocessArtifactFields(workspaceID, coll, art.Kind, schema, fields)
fieldsJSON, err := json.Marshal(fields)
norm, err := normalizeArtifact(art, coll, schema, func(slug string) (bool, error) {
return s.invocationSlugTaken(workspaceID, coll.ID, slug)
})
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to marshal fields")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to normalize the artifact")
return
}
// Normalize the field map through JSON so its nested types match what the
// normal create path validates. handleCreateItem builds its fieldMap by
// unmarshalling the JSON request body, so structured values are canonical
// JSON types ([]any, map[string]any). The artifact decode produces Go-native
// types (e.g. arguments as []map[string]any), which ValidateFields' json case
// rejects — round-tripping fixes that without special-casing any field.
normalizedFields := make(map[string]any)
if err := json.Unmarshal(fieldsJSON, &normalizedFields); err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to normalize fields")
return
}
body := art.Body
if footer := artifactProvenanceFooter(art.Provenance); footer != "" {
body = body + footer
}
warnings := norm.Warnings
input := models.ItemCreate{
Title: art.Title,
Content: body,
Fields: string(fieldsJSON),
Title: norm.Title,
Content: norm.Content,
Fields: norm.FieldsJSON,
}
// Attribution: a normal agent/api create. CreatedBy and Source are both
// left blank so createItemChecked stamps them from the request auth
@@ -203,7 +178,7 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) {
// ruling, day 57. An artifact was written elsewhere and the importer did
// not choose its field values; refusing would break the import of exactly
// the artifacts most likely to hold pre-validation junk.
item, cerr := s.createItemChecked(r, workspaceID, coll, schema, input, normalizedFields, "", relationsCarry)
item, cerr := s.createItemChecked(r, workspaceID, coll, schema, input, norm.Fields, "", relationsCarry)
if cerr != nil {
cerr.write(w, r)
return
@@ -265,7 +240,7 @@ func (s *Server) handleImportArtifact(w http.ResponseWriter, r *http.Request) {
// - status is forced to "draft" regardless of the artifact's value.
// - For playbooks, a non-empty invocation_slug that's already taken in the
// destination collection is suffixed (-2, -3, …) until free.
func (s *Server) preprocessArtifactFields(workspaceID string, coll *models.Collection, kind artifact.Kind, schema models.CollectionSchema, fields map[string]any) []string {
func preprocessArtifactFields(kind artifact.Kind, schema models.CollectionSchema, fields map[string]any, slugTaken func(string) (bool, error)) []string {
var warnings []string
// Blank foreign select values (trigger/scope/priority and any other
@@ -302,7 +277,7 @@ func (s *Server) preprocessArtifactFields(workspaceID string, coll *models.Colle
// De-collide invocation_slug for playbooks.
if kind == artifact.KindPlaybook {
if slug, _ := fields["invocation_slug"].(string); slug != "" {
free, changed := s.freeInvocationSlug(workspaceID, coll.ID, slug)
free, changed := freeInvocationSlug(slug, slugTaken)
if changed {
fields["invocation_slug"] = free
warnings = append(warnings,
@@ -315,12 +290,12 @@ func (s *Server) preprocessArtifactFields(workspaceID string, coll *models.Colle
}
// freeInvocationSlug returns an invocation_slug that's free in the destination
// collection. If the requested slug is already taken it appends -2, -3, …
// until an unused value is found. Returns (slug, changed).
func (s *Server) freeInvocationSlug(workspaceID, collectionID, requested string) (string, bool) {
// collection, per taken. If the requested slug is already taken it appends
// -2, -3, … until an unused value is found. Returns (slug, changed).
func freeInvocationSlug(requested string, isTaken func(string) (bool, error)) (string, bool) {
candidate := requested
for n := 2; ; n++ {
taken, err := s.invocationSlugTaken(workspaceID, collectionID, candidate)
taken, err := isTaken(candidate)
if err != nil {
// On a query error, fall back to the requested slug and let the
// create-time uniqueness precheck/constraint surface a conflict.
+70 -51
View File
@@ -861,56 +861,9 @@ const (
)
func (s *Server) createItemChecked(r *http.Request, workspaceID string, coll *models.Collection, schema models.CollectionSchema, input models.ItemCreate, fieldMap map[string]any, parentValue string, posture relationPosture) (*models.Item, *itemCreateError) {
// Coerce strings to their declared types before validating (BUG-2850).
fieldMap = items.CoerceFields(fieldMap, schema)
// BUG-3028: on an ordinary create every value is SUPPLIED, so a blank
// relation is removed before validation and a required one is refused as
// required. An artifact import (relationsCarry) carries values nobody typed
// in this request, so its blanks are removed after validation instead —
// the same carry-not-refuse posture it already has for unresolvable ones.
if posture == relationsRefuse {
items.DropBlankRelations(fieldMap, schema, nil)
}
// Snapshot before validation, which INJECTS schema defaults without
// type-checking them (codex round 7). The resolver below skips a
// non-string, so an injected `default: 42` on a relation field reached the
// blob unchallenged — the same hole the migrate doors had, by the same
// route, and closed with the same pass.
relBefore := store.RelationKeysPresent(schema, fieldMap)
// BUG-3079: a default that fails its own type check is DISCARDED and
// reported, not stored. The list joins the relation drops below in the one
// `warnings.dropped_fields` channel — they are the same event to a caller.
defaultDrops, err := items.ValidateFieldsWithDrops(fieldMap, schema)
if err != nil {
return nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: err.Error()}
}
items.DropBlankRelations(fieldMap, schema, nil)
// Referent validation for relation values (TASK-2878). AFTER the shape
// check, so "must be a string" and "names nothing" are never both reported
// for one value, and after coercion so the value is in its final form.
// The four steps live in one place — see resolveRelationsForWrite.
relRefusals, droppedDefaults, relErr := s.resolveRelationsForWrite(
r, workspaceID, workspaceRole(r), schema, fieldMap, relBefore, posture)
if relErr != nil {
return nil, &itemCreateError{status: http.StatusInternalServerError, code: "internal_error", message: "Failed to resolve relation references"}
}
droppedDefaults = append(defaultDrops, droppedDefaults...)
var unresolved []string
if len(relRefusals) > 0 {
if posture == relationsRefuse {
return nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: relationIssuesMessage(relRefusals)}
}
// Carry: the values stay in fieldMap exactly as supplied — the
// resolver leaves what it cannot resolve untouched — and the write
// says which ones, so an import is never silently lossy.
for _, ri := range relRefusals {
unresolved = append(unresolved, ri.Key)
}
}
undeclared := items.UndeclaredFieldKeys(fieldMap, schema)
if err := s.checkUniqueFields(workspaceID, coll.ID, "", schema, fieldMap); err != nil {
return nil, &itemCreateError{status: http.StatusConflict, code: "conflict", message: err.Error()}
fieldMap, droppedDefaults, unresolved, undeclared, cerr := s.prepareCreateFields(r, workspaceID, coll, schema, fieldMap, posture)
if cerr != nil {
return nil, cerr
}
// Marshal validated/defaulted fields back
@@ -1034,6 +987,67 @@ func (s *Server) createItemChecked(r *http.Request, workspaceID string, coll *mo
return item, nil
}
// prepareCreateFields is createItemChecked's field pipeline, in its order:
// coerce to declared types, drop blank relations (refuse posture), apply and
// validate schema defaults, resolve relation referents, name undeclared keys,
// and the unique-field check. It returns the fields as a create would store
// them, plus the three warning lists. Extracted so the app installer's
// preview (TASK-3397) computes its digest over exactly what the import
// stores; createItemChecked is unchanged in behaviour.
func (s *Server) prepareCreateFields(r *http.Request, workspaceID string, coll *models.Collection, schema models.CollectionSchema, fieldMap map[string]any, posture relationPosture) (fields map[string]any, droppedDefaults, unresolved, undeclared []string, cerr *itemCreateError) {
// Coerce strings to their declared types before validating (BUG-2850).
fieldMap = items.CoerceFields(fieldMap, schema)
// BUG-3028: on an ordinary create every value is SUPPLIED, so a blank
// relation is removed before validation and a required one is refused as
// required. An artifact import (relationsCarry) carries values nobody typed
// in this request, so its blanks are removed after validation instead —
// the same carry-not-refuse posture it already has for unresolvable ones.
if posture == relationsRefuse {
items.DropBlankRelations(fieldMap, schema, nil)
}
// Snapshot before validation, which INJECTS schema defaults without
// type-checking them (codex round 7). The resolver below skips a
// non-string, so an injected `default: 42` on a relation field reached the
// blob unchallenged — the same hole the migrate doors had, by the same
// route, and closed with the same pass.
relBefore := store.RelationKeysPresent(schema, fieldMap)
// BUG-3079: a default that fails its own type check is DISCARDED and
// reported, not stored. The list joins the relation drops below in the one
// `warnings.dropped_fields` channel — they are the same event to a caller.
defaultDrops, err := items.ValidateFieldsWithDrops(fieldMap, schema)
if err != nil {
return nil, nil, nil, nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: err.Error()}
}
items.DropBlankRelations(fieldMap, schema, nil)
// Referent validation for relation values (TASK-2878). AFTER the shape
// check, so "must be a string" and "names nothing" are never both reported
// for one value, and after coercion so the value is in its final form.
// The four steps live in one place — see resolveRelationsForWrite.
relRefusals, droppedDefaults, relErr := s.resolveRelationsForWrite(
r, workspaceID, workspaceRole(r), schema, fieldMap, relBefore, posture)
if relErr != nil {
return nil, nil, nil, nil, &itemCreateError{status: http.StatusInternalServerError, code: "internal_error", message: "Failed to resolve relation references"}
}
droppedDefaults = append(defaultDrops, droppedDefaults...)
if len(relRefusals) > 0 {
if posture == relationsRefuse {
return nil, nil, nil, nil, &itemCreateError{status: http.StatusBadRequest, code: "validation_error", message: relationIssuesMessage(relRefusals)}
}
// Carry: the values stay in fieldMap exactly as supplied — the
// resolver leaves what it cannot resolve untouched — and the write
// says which ones, so an import is never silently lossy.
for _, ri := range relRefusals {
unresolved = append(unresolved, ri.Key)
}
}
undeclared = items.UndeclaredFieldKeys(fieldMap, schema)
if err := s.checkUniqueFields(workspaceID, coll.ID, "", schema, fieldMap); err != nil {
return nil, nil, nil, nil, &itemCreateError{status: http.StatusConflict, code: "conflict", message: err.Error()}
}
return fieldMap, droppedDefaults, unresolved, undeclared, nil
}
// writeItemResolveError distinguishes a soft-deleted (archived) item from a
// genuinely-missing one for mutation handlers. An archived item still appears
// in include-archived list results and can be fetched read-only via GET, so a
@@ -3880,9 +3894,14 @@ func (s *Server) extractParentLink(
return parentValue, parentProvided, true
}
// uniqueEnforced reports whether checkUniqueFields enforces def. The app
// installer's preview uses the same predicate (TASK-3397), so the two cannot
// disagree about which fields are unique.
func uniqueEnforced(def models.FieldDef) bool { return def.UniqueScope == "workspace_collection" }
func (s *Server) checkUniqueFields(workspaceID, collectionID, excludeItemID string, schema models.CollectionSchema, fieldMap map[string]any) error {
for _, def := range schema.Fields {
if def.UniqueScope != "workspace_collection" {
if !uniqueEnforced(def) {
continue
}
raw, ok := fieldMap[def.Key]
+19
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"crypto/subtle"
"crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
@@ -169,6 +170,9 @@ type Server struct {
// build that doesn't want the dependency — still serves every
// other endpoint and stores originals untouched.
imageProcessor attachments.Processor
// appFetchTLS is the TLS config app manifest/artifact fetches use; nil
// (the system roots) in production, set by tests (SetAppFetchTLS).
appFetchTLS *tls.Config
// MCP Streamable HTTP transport (PLAN-943 TASK-950). Wired via
// SetMCPTransport at startup on every install (PLAN-2310 DR-4); nil
@@ -339,6 +343,10 @@ type Server struct {
// the loop via stopReminderTick.
reminderTick reminderTickConfig
// appPendingSweep deletes expired pending app installs and their staged
// bytes (TASK-3397). Started via StartAppPendingSweep.
appPendingSweep appPendingSweepConfig
// decisionTick holds the typed-decision runner and its evaluation loop
// (TASK-3117). Same shape as reminderTick; started via StartDecisionTick
// and stopped by Stop() via stopDecisionTick.
@@ -617,6 +625,7 @@ func (s *Server) Stop() {
// Item reminder tick (IDEA-2641). Same lifecycle pattern; an in-flight
// pass is tracked on s.bg and awaited below.
s.stopReminderTick()
s.stopAppPendingSweep()
s.stopDecisionTick()
// MCP audit writer / sweeper run on s.bg too. Signal first so
// the workers see the close BEFORE Wait() blocks; without the
@@ -1722,6 +1731,8 @@ func (s *Server) setupRouter() {
r.Put("/decision-provider", s.handleUpdateDecisionSettings)
r.Get("/mcp", s.handleGetMCPSettings)
r.Put("/mcp", s.handleUpdateMCPSettings)
r.Get("/apps", s.handleGetAppsSettings)
r.Put("/apps", s.handleUpdateAppsSettings)
r.Post("/test-email", s.handleTestEmail)
// Cloud sidecar endpoints — only exist in cloud mode. requireCloudMode
@@ -2180,6 +2191,14 @@ func (s *Server) setupRouter() {
// short TTL — see handleGetWorkspaceStorageUsage.
r.Get("/storage/usage", s.handleGetWorkspaceStorageUsage)
// App install staging (SPEC-6 U8a, TASK-3397). Owner-only and
// gated on appsAvailable inside each handler.
r.Route("/apps/install", func(r chi.Router) {
r.Post("/preview", s.handleAppInstallPreview)
r.Get("/pending/{pendingID}", s.handleGetAppInstallPending)
r.Delete("/pending/{pendingID}", s.handleDeleteAppInstallPending)
})
// Webhooks
r.Route("/webhooks", func(r chi.Router) {
r.Get("/", s.handleListWebhooks)
+129
View File
@@ -0,0 +1,129 @@
{
"blank title": {
"status": 400,
"title": "",
"content": "",
"fields": null,
"warnings": null
},
"convention, active, valid selects": {
"status": 201,
"title": "Clean Convention",
"content": "A clean convention.\n",
"fields": {
"priority": "must",
"scope": "all",
"status": "draft",
"trigger": "on-commit"
},
"warnings": [
"status \"active\" reset to \"draft\" on import"
]
},
"convention, foreign select value": {
"status": 201,
"title": "Foreign",
"content": "x\n",
"fields": {
"scope": "all",
"status": "draft",
"trigger": ""
},
"warnings": [
"field \"trigger\" value \"on-candidate-advance\" is not a valid option in this workspace; cleared on import",
"status \"active\" reset to \"draft\" on import"
]
},
"golden convention": {
"status": 201,
"title": "Run the test suite before pushing",
"content": "Always run `make test` before pushing to main.\n\n\n---\n\n_Imported artifact from workspace `demo`, exported by xarmian at 2026-06-22T00:00:00Z._\n",
"fields": {
"priority": "",
"role": "engineer",
"scope": "",
"status": "draft",
"trigger": "on-commit"
},
"warnings": [
"field \"priority\" value \"high\" is not a valid option in this workspace; cleared on import",
"field \"scope\" value \"repo\" is not a valid option in this workspace; cleared on import",
"status \"active\" reset to \"draft\" on import"
]
},
"golden playbook": {
"status": 201,
"title": "Ship a change",
"content": "## Steps\n\n1. Run the tests\n2. Open a PR\n\n\n---\n\n_Imported artifact from workspace `demo`, exported by xarmian at 2026-06-22T00:00:00Z._\n",
"fields": {
"arguments": [
{
"description": "the item to ship",
"name": "ref",
"required": true,
"type": "ref"
},
{
"default": "squash",
"name": "merge-strategy",
"required": false,
"type": "enum"
}
],
"invocation_slug": "ship",
"scope": "",
"status": "draft",
"trigger": ""
},
"warnings": [
"field \"scope\" value \"workspace\" is not a valid option in this workspace; cleared on import",
"field \"trigger\" value \"on-intent\" is not a valid option in this workspace; cleared on import",
"status \"active\" reset to \"draft\" on import"
]
},
"playbook with arguments": {
"status": 201,
"title": "Playbook With Args",
"content": "Do the thing.\n",
"fields": {
"arguments": [
{
"description": "the thing",
"name": "target",
"required": true,
"type": "ref"
}
],
"invocation_slug": "with-args",
"scope": "all",
"status": "draft",
"trigger": "manual"
},
"warnings": [
"status \"active\" reset to \"draft\" on import"
]
},
"playbook, slug collision": {
"status": 201,
"title": "Second",
"content": "second\n",
"fields": {
"invocation_slug": "ship-2",
"status": "draft"
},
"warnings": [
"invocation_slug \"ship\" was already in use; imported as \"ship-2\"",
"status \"active\" reset to \"draft\" on import"
]
},
"with provenance": {
"status": 201,
"title": "Provenanced",
"content": "Body.\n\n\n---\n\n_Imported artifact from workspace `elsewhere`, exported by someone at 2026-01-02T03:04:05Z._\n",
"fields": {
"status": "draft",
"trigger": "on-commit"
},
"warnings": null
}
}
+326
View File
@@ -0,0 +1,326 @@
package store
import (
"database/sql"
"errors"
"fmt"
"strings"
"time"
)
// Pending app installs (SPEC-6 U8a, DOC-3371 §2 "Staging caps"; TASK-3397).
//
// A reservation is taken BEFORE any network fetch, in one transaction that
// locks the owner's users row (FOR NO KEY UPDATE, the enforceUserLimitTx
// shape) and then the instance-wide pending-bytes advisory lock, and counts:
// at most PendingPerOwner live reservations per owner, and at most
// PendingInstanceBytes of live reserved bytes on the instance. It charges the
// FULL PendingInstallBytes. The charge is kept, whatever has been staged so
// far, until no further bytes can arrive (FinishPendingInstall); only then is
// it reduced to the bytes actually staged, under the instance lock. Releasing
// capacity mid-fetch would let the remaining artifacts of concurrent installs
// together exceed the instance cap.
//
// "Live" is expires_at in the future. A fetching reservation expires shortly
// after the fetch deadline, a staged one after an hour; the sweep deletes
// expired ones, and the blobs go with them (ON DELETE CASCADE). On SQLite
// BEGIN IMMEDIATE serializes all of this, and the lock clauses are skipped.
//
// LOCK ORDER: owner row, then the instance lock. The other transactions here
// take the instance lock alone, so no cycle forms.
// Staging caps (DOC-3371 §2).
const (
PendingInstallBytes int64 = 4 << 20
PendingInstanceBytes int64 = 64 << 20
PendingPerOwner = 3
PendingFetchTTL = 2 * time.Minute
PendingStagedTTL = time.Hour
)
// PendingManifestKey is the blob key the manifest is staged under.
const PendingManifestKey = "$manifest"
var (
// ErrPendingLimit: the owner already has PendingPerOwner live pending
// installs, or the instance's pending bytes are at their cap. The
// refusal carries no counts.
ErrPendingLimit = errors.New("app install: too many pending installs")
// ErrPendingOverCap: staging a blob would exceed the install's byte cap.
ErrPendingOverCap = errors.New("app install: the pack exceeds its size cap")
// ErrPendingNotFound: no live pending install with that id for that
// workspace and owner.
ErrPendingNotFound = errors.New("app install: pending install not found")
)
// PendingInstall is a pending-install record.
type PendingInstall struct {
ID string
WorkspaceID string
OwnerID string
Origin string
State string
ReservedBytes int64
ManifestSHA256 string
Preview string
ExpiresAt time.Time
}
// PendingBlob is one staged blob: the manifest or an artifact.
type PendingBlob struct {
Key string
URL string
SHA256 string
Data []byte
}
func (s *Store) lockPendingBytes(tx *sql.Tx) error {
if s.dialect.Driver() != DriverPostgres {
return nil
}
if _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtext('pad:app-pending-bytes'))`); err != nil {
return fmt.Errorf("pending install: instance lock: %w", err)
}
return nil
}
func timeText(t time.Time) string { return t.UTC().Format(time.RFC3339) }
// ReservePendingInstall takes a reservation for an install the owner is about
// to fetch, or refuses with ErrPendingLimit. No fetch may begin without one.
func (s *Store) ReservePendingInstall(workspaceID, ownerID, origin string) (*PendingInstall, error) {
tx, err := s.db.Begin()
if err != nil {
return nil, fmt.Errorf("pending install: begin: %w", err)
}
defer tx.Rollback()
ownerQ := `SELECT id FROM users WHERE id = ?`
if s.dialect.Driver() == DriverPostgres {
ownerQ += ` FOR NO KEY UPDATE`
}
var got string
if err := tx.QueryRow(s.q(ownerQ), ownerID).Scan(&got); err != nil {
return nil, fmt.Errorf("pending install: lock owner: %w", err)
}
if err := s.lockPendingBytes(tx); err != nil {
return nil, err
}
nowT := time.Now()
nowS := timeText(nowT)
var mine int
if err := tx.QueryRow(s.q(`SELECT COUNT(*) FROM app_install_pending WHERE owner_id = ? AND expires_at > ?`), ownerID, nowS).Scan(&mine); err != nil {
return nil, fmt.Errorf("pending install: count owner: %w", err)
}
var instance int64
if err := tx.QueryRow(s.q(`SELECT COALESCE(SUM(reserved_bytes), 0) FROM app_install_pending WHERE expires_at > ?`), nowS).Scan(&instance); err != nil {
return nil, fmt.Errorf("pending install: sum instance: %w", err)
}
if mine >= PendingPerOwner || instance+PendingInstallBytes > PendingInstanceBytes {
return nil, ErrPendingLimit
}
p := &PendingInstall{
ID: newID(), WorkspaceID: workspaceID, OwnerID: ownerID, Origin: origin, State: "fetching",
ReservedBytes: PendingInstallBytes, ExpiresAt: nowT.Add(PendingFetchTTL).UTC().Truncate(time.Second),
}
if _, err := tx.Exec(s.q(`INSERT INTO app_install_pending (id, workspace_id, owner_id, origin, state, reserved_bytes, expires_at, created_at, updated_at)
VALUES (?, ?, ?, ?, 'fetching', ?, ?, ?, ?)`),
p.ID, workspaceID, ownerID, origin, PendingInstallBytes, timeText(p.ExpiresAt), nowS, nowS); err != nil {
return nil, fmt.Errorf("pending install: insert: %w", err)
}
if err := tx.Commit(); err != nil {
return nil, fmt.Errorf("pending install: commit: %w", err)
}
return p, nil
}
// StagePendingBlob writes one fetched blob into a fetching reservation. The
// staged total may not exceed the reservation's charge (the per-install
// cap), which is never reduced while fetching.
func (s *Store) StagePendingBlob(pendingID string, b PendingBlob) error {
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("pending blob: begin: %w", err)
}
defer tx.Rollback()
q := `SELECT reserved_bytes FROM app_install_pending WHERE id = ? AND state = 'fetching' AND expires_at > ?`
if s.dialect.Driver() == DriverPostgres {
q += ` FOR UPDATE`
}
var reserved int64
switch err := tx.QueryRow(s.q(q), pendingID, timeText(time.Now())).Scan(&reserved); {
case errors.Is(err, sql.ErrNoRows):
return ErrPendingNotFound
case err != nil:
return fmt.Errorf("pending blob: lock: %w", err)
}
var staged int64
if err := tx.QueryRow(s.q(`SELECT COALESCE(SUM(size_bytes), 0) FROM app_install_pending_blobs WHERE pending_id = ?`), pendingID).Scan(&staged); err != nil {
return fmt.Errorf("pending blob: sum: %w", err)
}
if staged+int64(len(b.Data)) > reserved {
return ErrPendingOverCap
}
if _, err := tx.Exec(s.q(`INSERT INTO app_install_pending_blobs (pending_id, key, url, sha256, size_bytes, data) VALUES (?, ?, ?, ?, ?, ?)`),
pendingID, b.Key, b.URL, b.SHA256, len(b.Data), b.Data); err != nil {
return fmt.Errorf("pending blob: insert: %w", err)
}
return tx.Commit()
}
// FinishPendingInstall ends the fetch: under the instance lock, the charge is
// reduced to the bytes actually staged, the record is marked staged with its
// manifest hash and preview, and it is kept for PendingStagedTTL.
func (s *Store) FinishPendingInstall(pendingID, manifestSHA256, preview string) error {
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("pending install: begin: %w", err)
}
defer tx.Rollback()
if err := s.lockPendingBytes(tx); err != nil {
return err
}
var staged int64
if err := tx.QueryRow(s.q(`SELECT COALESCE(SUM(size_bytes), 0) FROM app_install_pending_blobs WHERE pending_id = ?`), pendingID).Scan(&staged); err != nil {
return fmt.Errorf("pending install: sum: %w", err)
}
nowT := time.Now()
res, err := tx.Exec(s.q(`UPDATE app_install_pending SET state = 'staged', reserved_bytes = ?, manifest_sha256 = ?, preview = ?, expires_at = ?, updated_at = ?
WHERE id = ? AND state = 'fetching' AND expires_at > ?`),
staged, manifestSHA256, preview, timeText(nowT.Add(PendingStagedTTL)), timeText(nowT), pendingID, timeText(nowT))
if err != nil {
return fmt.Errorf("pending install: finish: %w", err)
}
if n, _ := res.RowsAffected(); n != 1 {
return ErrPendingNotFound
}
return tx.Commit()
}
// DeletePendingInstall removes a reservation and every staged blob: a failed
// or cancelled fetch, or an owner discarding a preview. A missing record is
// not an error.
func (s *Store) DeletePendingInstall(pendingID string) error {
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("pending install: begin: %w", err)
}
defer tx.Rollback()
if err := s.lockPendingBytes(tx); err != nil {
return err
}
if _, err := tx.Exec(s.q(`DELETE FROM app_install_pending WHERE id = ?`), pendingID); err != nil {
return fmt.Errorf("pending install: delete: %w", err)
}
return tx.Commit()
}
// SweepExpiredPendingInstalls deletes every expired reservation and its
// blobs, returning how many records it removed.
func (s *Store) SweepExpiredPendingInstalls() (int64, error) {
tx, err := s.db.Begin()
if err != nil {
return 0, fmt.Errorf("pending sweep: begin: %w", err)
}
defer tx.Rollback()
if err := s.lockPendingBytes(tx); err != nil {
return 0, err
}
res, err := tx.Exec(s.q(`DELETE FROM app_install_pending WHERE expires_at <= ?`), timeText(time.Now()))
if err != nil {
return 0, fmt.Errorf("pending sweep: %w", err)
}
n, _ := res.RowsAffected()
return n, tx.Commit()
}
// GetPendingInstall returns a LIVE pending install of this owner in this
// workspace, or ErrPendingNotFound (another owner's, another workspace's, an
// expired and a missing one all answer the same).
func (s *Store) GetPendingInstall(pendingID, workspaceID, ownerID string) (*PendingInstall, error) {
var p PendingInstall
var manifest, preview sql.NullString
var expires string
err := s.db.QueryRow(s.q(`SELECT id, workspace_id, owner_id, origin, state, reserved_bytes, manifest_sha256, preview, expires_at
FROM app_install_pending WHERE id = ? AND workspace_id = ? AND owner_id = ? AND expires_at > ?`),
pendingID, workspaceID, ownerID, timeText(time.Now())).
Scan(&p.ID, &p.WorkspaceID, &p.OwnerID, &p.Origin, &p.State, &p.ReservedBytes, &manifest, &preview, &expires)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrPendingNotFound
}
if err != nil {
return nil, fmt.Errorf("get pending install: %w", err)
}
p.ManifestSHA256, p.Preview, p.ExpiresAt = manifest.String, preview.String, parseTime(expires)
return &p, nil
}
// PendingInstallBlobs returns a pending install's staged blobs, by key.
func (s *Store) PendingInstallBlobs(pendingID string) (map[string]PendingBlob, error) {
rows, err := s.db.Query(s.q(`SELECT key, url, sha256, data FROM app_install_pending_blobs WHERE pending_id = ?`), pendingID)
if err != nil {
return nil, fmt.Errorf("pending blobs: %w", err)
}
defer rows.Close()
out := map[string]PendingBlob{}
for rows.Next() {
var b PendingBlob
if err := rows.Scan(&b.Key, &b.URL, &b.SHA256, &b.Data); err != nil {
return nil, err
}
out[b.Key] = b
}
return out, rows.Err()
}
// CollectionSlugOwners reports, for each of slugs that already names a
// collection in the workspace (live or soft-deleted, since the unique index
// covers both), the origin of the app install that created it, or "" when no
// install did. Slugs absent from the result are free. It backs the install
// conflict check (DOC-3371 §2 step 5, L6).
func (s *Store) CollectionSlugOwners(workspaceID string, slugs []string) (map[string]string, error) {
out := map[string]string{}
if len(slugs) == 0 {
return out, nil
}
in := strings.TrimSuffix(strings.Repeat("?,", len(slugs)), ",")
args := []any{workspaceID}
for _, sl := range slugs {
args = append(args, sl)
}
rows, err := s.db.Query(s.q(`SELECT c.slug, COALESCE(a.origin, '') FROM collections c
LEFT JOIN app_installs a ON a.id = c.via_app
WHERE c.workspace_id = ? AND c.slug IN (`+in+`)`), args...)
if err != nil {
return nil, fmt.Errorf("collection slug owners: %w", err)
}
defer rows.Close()
for rows.Next() {
var slug, origin string
if err := rows.Scan(&slug, &origin); err != nil {
return nil, err
}
out[slug] = origin
}
return out, rows.Err()
}
// InvocationSlugIndexTaken reports whether a live item in collectionID would
// collide with slug in idx_items_invocation_slug_per_collection, using the
// index's OWN expression on each dialect (SQLite json_extract, Postgres
// ->>), so it agrees with the index on every stored shape, arrays and
// objects included (TASK-3397, codex round 8). It backs the app installer's
// preview of a defaulted slug.
func (s *Store) InvocationSlugIndexTaken(collectionID, slug string) (bool, error) {
expr := `json_extract(fields, '$.invocation_slug')`
if s.dialect.Driver() == DriverPostgres {
expr = `(fields->>'invocation_slug')`
}
var n int
err := s.db.QueryRow(s.q(`SELECT COUNT(*) FROM items WHERE collection_id = ? AND deleted_at IS NULL
AND `+expr+` IS NOT NULL AND `+expr+` != '' AND `+expr+` = ?`), collectionID, slug).Scan(&n)
if err != nil {
return false, fmt.Errorf("invocation slug index lookup: %w", err)
}
return n > 0, nil
}
+218
View File
@@ -0,0 +1,218 @@
package store
import (
"bytes"
"errors"
"fmt"
"sync"
"testing"
"time"
)
// TASK-3397 (U8a): pending app-install reservations and their caps.
func pendingOwner(t *testing.T, s *Store, n int) string {
t.Helper()
return createTestUser(t, s, fmt.Sprintf("owner-%d-%d@example.com", n, time.Now().UnixNano()), "Owner", "password123").ID
}
func TestPendingInstall_PerOwnerCap(t *testing.T) {
s := testStore(t)
ws := createTestWorkspace(t, s, "Apps")
owner := pendingOwner(t, s, 1)
for i := 0; i < PendingPerOwner; i++ {
if _, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example"); err != nil {
t.Fatalf("reservation %d: %v", i, err)
}
}
if _, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example"); !errors.Is(err, ErrPendingLimit) {
t.Fatalf("over the per-owner cap: %v", err)
}
// Another owner is unaffected.
if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, 2), "https://app.example"); err != nil {
t.Fatalf("another owner: %v", err)
}
}
func TestPendingInstall_InstanceBytesCap(t *testing.T) {
s := testStore(t)
ws := createTestWorkspace(t, s, "Apps")
max := int(PendingInstanceBytes / PendingInstallBytes)
for i := 0; i < max; i++ {
if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, i), "https://app.example"); err != nil {
t.Fatalf("reservation %d of %d: %v", i, max, err)
}
}
if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, max), "https://app.example"); !errors.Is(err, ErrPendingLimit) {
t.Fatalf("over the instance cap: %v", err)
}
}
// The full charge is held while fetching, however little has been staged, and
// reduced to the staged bytes only when the fetch ends.
func TestPendingInstall_ChargeIsHeldUntilTheFetchEnds(t *testing.T) {
s := testStore(t)
ws := createTestWorkspace(t, s, "Apps")
max := int(PendingInstanceBytes / PendingInstallBytes)
var first *PendingInstall
for i := 0; i < max; i++ {
p, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, i), "https://app.example")
if err != nil {
t.Fatal(err)
}
if first == nil {
first = p
}
}
if err := s.StagePendingBlob(first.ID, PendingBlob{Key: PendingManifestKey, URL: "https://app.example/m", SHA256: "x", Data: []byte("{}")}); err != nil {
t.Fatal(err)
}
// Staging 2 bytes did not release capacity.
if _, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, 99), "https://app.example"); !errors.Is(err, ErrPendingLimit) {
t.Fatalf("mid-fetch staging released capacity: %v", err)
}
reserved := func() int64 {
var sum int64
if err := s.db.QueryRow(`SELECT COALESCE(SUM(reserved_bytes), 0) FROM app_install_pending`).Scan(&sum); err != nil {
t.Fatal(err)
}
return sum
}
if got := reserved(); got != PendingInstanceBytes {
t.Fatalf("reserved %d mid-fetch, want the full %d", got, PendingInstanceBytes)
}
if err := s.FinishPendingInstall(first.ID, "sha", `{"p":1}`); err != nil {
t.Fatal(err)
}
if got, want := reserved(), PendingInstanceBytes-PendingInstallBytes+2; got != want {
t.Fatalf("reserved %d after finish, want %d (the charge reduced to the 2 staged bytes)", got, want)
}
got, err := s.GetPendingInstall(first.ID, ws.ID, first.OwnerID)
if err != nil {
t.Fatal(err)
}
if got.State != "staged" || got.ReservedBytes != 2 || got.Preview != `{"p":1}` {
t.Fatalf("after finish: %+v", got)
}
}
func TestPendingInstall_StagingCannotExceedTheInstallCap(t *testing.T) {
s := testStore(t)
ws := createTestWorkspace(t, s, "Apps")
p, err := s.ReservePendingInstall(ws.ID, pendingOwner(t, s, 1), "https://app.example")
if err != nil {
t.Fatal(err)
}
half := bytes.Repeat([]byte("a"), int(PendingInstallBytes/2))
for i, c := range []struct {
data []byte
want error
}{{half, nil}, {half, nil}, {[]byte("x"), ErrPendingOverCap}} {
err := s.StagePendingBlob(p.ID, PendingBlob{Key: fmt.Sprintf("a%d", i), URL: "https://app.example/a", SHA256: "x", Data: c.data})
if !errors.Is(err, c.want) {
t.Fatalf("blob %d: %v, want %v", i, err, c.want)
}
}
}
func TestPendingInstall_ScopingDeleteAndSweep(t *testing.T) {
s := testStore(t)
ws := createTestWorkspace(t, s, "Apps")
other := createTestWorkspace(t, s, "Other")
owner := pendingOwner(t, s, 1)
p, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example")
if err != nil {
t.Fatal(err)
}
data := []byte("manifest bytes\x00with a nul")
if err := s.StagePendingBlob(p.ID, PendingBlob{Key: PendingManifestKey, URL: "https://app.example/m", SHA256: "h", Data: data}); err != nil {
t.Fatal(err)
}
blobs, err := s.PendingInstallBlobs(p.ID)
if err != nil || !bytes.Equal(blobs[PendingManifestKey].Data, data) {
t.Fatalf("blob round trip: %v %q", err, blobs[PendingManifestKey].Data)
}
for name, args := range map[string][3]string{
"another owner": {p.ID, ws.ID, pendingOwner(t, s, 2)},
"another workspace": {p.ID, other.ID, owner},
"unknown": {newID(), ws.ID, owner},
} {
if _, err := s.GetPendingInstall(args[0], args[1], args[2]); !errors.Is(err, ErrPendingNotFound) {
t.Errorf("%s: %v", name, err)
}
}
// Expire it, then sweep: the record and its blobs go together.
if _, err := s.db.Exec(s.q(`UPDATE app_install_pending SET expires_at = ? WHERE id = ?`), timeText(time.Now().Add(-time.Minute)), p.ID); err != nil {
t.Fatal(err)
}
if _, err := s.GetPendingInstall(p.ID, ws.ID, owner); !errors.Is(err, ErrPendingNotFound) {
t.Errorf("expired record still readable: %v", err)
}
// Expired but NOT yet swept, it already no longer counts against its
// owner: the caps read expires_at, not the sweep.
for i := 0; i < PendingPerOwner; i++ {
if _, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example"); err != nil {
t.Fatalf("with an expired, unswept reservation, reservation %d: %v", i, err)
}
}
n, err := s.SweepExpiredPendingInstalls()
if err != nil || n != 1 {
t.Fatalf("sweep: %d %v", n, err)
}
var left int
if err := s.db.QueryRow(s.q(`SELECT COUNT(*) FROM app_install_pending_blobs WHERE pending_id = ?`), p.ID).Scan(&left); err != nil || left != 0 {
t.Fatalf("blobs left after sweep: %d %v", left, err)
}
}
// Concurrent reservations never exceed either cap. On SQLite BEGIN IMMEDIATE
// serializes them; on Postgres the owner row lock and the instance advisory
// lock do.
func TestPendingInstall_ConcurrentReservationsRespectBothCaps(t *testing.T) {
s := testStore(t)
ws := createTestWorkspace(t, s, "Apps")
shared := pendingOwner(t, s, 0)
owners := make([]string, 24)
for i := range owners {
owners[i] = pendingOwner(t, s, i+1)
}
var mu sync.Mutex
okShared, okTotal := 0, 0
var wg sync.WaitGroup
run := func(owner string, isShared bool) {
defer wg.Done()
_, err := s.ReservePendingInstall(ws.ID, owner, "https://app.example")
switch {
case err == nil:
mu.Lock()
okTotal++
if isShared {
okShared++
}
mu.Unlock()
case errors.Is(err, ErrPendingLimit):
default:
t.Errorf("unexpected: %v", err)
}
}
for i := 0; i < 8; i++ {
wg.Add(1)
go run(shared, true)
}
for _, o := range owners {
wg.Add(1)
go run(o, false)
}
wg.Wait()
if okShared > PendingPerOwner {
t.Errorf("one owner holds %d reservations, cap %d", okShared, PendingPerOwner)
}
if max := int(PendingInstanceBytes / PendingInstallBytes); okTotal > max {
t.Errorf("%d reservations, instance cap %d", okTotal, max)
}
var sum int64
if err := s.db.QueryRow(`SELECT COALESCE(SUM(reserved_bytes), 0) FROM app_install_pending`).Scan(&sum); err != nil || sum > PendingInstanceBytes {
t.Errorf("reserved bytes %d > %d (%v)", sum, PendingInstanceBytes, err)
}
}
+18
View File
@@ -160,3 +160,21 @@ func (d *postgresDialect) JSONFieldOrder(column, key, dir string) string {
num := fmt.Sprintf("(CASE WHEN %s = 'number' THEN (%s->'%s')::numeric END)", t, column, key)
return fmt.Sprintf("%s ASC, %s %s, %s %s", rank, num, dir, d.JSONFieldText(column, key), dir)
}
// NumericClaimForms is the form under which JSONFieldEquals could find two
// numeric values equal, on either dialect, for arg when numericArg accepts
// it: its float64 value. SQLite compares an INTEGER and a REAL as doubles,
// and Postgres compares numeric EXACTLY; two exactly equal values round to
// the same double, so equal doubles is a superset of both. The app
// installer's in-pack uniqueness claims key numbers on it (TASK-3397, codex
// rounds 10-12).
func NumericClaimForms(arg string) []string {
if _, ok := numericArg(arg); !ok {
return nil
}
f, err := strconv.ParseFloat(arg, 64)
if err != nil && !math.IsInf(f, 0) {
return nil
}
return []string{"f:" + strconv.FormatFloat(f, 'g', -1, 64)}
}
@@ -0,0 +1,65 @@
-- Migration 118: app install staging and provenance (SPEC-6 U8a, TASK-3397).
--
-- app_install_pending is the pending-install record (DOC-3371 §2 step 1): a
-- reservation taken BEFORE any network fetch, under the owner's users row and
-- an instance-wide lock, that charges the full per-install byte cap until the
-- fetch has ended. The staged bytes live in app_install_pending_blobs (the
-- manifest under the key '$manifest', then one row per artifact), so a
-- failure, the fetch deadline or expiry deletes the reservation and every
-- staged byte in one statement (ON DELETE CASCADE). A fetching record expires
-- two minutes after it is taken (the 30 s fetch deadline plus margin), so a
-- process that dies mid-fetch holds its charge only that long; a staged one
-- is kept for an hour awaiting consent.
--
-- The app_installs columns are written by provisioning (U8b): the install's
-- bot (the OAuth client is U5's, keyed by oauth_clients.app_install_id), the manifest's hash and version, the declared events
-- and item actions (stored now, provisioned when A5/A6 land), the access
-- levels, the owner-edited config and every raw and normalized digest.
--
-- collections.via_app is the install that created a companion collection. The
-- conflict check (§2 step 5, L6) adopts an existing slug only when the install
-- that created it has the same origin; any other existing slug is an error.
--
-- items.source_pack / source_artifact_sha256 / installed_sha256 stamp a
-- companion artifact with the pack it came from and both digests (§2 step 6),
-- so a later installer can adopt it.
CREATE TABLE IF NOT EXISTS app_install_pending (
id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
origin TEXT NOT NULL,
state TEXT NOT NULL DEFAULT 'fetching' CHECK (state IN ('fetching', 'staged')),
reserved_bytes INTEGER NOT NULL,
manifest_sha256 TEXT,
preview TEXT,
expires_at TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_app_install_pending_owner ON app_install_pending (owner_id, expires_at);
CREATE INDEX IF NOT EXISTS idx_app_install_pending_expires ON app_install_pending (expires_at);
CREATE TABLE IF NOT EXISTS app_install_pending_blobs (
pending_id TEXT NOT NULL REFERENCES app_install_pending(id) ON DELETE CASCADE,
key TEXT NOT NULL,
url TEXT NOT NULL,
sha256 TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
data BLOB NOT NULL,
PRIMARY KEY (pending_id, key)
);
ALTER TABLE app_installs ADD COLUMN bot_user_id TEXT REFERENCES users(id) ON DELETE SET NULL;
ALTER TABLE app_installs ADD COLUMN manifest_sha256 TEXT;
ALTER TABLE app_installs ADD COLUMN manifest_version TEXT;
ALTER TABLE app_installs ADD COLUMN manifest TEXT;
ALTER TABLE app_installs ADD COLUMN service_access TEXT;
ALTER TABLE app_installs ADD COLUMN delegated_access TEXT;
ALTER TABLE app_installs ADD COLUMN config TEXT NOT NULL DEFAULT '{}';
ALTER TABLE app_installs ADD COLUMN digests TEXT NOT NULL DEFAULT '{}';
ALTER TABLE collections ADD COLUMN via_app TEXT REFERENCES app_installs(id) ON DELETE SET NULL;
ALTER TABLE items ADD COLUMN source_pack TEXT;
ALTER TABLE items ADD COLUMN source_artifact_sha256 TEXT;
ALTER TABLE items ADD COLUMN installed_sha256 TEXT;
@@ -16,9 +16,32 @@ api_tokens.prefix
api_tokens.token_hash
api_tokens.user_id
api_tokens.workspace_id
app_install_pending.created_at
app_install_pending.expires_at
app_install_pending.id
app_install_pending.manifest_sha256
app_install_pending.origin
app_install_pending.owner_id
app_install_pending.preview
app_install_pending.state
app_install_pending.updated_at
app_install_pending.workspace_id
app_install_pending_blobs.data
app_install_pending_blobs.key
app_install_pending_blobs.pending_id
app_install_pending_blobs.sha256
app_install_pending_blobs.url
app_installs.bot_user_id
app_installs.config
app_installs.created_at
app_installs.delegated_access
app_installs.digests
app_installs.id
app_installs.manifest
app_installs.manifest_sha256
app_installs.manifest_version
app_installs.origin
app_installs.service_access
app_installs.state
app_installs.updated_at
app_installs.workspace_id
@@ -55,6 +78,7 @@ collections.created_at
collections.deleted_at
collections.id
collections.updated_at
collections.via_app
collections.workspace_id
comment_reactions.comment_id
comment_reactions.created_at
@@ -177,10 +201,13 @@ items.created_by_user_id
items.created_via_app
items.deleted_at
items.id
items.installed_sha256
items.last_modified_by_user_id
items.lease_acquired_at
items.lease_expires_at
items.parent_id
items.source_artifact_sha256
items.source_pack
items.updated_at
items.via_app
items.workspace_id
+5 -1
View File
@@ -34,7 +34,11 @@ func TestBinaryColumnCensus(t *testing.T) {
t.Parallel()
// item_yjs_updates_set_aside.update_data holds op-log frames moved verbatim
// by a schema rebuild (BUG-3244): the same raw Yjs bytes, NULs included.
want := map[string]bool{"item_yjs_updates.update_data": true, "item_yjs_updates_set_aside.update_data": true}
// app_install_pending_blobs.data holds an app's manifest and artifact
// bytes exactly as fetched (TASK-3397), before any decoding: arbitrary
// bytes, NULs included, bound only as []byte by StagePendingBlob.
want := map[string]bool{"item_yjs_updates.update_data": true, "item_yjs_updates_set_aside.update_data": true,
"app_install_pending_blobs.data": true}
// One pattern per dialect spelling, applied to the raw migration text so
// nothing depends on a Go-side model of the schema.
@@ -0,0 +1,42 @@
-- Migration 092: app install staging and provenance (SPEC-6 U8a, TASK-3397).
-- Mirrors SQLite migration 118; see it for the rationale.
CREATE TABLE IF NOT EXISTS app_install_pending (
id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
origin TEXT NOT NULL,
state TEXT NOT NULL DEFAULT 'fetching' CHECK (state IN ('fetching', 'staged')),
reserved_bytes BIGINT NOT NULL,
manifest_sha256 TEXT,
preview TEXT,
expires_at TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_app_install_pending_owner ON app_install_pending (owner_id, expires_at);
CREATE INDEX IF NOT EXISTS idx_app_install_pending_expires ON app_install_pending (expires_at);
CREATE TABLE IF NOT EXISTS app_install_pending_blobs (
pending_id TEXT NOT NULL REFERENCES app_install_pending(id) ON DELETE CASCADE,
key TEXT NOT NULL,
url TEXT NOT NULL,
sha256 TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
data BYTEA NOT NULL,
PRIMARY KEY (pending_id, key)
);
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS bot_user_id TEXT REFERENCES users(id) ON DELETE SET NULL;
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS manifest_sha256 TEXT;
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS manifest_version TEXT;
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS manifest TEXT;
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS service_access TEXT;
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS delegated_access TEXT;
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS config TEXT NOT NULL DEFAULT '{}';
ALTER TABLE app_installs ADD COLUMN IF NOT EXISTS digests TEXT NOT NULL DEFAULT '{}';
ALTER TABLE collections ADD COLUMN IF NOT EXISTS via_app TEXT REFERENCES app_installs(id) ON DELETE SET NULL;
ALTER TABLE items ADD COLUMN IF NOT EXISTS source_pack TEXT;
ALTER TABLE items ADD COLUMN IF NOT EXISTS source_artifact_sha256 TEXT;
ALTER TABLE items ADD COLUMN IF NOT EXISTS installed_sha256 TEXT;