diff --git a/internal/models/reserved_field_decode_guard_test.go b/internal/models/reserved_field_decode_guard_test.go index e43f3af85..0fd203aae 100644 --- a/internal/models/reserved_field_decode_guard_test.go +++ b/internal/models/reserved_field_decode_guard_test.go @@ -61,10 +61,11 @@ var rawDecodeAllowed = map[string]string{ // GENERIC decode helper whose callers choose the concrete type, and none // of them can be converted without breaking every other thing they decode. // A new one shows up here and has to be reasoned about, which is the point. - "internal/cli/client.go::result": "generic HTTP response decoder: the caller supplies the destination, so this call has no schema to strip — a caller that wants an item-field schema must decode one itself", - "internal/oauth/storage.go::session": "fosite session hydration: decodes an OAuth session blob, never a collection schema", - "internal/server/server.go::v": "the shared request-body decoder (BUG-2803): decodes a handler's own input struct, and a handler taking a schema goes through the collection-definition doors above", - "internal/server/handlers_bootstrap.go::s": "trimRedundantSchemaLabels decodes into the parallel bootstrapSchema struct and strips reserved keys in the same loop — the parallel-struct twin of UnmarshalItemFieldSchema, and it cannot call it because the whole point of that struct is a different FieldDef shape", + "internal/cli/client.go::result": "generic HTTP response decoder: the caller supplies the destination, so this call has no schema to strip — a caller that wants an item-field schema must decode one itself", + "internal/oauth/storage.go::session": "fosite session hydration: decodes an OAuth session blob, never a collection schema", + "internal/server/server.go::v": "the shared request-body decoder (BUG-2803): decodes a handler's own input struct, and a handler taking a schema goes through the collection-definition doors above", + "scripts/decision-eval/attention/main.go::v": "offline eval tool (TASK-3137): readJSON decodes local `pad item show|comments|history` and activity dumps into items, comments, versions and activity rows; nothing it reads is a collection definition", + "internal/server/handlers_bootstrap.go::s": "trimRedundantSchemaLabels decodes into the parallel bootstrapSchema struct and strips reserved keys in the same loop — the parallel-struct twin of UnmarshalItemFieldSchema, and it cannot call it because the whole point of that struct is a different FieldDef shape", "internal/server/handlers_collections.go::schema": "collection create/update INPUT — the declaration is what is being validated, by validateNoReservedFieldKeys", "internal/server/handlers_collections.go::prevSchema": "the grandfather test's own baseline: a stripped prevSchema would reclassify every EXISTING declaration as newly introduced and refuse every update to such a collection", @@ -551,7 +552,7 @@ func TestGuardAllowListHasNoDeadEntries(t *testing.T) { // two — so key-set equality alone would pass if the guard went blind to one of // a pair. A DROP with no conversion in the same change means the guard stopped // seeing sites, not that the sites stopped existing. -const rawDecodeSiteCount = 20 +const rawDecodeSiteCount = 21 func TestGuardSeesEveryKnownDecodeSite(t *testing.T) { sites := collectSchemaDecodes(t, repoRoot(t)) diff --git a/scripts/decision-eval/attention/RESULTS.md b/scripts/decision-eval/attention/RESULTS.md new file mode 100644 index 000000000..8b6c84c46 --- /dev/null +++ b/scripts/decision-eval/attention/RESULTS.md @@ -0,0 +1,70 @@ +# Attention threshold eval — TASK-3137 (2026-09-22) + +Re-run of the day-73 decision-detector eval (eval2.py section D) on the SAME +140 items: 70 human-tasks as positives, and the day-73 run's 70 randomly drawn +tasks as negatives. Model jev-1.13.0, question `needs_human_decision` verbatim. +One call per item per mode, 0 errors. Per-item JSON is kept locally, not +committed. + +| mode | state sent | AUC | P / R at 0.5 | P / R at 0.6 | P / R at 0.7 | P / R at 0.8 | +|---|---|---|---|---|---|---| +| day-73 (recorded) | title + body | 0.939 | — | — | — | — | +| titlebody (control, today) | title + body | 0.935 | 0.945 / 0.743 | 0.930 / 0.571 | 0.944 / 0.486 | 0.958 / 0.329 | +| production | `BuildItemState` | 0.854 | 0.897 / 0.371 | 0.833 / 0.214 | 0.750 / 0.129 | 0.750 / 0.043 | +| nostatus (ablation) | production − fields.status | 0.887 | 0.881 / 0.529 | 0.889 / 0.457 | 0.885 / 0.329 | 0.778 / 0.100 | +| nostatus-notrail (ablation) | production − status − trail | 0.935 | 0.927 / 0.729 | 0.936 / 0.629 | 0.949 / 0.529 | 1.000 / 0.371 | +| **r1, replay at last open moment (DECIDES)** | `BuildItemState` over the item as of just before its first logged terminal transition | 0.885 | 0.850 / 0.750 | 0.846 / 0.647 | 0.825 / 0.485 | 0.833 / 0.294 | +| r0, replay at filing | `BuildItemState` over the oldest version, initial status, no trail | 0.931 | 0.926 / 0.735 | 0.935 / 0.632 | 0.971 / 0.485 | 1.000 / 0.368 | + +The r0 and r1 rows cover 138 items: 68 positives and 70 negatives. Two positives are +excluded from each: in r1, items closed with no logged terminal transition; in +r0, items created terminal. The spec and rule were pre-registered on TASK-3137, +checkpoint 3, before any replay call. + +## Outcome (lead-ruled rule: highest t with P >= 90% and R >= 60% on r1) + +**No threshold passes on r1**: precision never reaches 90%. Per the ruling, this +is a finding about the question set, and the constant is NOT changed. r0 would +pass at 0.6 (P 0.935 / R 0.632), but it does not decide: production asks +throughout an item's open life, not only at filing. + +What the r1 false positives are: 6 of the 9 at t=0.5 are negatives that RISE +by more than 0.2 from r0 to r1, all with a trail, and 5 of them are in-progress. +At its last open moment an ordinary task often IS waiting on a human or +someone external. TASK-1717's last comment before closing is "leaving +in-progress until the shared E2E device matrix", a device test only a person +can run. TASK-1668 was "paused before merge — Codex CLI hit its usage limit", +an external wait. The labels say which collection an item was FILED in, and +they cannot see such transient waits. Setting a threshold for the dashboard +needs present-tense labels at the replayed moment, not collection membership. + +## Reading (the first four modes) + +- **No model drift.** The control reproduces day-73 (0.935 vs 0.939). +- **The production gap is the labels, not the question.** 68 of the 70 + positives, and 50 of the 70 negatives, are now `status: done`. The production + state carries the present status and the resolution trail, and the model + answers what it is asked: a finished item is not waiting on anyone NOW. The + ablation was pre-registered on TASK-3137. Removing status and trail recovers + AUC fully, 0.935, and removing status alone recovers it part way, 0.887. +- **So this set cannot validate a threshold for production.** The labels say + "was filed as a human-task", which is historical; the `attention` set asks + only about OPEN items and asks about the present. The open subset is 22 + items, only 2 of them positive. +- The pre-registered choice rule (precision >= 90% and recall >= 60% on the + production mode) is met by no threshold, so `AttentionThreshold` is NOT + changed by this run. +- Probe items HT-1177 and HT-544 score 0.70 and 0.76 on production, but both + are done, and production never asks about them. TASK-3118's 0.71 probe + measured the same confound. + +## Reproduce + + go run ./scripts/decision-eval/attention -dump -mode -out run.json + go run ./scripts/decision-eval/attention -dump -out /dev/null -dry # instrument check + +`` holds `population.json` (ref, truth, day73_p, from the day-73 +results-decision.json) and `items/.json` + `items/.comments.json` +from `pad item show|comments --format json`. `-dry` builds every state without +calling the provider, and must reproduce the day-73 AUC of 0.939 from the +recorded probabilities. diff --git a/scripts/decision-eval/attention/main.go b/scripts/decision-eval/attention/main.go new file mode 100644 index 000000000..02eaacdd3 --- /dev/null +++ b/scripts/decision-eval/attention/main.go @@ -0,0 +1,353 @@ +// Command attention re-runs the day-73 decision-detector eval (eval2.py +// section D) through the PRODUCTION state builder, to set +// decision.AttentionThreshold from a measurement (TASK-3137). +// +// Inputs are local and fixed: population.json (ref, truth, day73_p) and, per +// ref, .json and .comments.json as `pad item show|comments +// --format json` wrote them. Nothing here reads the database. +// +// -mode production state = decision.BuildItemState(item, recent trail), +// questions = decision.AttentionSet() — the bytes and +// questions the `attention` set sends. +// -mode nostatus production state minus fields.status (ablation) +// -mode nostatus-notrail ...and minus recent_trail (ablation) +// -mode r1 / r0 production builder over the item REPLAYED to its last +// open moment / its filing (replay.go) +// -mode titlebody state = {"title","body"[:5000 runes]} with the same two +// eval questions — the day-73 shape, re-run today, as the +// control that separates model drift from state shape. +// +// The key is read from PAD_TYPESAFE_API_KEY or TYPESAFE_API_KEY and is never +// printed or written. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/PerpetualSoftware/pad/internal/decision" + "github.com/PerpetualSoftware/pad/internal/models" +) + +type member struct { + Ref string `json:"ref"` + Truth bool `json:"truth"` + Day73P float64 `json:"day73_p"` +} + +// Row is one item's result, written to the local run JSON. +type Row struct { + Ref string `json:"ref"` + Truth bool `json:"truth"` + Day73P float64 `json:"day73_p"` + P map[string]float64 `json:"p"` + Tokens int `json:"input_tokens"` + Truncated bool `json:"truncated"` + TrailLen int `json:"trail_len"` + Status string `json:"status"` + Err string `json:"error,omitempty"` +} + +func main() { + dump := flag.String("dump", "", "directory holding population.json and items/") + mode := flag.String("mode", "production", "production | titlebody") + out := flag.String("out", "", "run JSON path") + dry := flag.Bool("dry", false, "build every state and report its shape; no provider call") + flag.Parse() + if *dump == "" || *out == "" { + fmt.Fprintln(os.Stderr, "need -dump and -out") + os.Exit(2) + } + if *dry { + if *mode == "r0" || *mode == "r1" { + replayDry(*dump, *mode) + return + } + dryRun(*dump) + return + } + key := os.Getenv("PAD_TYPESAFE_API_KEY") + if key == "" { + key = os.Getenv("TYPESAFE_API_KEY") + } + p, err := decision.New(decision.Config{Provider: decision.ProviderTypesafe, APIKey: key, Model: "jev-1.13.0"}) + if err != nil || p == nil { + fmt.Fprintln(os.Stderr, "provider:", err) + os.Exit(2) + } + var pop []member + mustRead(filepath.Join(*dump, "population.json"), &pop) + + all := decision.AttentionSet().Questions + questions := all + if *mode == "titlebody" { + questions = map[string]decision.Question{ + decision.AttentionNeedsHuman: all[decision.AttentionNeedsHuman], + decision.AttentionBlocked: all[decision.AttentionBlocked], + } + } else if *mode != "production" && *mode != "nostatus" && *mode != "nostatus-notrail" && *mode != "r0" && *mode != "r1" { + fmt.Fprintln(os.Stderr, "unknown -mode", *mode) + os.Exit(2) + } + + rows := make([]Row, len(pop)) + var wg sync.WaitGroup + sem := make(chan struct{}, 6) + for i, m := range pop { + wg.Add(1) + go func(i int, m member) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + rows[i] = evalOne(p, *dump, *mode, m, questions) + }(i, m) + } + wg.Wait() + + b, _ := json.MarshalIndent(rows, "", " ") + if err := os.WriteFile(*out, b, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + summarize(*mode, rows) +} + +func evalOne(p decision.Provider, dump, mode string, m member, qs map[string]decision.Question) Row { + row := Row{Ref: m.Ref, Truth: m.Truth, Day73P: m.Day73P} + var item models.Item + var comments []models.Comment + if err := readJSON(filepath.Join(dump, "items", m.Ref+".json"), &item); err != nil { + row.Err = err.Error() + return row + } + if err := readJSON(filepath.Join(dump, "items", m.Ref+".comments.json"), &comments); err != nil { + row.Err = err.Error() + return row + } + row.Status = statusOf(item.Fields) + // Store.RecentComments order: (created_at, id) ascending; the builder + // keeps the newest RecentTrailWindow of them. + sort.Slice(comments, func(i, j int) bool { + if !comments[i].CreatedAt.Equal(comments[j].CreatedAt) { + return comments[i].CreatedAt.Before(comments[j].CreatedAt) + } + return comments[i].ID < comments[j].ID + }) + row.TrailLen = min(len(comments), decision.RecentTrailWindow) + + var state json.RawMessage + switch mode { + case "production": + st, err := decision.BuildItemState(&item, comments) + if err != nil { + row.Err = err.Error() + return row + } + state, row.Truncated = st.Bytes, st.Truncated + case "r0", "r1": + it, cs, excluded := replayed(dump, mode, item, comments) + if excluded != "" { + row.Err = "excluded: " + excluded + return row + } + row.Status = statusOf(it.Fields) + row.TrailLen = min(len(cs), decision.RecentTrailWindow) + st, err := decision.BuildItemState(&it, cs) + if err != nil { + row.Err = err.Error() + return row + } + state, row.Truncated = st.Bytes, st.Truncated + case "nostatus", "nostatus-notrail": + // Ablations (TASK-3137 checkpoint 1): the production builder with the + // item's present status removed, and additionally its trail — to test + // whether the gap is the state saying the item is already done. + st, err := decision.BuildItemState(&item, comments) + if err != nil { + row.Err = err.Error() + return row + } + var is decision.ItemState + if err := json.Unmarshal(st.Bytes, &is); err != nil { + row.Err = err.Error() + return row + } + delete(is.Fields, "status") + if mode == "nostatus-notrail" { + is.Trail = []decision.ItemStateRemark{} + } + state, _ = json.Marshal(is) + row.Truncated = st.Truncated + case "titlebody": + body := []rune(item.Content) + if len(body) > 5000 { + body = body[:5000] + } + state, _ = json.Marshal(map[string]string{"title": item.Title, "body": string(body)}) + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + answers, usage, err := p.Ask(ctx, state, qs) + if err != nil { + row.Err = err.Error() + return row + } + row.Tokens = usage.InputTokens + row.Truncated = row.Truncated || usage.StateTruncated + row.P = map[string]float64{} + for k, a := range answers { + row.P[k] = a.Noul + } + return row +} + +func summarize(mode string, rows []Row) { + var ok []Row + errs := 0 + for _, r := range rows { + if r.Err != "" { + errs++ + continue + } + ok = append(ok, r) + } + exPos, exNeg := 0, 0 + for _, r := range rows { + if strings.HasPrefix(r.Err, "excluded: ") { + if r.Truth { + exPos++ + } else { + exNeg++ + } + } + } + fmt.Printf("mode=%s items=%d errors=%d (excluded: %d positives, %d negatives)\n", mode, len(ok), errs, exPos, exNeg) + fmt.Printf("needs_human_decision AUC=%.3f (day-73 recorded p on the same items: AUC=%.3f)\n", + auc(ok, func(r Row) float64 { return r.P[decision.AttentionNeedsHuman] }), + auc(ok, func(r Row) float64 { return r.Day73P })) + for _, t := range []float64{0.5, 0.6, 0.7, 0.8} { + tp, fp, fn := 0, 0, 0 + for _, r := range ok { + flag := r.P[decision.AttentionNeedsHuman] >= t + switch { + case flag && r.Truth: + tp++ + case flag && !r.Truth: + fp++ + case !flag && r.Truth: + fn++ + } + } + prec, rec := 0.0, 0.0 + if tp+fp > 0 { + prec = float64(tp) / float64(tp+fp) + } + if tp+fn > 0 { + rec = float64(tp) / float64(tp+fn) + } + fmt.Printf(" t=%.1f precision=%.3f (%d/%d) recall=%.3f (%d/%d)\n", t, prec, tp, tp+fp, rec, tp, tp+fn) + } + for _, r := range ok { + if r.Ref == "HT-1177" || r.Ref == "HT-544" { + fmt.Printf(" %s needs_human=%.2f (day-73 %.2f)\n", r.Ref, r.P[decision.AttentionNeedsHuman], r.Day73P) + } + } +} + +// auc is the Mann-Whitney U statistic over positives vs negatives, ties +// counted half. +func auc(rows []Row, score func(Row) float64) float64 { + var pos, neg []float64 + for _, r := range rows { + if r.Truth { + pos = append(pos, score(r)) + } else { + neg = append(neg, score(r)) + } + } + if len(pos) == 0 || len(neg) == 0 { + return 0 + } + var u float64 + for _, a := range pos { + for _, b := range neg { + switch { + case a > b: + u++ + case a == b: + u += 0.5 + } + } + } + return u / float64(len(pos)*len(neg)) +} + +func statusOf(fields string) string { + var f map[string]any + _ = json.Unmarshal([]byte(fields), &f) + s, _ := f["status"].(string) + return s +} + +func readJSON(path string, v any) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + return json.Unmarshal(b, v) +} + +func mustRead(path string, v any) { + if err := readJSON(path, v); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } +} + +// dryRun is the instrument check: every item parses, every state builds, and +// the recorded day-73 probabilities reproduce the eval's AUC through this +// file's auc — so a wrong parse or a wrong statistic shows before any call. +func dryRun(dump string) { + var pop []member + mustRead(filepath.Join(dump, "population.json"), &pop) + var rows []Row + emptyBody, emptyColl, trail := 0, 0, 0 + for _, m := range pop { + var item models.Item + var comments []models.Comment + if err := readJSON(filepath.Join(dump, "items", m.Ref+".json"), &item); err != nil { + fmt.Println("parse", m.Ref, err) + continue + } + if err := readJSON(filepath.Join(dump, "items", m.Ref+".comments.json"), &comments); err != nil { + fmt.Println("parse comments", m.Ref, err) + continue + } + st, err := decision.BuildItemState(&item, comments) + if err != nil { + fmt.Println("build", m.Ref, err) + continue + } + var is decision.ItemState + _ = json.Unmarshal(st.Bytes, &is) + if is.Body == "" { + emptyBody++ + } + if is.Collection == "" { + emptyColl++ + } + trail += len(is.Trail) + rows = append(rows, Row{Ref: m.Ref, Truth: m.Truth, Day73P: m.Day73P}) + } + fmt.Printf("dry: built=%d of %d, empty body=%d, empty collection=%d, trail entries=%d\n", len(rows), len(pop), emptyBody, emptyColl, trail) + fmt.Printf("dry: recorded day-73 AUC through auc() = %.3f (eval reported 0.939)\n", auc(rows, func(r Row) float64 { return r.Day73P })) +} diff --git a/scripts/decision-eval/attention/replay.go b/scripts/decision-eval/attention/replay.go new file mode 100644 index 000000000..15bc8e32f --- /dev/null +++ b/scripts/decision-eval/attention/replay.go @@ -0,0 +1,187 @@ +package main + +import ( + "encoding/json" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/PerpetualSoftware/pad/internal/models" +) + +// Replay (TASK-3137 checkpoint 3): rebuild an item as it stood at a moment it +// was OPEN, so the 70/70 labels — which describe the need, not the +// resolution — are compared with the production question asked at a time +// production would actually ask it. + +type version struct { + Content string `json:"content"` + CreatedAt time.Time `json:"created_at"` +} + +type activity struct { + Action string `json:"action"` + CreatedAt time.Time `json:"created_at"` + Metadata string `json:"metadata"` +} + +type transition struct { + At time.Time + From, To string +} + +// terminal is each collection's terminal_options, read from the workspace +// schema on 2026-09-22 (human-tasks: done; tasks: done, cancelled). +var terminal = map[string]map[string]bool{ + "human-tasks": {"done": true}, + "tasks": {"done": true, "cancelled": true}, +} + +func statusTransitions(acts []activity) []transition { + var out []transition + for _, a := range acts { + var md struct { + Changes string `json:"changes"` + } + _ = json.Unmarshal([]byte(a.Metadata), &md) + for _, part := range strings.Split(md.Changes, "; ") { + rest, ok := strings.CutPrefix(part, "status: ") + if !ok { + continue + } + from, to, ok := strings.Cut(rest, " → ") + if !ok { + continue + } + out = append(out, transition{At: a.CreatedAt, From: strings.TrimSpace(from), To: strings.TrimSpace(to)}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].At.Before(out[j].At) }) + return out +} + +// replayed returns the item and comments as they stood at the replay point, +// or excluded=true with a reason. +func replayed(dump, mode string, item models.Item, comments []models.Comment) (models.Item, []models.Comment, string) { + var versions []version + var acts []activity + if err := readJSON(filepath.Join(dump, "items", item.Ref+".history.json"), &versions); err != nil { + return item, nil, "history: " + err.Error() + } + if err := readJSON(filepath.Join(dump, "items", item.Ref+".activity.json"), &acts); err != nil { + return item, nil, "activity: " + err.Error() + } + sort.Slice(versions, func(i, j int) bool { return versions[i].CreatedAt.Before(versions[j].CreatedAt) }) + trans := statusTransitions(acts) + term := terminal[item.CollectionSlug] + cur := statusOf(item.Fields) + + setStatus := func(it models.Item, s string) models.Item { + var f map[string]any + _ = json.Unmarshal([]byte(it.Fields), &f) + if f == nil { + f = map[string]any{} + } + f["status"] = s + b, _ := json.Marshal(f) + it.Fields = string(b) + return it + } + + switch mode { + case "r0": + if len(versions) > 0 { + item.Content = versions[0].Content + } + initial := cur + if len(trans) > 0 { + initial = trans[0].From + } + if term[initial] { + return item, nil, "created terminal" + } + return setStatus(item, initial), nil, "" + case "r1": + var t1 *transition + for i := range trans { + if term[trans[i].To] && !term[trans[i].From] { + t1 = &trans[i] + break + } + } + if t1 == nil { + if term[cur] { + return item, nil, "closed, no logged terminal transition" + } + return item, comments, "" // still open: its current state + } + content := "" + found := false + for _, v := range versions { + if v.CreatedAt.Before(t1.At) { + content, found = v.Content, true + } + } + if !found && len(versions) > 0 { + content = versions[0].Content + } + item.Content = content + var before []models.Comment + for _, c := range comments { + if c.CreatedAt.Before(t1.At) { + before = append(before, c) + } + } + return setStatus(item, t1.From), before, "" + } + return item, comments, "unknown replay mode" +} + +// replayDry is the replay's instrument check: exclusions by class, the +// replayed status distribution (none may be terminal), trail sizes, and a +// sample line per item for eyeballing. +func replayDry(dump, mode string) { + var pop []member + mustRead(filepath.Join(dump, "population.json"), &pop) + exPos, exNeg, keptPos, keptNeg, terminalLeak := 0, 0, 0, 0, 0 + statuses := map[string]int{} + reasons := map[string]int{} + for i, m := range pop { + var item models.Item + var comments []models.Comment + mustRead(filepath.Join(dump, "items", m.Ref+".json"), &item) + mustRead(filepath.Join(dump, "items", m.Ref+".comments.json"), &comments) + sort.Slice(comments, func(i, j int) bool { return comments[i].CreatedAt.Before(comments[j].CreatedAt) }) + it, cs, ex := replayed(dump, mode, item, comments) + if ex != "" { + reasons[ex]++ + if m.Truth { + exPos++ + } else { + exNeg++ + } + continue + } + if m.Truth { + keptPos++ + } else { + keptNeg++ + } + s := statusOf(it.Fields) + statuses[item.CollectionSlug+":"+s]++ + if terminal[item.CollectionSlug][s] { + terminalLeak++ + } + if i%20 == 0 { + println(m.Ref, "now:", statusOf(item.Fields), "replayed:", s, "body:", len(it.Content), "of", len(item.Content), "trail:", len(cs), "of", len(comments)) + } + } + println("kept pos", keptPos, "neg", keptNeg, "| excluded pos", exPos, "neg", exNeg, "| terminal leaks", terminalLeak) + for k, v := range reasons { + println(" excluded:", k, v) + } + for k, v := range statuses { + println(" replayed status", k, v) + } +}