mirror of
https://github.com/PerpetualSoftware/pad.git
synced 2026-10-25 05:49:50 +00:00
chore(decision-eval): attention threshold eval — no threshold passes the pre-registered rule (TASK-3137) (#1424)
* chore(decision-eval): attention threshold eval through the production state builder (TASK-3137) The instrument and its recorded results. The pre-registered rule is met by no threshold on the production state, so AttentionThreshold is unchanged; the ablation shows the 70/70 labels are historical while production asks about open items in the present. See RESULTS.md. Claude-Session: https://claude.ai/code/session_01ByMjAhvdWgbAWWeGE9jwWd * chore(decision-eval): replay each item to its last open moment (TASK-3137) The lead-ruled instrument: BuildItemState over the item as of just before its first logged terminal transition (r1), and at filing (r0), with the same 140 labels. The pre-registered rule (P >= 90%, R >= 60% on r1) is met by no threshold, so AttentionThreshold is unchanged. RESULTS.md records the finding: collection-membership labels cannot see the transient human waits that a task at its last open moment really has. Claude-Session: https://claude.ai/code/session_01ByMjAhvdWgbAWWeGE9jwWd * test(models): classify the eval tool's generic decode site (TASK-3137) The BUG-2685 guard found the new scripts/decision-eval readJSON helper as an unclassified interface-destination decode. It decodes item, comment, history and activity dumps, never a collection definition. Claude-Session: https://claude.ai/code/session_01ByMjAhvdWgbAWWeGE9jwWd
This commit is contained in:
xarmian
@@ -61,10 +61,11 @@ var rawDecodeAllowed = map[string]string{
|
||||
// GENERIC decode helper whose callers choose the concrete type, and none
|
||||
// of them can be converted without breaking every other thing they decode.
|
||||
// A new one shows up here and has to be reasoned about, which is the point.
|
||||
"internal/cli/client.go::result": "generic HTTP response decoder: the caller supplies the destination, so this call has no schema to strip — a caller that wants an item-field schema must decode one itself",
|
||||
"internal/oauth/storage.go::session": "fosite session hydration: decodes an OAuth session blob, never a collection schema",
|
||||
"internal/server/server.go::v": "the shared request-body decoder (BUG-2803): decodes a handler's own input struct, and a handler taking a schema goes through the collection-definition doors above",
|
||||
"internal/server/handlers_bootstrap.go::s": "trimRedundantSchemaLabels decodes into the parallel bootstrapSchema struct and strips reserved keys in the same loop — the parallel-struct twin of UnmarshalItemFieldSchema, and it cannot call it because the whole point of that struct is a different FieldDef shape",
|
||||
"internal/cli/client.go::result": "generic HTTP response decoder: the caller supplies the destination, so this call has no schema to strip — a caller that wants an item-field schema must decode one itself",
|
||||
"internal/oauth/storage.go::session": "fosite session hydration: decodes an OAuth session blob, never a collection schema",
|
||||
"internal/server/server.go::v": "the shared request-body decoder (BUG-2803): decodes a handler's own input struct, and a handler taking a schema goes through the collection-definition doors above",
|
||||
"scripts/decision-eval/attention/main.go::v": "offline eval tool (TASK-3137): readJSON decodes local `pad item show|comments|history` and activity dumps into items, comments, versions and activity rows; nothing it reads is a collection definition",
|
||||
"internal/server/handlers_bootstrap.go::s": "trimRedundantSchemaLabels decodes into the parallel bootstrapSchema struct and strips reserved keys in the same loop — the parallel-struct twin of UnmarshalItemFieldSchema, and it cannot call it because the whole point of that struct is a different FieldDef shape",
|
||||
|
||||
"internal/server/handlers_collections.go::schema": "collection create/update INPUT — the declaration is what is being validated, by validateNoReservedFieldKeys",
|
||||
"internal/server/handlers_collections.go::prevSchema": "the grandfather test's own baseline: a stripped prevSchema would reclassify every EXISTING declaration as newly introduced and refuse every update to such a collection",
|
||||
@@ -551,7 +552,7 @@ func TestGuardAllowListHasNoDeadEntries(t *testing.T) {
|
||||
// two — so key-set equality alone would pass if the guard went blind to one of
|
||||
// a pair. A DROP with no conversion in the same change means the guard stopped
|
||||
// seeing sites, not that the sites stopped existing.
|
||||
const rawDecodeSiteCount = 20
|
||||
const rawDecodeSiteCount = 21
|
||||
|
||||
func TestGuardSeesEveryKnownDecodeSite(t *testing.T) {
|
||||
sites := collectSchemaDecodes(t, repoRoot(t))
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# Attention threshold eval — TASK-3137 (2026-09-22)
|
||||
|
||||
Re-run of the day-73 decision-detector eval (eval2.py section D) on the SAME
|
||||
140 items: 70 human-tasks as positives, and the day-73 run's 70 randomly drawn
|
||||
tasks as negatives. Model jev-1.13.0, question `needs_human_decision` verbatim.
|
||||
One call per item per mode, 0 errors. Per-item JSON is kept locally, not
|
||||
committed.
|
||||
|
||||
| mode | state sent | AUC | P / R at 0.5 | P / R at 0.6 | P / R at 0.7 | P / R at 0.8 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| day-73 (recorded) | title + body | 0.939 | — | — | — | — |
|
||||
| titlebody (control, today) | title + body | 0.935 | 0.945 / 0.743 | 0.930 / 0.571 | 0.944 / 0.486 | 0.958 / 0.329 |
|
||||
| production | `BuildItemState` | 0.854 | 0.897 / 0.371 | 0.833 / 0.214 | 0.750 / 0.129 | 0.750 / 0.043 |
|
||||
| nostatus (ablation) | production − fields.status | 0.887 | 0.881 / 0.529 | 0.889 / 0.457 | 0.885 / 0.329 | 0.778 / 0.100 |
|
||||
| nostatus-notrail (ablation) | production − status − trail | 0.935 | 0.927 / 0.729 | 0.936 / 0.629 | 0.949 / 0.529 | 1.000 / 0.371 |
|
||||
| **r1, replay at last open moment (DECIDES)** | `BuildItemState` over the item as of just before its first logged terminal transition | 0.885 | 0.850 / 0.750 | 0.846 / 0.647 | 0.825 / 0.485 | 0.833 / 0.294 |
|
||||
| r0, replay at filing | `BuildItemState` over the oldest version, initial status, no trail | 0.931 | 0.926 / 0.735 | 0.935 / 0.632 | 0.971 / 0.485 | 1.000 / 0.368 |
|
||||
|
||||
The r0 and r1 rows cover 138 items: 68 positives and 70 negatives. Two positives are
|
||||
excluded from each: in r1, items closed with no logged terminal transition; in
|
||||
r0, items created terminal. The spec and rule were pre-registered on TASK-3137,
|
||||
checkpoint 3, before any replay call.
|
||||
|
||||
## Outcome (lead-ruled rule: highest t with P >= 90% and R >= 60% on r1)
|
||||
|
||||
**No threshold passes on r1**: precision never reaches 90%. Per the ruling, this
|
||||
is a finding about the question set, and the constant is NOT changed. r0 would
|
||||
pass at 0.6 (P 0.935 / R 0.632), but it does not decide: production asks
|
||||
throughout an item's open life, not only at filing.
|
||||
|
||||
What the r1 false positives are: 6 of the 9 at t=0.5 are negatives that RISE
|
||||
by more than 0.2 from r0 to r1, all with a trail, and 5 of them are in-progress.
|
||||
At its last open moment an ordinary task often IS waiting on a human or
|
||||
someone external. TASK-1717's last comment before closing is "leaving
|
||||
in-progress until the shared E2E device matrix", a device test only a person
|
||||
can run. TASK-1668 was "paused before merge — Codex CLI hit its usage limit",
|
||||
an external wait. The labels say which collection an item was FILED in, and
|
||||
they cannot see such transient waits. Setting a threshold for the dashboard
|
||||
needs present-tense labels at the replayed moment, not collection membership.
|
||||
|
||||
## Reading (the first four modes)
|
||||
|
||||
- **No model drift.** The control reproduces day-73 (0.935 vs 0.939).
|
||||
- **The production gap is the labels, not the question.** 68 of the 70
|
||||
positives, and 50 of the 70 negatives, are now `status: done`. The production
|
||||
state carries the present status and the resolution trail, and the model
|
||||
answers what it is asked: a finished item is not waiting on anyone NOW. The
|
||||
ablation was pre-registered on TASK-3137. Removing status and trail recovers
|
||||
AUC fully, 0.935, and removing status alone recovers it part way, 0.887.
|
||||
- **So this set cannot validate a threshold for production.** The labels say
|
||||
"was filed as a human-task", which is historical; the `attention` set asks
|
||||
only about OPEN items and asks about the present. The open subset is 22
|
||||
items, only 2 of them positive.
|
||||
- The pre-registered choice rule (precision >= 90% and recall >= 60% on the
|
||||
production mode) is met by no threshold, so `AttentionThreshold` is NOT
|
||||
changed by this run.
|
||||
- Probe items HT-1177 and HT-544 score 0.70 and 0.76 on production, but both
|
||||
are done, and production never asks about them. TASK-3118's 0.71 probe
|
||||
measured the same confound.
|
||||
|
||||
## Reproduce
|
||||
|
||||
go run ./scripts/decision-eval/attention -dump <dir> -mode <mode> -out run.json
|
||||
go run ./scripts/decision-eval/attention -dump <dir> -out /dev/null -dry # instrument check
|
||||
|
||||
`<dir>` holds `population.json` (ref, truth, day73_p, from the day-73
|
||||
results-decision.json) and `items/<ref>.json` + `items/<ref>.comments.json`
|
||||
from `pad item show|comments --format json`. `-dry` builds every state without
|
||||
calling the provider, and must reproduce the day-73 AUC of 0.939 from the
|
||||
recorded probabilities.
|
||||
@@ -0,0 +1,353 @@
|
||||
// Command attention re-runs the day-73 decision-detector eval (eval2.py
|
||||
// section D) through the PRODUCTION state builder, to set
|
||||
// decision.AttentionThreshold from a measurement (TASK-3137).
|
||||
//
|
||||
// Inputs are local and fixed: population.json (ref, truth, day73_p) and, per
|
||||
// ref, <ref>.json and <ref>.comments.json as `pad item show|comments
|
||||
// --format json` wrote them. Nothing here reads the database.
|
||||
//
|
||||
// -mode production state = decision.BuildItemState(item, recent trail),
|
||||
// questions = decision.AttentionSet() — the bytes and
|
||||
// questions the `attention` set sends.
|
||||
// -mode nostatus production state minus fields.status (ablation)
|
||||
// -mode nostatus-notrail ...and minus recent_trail (ablation)
|
||||
// -mode r1 / r0 production builder over the item REPLAYED to its last
|
||||
// open moment / its filing (replay.go)
|
||||
// -mode titlebody state = {"title","body"[:5000 runes]} with the same two
|
||||
// eval questions — the day-73 shape, re-run today, as the
|
||||
// control that separates model drift from state shape.
|
||||
//
|
||||
// The key is read from PAD_TYPESAFE_API_KEY or TYPESAFE_API_KEY and is never
|
||||
// printed or written.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/PerpetualSoftware/pad/internal/decision"
|
||||
"github.com/PerpetualSoftware/pad/internal/models"
|
||||
)
|
||||
|
||||
type member struct {
|
||||
Ref string `json:"ref"`
|
||||
Truth bool `json:"truth"`
|
||||
Day73P float64 `json:"day73_p"`
|
||||
}
|
||||
|
||||
// Row is one item's result, written to the local run JSON.
|
||||
type Row struct {
|
||||
Ref string `json:"ref"`
|
||||
Truth bool `json:"truth"`
|
||||
Day73P float64 `json:"day73_p"`
|
||||
P map[string]float64 `json:"p"`
|
||||
Tokens int `json:"input_tokens"`
|
||||
Truncated bool `json:"truncated"`
|
||||
TrailLen int `json:"trail_len"`
|
||||
Status string `json:"status"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
dump := flag.String("dump", "", "directory holding population.json and items/")
|
||||
mode := flag.String("mode", "production", "production | titlebody")
|
||||
out := flag.String("out", "", "run JSON path")
|
||||
dry := flag.Bool("dry", false, "build every state and report its shape; no provider call")
|
||||
flag.Parse()
|
||||
if *dump == "" || *out == "" {
|
||||
fmt.Fprintln(os.Stderr, "need -dump and -out")
|
||||
os.Exit(2)
|
||||
}
|
||||
if *dry {
|
||||
if *mode == "r0" || *mode == "r1" {
|
||||
replayDry(*dump, *mode)
|
||||
return
|
||||
}
|
||||
dryRun(*dump)
|
||||
return
|
||||
}
|
||||
key := os.Getenv("PAD_TYPESAFE_API_KEY")
|
||||
if key == "" {
|
||||
key = os.Getenv("TYPESAFE_API_KEY")
|
||||
}
|
||||
p, err := decision.New(decision.Config{Provider: decision.ProviderTypesafe, APIKey: key, Model: "jev-1.13.0"})
|
||||
if err != nil || p == nil {
|
||||
fmt.Fprintln(os.Stderr, "provider:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
var pop []member
|
||||
mustRead(filepath.Join(*dump, "population.json"), &pop)
|
||||
|
||||
all := decision.AttentionSet().Questions
|
||||
questions := all
|
||||
if *mode == "titlebody" {
|
||||
questions = map[string]decision.Question{
|
||||
decision.AttentionNeedsHuman: all[decision.AttentionNeedsHuman],
|
||||
decision.AttentionBlocked: all[decision.AttentionBlocked],
|
||||
}
|
||||
} else if *mode != "production" && *mode != "nostatus" && *mode != "nostatus-notrail" && *mode != "r0" && *mode != "r1" {
|
||||
fmt.Fprintln(os.Stderr, "unknown -mode", *mode)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
rows := make([]Row, len(pop))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, 6)
|
||||
for i, m := range pop {
|
||||
wg.Add(1)
|
||||
go func(i int, m member) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
rows[i] = evalOne(p, *dump, *mode, m, questions)
|
||||
}(i, m)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
b, _ := json.MarshalIndent(rows, "", " ")
|
||||
if err := os.WriteFile(*out, b, 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
summarize(*mode, rows)
|
||||
}
|
||||
|
||||
func evalOne(p decision.Provider, dump, mode string, m member, qs map[string]decision.Question) Row {
|
||||
row := Row{Ref: m.Ref, Truth: m.Truth, Day73P: m.Day73P}
|
||||
var item models.Item
|
||||
var comments []models.Comment
|
||||
if err := readJSON(filepath.Join(dump, "items", m.Ref+".json"), &item); err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
if err := readJSON(filepath.Join(dump, "items", m.Ref+".comments.json"), &comments); err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
row.Status = statusOf(item.Fields)
|
||||
// Store.RecentComments order: (created_at, id) ascending; the builder
|
||||
// keeps the newest RecentTrailWindow of them.
|
||||
sort.Slice(comments, func(i, j int) bool {
|
||||
if !comments[i].CreatedAt.Equal(comments[j].CreatedAt) {
|
||||
return comments[i].CreatedAt.Before(comments[j].CreatedAt)
|
||||
}
|
||||
return comments[i].ID < comments[j].ID
|
||||
})
|
||||
row.TrailLen = min(len(comments), decision.RecentTrailWindow)
|
||||
|
||||
var state json.RawMessage
|
||||
switch mode {
|
||||
case "production":
|
||||
st, err := decision.BuildItemState(&item, comments)
|
||||
if err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
state, row.Truncated = st.Bytes, st.Truncated
|
||||
case "r0", "r1":
|
||||
it, cs, excluded := replayed(dump, mode, item, comments)
|
||||
if excluded != "" {
|
||||
row.Err = "excluded: " + excluded
|
||||
return row
|
||||
}
|
||||
row.Status = statusOf(it.Fields)
|
||||
row.TrailLen = min(len(cs), decision.RecentTrailWindow)
|
||||
st, err := decision.BuildItemState(&it, cs)
|
||||
if err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
state, row.Truncated = st.Bytes, st.Truncated
|
||||
case "nostatus", "nostatus-notrail":
|
||||
// Ablations (TASK-3137 checkpoint 1): the production builder with the
|
||||
// item's present status removed, and additionally its trail — to test
|
||||
// whether the gap is the state saying the item is already done.
|
||||
st, err := decision.BuildItemState(&item, comments)
|
||||
if err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
var is decision.ItemState
|
||||
if err := json.Unmarshal(st.Bytes, &is); err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
delete(is.Fields, "status")
|
||||
if mode == "nostatus-notrail" {
|
||||
is.Trail = []decision.ItemStateRemark{}
|
||||
}
|
||||
state, _ = json.Marshal(is)
|
||||
row.Truncated = st.Truncated
|
||||
case "titlebody":
|
||||
body := []rune(item.Content)
|
||||
if len(body) > 5000 {
|
||||
body = body[:5000]
|
||||
}
|
||||
state, _ = json.Marshal(map[string]string{"title": item.Title, "body": string(body)})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
answers, usage, err := p.Ask(ctx, state, qs)
|
||||
if err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
row.Tokens = usage.InputTokens
|
||||
row.Truncated = row.Truncated || usage.StateTruncated
|
||||
row.P = map[string]float64{}
|
||||
for k, a := range answers {
|
||||
row.P[k] = a.Noul
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func summarize(mode string, rows []Row) {
|
||||
var ok []Row
|
||||
errs := 0
|
||||
for _, r := range rows {
|
||||
if r.Err != "" {
|
||||
errs++
|
||||
continue
|
||||
}
|
||||
ok = append(ok, r)
|
||||
}
|
||||
exPos, exNeg := 0, 0
|
||||
for _, r := range rows {
|
||||
if strings.HasPrefix(r.Err, "excluded: ") {
|
||||
if r.Truth {
|
||||
exPos++
|
||||
} else {
|
||||
exNeg++
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("mode=%s items=%d errors=%d (excluded: %d positives, %d negatives)\n", mode, len(ok), errs, exPos, exNeg)
|
||||
fmt.Printf("needs_human_decision AUC=%.3f (day-73 recorded p on the same items: AUC=%.3f)\n",
|
||||
auc(ok, func(r Row) float64 { return r.P[decision.AttentionNeedsHuman] }),
|
||||
auc(ok, func(r Row) float64 { return r.Day73P }))
|
||||
for _, t := range []float64{0.5, 0.6, 0.7, 0.8} {
|
||||
tp, fp, fn := 0, 0, 0
|
||||
for _, r := range ok {
|
||||
flag := r.P[decision.AttentionNeedsHuman] >= t
|
||||
switch {
|
||||
case flag && r.Truth:
|
||||
tp++
|
||||
case flag && !r.Truth:
|
||||
fp++
|
||||
case !flag && r.Truth:
|
||||
fn++
|
||||
}
|
||||
}
|
||||
prec, rec := 0.0, 0.0
|
||||
if tp+fp > 0 {
|
||||
prec = float64(tp) / float64(tp+fp)
|
||||
}
|
||||
if tp+fn > 0 {
|
||||
rec = float64(tp) / float64(tp+fn)
|
||||
}
|
||||
fmt.Printf(" t=%.1f precision=%.3f (%d/%d) recall=%.3f (%d/%d)\n", t, prec, tp, tp+fp, rec, tp, tp+fn)
|
||||
}
|
||||
for _, r := range ok {
|
||||
if r.Ref == "HT-1177" || r.Ref == "HT-544" {
|
||||
fmt.Printf(" %s needs_human=%.2f (day-73 %.2f)\n", r.Ref, r.P[decision.AttentionNeedsHuman], r.Day73P)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// auc is the Mann-Whitney U statistic over positives vs negatives, ties
|
||||
// counted half.
|
||||
func auc(rows []Row, score func(Row) float64) float64 {
|
||||
var pos, neg []float64
|
||||
for _, r := range rows {
|
||||
if r.Truth {
|
||||
pos = append(pos, score(r))
|
||||
} else {
|
||||
neg = append(neg, score(r))
|
||||
}
|
||||
}
|
||||
if len(pos) == 0 || len(neg) == 0 {
|
||||
return 0
|
||||
}
|
||||
var u float64
|
||||
for _, a := range pos {
|
||||
for _, b := range neg {
|
||||
switch {
|
||||
case a > b:
|
||||
u++
|
||||
case a == b:
|
||||
u += 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
return u / float64(len(pos)*len(neg))
|
||||
}
|
||||
|
||||
func statusOf(fields string) string {
|
||||
var f map[string]any
|
||||
_ = json.Unmarshal([]byte(fields), &f)
|
||||
s, _ := f["status"].(string)
|
||||
return s
|
||||
}
|
||||
|
||||
func readJSON(path string, v any) error {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(b, v)
|
||||
}
|
||||
|
||||
func mustRead(path string, v any) {
|
||||
if err := readJSON(path, v); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
// dryRun is the instrument check: every item parses, every state builds, and
|
||||
// the recorded day-73 probabilities reproduce the eval's AUC through this
|
||||
// file's auc — so a wrong parse or a wrong statistic shows before any call.
|
||||
func dryRun(dump string) {
|
||||
var pop []member
|
||||
mustRead(filepath.Join(dump, "population.json"), &pop)
|
||||
var rows []Row
|
||||
emptyBody, emptyColl, trail := 0, 0, 0
|
||||
for _, m := range pop {
|
||||
var item models.Item
|
||||
var comments []models.Comment
|
||||
if err := readJSON(filepath.Join(dump, "items", m.Ref+".json"), &item); err != nil {
|
||||
fmt.Println("parse", m.Ref, err)
|
||||
continue
|
||||
}
|
||||
if err := readJSON(filepath.Join(dump, "items", m.Ref+".comments.json"), &comments); err != nil {
|
||||
fmt.Println("parse comments", m.Ref, err)
|
||||
continue
|
||||
}
|
||||
st, err := decision.BuildItemState(&item, comments)
|
||||
if err != nil {
|
||||
fmt.Println("build", m.Ref, err)
|
||||
continue
|
||||
}
|
||||
var is decision.ItemState
|
||||
_ = json.Unmarshal(st.Bytes, &is)
|
||||
if is.Body == "" {
|
||||
emptyBody++
|
||||
}
|
||||
if is.Collection == "" {
|
||||
emptyColl++
|
||||
}
|
||||
trail += len(is.Trail)
|
||||
rows = append(rows, Row{Ref: m.Ref, Truth: m.Truth, Day73P: m.Day73P})
|
||||
}
|
||||
fmt.Printf("dry: built=%d of %d, empty body=%d, empty collection=%d, trail entries=%d\n", len(rows), len(pop), emptyBody, emptyColl, trail)
|
||||
fmt.Printf("dry: recorded day-73 AUC through auc() = %.3f (eval reported 0.939)\n", auc(rows, func(r Row) float64 { return r.Day73P }))
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PerpetualSoftware/pad/internal/models"
|
||||
)
|
||||
|
||||
// Replay (TASK-3137 checkpoint 3): rebuild an item as it stood at a moment it
|
||||
// was OPEN, so the 70/70 labels — which describe the need, not the
|
||||
// resolution — are compared with the production question asked at a time
|
||||
// production would actually ask it.
|
||||
|
||||
type version struct {
|
||||
Content string `json:"content"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type activity struct {
|
||||
Action string `json:"action"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Metadata string `json:"metadata"`
|
||||
}
|
||||
|
||||
type transition struct {
|
||||
At time.Time
|
||||
From, To string
|
||||
}
|
||||
|
||||
// terminal is each collection's terminal_options, read from the workspace
|
||||
// schema on 2026-09-22 (human-tasks: done; tasks: done, cancelled).
|
||||
var terminal = map[string]map[string]bool{
|
||||
"human-tasks": {"done": true},
|
||||
"tasks": {"done": true, "cancelled": true},
|
||||
}
|
||||
|
||||
func statusTransitions(acts []activity) []transition {
|
||||
var out []transition
|
||||
for _, a := range acts {
|
||||
var md struct {
|
||||
Changes string `json:"changes"`
|
||||
}
|
||||
_ = json.Unmarshal([]byte(a.Metadata), &md)
|
||||
for _, part := range strings.Split(md.Changes, "; ") {
|
||||
rest, ok := strings.CutPrefix(part, "status: ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
from, to, ok := strings.Cut(rest, " → ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, transition{At: a.CreatedAt, From: strings.TrimSpace(from), To: strings.TrimSpace(to)})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out
|
||||
}
|
||||
|
||||
// replayed returns the item and comments as they stood at the replay point,
|
||||
// or excluded=true with a reason.
|
||||
func replayed(dump, mode string, item models.Item, comments []models.Comment) (models.Item, []models.Comment, string) {
|
||||
var versions []version
|
||||
var acts []activity
|
||||
if err := readJSON(filepath.Join(dump, "items", item.Ref+".history.json"), &versions); err != nil {
|
||||
return item, nil, "history: " + err.Error()
|
||||
}
|
||||
if err := readJSON(filepath.Join(dump, "items", item.Ref+".activity.json"), &acts); err != nil {
|
||||
return item, nil, "activity: " + err.Error()
|
||||
}
|
||||
sort.Slice(versions, func(i, j int) bool { return versions[i].CreatedAt.Before(versions[j].CreatedAt) })
|
||||
trans := statusTransitions(acts)
|
||||
term := terminal[item.CollectionSlug]
|
||||
cur := statusOf(item.Fields)
|
||||
|
||||
setStatus := func(it models.Item, s string) models.Item {
|
||||
var f map[string]any
|
||||
_ = json.Unmarshal([]byte(it.Fields), &f)
|
||||
if f == nil {
|
||||
f = map[string]any{}
|
||||
}
|
||||
f["status"] = s
|
||||
b, _ := json.Marshal(f)
|
||||
it.Fields = string(b)
|
||||
return it
|
||||
}
|
||||
|
||||
switch mode {
|
||||
case "r0":
|
||||
if len(versions) > 0 {
|
||||
item.Content = versions[0].Content
|
||||
}
|
||||
initial := cur
|
||||
if len(trans) > 0 {
|
||||
initial = trans[0].From
|
||||
}
|
||||
if term[initial] {
|
||||
return item, nil, "created terminal"
|
||||
}
|
||||
return setStatus(item, initial), nil, ""
|
||||
case "r1":
|
||||
var t1 *transition
|
||||
for i := range trans {
|
||||
if term[trans[i].To] && !term[trans[i].From] {
|
||||
t1 = &trans[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if t1 == nil {
|
||||
if term[cur] {
|
||||
return item, nil, "closed, no logged terminal transition"
|
||||
}
|
||||
return item, comments, "" // still open: its current state
|
||||
}
|
||||
content := ""
|
||||
found := false
|
||||
for _, v := range versions {
|
||||
if v.CreatedAt.Before(t1.At) {
|
||||
content, found = v.Content, true
|
||||
}
|
||||
}
|
||||
if !found && len(versions) > 0 {
|
||||
content = versions[0].Content
|
||||
}
|
||||
item.Content = content
|
||||
var before []models.Comment
|
||||
for _, c := range comments {
|
||||
if c.CreatedAt.Before(t1.At) {
|
||||
before = append(before, c)
|
||||
}
|
||||
}
|
||||
return setStatus(item, t1.From), before, ""
|
||||
}
|
||||
return item, comments, "unknown replay mode"
|
||||
}
|
||||
|
||||
// replayDry is the replay's instrument check: exclusions by class, the
|
||||
// replayed status distribution (none may be terminal), trail sizes, and a
|
||||
// sample line per item for eyeballing.
|
||||
func replayDry(dump, mode string) {
|
||||
var pop []member
|
||||
mustRead(filepath.Join(dump, "population.json"), &pop)
|
||||
exPos, exNeg, keptPos, keptNeg, terminalLeak := 0, 0, 0, 0, 0
|
||||
statuses := map[string]int{}
|
||||
reasons := map[string]int{}
|
||||
for i, m := range pop {
|
||||
var item models.Item
|
||||
var comments []models.Comment
|
||||
mustRead(filepath.Join(dump, "items", m.Ref+".json"), &item)
|
||||
mustRead(filepath.Join(dump, "items", m.Ref+".comments.json"), &comments)
|
||||
sort.Slice(comments, func(i, j int) bool { return comments[i].CreatedAt.Before(comments[j].CreatedAt) })
|
||||
it, cs, ex := replayed(dump, mode, item, comments)
|
||||
if ex != "" {
|
||||
reasons[ex]++
|
||||
if m.Truth {
|
||||
exPos++
|
||||
} else {
|
||||
exNeg++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if m.Truth {
|
||||
keptPos++
|
||||
} else {
|
||||
keptNeg++
|
||||
}
|
||||
s := statusOf(it.Fields)
|
||||
statuses[item.CollectionSlug+":"+s]++
|
||||
if terminal[item.CollectionSlug][s] {
|
||||
terminalLeak++
|
||||
}
|
||||
if i%20 == 0 {
|
||||
println(m.Ref, "now:", statusOf(item.Fields), "replayed:", s, "body:", len(it.Content), "of", len(item.Content), "trail:", len(cs), "of", len(comments))
|
||||
}
|
||||
}
|
||||
println("kept pos", keptPos, "neg", keptNeg, "| excluded pos", exPos, "neg", exNeg, "| terminal leaks", terminalLeak)
|
||||
for k, v := range reasons {
|
||||
println(" excluded:", k, v)
|
||||
}
|
||||
for k, v := range statuses {
|
||||
println(" replayed status", k, v)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user