From 5aa9e49ab45a0ce714caece593ddbbade035f65e Mon Sep 17 00:00:00 2001 From: xarmian Date: Wed, 21 Oct 2026 10:46:44 -0400 Subject: [PATCH] feat(needs): importance and simplicity scorers on /me/needs (TASK-3959, part 1b) (#2273) * feat(needs): importance and simplicity scorers on /me/needs (TASK-3959, part 1b) Registers the two needs scorers in server.New. Each fills needs.Score {Value, Hint, Why} on every row, reading through the part 1a batch reads and a request-scoped memo, so one request does each read once. - Importance: round(100 x (.25P + .20B + .20D + .15S + .10H + .10A)). B is a breadth-first fan-out over blocks links (visited set, depth 5, cap 99). It counts and expands only targets visible under the scope and still open, and returns a count, never refs. S is 0 until the high_stakes eval (TASK-4011, ruling 440f5c63), so importance is capped at 85 and not renormalised. - Simplicity: (3 - effort)/3 x 100. The effort level is the argmax and the hint comes from a fixed table (cap 40). With no decision runner, effort is the text heuristic, with no hint. - needs.Viewer gains Now, set from asOf, as the scorers' clock. - Why emits only a fixed vocabulary, listed in docs/api.md; a test holds it to that list. The PLAN-3535 guard classes the new IsTerminalItem call as "keep". Rulings 99c2bcfe and 6901f2f5 on the TASK-3959 trail. Claude-Session: https://claude.ai/code/session_01UghmjfBZb6A8UbRgi4rPaw * fix(needs): pass the workspace to the scoped decision and job reads (TASK-3959, part 1b) Follows part 1a's round-1 fix, which scopes LatestDecisionsForItems and OwedDecisionJobs to a workspace. The needs memo passes its own workspace id. Claude-Session: https://claude.ai/code/session_01UghmjfBZb6A8UbRgi4rPaw * fix(needs): fail closed on scope and unknown collections, local-day deadline, request ctx (TASK-3959, part 1b r1) Leak test through the router for a restricted viewer, fail-closed fan-out pins, isOpen fails closed for an unknown collection, the question-hash read takes the request ctx, D measures the server-local day like the dashboard, H is clamped to [0,1], and docs/api.md's degraded prose names the real causes. Claude-Session: https://claude.ai/code/session_01UghmjfBZb6A8UbRgi4rPaw * fix(needs): pin every importance input, overdue parity with the dashboard, snapped rounding (TASK-3959, part 1b r2) Per-input contribution table; overdue decided by itemOverdue; weighted sum snapped before rounding; injectable location instead of mutating time.Local; restricted-viewer score compared exactly to the hidden-item-deleted score; docs: overdue rule and the WorkspaceFlagsForItems degraded path. Claude-Session: https://claude.ai/code/session_01UghmjfBZb6A8UbRgi4rPaw * test(needs): pin the deadline key pick, priority case and unreadable effort; scores at as_of (TASK-3959, part 1b r3) - The importance parts table pins the earlier of due_date and end_date (either key earlier, end_date alone) and the case-insensitive priority. - A current effort answer whose level cannot be read falls back to the text heuristic with no hint; a test now pins it, and docs/api.md says so. - docs/api.md: scores are computed at the request's as_of, which the cursor carries to later pages. Claude-Session: https://claude.ai/code/session_014QJdFAhYDf79NBf2udvfdY * test(needs): as_of reaches the scorers through the handler; threshold, key and date pins (TASK-3959, part 1b r4) - A request with a cursor scores at the cursor's as_of, end to end through ServeHTTP (page 1 at the wall clock is 33; a cursor a week after waiting_since is 28). - "judged to need a human" from the attention threshold up, not below. - Unreadable effort keys -1 and x fall back to the heuristic, like 7. - A date too short to parse scores D=0. - blocks() states the lock it runs under. Claude-Session: https://claude.ai/code/session_014QJdFAhYDf79NBf2udvfdY * test(needs): the as_of check is its own test, so the binding test stays one claim (TASK-3959, part 1b r4) Claude-Session: https://claude.ai/code/session_014QJdFAhYDf79NBf2udvfdY * test(needs): a failed read degrades both real scorers; short-date overdue; effort without a medium (TASK-3959, part 1b r5) Claude-Session: https://claude.ai/code/session_014QJdFAhYDf79NBf2udvfdY * docs(needs): every scorer sharing a memo passes the same rows (TASK-3959, part 1b r6) Claude-Session: https://claude.ai/code/session_014QJdFAhYDf79NBf2udvfdY * fix(needs): needs-human weighs only an answer the inbox's flag rule holds current (TASK-3959, part 1b r7) H and "judged to need a human" read needs_human through the flag path (WorkspaceFlagsForItems at threshold 0 over the rows), so an answer made stale by an edit, with no refresh owed, no longer weighs or says so while the router does not flag the item. Effort keeps question-hash currency; docs/api.md states both rules. EffortMediumMinProb is pinned at exactly .5. Claude-Session: https://claude.ai/code/session_014QJdFAhYDf79NBf2udvfdY * test(needs): a needs-human answer held while its refresh is owed; the asked-row exception in the docs (TASK-3959, part 1b r8) Claude-Session: https://claude.ai/code/session_014QJdFAhYDf79NBf2udvfdY --- docs/api.md | 5 +- internal/decision/attention.go | 12 +- .../plan3535_lifecycle_sites_guard_test.go | 1 + internal/needs/needs.go | 11 +- internal/server/handlers_me_needs.go | 5 +- internal/server/needs_scorers.go | 569 +++++++++++++ internal/server/needs_scorers_test.go | 774 ++++++++++++++++++ internal/server/server.go | 11 +- 8 files changed, 1375 insertions(+), 13 deletions(-) create mode 100644 internal/server/needs_scorers.go create mode 100644 internal/server/needs_scorers_test.go diff --git a/docs/api.md b/docs/api.md index 728f944eb..26da4b33d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -111,13 +111,14 @@ One browser tab's content writes to an item are ordered by the server, also **pe - `POST /workspaces/{ws}/members/invite` — invite user to workspace. `role` defaults to `editor`; any value other than `owner`, `editor` or `viewer` answers `400 bad_request` ("role must be owner, editor or viewer"), the same answer the member PATCH gives (BUG-3861; it used to be a 500 from the column's CHECK constraint). An invitation records its inviter, so a request with no resolved user is refused (BUG-3544): before the first account exists it answers `409 setup_required` ("This Pad instance must be initialized with pad auth setup"), as sign-in does, and otherwise `401 unauthorized`. It used to answer 500 from the inviter foreign key. A caller restricted to specific collections gets `403 forbidden` (BUG-3634): an invitation carries no scope, and the member it makes sees every collection. The same rule applies to `PATCH /workspaces/{ws}/members/{userID}`: a restricted owner may change the role only of a member whose collection scope lies within its own, and otherwise gets `403` (the target sees every collection) or `404` (a collection the caller cannot see). The scope is checked again under the row lock the write takes: if another owner widened the target in between, the change answers `409 member_scope_changed` (reload and retry) - `GET /api/v1/me/invitations`, `POST /api/v1/me/invitations/{id}/accept` — the caller's pending, unexpired invitations to live workspaces they are not already in (PLAN-3002 U4b / TASK-3277), backing the Invitations section of the TopBar "+". Matched on the caller's email ONLY when it is VERIFIED, since an unverified address is just a string typed at signup; every self-hosted account is verified (migration 070 + `CreateUser`'s default), so this bites only on Pad Cloud self-serve accounts that have not confirmed, which get an empty list with `email_verified: false`. Entries carry no code (only its hash is stored), hence the by-id accept, which shares `acceptInvitationCore` with `/invitations/{code}/accept` (membership, role, `accepted_at`, the `workspace_access_changed` gain). It differs in admission only: the caller must already be verified, and it never verifies them. Neither does the code route, since BUG-3348: the inviter also sees the code. What verifies is the mailbox-only proof the invitation EMAIL's link carries in its fragment (`/join/#proof=…`, TASK-3352), sent as `proof` on `POST /invitations/{code}/accept` or `invitation_proof` on register. It is stored hashed, single-use and bound to the invitation, and never appears in the invite response, the members or admin invitation lists, or copy-link; a wrong or spent one is ignored, not refused. An id that is not a pending invitation to the caller answers 404, byte-identical to an unknown id. Both accept routes' success bodies now also carry `workspace_slug` and `owner_username` (additive). An accept by a caller who is ALREADY a member, and the loser of two concurrent accepts, answers 200 and marks the invitation accepted without changing the member's role in either direction; the body's `role` is the role held, and nothing is published (BUG-3281; it used to be a 500). Every accept door (both of these and register-with-code) re-reads the inviter's authority: if the inviter is now restricted to specific collections (BUG-3634), or is no longer an owner of the workspace because they were removed or demoted (BUG-3770; there is no self-leave door), the accept is refused with `403 inviter_scope_changed`, and the invitation stays pending. It is refused rather than narrowed, because the invitation has no scope to narrow to. Instance admins are exempt. A caller who is ALREADY a member is not refused (BUG-3870): the invitation grants them nothing, so the accept is BUG-3281's idempotent 200 (marked accepted, role unchanged). The member check runs only after the inviter check refuses, and it never inserts; on Postgres it share-locks the member row, so a concurrent removal of the invitee is ordered against it. The list above HIDES an invitation the accept would refuse (hidden, not deleted: it returns if the inviter's authority does), and `GET /api/v1/invitations/{code}/preview` reports `inviter_can_grant: false` for one (additive; absent from older servers, which reads as true), except to the invitee's own signed-in session when they are already a member, who reads `true` and is offered the Accept (BUG-3870; an anonymous code holder or another signed-in user reads `false`, so membership is not disclosed before auth), which `/join` shows as "can no longer grant access" with no Accept and no form. The web client (the TopBar "+" and its count badge) and the CLI (`pad workspace invitations` / `accept ` / `decline `, BUG-2136) use them; not on MCP - `GET /api/v1/tutorials`, `GET /api/v1/tutorials/posters/{slug}` — in-app tutorials (TASK-3452), **Pad Cloud only**: a self-hosted server answers both with 404 and makes no outbound request, and its UI links to getpad.dev/learn instead. On Cloud the server fetches `https://www.getpad.dev/learn/catalog.json` (a constant; pad-web builds it from the same catalog as getpad.dev/learn, so a new tutorial needs no Pad release), caches it for an hour, keeps the last good copy, backs off 5 minutes after a failure, and runs one refresh at a time without holding the lock: a caller with a copy gets it at once. `available: false` (no copy at all) makes the UI fall back to links. Posters are proxied same-origin from `https://www.getpad.dev/` only, never following a redirect, because the page CSP keeps `img-src` and `connect-src` to itself. The page CSP gains `frame-src https://www.youtube-nocookie.com` on Cloud only, for the click-to-load player at `/console/tutorials`. Web client only -- `GET /api/v1/me/needs?sort=age&workspace=&view=active|hidden&limit=50&cursor=` — the cross-workspace needs-you inbox (TASK-3920, IDEA-3906 half A unit A4): the open items that need THE CALLER, oldest-waiting first, across every workspace they are a member or guest of. Returns `{rows, next_cursor?, total, hidden, workspaces: [{slug, name, owner_username, count}], degraded?}`; a row is `{workspace, item: {ref, title, collection, collection_slug, status?, priority?, created_at, updated_at, path}, why: {rule, text, by?, at, comment_id?}, because: [{kind: human_task|needs_human|mention, at, by?, comment_id?}], needs_human?, waiting_since, marker, snooze?, importance?, simplicity?}` (the last two are scores, `{value: 0-100, hint?, why?}`, absent until TASK-3959 registers a scorer). `401` with no user; `404` for an app principal (it has its own door). `sort` takes `age` only, `view` `active` (default) or `hidden` (the caller's holding snoozes), `limit` defaults to 50 and is clamped to 200; a bad value is `400 validation_error`. A workspace-pinned token (BUG-3982) is shown that workspace only; that filter predates the general pin rule below. +- `GET /api/v1/me/needs?sort=age&workspace=&view=active|hidden&limit=50&cursor=` — the cross-workspace needs-you inbox (TASK-3920, IDEA-3906 half A unit A4): the open items that need THE CALLER, oldest-waiting first, across every workspace they are a member or guest of. Returns `{rows, next_cursor?, total, hidden, workspaces: [{slug, name, owner_username, count}], degraded?}`; a row is `{workspace, item: {ref, title, collection, collection_slug, status?, priority?, created_at, updated_at, path}, why: {rule, text, by?, at, comment_id?}, because: [{kind: human_task|needs_human|mention, at, by?, comment_id?}], needs_human?, waiting_since, marker, snooze?, importance?, simplicity?}` (the last two are scores, `{value: 0-100, hint?, why?}`, described under **Scores** below; absent on a row of a `degraded` workspace whose scorer failed). `401` with no user; `404` for an app principal (it has its own door). `sort` takes `age` only, `view` `active` (default) or `hidden` (the caller's holding snoozes), `limit` defaults to 50 and is clamped to 200; a bad value is `400 validation_error`. A workspace-pinned token (BUG-3982) is shown that workspace only; that filter predates the general pin rule below. - **Routing** (`internal/needs.Resolve`): a candidate is an open item in a live, non-system collection that is an ASK (the collection's `awaits_human` setting), carries a `needs_human` attention answer, or holds an OPEN @mention (reference collections included). The first rule that yields an ELIGIBLE person wins, and only that rule's people are routed (exclusive): `asked` (an ask whose assigned person is eligible), `mentioned` (eligible people with an open mention), `assigned` (the assigned person; a role-only assignment falls through), `admin_fallback` (the eligible workspace owners). Nobody eligible drops the item, never "everyone". ELIGIBLE means a live human account (not disabled, not an app principal) who may see the item under membership and grants alone, the bearer rule: a platform admin counts by membership, not by role. `why.text` is written from the winning rule only and never names a person the rules skipped. A mention is open until the mentioned person writes a later human comment on the item (a reply in the same second counts); editing the mention away and back is a new mention. - **waiting_since** is the earliest open claim, and `sort=age` orders by it, then the item's `created_at`, then the item id (a total order). The claims: for `asked` and `admin_fallback`, the item's `created_at`; for `assigned`, the latest assignment activity row naming that assignee (the trail stores no user id, so the assignee is matched by DISPLAY NAME and two users sharing one can be confused; the effect is the row's age and sort position only, never who can see it), falling back to `created_at` when there is none (an item created already assigned has no row), so an old item assigned today does not sort as the oldest need; for `mentioned`, the mention's `created_at` (the edit time when an edit added it); and, for any rule, the `needs_human` answer's `evaluated_at`, so a re-ask that writes a NEW decision row moves it. - **Visibility comes first.** Candidates are read under the caller's own visibility (collection access, item grants), so a mention, a snooze, a flag or a count is only ever read for an item the caller can see, and `total`, `hidden` and `workspaces[].count` are derived from the filtered set. A snooze is joined through `items.workspace_id` (never trusted from the snooze row), so an item that moved workspaces leaves its snooze on the archived source and the clone is unaffected; a snooze on an item the caller can no longer see is neither shown nor counted. `hidden` counts the caller's holding snoozes; `workspaces` lists only workspaces with rows in the requested view. - **Admission.** Workspaces come from the caller's memberships plus their guest workspaces, narrowed to a token's consent allow-list (BUG-2102), and, for a token bound to one workspace, to that workspace alone (stricter than `/search`, which ignores the binding for a user-bound token: BUG-3982). `?workspace=` that is unknown, not admitted, token-excluded or one the caller is not in all return the SAME empty `200` (`{rows: [], total: 0, hidden: 0, workspaces: []}`), byte for byte, as a workspace with nothing waiting. - **Pagination.** `next_cursor` is an opaque token, present only when more rows follow; omit it to start. It carries a position (`v`, sort, view, workspace filter, `as_of`, the sort key and the item id) and no authority: every page re-runs the whole filtered pipeline and takes the rows strictly after the key, and `as_of` (set on page 1) is the time defers are tested at, so one expiring between pages does not reshuffle them; an `as_of` later than the server's now is clamped to now. A malformed cursor, or one minted for a different sort, view or `workspace`, is `400 invalid_cursor`. A row's `marker` is the item's state now, to send back when snoozing it. - - **`degraded`** lists, by slug, the admitted workspaces whose score provider failed (or whose attention decisions could not be read). It is set only on a provider ERROR, never from the decision provider's failing bit: that bit covers items the caller cannot see, and exposing it would leak one bit about them. The workspace's other rows still return. + - **Scores** (TASK-3959, IDEA-3906 B4b). Both are computed per request for the caller, from what the caller may see, and stored nowhere. They are computed at the request's `as_of` instant: the first page's, which the cursor carries to later pages, so every page measures "today", deadlines and age from the same moment (and the dashboard agreement below holds at that instant). `importance` is `round(100 * (.25 P + .20 B + .20 D + .15 S + .10 H + .10 A))`, each part 0-1: P priority (critical 1, high .75, medium or unset .5, low .25); B blocks, the count n of distinct open items the row's item blocks directly or through others (breadth-first, depth 5, stops at 99, a hidden or terminal item, or one whose collection cannot be found, is neither counted nor walked through), `min(1, log2(1+n)/log2(17))`; D deadline, the earlier of `due_date` and `end_date` (1 when overdue or today, falling to 0 over 14 days, 0 with no date; "overdue" is decided by the dashboard's own rule (`itemOverdue` against the server-local calendar day), so the two agree for every stored value: a timestamp dated today is not overdue (it is "due today"), and a malformed date such as `2026-02-30` compares as a string and is overdue; only "due today" and the slope parse the date); S stakes; H whether a human is needed (1 for an `asked` row, else the `needs_human_decision` probability clamped to 0-1 from an answer the inbox's own flag rule holds current, by question hash AND item state (or held while its refresh is owed), .5 with none; so on a row that is not `asked`, `judged to need a human` appears exactly when that rule flags the item); A age, days since `waiting_since` over 14, capped at 1. **S is held at 0** (the `high_stakes` answer is stored but not read, pending its labelled eval, TASK-4011), so importance tops out at 85 and is NOT renormalised: rescaling would move every score the day S switches on. `simplicity` is `round((3 - effort) / 3 * 100)` for the effort level 0-3 (100, 67, 33, 0). The level is the highest-probability level of the stored `effort` answer, so `value` and `hint` always describe the same level, when that answer is to the question as asked now (its question hash matches; a re-pinned model or reworded question makes it stale). **For effort, currency is the question hash only**: an answer one edit behind the item is still used, and while a job is owed for the item `why` says the estimate is being refreshed. With no decision provider, or no current answer whose level it can read (0 to 3), the level is a text heuristic (0 for a title ending in `?` with a body of at most 600 characters; otherwise 1 to 1,500 characters, 2 to 6,000, else 3) and `hint` is absent. No provider is called during the request. `hint` (at most 40 characters) is a fixed phrase for the level, `yes/no`, `5 min`, `under an hour` or `research`, plus, from a current `effort_medium` answer with probability of at least .5, ` at desk`, `, needs a device` or `, needs someone else`. `why` is a FIXED vocabulary, never a title, ref, count or other free text. Importance: `priority critical`, `priority high`, `priority low`, `blocks other open work`, `overdue`, `due today`, `due within two weeks`, `waiting a week or more`, `waiting two weeks or more`, `asked of you`, `judged to need a human`. Simplicity: `estimated from the text`, `estimate is being refreshed`. + - **`degraded`** lists, by slug, the admitted workspaces whose scorer failed or whose attention-flag read (`needs_human`, the provider's per-item answers) errored: a scorer's batch read (items, blocks links, collections, attention answers, owed jobs, the flag rule's currency verdict on the rows' answers) or its question-hash read errored. The importance and simplicity scorers call no decision provider. It is set only on such a read ERROR, never from the decision provider's failing bit: that bit covers items the caller cannot see, and exposing it would leak one bit about them. The workspace's other rows still return. - `GET /api/v1/me/ui-dismissals`, `PUT /api/v1/me/ui-dismissals/{key}` — one-time UI suggestions the caller dismissed (TASK-3452: the first-time tutorial cards on /console and the workspace launchpad), stored on the user (`users.ui_dismissals`, a JSON set) so a dismissal holds on every device. Keys are a closed set (`models.UIDismissalKeys`); any other answers 400. A column rather than a table by ruling, until a third kind of dismissal appears. Web client only - `PUT|DELETE /api/v1/me/push-devices/{connection_id}` — register or remove the caller's push device (TASK-3938, PLAN-3908 phase 2). **Session-only**: a cookie or a `padsess_` bearer. An API token, an OAuth bearer (also with a session cookie riding along), a token with a cookie, and an app principal all get `403 session_required`; a workspace-pinned API token is refused before that by the BUG-3982 gate (`403 permission_denied`, `details.reason: "token_workspace_pinned"`); no credential is `401`. After that gate, `404 push_unavailable` when the server has no gateway (the app stops trying that server until its next start, and treats any other 404 as a retry, so the error code is a contract), then `400 validation_error` unless `connection_id` matches `^[A-Za-z0-9-]{1,64}$`. PUT body is strict: `{platform: "ios"|"android", token, app_version?, environment?}`; `token` is non-empty, at most 4096 bytes and NUL-free, `app_version` at most 64 characters, an unknown or repeated member is `400`. `environment` is accepted for client compatibility and IGNORED: every token is an FCM token and FCM routes APNs itself. Success is `200 {id, connection_id, platform, app_version, updated_at}`; DELETE is `204`, also when nothing was registered, and removes only the device bound to the caller's own session under that connection id. **Register is one transactional replace**: it deletes every row on EITHER unique key (the caller's session's own row, whatever its token, and this user's row for this token, whatever its session, even one whose session is still alive), evicts, then inserts, so re-registering after a re-sign-in or an FCM token rotation leaves exactly one row, on the caller's session. **Cap: 20 devices per user.** At the cap the device with the oldest `updated_at` (ties: `created_at`, then id) is evicted SILENTLY (a debug log of the device id, never the token); replacing the caller's own row evicts nothing. A session that is gone or expired between admission and the write is `401`. **A device lives and dies with its session**: logout, session expiry and the expired-session sweep delete the row (`push_devices.session_id` cascades), and so does deleting the account. **A credential change** (password change, 2FA disable, provider unlink) rotates sessions in one transaction (`Store.RotateUserSessions`): the CALLER's devices are re-bound to the replacement session and survive, while devices on the user's OTHER sessions die with those sessions, since the change exists to cut off stolen sessions and their push tokens go with them. If that transaction fails every session is revoked and the call answers 500 (fail closed). Self-hosted servers send nothing unless they configure a gateway. - `GET|PUT /api/v1/me/notification-settings` — the caller's recipient-side push filters (TASK-3938): `{push_enabled, notify_assigned, notify_comments, notify_replies, notify_reminders, notify_mentions, include_agent_activity}`, seven booleans. GET returns the defaults (everything on except `include_agent_activity`) when the user never saved any. PUT is a full replace: all seven keys are required, and a missing, unknown, null or non-boolean one is `400`, so nothing is silently reset. Available to any authenticated person, an API token included; an app principal is `403`, and so is a token pinned to one workspace, refused by the BUG-3982 gate (`403 permission_denied`, `details.reason: "token_workspace_pinned"`) because these are account-level settings. Works without a push gateway. diff --git a/internal/decision/attention.go b/internal/decision/attention.go index bb44ac197..202f28555 100644 --- a/internal/decision/attention.go +++ b/internal/decision/attention.go @@ -31,11 +31,13 @@ import ( // THREE MORE QUESTIONS are STAGED here (TASK-3921, IDEA-3906 half B4a): // effort, effort_medium and high_stakes. They ride the same call as // needs_human_decision, so the cost is output tokens, not a second request. -// Nothing READS them yet: their reader is TASK-3959 (B4b), so the set lists -// them in Unsurfaced and no flag, chip or brief entry is made from them. Their -// answers are stored, and the item's decisions list returns them. The read -// rule that belongs to their reader, not to the set: an effort_medium answer -// whose top probability is under .5 means "unclear". +// Their reader is TASK-3959 (B4b): the needs-you scorers read effort and +// effort_medium (effort_read.go), so the set lists all three in Unsurfaced and +// no flag, chip or brief entry is made from them. high_stakes is read by +// nothing yet (its labelled eval is TASK-4011). Their answers are stored, and +// the item's decisions list returns them. The read rule that belongs to their +// reader, not to the set: an effort_medium answer whose top probability is +// under .5 means "unclear" (EffortMediumMinProb). // // The eval's state was {title, body}; the runner sends the richer // BuildItemState (collection, fields, and the recent trail as well), so the diff --git a/internal/models/plan3535_lifecycle_sites_guard_test.go b/internal/models/plan3535_lifecycle_sites_guard_test.go index 06d431c85..4742aa2a7 100644 --- a/internal/models/plan3535_lifecycle_sites_guard_test.go +++ b/internal/models/plan3535_lifecycle_sites_guard_test.go @@ -83,6 +83,7 @@ var lifecycleSites = map[string]string{ "internal/server/handlers_reminders.go::collectPendingRemindersBounded::isItemDone": "keep: a reminder is the user's own instruction, on any item", "internal/store/item_stars.go::isTerminalWithContext::IsTerminalItem": "keep: starred items are the user's own picks", "internal/server/handlers_me_needs.go::needsForWorkspace::IsTerminalItem": "keep: an @mention is a real ask of a person in any non-system collection, reference collections included (TASK-3920, lead ruling Q2); only a terminal item is dropped", + "internal/server/needs_scorers.go::isOpen::IsTerminalItem": "keep: the importance scorer counts what an item BLOCKS over explicit blocks links, as the dashboard keeps a reference blocker; an explicit link names held-up work in any collection, and only a terminal item is left out (TASK-3959)", } func TestPLAN3535_LifecycleHelperCallSitesAreClassified(t *testing.T) { diff --git a/internal/needs/needs.go b/internal/needs/needs.go index d472f4ac0..edae1aa32 100644 --- a/internal/needs/needs.go +++ b/internal/needs/needs.go @@ -235,6 +235,10 @@ type Score struct { type Viewer struct { UserID string AuthIsBearer bool + // Now is the clock a scorer measures age and deadlines against: the + // request's as_of, so every page of one listing scores at one instant. + // The zero value means the wall clock. + Now time.Time } // Scope is what the viewer may see in one workspace, in the shape the stores @@ -247,9 +251,10 @@ type Scope struct { ItemIDs []string } -// Scorer fills a score on rows (TASK-3959 registers the real ones). It is -// called once per workspace, after visibility, routing and snooze and before -// sort and pagination, and must not call a provider synchronously. degraded +// Scorer fills a score on rows (the importance and simplicity scorers of +// TASK-3959 are the registered ones). It is called once per workspace, after +// visibility, routing and snooze and before sort and pagination, and must not +// call a provider synchronously. degraded // reports that its source failed and the scores came back partial or empty: // the server flags that workspace only. type Scorer interface { diff --git a/internal/server/handlers_me_needs.go b/internal/server/handlers_me_needs.go index 230053519..45834a167 100644 --- a/internal/server/handlers_me_needs.go +++ b/internal/server/handlers_me_needs.go @@ -536,8 +536,11 @@ func (s *Server) needsForWorkspace(r *http.Request, user *models.User, ws *model if nscope.ItemIDs == nil { nscope.ItemIDs = []string{} } + // The scorers share this workspace's batch reads through a memo that lives + // for this call alone (see needsMemo). + sctx := withNeedsMemo(r.Context(), s.newNeedsMemo(ws.ID)) for _, sc := range s.needsScorers { - deg, serr := sc.Score(r.Context(), needs.Viewer{UserID: user.ID, AuthIsBearer: isBearerAuth(r)}, nscope, rows) + deg, serr := sc.Score(sctx, needs.Viewer{UserID: user.ID, AuthIsBearer: isBearerAuth(r), Now: asOf}, nscope, rows) if serr != nil { slog.Error("needs: scorer failed; workspace degraded", "scorer", sc.Name(), "workspace_id", ws.ID, "error", serr) } diff --git a/internal/server/needs_scorers.go b/internal/server/needs_scorers.go new file mode 100644 index 000000000..7e7a2e82e --- /dev/null +++ b/internal/server/needs_scorers.go @@ -0,0 +1,569 @@ +package server + +import ( + "context" + "encoding/json" + "math" + "strings" + "sync" + "time" + + "github.com/PerpetualSoftware/pad/internal/decision" + "github.com/PerpetualSoftware/pad/internal/models" + "github.com/PerpetualSoftware/pad/internal/needs" + "github.com/PerpetualSoftware/pad/internal/store" +) + +// The importance and simplicity scorers on GET /me/needs (TASK-3959, IDEA-3906 +// B4b). Both fill needs.Row scores after visibility, routing and snooze, and +// neither calls a provider: simplicity reads the effort answers the decision +// runner stored, and falls back to a text heuristic. The scores are computed at +// read time for the viewer and stored nowhere. +// +// Score.Why carries a FIXED vocabulary (needsWhyVocabulary, listed in +// docs/api.md): no title, ref, count or other free text ever enters it. + +// Why strings. Every string a scorer may emit is in needsWhyVocabulary. +const ( + whyPriorityCritical = "priority critical" + whyPriorityHigh = "priority high" + whyPriorityLow = "priority low" + whyBlocksWork = "blocks other open work" + whyOverdue = "overdue" + whyDueToday = "due today" + whyDueSoon = "due within two weeks" + whyWaitingWeek = "waiting a week or more" + whyWaitingTwoWeeks = "waiting two weeks or more" + whyAsked = "asked of you" + whyNeedsHuman = "judged to need a human" + whyEstimated = "estimated from the text" + whyRefreshing = "estimate is being refreshed" +) + +var needsWhyVocabulary = []string{ + whyPriorityCritical, whyPriorityHigh, whyPriorityLow, whyBlocksWork, + whyOverdue, whyDueToday, whyDueSoon, whyWaitingWeek, whyWaitingTwoWeeks, + whyAsked, whyNeedsHuman, whyEstimated, whyRefreshing, +} + +// Importance weights (the design's B3): priority, blocks, deadline, stakes, +// needs-human, age. +const ( + wPriority = 0.25 + wBlocks = 0.20 + wDeadline = 0.20 + wStakes = 0.15 + wHuman = 0.10 + wAge = 0.10 +) + +// stakesPart is the high_stakes component (S). It is held at 0 and nothing +// reads or surfaces the high_stakes answer: the question is not trusted until +// its labelled eval has run (lead ruling 440f5c63, filed as TASK-4011). Until +// then importance tops out at 85 and is NOT renormalised, because rescaling +// would move every score the day S switches on. Reading the answer into this +// constant is the one change that switch makes. +const stakesPart = 0.0 + +// The fan-out bounds: breadth-first over blocks, depth at most 5, count at most 99. +const ( + fanoutDepth = 5 + fanoutCap = 99 +) + +// needsHintMax is the longest hint, in bytes (all ASCII). +const needsHintMax = 40 + +// effortHints are the hint heads by effort level; the medium suffix follows. +var effortHints = [4]string{"yes/no", "5 min", "under an hour", "research"} + +// effortMediumSuffix is appended to the hint for a confident effort_medium +// answer; "unclear", an unknown option and a low-probability answer add none. +var effortMediumSuffix = map[string]string{ + "desk": " at desk", + "device": ", needs a device", + "other_person": ", needs someone else", +} + +// needsMemo holds ONE workspace's batch reads for ONE call of the scorers, so +// importance and simplicity read each once. It is built per request per +// workspace and dropped with it: never cached across requests, since every +// input (links, answers, jobs, items) changes between them. +type needsMemo struct { + s *Server + wsID string + runner *decision.Runner + + mu sync.Mutex + items memoSlot[map[string]store.NeedsScoringItem] + edges memoSlot[[]store.BlocksEdge] + colls memoSlot[[]models.Collection] + decs memoSlot[map[string][]models.ItemDecision] + owed memoSlot[map[string]bool] + qhash memoSlot[map[string]string] + human memoSlot[map[string]map[string]float64] + graph *blocksGraph + graphE error +} + +type memoSlot[T any] struct { + done bool + val T + err error +} + +func memoLoad[T any](m *needsMemo, read string, slot *memoSlot[T], fn func() (T, error)) (T, error) { + if !slot.done { + slot.done = true + if m.s.needsMemoLoad != nil { + m.s.needsMemoLoad(m.wsID + ":" + read) + } + slot.val, slot.err = fn() + } + return slot.val, slot.err +} + +func (s *Server) newNeedsMemo(wsID string) *needsMemo { + return &needsMemo{s: s, wsID: wsID, runner: s.decisionRunner()} +} + +type needsMemoKey struct{} + +func withNeedsMemo(ctx context.Context, m *needsMemo) context.Context { + return context.WithValue(ctx, needsMemoKey{}, m) +} + +// memoFor is the memo the handler put in ctx for this workspace, or a fresh one +// for a caller (a test) that did not. +func (s *Server) memoFor(ctx context.Context, wsID string) *needsMemo { + if m, ok := ctx.Value(needsMemoKey{}).(*needsMemo); ok && m.wsID == wsID { + return m + } + return s.newNeedsMemo(wsID) +} + +func rowIDs(rows []*needs.Row) []string { + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.ItemID + } + return ids +} + +// itemsFor reads the rows' items once and returns that read to every later +// caller, whatever rows it passes: every scorer sharing a memo must pass the +// same row slice (the handler does). The same holds for answers. +func (m *needsMemo) itemsFor(rows []*needs.Row) (map[string]store.NeedsScoringItem, error) { + return memoLoad(m, "items", &m.items, func() (map[string]store.NeedsScoringItem, error) { + return m.s.store.NeedsScoringItems(m.wsID, rowIDs(rows)) + }) +} + +// blocks is the workspace's blocks graph, built once. Like every memo read it +// runs under m.mu, which both scorers' Score methods hold for their whole call. +func (m *needsMemo) blocks() (*blocksGraph, error) { + if m.graph != nil || m.graphE != nil { + return m.graph, m.graphE + } + edges, err := memoLoad(m, "blocks_edges", &m.edges, func() ([]store.BlocksEdge, error) { return m.s.store.GetBlocksEdges(m.wsID) }) + if err == nil { + var colls []models.Collection + colls, err = memoLoad(m, "collections", &m.colls, func() ([]models.Collection, error) { return m.s.store.ListCollections(m.wsID) }) + if err == nil { + m.graph = newBlocksGraph(edges, colls) + } + } + m.graphE = err + return m.graph, err +} + +// answers returns the latest attention answers of the rows' items, the owed +// jobs, and the question hashes as the set asks them now. All three are nil +// without a runner: no provider, no answers worth trusting. +func (m *needsMemo) answers(ctx context.Context, rows []*needs.Row) (decs map[string][]models.ItemDecision, owed map[string]bool, qh map[string]string, err error) { + if m.runner == nil { + return nil, nil, nil, nil + } + if qh, err = memoLoad(m, "question_hashes", &m.qhash, func() (map[string]string, error) { + return m.runner.CurrentQuestionHashes(ctx, m.wsID, decision.AttentionSetName) + }); err != nil { + return + } + if decs, err = memoLoad(m, "decisions", &m.decs, func() (map[string][]models.ItemDecision, error) { + return m.s.store.LatestDecisionsForItems(m.wsID, rowIDs(rows), decision.AttentionSetName) + }); err != nil { + return + } + owed, err = memoLoad(m, "owed_jobs", &m.owed, func() (map[string]bool, error) { + return m.s.store.OwedDecisionJobs(m.wsID, rowIDs(rows), decision.AttentionSetName) + }) + return +} + +// humanCurrent is the inbox's own verdict on which of the rows' attention +// answers are current: the flag path (WorkspaceFlagsForItems) at threshold 0 +// returns every answer current by question hash AND item state, or held while +// its refresh is owed, by item and key. H and "judged to need a human" read +// needs_human only through it, so they agree with whether the router counts +// the item as flagged. Its failing bit is not read. Nil without a runner. +func (m *needsMemo) humanCurrent(rows []*needs.Row) (map[string]map[string]float64, error) { + if m.runner == nil { + return nil, nil + } + return memoLoad(m, "attention_current", &m.human, func() (map[string]map[string]float64, error) { + cur, _, err := m.runner.WorkspaceFlagsForItems(m.wsID, decision.AttentionSetName, 0, rowIDs(rows)) + return cur, err + }) +} + +// withCurrentHuman is decs without the needs_human answer unless the flag path +// still holds it current (cur, from humanCurrent). It copies; decs is shared. +func withCurrentHuman(decs []models.ItemDecision, cur map[string]float64) []models.ItemDecision { + if _, ok := cur[decision.AttentionNeedsHuman]; ok { + return decs + } + out := make([]models.ItemDecision, 0, len(decs)) + for _, d := range decs { + if d.QuestionKey != decision.AttentionNeedsHuman { + out = append(out, d) + } + } + return out +} + +// currentAnswer is the item's latest answer for a question key, if it answers +// the question as asked now. +func currentAnswer(decs []models.ItemDecision, qh map[string]string, key string) (models.ItemDecision, bool) { + for _, d := range decs { + if d.QuestionKey == key && qh[key] != "" && d.QuestionHash == qh[key] { + return d, true + } + } + return models.ItemDecision{}, false +} + +// newNeedsScorers is the pair New registers. The runner is read per call, not +// here: it is attached after New (SetDecisionRunner). +func (s *Server) newNeedsScorers() []needs.Scorer { + return []needs.Scorer{&importanceScorer{s: s}, &simplicityScorer{s}} +} + +func nowOr(v needs.Viewer) time.Time { + if v.Now.IsZero() { + return time.Now().UTC() + } + return v.Now +} + +// ---- importance ---- + +// loc is the calendar the deadline part reads "today" in; nil means time.Local, +// the dashboard's own (overdueToday). A field so a test needs no global. +type importanceScorer struct { + s *Server + loc *time.Location +} + +func (*importanceScorer) Name() string { return "importance" } + +func (sc *importanceScorer) Score(ctx context.Context, v needs.Viewer, scope needs.Scope, rows []*needs.Row) (bool, error) { + if len(rows) == 0 { + return false, nil + } + m := sc.s.memoFor(ctx, scope.WorkspaceID) + m.mu.Lock() + defer m.mu.Unlock() + items, err := m.itemsFor(rows) + if err != nil { + return true, err + } + g, err := m.blocks() + if err != nil { + return true, err + } + decs, _, qh, err := m.answers(ctx, rows) + if err != nil { + return true, err + } + hc, err := m.humanCurrent(rows) + if err != nil { + return true, err + } + now := nowOr(v) + vis := newScopeSet(scope) + for _, r := range rows { + r.Importance = importanceIn(sc.loc, r, items[r.ItemID], g.fanout(r.ItemID, vis), withCurrentHuman(decs[r.ItemID], hc[r.ItemID]), qh, now) + } + return false, nil +} + +func importanceOf(r *needs.Row, it store.NeedsScoringItem, blocked int, decs []models.ItemDecision, qh map[string]string, now time.Time) *needs.Score { + return importanceIn(nil, r, it, blocked, decs, qh, now) +} + +// importanceParts are the unweighted inputs, each in [0,1]. +type importanceParts struct{ p, b, d, s, h, a float64 } + +func importanceIn(loc *time.Location, r *needs.Row, it store.NeedsScoringItem, blocked int, decs []models.ItemDecision, qh map[string]string, now time.Time) *needs.Score { + sc, _ := importanceDetail(loc, r, it, blocked, decs, qh, now) + return sc +} + +func importanceDetail(loc *time.Location, r *needs.Row, it store.NeedsScoringItem, blocked int, decs []models.ItemDecision, qh map[string]string, now time.Time) (*needs.Score, importanceParts) { + var why []string + p := 0.5 + switch strings.ToLower(r.Item.Priority) { + case "critical": + p, why = 1, append(why, whyPriorityCritical) + case "high": + p, why = 0.75, append(why, whyPriorityHigh) + case "low": + p, why = 0.25, append(why, whyPriorityLow) + } + b := math.Min(1, math.Log2(1+float64(blocked))/math.Log2(17)) + if blocked > 0 { + why = append(why, whyBlocksWork) + } + d, dwhy := deadlinePart(it.Fields, now, loc) + if dwhy != "" { + why = append(why, dwhy) + } + days := now.Sub(r.WaitingSince).Hours() / 24 + a := math.Max(0, math.Min(1, days/14)) + switch { + case days >= 14: + why = append(why, whyWaitingTwoWeeks) + case days >= 7: + why = append(why, whyWaitingWeek) + } + h := 0.5 + if r.Why.Rule == needs.RuleAsked { + h = 1 + why = append(why, whyAsked) + } else if d, ok := currentAnswer(decs, qh, decision.AttentionNeedsHuman); ok { + if pr, ok := decision.NoulValue(d); ok { + h = math.Max(0, math.Min(1, pr)) + // The router's own threshold (handlers_me_needs.go), so the Why + // agrees with whether the item is flagged. + if h >= decision.AttentionThreshold { + why = append(why, whyNeedsHuman) + } + } + } + val := wPriority*p + wBlocks*b + wDeadline*d + wStakes*stakesPart + wHuman*h + wAge*a + // Snap before rounding: the weighted sum is a float, and a value that is + // exactly x.5 mathematically must not land either side by summation order. + v100 := math.Round(100*val*1e9) / 1e9 + return &needs.Score{Value: int(math.Round(v100)), Why: why}, importanceParts{p, b, d, stakesPart, h, a} +} + +// deadlinePart is D: 1 when overdue or due today, falling to 0 over 14 days, +// 0 with no date. "Overdue" is decided by the dashboard's own itemOverdue over +// the server-local day (overdueToday), so the two agree for every stored value, +// timestamps and malformed dates included. Date parsing only separates "due +// today" from the slope among rows that are not overdue; the earliest of +// due_date and end_date counts. A nil loc is time.Local. +func deadlinePart(fieldsJSON string, now time.Time, loc *time.Location) (float64, string) { + if loc == nil { + loc = time.Local + } + todayStr := overdueToday(now.In(loc)) + if _, _, ok := itemOverdue(fieldsJSON, todayStr); ok { + return 1, whyOverdue + } + today, _ := time.Parse("2006-01-02", todayStr) + best, found := time.Time{}, false + for _, key := range overdueDateFields { + v := extractFieldValue(fieldsJSON, key) + if len(v) < 10 { + continue + } + t, err := time.Parse("2006-01-02", v[:10]) + if err != nil { + continue + } + if !found || t.Before(best) { + best, found = t, true + } + } + if !found { + return 0, "" + } + days := best.Sub(today).Hours() / 24 + switch { + case days <= 0: + return 1, whyDueToday + case days < 14: + return 1 - days/14, whyDueSoon + } + return 0, "" +} + +// ---- blocks fan-out ---- + +// scopeSet is what the viewer may see in one workspace, as sets. +type scopeSet struct { + all bool + collections map[string]bool + items map[string]bool +} + +func newScopeSet(sc needs.Scope) scopeSet { + v := scopeSet{all: sc.Unrestricted, collections: map[string]bool{}, items: map[string]bool{}} + for _, id := range sc.CollectionIDs { + v.collections[id] = true + } + for _, id := range sc.ItemIDs { + v.items[id] = true + } + return v +} + +func (v scopeSet) sees(collectionID, itemID string) bool { + return v.all || v.collections[collectionID] || v.items[itemID] +} + +// blocksGraph is a workspace's blocks links by blocker, with what it takes to +// tell whether a blocked item is still open. +type blocksGraph struct { + bySource map[string][]store.BlocksEdge + colls map[string]*models.Collection + open map[string]bool +} + +func newBlocksGraph(edges []store.BlocksEdge, colls []models.Collection) *blocksGraph { + g := &blocksGraph{bySource: map[string][]store.BlocksEdge{}, colls: map[string]*models.Collection{}, open: map[string]bool{}} + for _, e := range edges { + g.bySource[e.SourceID] = append(g.bySource[e.SourceID], e) + } + for i := range colls { + g.colls[colls[i].ID] = &colls[i] + } + return g +} + +func (g *blocksGraph) isOpen(e store.BlocksEdge) bool { + if o, ok := g.open[e.TargetID]; ok { + return o + } + var fields map[string]any + _ = json.Unmarshal([]byte(e.TargetFields), &fields) + // A target whose collection is not in the map cannot be judged terminal or + // not, so it fails closed: neither counted nor walked through. + open := false + if c := g.colls[e.TargetCollectionID]; c != nil { + var schema models.CollectionSchema + _ = models.UnmarshalItemFieldSchema([]byte(c.Schema), &schema) + var settings models.CollectionSettings + if c.Settings != "" { + _ = json.Unmarshal([]byte(c.Settings), &settings) + } + open = !models.IsTerminalItem(fields, schema, settings) + } + g.open[e.TargetID] = open + return open +} + +// fanout counts the distinct items the root blocks, directly or through other +// blocked items: breadth-first with a visited set (the root starts in it, so a +// cycle never counts the root), at most fanoutDepth links deep, stopping at +// fanoutCap. An item the viewer cannot see, or that is terminal, is neither +// counted nor walked through, and the answer is a count only. +func (g *blocksGraph) fanout(root string, vis scopeSet) int { + visited := map[string]bool{root: true} + frontier := []string{root} + n := 0 + for depth := 0; depth < fanoutDepth && len(frontier) > 0; depth++ { + var next []string + for _, id := range frontier { + for _, e := range g.bySource[id] { + if visited[e.TargetID] { + continue + } + visited[e.TargetID] = true + if !vis.sees(e.TargetCollectionID, e.TargetID) || !g.isOpen(e) { + continue + } + if n++; n >= fanoutCap { + return fanoutCap + } + next = append(next, e.TargetID) + } + } + frontier = next + } + return n +} + +// ---- simplicity ---- + +type simplicityScorer struct{ s *Server } + +func (*simplicityScorer) Name() string { return "simplicity" } + +func (sc *simplicityScorer) Score(ctx context.Context, _ needs.Viewer, scope needs.Scope, rows []*needs.Row) (bool, error) { + if len(rows) == 0 { + return false, nil + } + m := sc.s.memoFor(ctx, scope.WorkspaceID) + m.mu.Lock() + defer m.mu.Unlock() + items, err := m.itemsFor(rows) + if err != nil { + return true, err + } + decs, owed, qh, err := m.answers(ctx, rows) + if err != nil { + return true, err + } + for _, r := range rows { + r.Simplicity = simplicityOf(r, items[r.ItemID], decs[r.ItemID], qh, owed[r.ItemID]) + } + return false, nil +} + +// simplicityOf is (3 - effort)/3 of 100. Effort is the level the stored effort +// answer picked (argmax, so the value and the hint cannot disagree), when that +// answer is to the question as asked now and its level reads as 0-3; otherwise +// a heuristic on the text. +// "Current" is the question hash only, not the item's state: an answer one edit +// behind is used, and a job owed for the item says so in Why. +func simplicityOf(r *needs.Row, it store.NeedsScoringItem, decs []models.ItemDecision, qh map[string]string, refreshing bool) *needs.Score { + var why []string + level, hint := 0, "" + if d, ok := currentAnswer(decs, qh, decision.AttentionEffort); ok { + if l, ok := decision.EffortLevel(d); ok { + level, hint = l, effortHints[l] + if md, ok := currentAnswer(decs, qh, decision.AttentionEffortMedium); ok { + if c, p, ok := decision.ChoiceValue(md); ok && p >= decision.EffortMediumMinProb { + hint += effortMediumSuffix[c] + } + } + } else { + level, hint = heuristicEffort(r.Item.Title, it.ContentRunes), "" + } + } else { + level = heuristicEffort(r.Item.Title, it.ContentRunes) + } + if hint == "" { + why = append(why, whyEstimated) + } + if refreshing { + why = append(why, whyRefreshing) + } + return &needs.Score{Value: int(math.Round(float64(3-level) / 3 * 100)), Hint: hint, Why: why} +} + +// heuristicEffort is the no-answer estimate: 0 for a question title with a +// short body, else by body length. +func heuristicEffort(title string, bodyRunes int) int { + switch { + case strings.HasSuffix(strings.TrimSpace(title), "?") && bodyRunes <= 600: + return 0 + case bodyRunes <= 1500: + return 1 + case bodyRunes <= 6000: + return 2 + } + return 3 +} diff --git a/internal/server/needs_scorers_test.go b/internal/server/needs_scorers_test.go new file mode 100644 index 000000000..36f4dd1de --- /dev/null +++ b/internal/server/needs_scorers_test.go @@ -0,0 +1,774 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "os" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/PerpetualSoftware/pad/internal/decision" + "github.com/PerpetualSoftware/pad/internal/models" + "github.com/PerpetualSoftware/pad/internal/needs" + "github.com/PerpetualSoftware/pad/internal/store" +) + +// TASK-3959: the importance and simplicity scorers on GET /me/needs. + +func edge(src, tgt, coll string) store.BlocksEdge { + return store.BlocksEdge{SourceID: src, TargetID: tgt, TargetCollectionID: coll, TargetFields: `{"status":"open"}`} +} + +// colls are collections with the open/done schema, so isOpen can judge a +// target; a target whose collection is missing is not open (fail closed). +func colls(ids ...string) []models.Collection { + out := make([]models.Collection, len(ids)) + for i, id := range ids { + out[i] = models.Collection{ID: id, Schema: nfxSchema} + } + return out +} + +func allSee() scopeSet { return newScopeSet(needs.Scope{Unrestricted: true}) } + +func TestNeedsYouSignals_FanoutCycleTerminates(t *testing.T) { + // A blocks B, B blocks A, A blocks C: two items, and the cycle ends. + g := newBlocksGraph([]store.BlocksEdge{edge("A", "B", "c"), edge("B", "A", "c"), edge("A", "C", "c")}, colls("c")) + if n := g.fanout("A", allSee()); n != 2 { + t.Fatalf("fan-out of A = %d, want 2 (B and C; A never counts itself)", n) + } + // A chain is walked five links deep and no further. + var chain []store.BlocksEdge + for i := 0; i < 9; i++ { + chain = append(chain, edge(fmt.Sprint("n", i), fmt.Sprint("n", i+1), "c")) + } + if n := newBlocksGraph(chain, colls("c")).fanout("n0", allSee()); n != fanoutDepth { + t.Fatalf("chain fan-out = %d, want the depth cap %d", n, fanoutDepth) + } + // A star of 150 stops at 99. + var star []store.BlocksEdge + for i := 0; i < 150; i++ { + star = append(star, edge("hub", fmt.Sprint("s", i), "c")) + } + if n := newBlocksGraph(star, colls("c")).fanout("hub", allSee()); n != fanoutCap { + t.Fatalf("star fan-out = %d, want the cap %d", n, fanoutCap) + } + // The blocks component follows: 0 -> 0, and the cap saturates at 1. + it := store.NeedsScoringItem{} + r := &needs.Row{WaitingSince: nfxT0} + now := nfxT0 + none, full := importanceOf(r, it, 0, nil, nil, now), importanceOf(r, it, 99, nil, nil, now) + if full.Value-none.Value != 20 { + t.Fatalf("saturated blocks adds %d, want 20 (weight .20)", full.Value-none.Value) + } +} + +func TestNeedsYouSignals_FanoutSkipsHidden(t *testing.T) { + edges := []store.BlocksEdge{ + edge("A", "B", "hiddenColl"), edge("B", "C", "seenColl"), // C is behind a hidden B + edge("A", "D", "seenColl"), edge("D", "E", "seenColl"), // E behind D, all seen + edge("A", "T", "cdone"), edge("T", "F", "seenColl"), // T terminal: not counted, not walked + edge("A", "G", "otherColl"), // visible only by an item grant + } + edges[5].TargetFields = `{"status":"open"}` + for i := range edges { + if edges[i].TargetID == "T" { + edges[i].TargetFields = `{"status":"done"}` + } + } + g := newBlocksGraph(edges, colls("cdone", "hiddenColl", "seenColl", "otherColl")) + restricted := newScopeSet(needs.Scope{CollectionIDs: []string{"seenColl", "cdone"}, ItemIDs: []string{"G"}}) + // Counted: D, E, G. Not: B (hidden), C (behind B), T (terminal), F (behind T). + if n := g.fanout("A", restricted); n != 3 { + t.Fatalf("restricted fan-out = %d, want 3", n) + } + // Unrestricted sees B and C too; the terminal item and what only it blocks stay out. + if n := g.fanout("A", allSee()); n != 5 { + t.Fatalf("unrestricted fan-out = %d, want 5 (B C D E G and not T or F)", n) + } +} + +// Every form of scope that does not grant anything fails closed: nothing is +// counted, whatever the graph holds. +func TestNeedsYouSignals_FanoutFailsClosed(t *testing.T) { + g := newBlocksGraph([]store.BlocksEdge{edge("A", "B", "c"), edge("B", "C", "c")}, colls("c")) + for name, sc := range map[string]needs.Scope{ + "both lists nil": {}, + "both lists empty": {CollectionIDs: []string{}, ItemIDs: []string{}}, + "items only, none of the targets": {ItemIDs: []string{"Z"}}, + } { + if n := g.fanout("A", newScopeSet(sc)); n != 0 { + t.Errorf("%s: fan-out = %d, want 0", name, n) + } + } + // A node visible only by an item grant is walked THROUGH to a visible child. + g = newBlocksGraph([]store.BlocksEdge{edge("A", "B", "hidden"), edge("B", "C", "seen")}, colls("hidden", "seen")) + sc := newScopeSet(needs.Scope{CollectionIDs: []string{"seen"}, ItemIDs: []string{"B"}}) + if n := g.fanout("A", sc); n != 2 { + t.Errorf("item-grant node: fan-out = %d, want 2 (B by grant, and C through it)", n) + } +} + +// A target whose collection is not known is neither counted nor expanded. +func TestNeedsYouSignals_FanoutUnknownCollectionFailsClosed(t *testing.T) { + g := newBlocksGraph([]store.BlocksEdge{edge("A", "B", "ghost"), edge("B", "C", "c")}, colls("c")) + if n := g.fanout("A", allSee()); n != 0 { + t.Fatalf("fan-out = %d, want 0 (B's collection is unknown, so B and C stay out)", n) + } +} + +// D agrees with the dashboard's server-local day (overdueToday), not the UTC day. +func TestNeedsYouSignals_DeadlineUsesServerLocalDay(t *testing.T) { + loc := time.FixedZone("UTC-8", -8*3600) + // 03:00 UTC on the 11th is 19:00 on the 10th locally. + now := time.Date(2026, 3, 11, 3, 0, 0, 0, time.UTC) + if overdueToday(now.In(loc)) != "2026-03-10" { + t.Fatal("fixture: local day should be the 10th") + } + if d, why := deadlinePart(`{"due_date":"2026-03-10"}`, now, loc); d != 1 || why != whyDueToday { + t.Errorf("due the local day: %v %q, want 1 %q (the UTC day would call it overdue)", d, why, whyDueToday) + } + if d, why := deadlinePart(`{"due_date":"2026-03-09"}`, now, loc); d != 1 || why != whyOverdue { + t.Errorf("due the day before: %v %q, want 1 %q", d, why, whyOverdue) + } + if d, why := deadlinePart(`{"due_date":"2026-03-11"}`, now, loc); why != whyDueSoon || d != 1-1.0/14 { + t.Errorf("due the next local day: %v %q, want %v %q", d, why, 1-1.0/14, whyDueSoon) + } +} + +// The scorer's overdue verdict is the dashboard's, for every stored shape. +func TestNeedsYouSignals_DeadlineOverdueMatchesDashboard(t *testing.T) { + now := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + today := overdueToday(now.In(time.UTC)) + for _, v := range []string{ + "2026-03-10T09:30:00Z", // timestamped today: not overdue by string order + "2026-03-10T00:00:00Z", + "2026-02-30", // malformed: string-orders before today, so overdue + "2026-03-09", + "2026-03-10", + "2026-03-11T08:00:00Z", + } { + fields := `{"due_date":"` + v + `"}` + _, _, dash := itemOverdue(fields, today) + d, why := deadlinePart(fields, now, time.UTC) + if got := why == whyOverdue; got != dash { + t.Errorf("%s: scorer overdue=%v, dashboard=%v", v, got, dash) + } + if dash && d != 1 { + t.Errorf("%s: overdue D = %v, want 1", v, d) + } + } + if _, why := deadlinePart(`{"due_date":"2026-03-10T09:30:00Z"}`, now, time.UTC); why != whyDueToday { + t.Errorf("timestamp dated today: %q, want %q", why, whyDueToday) + } +} + +// Every input's exact contribution (hand-computed in the comments). The parts +// are the unweighted inputs; the values are round(100 * weighted sum), chosen +// away from .5 boundaries. +func TestNeedsYouSignals_ImportancePartsPinned(t *testing.T) { + now := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + day := func(n float64) time.Time { return now.Add(-time.Duration(n * 24 * float64(time.Hour))) } + qh := map[string]string{decision.AttentionNeedsHuman: "h"} + ans := func(p string) []models.ItemDecision { + return []models.ItemDecision{{QuestionKey: decision.AttentionNeedsHuman, Kind: "noul", QuestionHash: "h", Answer: json.RawMessage(`{"type":"noul","noul":` + p + `}`)}} + } + due := func(d string) store.NeedsScoringItem { + return store.NeedsScoringItem{Fields: `{"due_date":"` + d + `"}`} + } + const eps = 1e-12 + b1 := 1 / math.Log2(17) // one blocked item: log2(2)/log2(17) + type row struct { + name string + prio string + waited float64 + item store.NeedsScoringItem + blocked int + decs []models.ItemDecision + asked bool + part func(importanceParts) float64 + want float64 // the part + value int // 0 = not pinned for this row + } + P := func(p importanceParts) float64 { return p.p } + B := func(p importanceParts) float64 { return p.b } + D := func(p importanceParts) float64 { return p.d } + A := func(p importanceParts) float64 { return p.a } + H := func(p importanceParts) float64 { return p.h } + S := func(p importanceParts) float64 { return p.s } + rows := []row{ + // Value = round(100*(.25P + .10H(.5) + .10A)), others 0. + {name: "P critical", prio: "critical", part: P, want: 1, value: 30}, // .25+.05 = .30 + {name: "P high", prio: "high", waited: 3.5, part: P, want: .75, value: 26}, // .1875+.05+.025 = .2625 + {name: "P medium", prio: "medium", waited: 3.5, part: P, want: .5, value: 20}, // .125+.05+.025 = .20 + {name: "P unset", prio: "", waited: 3.5, part: P, want: .5, value: 20}, // same as medium + {name: "P low", prio: "low", waited: 3.5, part: P, want: .25, value: 14}, // .0625+.05+.025 = .1375 + {name: "P HIGH", prio: "HIGH", waited: 3.5, part: P, want: .75, value: 26}, // the match ignores case + {name: "P Critical", prio: "Critical", part: P, want: 1, value: 30}, + // D on a low-priority row: .0625 + .05 + .20D. + {name: "D overdue", prio: "low", item: due("2026-03-09"), part: D, want: 1, value: 31}, // .3125 + {name: "D today", prio: "low", item: due("2026-03-10"), part: D, want: 1, value: 31}, + {name: "D 1 day", prio: "low", item: due("2026-03-11"), part: D, want: 13.0 / 14}, // .1125+.2*13/14 + {name: "D 7 days", prio: "low", item: due("2026-03-17"), part: D, want: .5, value: 21}, // .1125+.10 = .2125 + {name: "D 13 days", prio: "low", item: due("2026-03-23"), part: D, want: 1.0 / 14}, + {name: "D 14 days", prio: "low", item: due("2026-03-24"), part: D, want: 0, value: 11}, // .1125 + {name: "D 15 days", prio: "low", item: due("2026-03-25"), part: D, want: 0}, + {name: "D none", prio: "low", part: D, want: 0, value: 11}, + // A date too short to parse follows the dashboard's string rule: "2026-3-5" + // sorts after today, so not overdue; "2026-02" sorts before it, so overdue. + {name: "D short date", prio: "low", item: due("2026-3-5"), part: D, want: 0, value: 11}, + {name: "D short date overdue", prio: "low", item: due("2026-02"), part: D, want: 1, value: 31}, + // The earlier of due_date and end_date counts, whichever key holds it. + {name: "D end_date earlier", prio: "low", item: store.NeedsScoringItem{Fields: `{"due_date":"2026-03-17","end_date":"2026-03-12"}`}, part: D, want: 12.0 / 14}, + {name: "D due_date earlier", prio: "low", item: store.NeedsScoringItem{Fields: `{"due_date":"2026-03-12","end_date":"2026-03-17"}`}, part: D, want: 12.0 / 14}, + {name: "D end_date only", prio: "low", item: store.NeedsScoringItem{Fields: `{"end_date":"2026-03-17"}`}, part: D, want: .5, value: 21}, + // A: .1125 + .10A. + {name: "A 0 days", prio: "low", waited: 0, part: A, want: 0, value: 11}, + {name: "A 7 days", prio: "low", waited: 7, part: A, want: .5, value: 16}, // .1625 + {name: "A 14 days", prio: "low", waited: 14, part: A, want: 1, value: 21}, // .2125 + {name: "A 30 days", prio: "low", waited: 30, part: A, want: 1, value: 21}, + // H: .0625 + .10H. + {name: "H no answer", prio: "low", part: H, want: .5, value: 11}, // .1125 + {name: "H answer .8", prio: "low", decs: ans("0.8"), part: H, want: .8, value: 14}, // .1425 + {name: "H asked", prio: "low", asked: true, part: H, want: 1, value: 16}, // .1625 + // B: .1125 + .20B. + {name: "B none", prio: "low", part: B, want: 0, value: 11}, + {name: "B one", prio: "low", blocked: 1, part: B, want: b1, value: 16}, // .1125+.2*.24465 = .16143 + // S is held at zero. + {name: "S", prio: "low", part: S, want: 0}, + } + for _, c := range rows { + r := &needs.Row{WaitingSince: day(c.waited)} + r.Item.Priority = c.prio + if c.asked { + r.Why.Rule = needs.RuleAsked + } + sc, parts := importanceDetail(time.UTC, r, c.item, c.blocked, c.decs, qh, now) + if got := c.part(parts); math.Abs(got-c.want) > eps { + t.Errorf("%s: part = %v, want %v", c.name, got, c.want) + } + if c.value != 0 && sc.Value != c.value { + t.Errorf("%s: importance = %d, want %d", c.name, sc.Value, c.value) + } + } +} + +// A weighted sum that is exactly .325 rounds up whatever order it was summed in. +func TestNeedsYouSignals_ImportanceRoundingSnapped(t *testing.T) { + now := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + // high .1875 + H .05 + A (12.25 days / 14 = .875) .0875 = .325 -> 32.5 -> 33. + r := &needs.Row{WaitingSince: now.Add(-294 * time.Hour)} + r.Item.Priority = "high" + if got := importanceIn(time.UTC, r, store.NeedsScoringItem{}, 0, nil, nil, now).Value; got != 33 { + t.Errorf("exactly .325 scored %d, want 33", got) + } + // unset priority .125 + H .05 + A (7 days / 14 = .5) .05 = .225 -> 22.5 -> 23; + // summed as floats this is 22.499999999999996 without the snap. + r = &needs.Row{WaitingSince: now.Add(-7 * 24 * time.Hour)} + if got := importanceIn(time.UTC, r, store.NeedsScoringItem{}, 0, nil, nil, now).Value; got != 23 { + t.Errorf("exactly .225 scored %d, want 23", got) + } +} + +// H is a probability: a stored value outside [0,1] cannot push importance past its bounds. +func TestNeedsYouSignals_NeedsHumanProbabilityClamped(t *testing.T) { + r := &needs.Row{WaitingSince: nfxT0} + qh := map[string]string{decision.AttentionNeedsHuman: "h"} + ans := func(p string) []models.ItemDecision { + return []models.ItemDecision{{QuestionKey: decision.AttentionNeedsHuman, Kind: "noul", QuestionHash: "h", Answer: json.RawMessage(`{"type":"noul","noul":` + p + `}`)}} + } + base := importanceOf(r, store.NeedsScoringItem{}, 0, ans("1"), qh, nfxT0).Value + if got := importanceOf(r, store.NeedsScoringItem{}, 0, ans("7"), qh, nfxT0).Value; got != base { + t.Errorf("probability 7 scored %d, want the value for 1 (%d)", got, base) + } + low := importanceOf(r, store.NeedsScoringItem{}, 0, ans("0"), qh, nfxT0).Value + if got := importanceOf(r, store.NeedsScoringItem{}, 0, ans("-4"), qh, nfxT0).Value; got != low { + t.Errorf("probability -4 scored %d, want the value for 0 (%d)", got, low) + } +} + +// "judged to need a human" is given from the attention threshold up, not below it. +func TestNeedsYouSignals_NeedsHumanWhyAtThreshold(t *testing.T) { + r := &needs.Row{WaitingSince: nfxT0} + qh := map[string]string{decision.AttentionNeedsHuman: "h"} + for _, c := range []struct { + p float64 + want bool + }{{decision.AttentionThreshold, true}, {decision.AttentionThreshold - 0.01, false}, {1, true}} { + decs := []models.ItemDecision{{QuestionKey: decision.AttentionNeedsHuman, Kind: "noul", QuestionHash: "h", Answer: json.RawMessage(fmt.Sprintf(`{"type":"noul","noul":%v}`, c.p))}} + why := importanceOf(r, store.NeedsScoringItem{}, 0, decs, qh, nfxT0).Why + if got := slices.Contains(why, whyNeedsHuman); got != c.want { + t.Errorf("probability %v: Why %v, want %q present = %v", c.p, why, whyNeedsHuman, c.want) + } + } +} + +func TestNeedsYouSignals_HeuristicWithoutRunner(t *testing.T) { + for _, c := range []struct { + title string + runes int + want int + }{ + {"Can we ship?", 0, 0}, {"Can we ship?", 600, 0}, {"Can we ship?", 601, 1}, + {"Ship it", 0, 1}, {"Ship it", 1500, 1}, {"Ship it", 1501, 2}, {"Ship it", 6000, 2}, {"Ship it", 6001, 3}, + {"Can we ship?", 1501, 2}, {"Can we ship? ", 10, 0}, + } { + if got := heuristicEffort(c.title, c.runes); got != c.want { + t.Errorf("heuristicEffort(%q, %d) = %d, want %d", c.title, c.runes, got, c.want) + } + } + // End to end with no runner attached: heuristic value, no hint, "estimated". + needsBackends(t, func(t *testing.T, srv *Server) { + f := newNfx(t, srv) + owner, alice := f.user("owner"), f.user("alice") + ws := f.workspace("One", owner) + f.member(ws, alice, "editor") + f.item(ws, f.collection(ws, "Asks", true), "Can we ship?", alice, nfxDay(1)) + resp := f.need(alice, "") + if len(resp.Rows) != 1 || srv.decisionRunner() != nil { + t.Fatalf("fixture: %d rows, runner %v", len(resp.Rows), srv.decisionRunner()) + } + sim, imp := resp.Rows[0].Simplicity, resp.Rows[0].Importance + if sim == nil || imp == nil || sim.Value != 100 || sim.Hint != "" || strings.Join(sim.Why, "|") != whyEstimated { + t.Fatalf("simplicity %+v", sim) + } + // asked: H = 1; priority unset: P = .5; waiting far over two weeks: A = 1. + // .25*.5 + .10*1 + .10*1 = .325 -> 33. + if imp.Value != 33 { + t.Fatalf("importance %+v, want 33", imp) + } + }) +} + +func effortAnswer(probs string, qhash string) models.ItemDecision { + return models.ItemDecision{QuestionKey: decision.AttentionEffort, Kind: "score", QuestionHash: qhash, + Answer: json.RawMessage(`{"type":"score","score":1.5,"probabilities":` + probs + `}`)} +} + +func mediumAnswer(choice string, p float64, qhash string) models.ItemDecision { + return models.ItemDecision{QuestionKey: decision.AttentionEffortMedium, Kind: "choice", QuestionHash: qhash, + Answer: json.RawMessage(fmt.Sprintf(`{"type":"choice","choice":%q,"probabilities":{%q:%v}}`, choice, choice, p))} +} + +func TestNeedsYouSignals_HintTable(t *testing.T) { + qh := map[string]string{decision.AttentionEffort: "qe", decision.AttentionEffortMedium: "qm", decision.AttentionNeedsHuman: "qn", decision.AttentionHighStakes: "qs"} + row := &needs.Row{Item: needs.ItemInfo{Title: "Ship it"}} + level := func(l int) string { + return fmt.Sprintf(`{"%d":0.9,"%d":0.1}`, l, (l+1)%4) + } + heads := []string{"yes/no", "5 min", "under an hour", "research"} + suffixes := map[string]string{"desk": " at desk", "device": ", needs a device", "other_person": ", needs someone else", "unclear": ""} + longest := 0 + for l := 0; l < 4; l++ { + for choice, suf := range suffixes { + s := simplicityOf(row, store.NeedsScoringItem{}, []models.ItemDecision{effortAnswer(level(l), "qe"), mediumAnswer(choice, 0.8, "qm")}, qh, false) + if want := heads[l] + suf; s.Hint != want { + t.Errorf("level %d %s: hint %q, want %q", l, choice, s.Hint, want) + } + if want := []int{100, 67, 33, 0}[l]; s.Value != want { + t.Errorf("level %d: value %d, want %d", l, s.Value, want) + } + if len(s.Why) != 0 { + t.Errorf("an answered, fresh estimate carries Why %v", s.Why) + } + longest = max(longest, len(s.Hint)) + } + } + if longest > needsHintMax || longest != len("under an hour, needs someone else") { + t.Fatalf("longest hint is %d bytes against a cap of %d", longest, needsHintMax) + } + + // Value and Hint come from one level: the argmax, even where the + // probability-weighted mean score sits on another level. + skew := effortAnswer(`{"0":0.4,"1":0,"2":0,"3":0.6}`, "qe") // mean 1.8, argmax 3 + skew.Answer = json.RawMessage(`{"type":"score","score":1.8,"probabilities":{"0":0.4,"3":0.6}}`) + if s := simplicityOf(row, store.NeedsScoringItem{}, []models.ItemDecision{skew}, qh, false); s.Value != 0 || s.Hint != "research" { + t.Fatalf("argmax: %+v, want value 0 and hint research", s) + } + + // No effort_medium answer at all: the level's phrase alone. + if s := simplicityOf(row, store.NeedsScoringItem{}, []models.ItemDecision{effortAnswer(level(2), "qe")}, qh, false); s.Hint != "under an hour" || s.Value != 33 { + t.Fatalf("no medium: %+v", s) + } + // EffortMediumMinProb is inclusive: exactly .5 adds the suffix. + if s := simplicityOf(row, store.NeedsScoringItem{}, []models.ItemDecision{effortAnswer(level(1), "qe"), mediumAnswer("device", decision.EffortMediumMinProb, "qm")}, qh, false); s.Hint != "5 min, needs a device" { + t.Fatalf("medium at exactly .5: hint %q", s.Hint) + } + // An unsure medium (top probability under .5) adds nothing. + s := simplicityOf(row, store.NeedsScoringItem{}, []models.ItemDecision{effortAnswer(level(1), "qe"), mediumAnswer("device", 0.49, "qm")}, qh, false) + if s.Hint != "5 min" { + t.Fatalf("unsure medium: hint %q", s.Hint) + } + // A medium answered to an older question is ignored; an effort answer to an + // older question falls back to the heuristic, with no hint. + s = simplicityOf(row, store.NeedsScoringItem{}, []models.ItemDecision{effortAnswer(level(1), "qe"), mediumAnswer("device", 0.9, "old")}, qh, false) + if s.Hint != "5 min" { + t.Fatalf("stale medium: hint %q", s.Hint) + } + s = simplicityOf(row, store.NeedsScoringItem{ContentRunes: 3000}, []models.ItemDecision{effortAnswer(level(0), "old")}, qh, false) + if s.Hint != "" || s.Value != 33 || strings.Join(s.Why, "|") != whyEstimated { + t.Fatalf("stale effort: %+v, want the heuristic (level 2)", s) + } + // A current effort answer whose level cannot be read (here 7, outside 0-3) + // is no answer: the heuristic, with no hint. + for _, probs := range []string{`{"7":0.9}`, `{"-1":0.9}`, `{"x":0.9}`} { + s = simplicityOf(row, store.NeedsScoringItem{ContentRunes: 3000}, []models.ItemDecision{effortAnswer(probs, "qe"), mediumAnswer("device", 0.9, "qm")}, qh, false) + if s.Hint != "" || s.Value != 33 || strings.Join(s.Why, "|") != whyEstimated { + t.Fatalf("unreadable effort %s: %+v, want the heuristic (level 2)", probs, s) + } + } + // An owed job adds the refreshing note, over the answer it still uses. + s = simplicityOf(row, store.NeedsScoringItem{}, []models.ItemDecision{effortAnswer(level(1), "qe")}, qh, true) + if s.Hint != "5 min" || strings.Join(s.Why, "|") != whyRefreshing { + t.Fatalf("owed job: %+v", s) + } + + // high_stakes is never read: an answer for it moves nothing. + stakes := models.ItemDecision{QuestionKey: decision.AttentionHighStakes, Kind: "noul", QuestionHash: "qs", Answer: json.RawMessage(`{"type":"noul","noul":1}`)} + now := nfxT0 + r := &needs.Row{WaitingSince: now} + if a, b := importanceOf(r, store.NeedsScoringItem{}, 0, nil, qh, now), importanceOf(r, store.NeedsScoringItem{}, 0, []models.ItemDecision{stakes}, qh, now); a.Value != b.Value || fmt.Sprint(a.Why) != fmt.Sprint(b.Why) { + t.Fatalf("high_stakes moved importance: %+v vs %+v", a, b) + } + // And importance tops out at 85 while S is 0 (not renormalised). + top := &needs.Row{Item: needs.ItemInfo{Priority: "critical"}, Why: needs.Why{Rule: needs.RuleAsked}, WaitingSince: now.AddDate(0, -1, 0)} + if v := importanceOf(top, store.NeedsScoringItem{Fields: `{"due_date":"2020-01-01"}`}, 99, nil, qh, now).Value; v != 85 { + t.Fatalf("maximum importance = %d, want 85", v) + } +} + +// Why is a fixed vocabulary: whatever the inputs, a scorer emits only strings +// from needsWhyVocabulary (the list docs/api.md publishes), never the title. +func TestNeedsYouSignals_WhyOnlyFromTheVocabulary(t *testing.T) { + vocab := map[string]bool{} + for _, w := range needsWhyVocabulary { + vocab[w] = true + } + qh := map[string]string{decision.AttentionEffort: "qe", decision.AttentionNeedsHuman: "qn"} + human := models.ItemDecision{QuestionKey: decision.AttentionNeedsHuman, Kind: "noul", QuestionHash: "qn", Answer: json.RawMessage(`{"type":"noul","noul":0.95}`)} + now := time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC) + seen := map[string]bool{} + for _, prio := range []string{"", "low", "medium", "high", "critical"} { + for _, due := range []string{"", "2024-05-01", "2024-06-01", "2024-06-05", "2024-09-01", "garbage"} { + for _, waited := range []int{0, 8, 30} { + for _, rule := range []string{needs.RuleAsked, needs.RuleAssigned} { + for _, blocked := range []int{0, 7} { + for _, ds := range [][]models.ItemDecision{nil, {human}, {effortAnswer(`{"1":1}`, "qe")}} { + for _, owed := range []bool{false, true} { + r := &needs.Row{Item: needs.ItemInfo{Title: "SECRET-TITLE ENG-1", Priority: prio}, Why: needs.Why{Rule: rule}, WaitingSince: now.AddDate(0, 0, -waited)} + it := store.NeedsScoringItem{Fields: `{"due_date":"` + due + `"}`, ContentRunes: waited * 300} + for _, sc := range []*needs.Score{importanceOf(r, it, blocked, ds, qh, now), simplicityOf(r, it, ds, qh, owed)} { + for _, w := range sc.Why { + seen[w] = true + if !vocab[w] || strings.Contains(w, "SECRET") { + t.Fatalf("Why %q is not in the published vocabulary", w) + } + } + } + } + } + } + } + } + } + } + for w := range vocab { + if !seen[w] { + t.Errorf("vocabulary entry %q is never emitted: drop it from the list and docs/api.md, or test it", w) + } + } +} + +// A request does each of a workspace's batch reads once, though two scorers +// want them, and the next request does them again: the memo is request-scoped. +func TestNeedsYouSignals_MemoReadsOncePerRequest(t *testing.T) { + needsBackends(t, func(t *testing.T, srv *Server) { + f := newNfx(t, srv) + owner, alice := f.user("owner"), f.user("alice") + w1, w2 := f.workspace("One", owner), f.workspace("Two", owner) + f.member(w1, alice, "editor") + f.member(w2, alice, "editor") + attachAttentionRunner(t, srv) + f.item(w1, f.collection(w1, "Asks", true), "in one", alice, nfxDay(1)) + f.item(w2, f.collection(w2, "Asks", true), "in two", alice, nfxDay(2)) + var mu sync.Mutex + loads := map[string]int{} + srv.needsMemoLoad = func(read string) { + mu.Lock() + defer mu.Unlock() + loads[read]++ + } + for round := 1; round <= 2; round++ { + if resp := f.need(alice, ""); len(resp.Rows) != 2 || resp.Rows[0].Importance == nil || resp.Rows[0].Simplicity == nil { + t.Fatalf("round %d: rows %+v", round, resp.Rows) + } + for _, ws := range []*models.Workspace{w1, w2} { + for _, read := range []string{"items", "blocks_edges", "collections", "question_hashes", "decisions", "owed_jobs", "attention_current"} { + if n := loads[ws.ID+":"+read]; n != round { + t.Errorf("round %d: %s read %d times, want %d (once per request, never cached across them)", round, read, n, round) + } + } + } + } + }) +} + +// Through a real server.New and the router (team CONVE-19): the scorers New +// registers fill both scores on /me/needs. +func TestNeedsYouSignals_BindingThroughServeHTTP(t *testing.T) { + srv := testServer(t) + f := newNfx(t, srv) + owner, alice := f.user("owner"), f.user("alice") + ws := f.workspace("One", owner) + f.member(ws, alice, "editor") + asks := f.collection(ws, "Asks", true) + f.item(ws, asks, "Is this OK?", alice, nfxDay(1)) + tok, err := srv.store.CreateAPIToken(alice.ID, models.APITokenCreate{Name: "pat"}, 30, 365) + if err != nil { + t.Fatal(err) + } + rr := doRequestWithBearer(srv, "GET", "/api/v1/me/needs", tok.Token, nil) + if rr.Code != http.StatusOK { + t.Fatalf("%d %s", rr.Code, rr.Body.String()) + } + var resp needsResponse + parseJSON(t, rr, &resp) + if len(resp.Rows) != 1 || resp.Rows[0].Importance == nil || resp.Rows[0].Simplicity == nil || len(resp.Degraded) != 0 { + t.Fatalf("rows %+v degraded %v (the scorers New registers must fill both scores)", resp.Rows, resp.Degraded) + } + if resp.Rows[0].Simplicity.Value != 100 || resp.Rows[0].Simplicity.Hint != "" { + t.Fatalf("simplicity %+v", resp.Rows[0].Simplicity) + } +} + +// Through the router: a request with a cursor scores at the cursor's as_of, +// not the wall clock, so every page measures age and deadlines from one moment. +func TestNeedsYouSignals_CursorPageScoresAtAsOf(t *testing.T) { + srv := testServer(t) + f := newNfx(t, srv) + owner, alice := f.user("owner"), f.user("alice") + ws := f.workspace("One", owner) + f.member(ws, alice, "editor") + asks := f.collection(ws, "Asks", true) + f.item(ws, asks, "Is this OK?", alice, nfxDay(1)) + tok, err := srv.store.CreateAPIToken(alice.ID, models.APITokenCreate{Name: "pat"}, 30, 365) + if err != nil { + t.Fatal(err) + } + rr := doRequestWithBearer(srv, "GET", "/api/v1/me/needs", tok.Token, nil) + if rr.Code != http.StatusOK { + t.Fatalf("%d %s", rr.Code, rr.Body.String()) + } + var resp needsResponse + parseJSON(t, rr, &resp) + if len(resp.Rows) != 1 || resp.Rows[0].Importance == nil { + t.Fatalf("rows %+v", resp.Rows) + } + if !resp.Rows[0].WaitingSince.Equal(nfxDay(1)) { + t.Fatalf("waiting_since %v, want %v", resp.Rows[0].WaitingSince, nfxDay(1)) + } + // Page 1 scores at the wall clock: medium .125 + asked .10 + age (years, capped) .10 = .325 -> 33. + if v := resp.Rows[0].Importance.Value; v != 33 { + t.Fatalf("page-1 importance %d, want 33", v) + } + // A later page scores at the cursor's as_of, one week after waiting_since: + // age .5, so .125 + .10 + .05 = .275 -> 28. + cur := needs.NewCursor(needs.SortAge, needsViewActive, "", nfxDay(8), &needs.Row{ItemID: "0", WaitingSince: nfxDay(0), Item: needs.ItemInfo{CreatedAt: nfxDay(0)}}).Encode() + rr = doRequestWithBearer(srv, "GET", "/api/v1/me/needs?cursor="+cur, tok.Token, nil) + if rr.Code != http.StatusOK { + t.Fatalf("%d %s", rr.Code, rr.Body.String()) + } + resp = needsResponse{} + parseJSON(t, rr, &resp) + if len(resp.Rows) != 1 || resp.Rows[0].Importance == nil || resp.Rows[0].Importance.Value != 28 { + t.Fatalf("cursor page rows %+v, want importance 28 at as_of %v", resp.Rows, nfxDay(8)) + } +} + +// docs/api.md publishes the Why vocabulary; the code's list and the doc agree. +func TestNeedsYouSignals_VocabularyIsDocumented(t *testing.T) { + doc, err := os.ReadFile("../../docs/api.md") + if err != nil { + t.Fatal(err) + } + for _, w := range needsWhyVocabulary { + if !strings.Contains(string(doc), "`"+w+"`") { + t.Errorf("Why string %q is not listed in docs/api.md", w) + } + } +} + +// Through the router (team CONVE-19): a restricted member's score is built from +// what that member may see. The row blocks an item they cannot see; the hidden +// item must add nothing, and the same viewer with full access scores higher. +func TestNeedsYouSignals_RestrictedViewerScoreHasNoHiddenBlocks(t *testing.T) { + needsBackends(t, func(t *testing.T, srv *Server) { + f := newNfx(t, srv) + owner, alice, guest := f.user("owner"), f.user("alice"), f.user("guest") + ws := f.workspace("One", owner) + f.member(ws, alice, "editor") + asks, hidden := f.collection(ws, "Asks", true), f.collection(ws, "Hidden tasks", false) + row := f.item(ws, asks, "blocking ask", alice, nfxDay(1)) + secret := f.item(ws, hidden, "secret work", nil, nfxDay(2)) + grantRow := f.item(ws, asks, "blocking ask for the guest", nil, nfxDay(1)) + f.forceAssign(grantRow, guest) + for _, id := range []string{row.ID, grantRow.ID} { + if _, err := srv.store.CreateItemLink(ws.ID, models.ItemLinkCreate{TargetID: secret.ID, LinkType: "blocks", CreatedBy: "user"}, id); err != nil { + t.Fatal(err) + } + } + token := func(u *models.User) string { + tok, err := srv.store.CreateAPIToken(u.ID, models.APITokenCreate{Name: "pat"}, 30, 365) + if err != nil { + t.Fatal(err) + } + return tok.Token + } + score := func(bearer string) *needs.Score { + rr := doRequestWithBearer(srv, "GET", "/api/v1/me/needs", bearer, nil) + if rr.Code != http.StatusOK { + t.Fatalf("%d %s", rr.Code, rr.Body.String()) + } + var resp needsResponse + parseJSON(t, rr, &resp) + if len(resp.Rows) != 1 || resp.Rows[0].Importance == nil { + t.Fatalf("rows %+v", resp.Rows) + } + return resp.Rows[0].Importance + } + hasBlocks := func(s *needs.Score) bool { + for _, w := range s.Why { + if w == whyBlocksWork { + return true + } + } + return false + } + aliceTok := token(alice) + // Control: with access to every collection the blocked item counts. + full := score(aliceTok) + if !hasBlocks(full) { + t.Fatalf("control: an unrestricted viewer should see the blocks part: %+v", full) + } + // Restricted to the asks collection alone, the hidden item vanishes. + if err := srv.store.SetMemberCollectionAccess(ws.ID, alice.ID, "specific", []string{asks.ID}); err != nil { + t.Fatal(err) + } + restricted := score(aliceTok) + if hasBlocks(restricted) || restricted.Value >= full.Value { + t.Fatalf("collection-restricted: %+v leaks the hidden blocked item (control %+v)", restricted, full) + } + // Only an item grant on the row: the same. + if _, err := srv.store.CreateItemGrant(ws.ID, grantRow.ID, guest.ID, "view", owner.ID); err != nil { + t.Fatal(err) + } + granted := score(token(guest)) + if hasBlocks(granted) || granted.Value != restricted.Value { + t.Fatalf("item-grant-only: %+v, want the restricted score %+v", granted, restricted) + } + // Exactness: each restricted score is the one the same viewer gets once the + // hidden blocked item does not exist at all, so no contribution too small + // to move the rounded value can have leaked either. + if err := srv.store.DeleteItem(secret.ID); err != nil { + t.Fatal(err) + } + same := func(name string, before, after *needs.Score) { + if before.Value != after.Value || fmt.Sprint(before.Why) != fmt.Sprint(after.Why) { + t.Fatalf("%s: %+v differs from the score once the hidden item is gone %+v", name, before, after) + } + } + same("collection-restricted", restricted, score(aliceTok)) + same("item-grant-only", granted, score(token(guest))) + }) +} + +// A batch read that fails leaves the rows unscored and says so: each real +// scorer reports degraded with the error, and sets no score from partial data. +func TestNeedsYouSignals_ReadFailureDegrades(t *testing.T) { + srv := testServer(t) + if err := srv.store.DB().Close(); err != nil { + t.Fatal(err) + } + for _, sc := range srv.needsScorers { + rows := []*needs.Row{{ItemID: "item-1", WorkspaceID: "ws-1"}} + deg, err := sc.Score(context.Background(), needs.Viewer{UserID: "u", Now: nfxT0}, needs.Scope{WorkspaceID: "ws-1", Unrestricted: true, CollectionIDs: []string{}, ItemIDs: []string{}}, rows) + if !deg || err == nil { + t.Errorf("%s: degraded %v err %v, want true and an error", sc.Name(), deg, err) + } + if rows[0].Importance != nil || rows[0].Simplicity != nil { + t.Errorf("%s: scored from a failed read: %+v %+v", sc.Name(), rows[0].Importance, rows[0].Simplicity) + } + } + if len(srv.needsScorers) != 2 { + t.Fatalf("%d scorers registered, want 2", len(srv.needsScorers)) + } +} + +// H and "judged to need a human" read needs_human through the inbox's own flag +// rule, which wants the answer current by item state as well as question: an +// answer of .9 to the question as asked now, but on an edited item (state hash +// moved, no refresh owed), weighs nothing, while the same answer on the item as +// it stands counts and says so. A stale answer whose refresh is owed is held, +// as the flag rule holds it. +func TestNeedsYouSignals_NeedsHumanNeedsFullCurrency(t *testing.T) { + srv := testServer(t) + f := newNfx(t, srv) + owner := f.user("owner") + ws := f.workspace("One", owner) + tasks := f.collection(ws, "Tasks", false) + current := f.item(ws, tasks, "Pick a vendor", nil, nfxDay(1)) + stale := f.item(ws, tasks, "Pick a host", nil, nfxDay(1)) + held := f.item(ws, tasks, "Pick a region", nil, nfxDay(1)) + attachAttentionRunner(t, srv) // after the items: no job is queued for either + runner := srv.decisionRunner() + qh, err := runner.CurrentQuestionHashes(context.Background(), ws.ID, decision.AttentionSetName) + if err != nil || qh[decision.AttentionNeedsHuman] == "" { + t.Fatalf("question hashes %v %v", qh, err) + } + _, st, err := runner.State(current.ID) + if err != nil { + t.Fatal(err) + } + answer := func(it *models.Item, stateHash string) models.ItemDecision { + return models.ItemDecision{ItemID: it.ID, QuestionSet: decision.AttentionSetName, QuestionKey: decision.AttentionNeedsHuman, Kind: "noul", + Answer: json.RawMessage(`{"type":"noul","noul":0.9}`), Provider: "stub", Model: "stub-1", + StateHash: stateHash, QuestionHash: qh[decision.AttentionNeedsHuman], ItemSeq: 1} + } + if err := srv.store.InsertItemDecisions(ws.ID, []models.ItemDecision{answer(current, st.Hash), answer(stale, "an-older-state"), answer(held, "an-older-state")}); err != nil { + t.Fatal(err) + } + if job, err := srv.store.GetDecisionJob(stale.ID, decision.AttentionSetName); err != nil || job != nil { + t.Fatalf("a job is owed for the stale item (%+v, %v): the flag rule would hold its answer", job, err) + } + if err := srv.store.OweDecisionJobs(ws.ID, decision.AttentionSetName, []string{held.ID}); err != nil { + t.Fatal(err) + } + now := time.Now() + row := func(it *models.Item) *needs.Row { + return &needs.Row{ItemID: it.ID, WorkspaceID: ws.ID, WaitingSince: now, Why: needs.Why{Rule: needs.RuleMentioned}} + } + rows := []*needs.Row{row(current), row(stale), row(held)} + deg, err := (&importanceScorer{s: srv, loc: time.UTC}).Score(context.Background(), needs.Viewer{UserID: owner.ID, Now: now}, + needs.Scope{WorkspaceID: ws.ID, Unrestricted: true, CollectionIDs: []string{}, ItemIDs: []string{}}, rows) + if deg || err != nil { + t.Fatalf("degraded %v err %v", deg, err) + } + // Control, current: medium .125 + H .9 -> .09 = .215 -> 22, and the Why. + if c := rows[0].Importance; c.Value != 22 || !slices.Contains(c.Why, whyNeedsHuman) { + t.Fatalf("current answer: %+v, want 22 with %q", c, whyNeedsHuman) + } + // Stale: H is the no-answer .5, so .125 + .05 = .175 -> 18, and no Why. + if s := rows[1].Importance; s.Value != 18 || slices.Contains(s.Why, whyNeedsHuman) { + t.Fatalf("stale answer: %+v, want 18 without %q", s, whyNeedsHuman) + } + // Stale but owed a refresh: held, like the current one. + if h := rows[2].Importance; h.Value != 22 || !slices.Contains(h.Why, whyNeedsHuman) { + t.Fatalf("stale answer with a refresh owed: %+v, want 22 with %q", h, whyNeedsHuman) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 38d806245..685107917 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -124,8 +124,13 @@ type Server struct { // to leaving it nil. Same synchronisation rule as afterItemPreRead. visibleCollectionIDsFault func() error // needsScorers are the importance and simplicity scorers GET /me/needs - // runs per workspace (TASK-3920). None ship yet: TASK-3959 registers them. + // runs per workspace (TASK-3920). New registers the two real ones + // (TASK-3959); tests replace the slice. needsScorers []needs.Scorer + // needsMemoLoad is a TEST-ONLY seam, nil in production: the scorers' shared + // memo calls it with the name of each batch read it actually performs, so a + // test can hold a request to one read of each kind (TASK-3959). + needsMemoLoad func(read string) // itemDoorAfterRowsHook is a TEST-ONLY seam, nil in production: the item // doors (/items-index, /items-changes) call it right after their row // query, so a test can commit a write between the rows and the access @@ -825,13 +830,15 @@ func New(s *store.Store) *Server { "A leaked admin token can then administer the instance; prefer a CLI session (`pad auth login`). " + "Ignored in cloud mode.") } - return &Server{ + srv := &Server{ store: s, rateLimiters: rl, storageInfoCache: newStorageInfoCache(storageInfoTTL), clientWrites: newClientWriteMarks(), adminAPITokensAllowed: adminTokens, } + srv.needsScorers = srv.newNeedsScorers() + return srv } // adminAcceptsAPITokens reports whether platform administration accepts an