From c775eed7394ec313c0aeefb01a987e649c49e07a Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 15 Jul 2026 15:04:26 +0100 Subject: [PATCH] Add branch-coverage tests for qualification scorer and report helpers New table-driven tests raise branch coverage on applyGates, ApplyProTrackGates, findFault, bestFindingMatch, validatePredicates, ApplyQualificationGates, canonicalToolInput, sanitizeArtifactText and allObservationsPassed, covering previously uncovered gate arms, grounding misses, redaction and JSON edge cases. Test-only, no source changes. --- .../manifest_predicates_branchcov_test.go | 319 +++++++++ .../replay_canonical_branchcov_test.go | 90 +++ .../report_helpers_branchcov_test.go | 167 +++++ .../report_qualgates_branchcov_test.go | 609 ++++++++++++++++++ .../scorer_gates_branchcov_test.go | 594 +++++++++++++++++ .../scorer_protrack_branchcov_test.go | 594 +++++++++++++++++ 6 files changed, 2373 insertions(+) create mode 100644 internal/ai/qualification/manifest_predicates_branchcov_test.go create mode 100644 internal/ai/qualification/replay_canonical_branchcov_test.go create mode 100644 internal/ai/qualification/report_helpers_branchcov_test.go create mode 100644 internal/ai/qualification/report_qualgates_branchcov_test.go create mode 100644 internal/ai/qualification/scorer_gates_branchcov_test.go create mode 100644 internal/ai/qualification/scorer_protrack_branchcov_test.go diff --git a/internal/ai/qualification/manifest_predicates_branchcov_test.go b/internal/ai/qualification/manifest_predicates_branchcov_test.go new file mode 100644 index 000000000..9f22822e7 --- /dev/null +++ b/internal/ai/qualification/manifest_predicates_branchcov_test.go @@ -0,0 +1,319 @@ +package qualification + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// This file is a white-box table-test suite for the pure manifest predicate +// helpers in manifest.go: validatePredicates and positiveDuration. Every +// helper introduced here is prefixed qualmp so it cannot collide with +// identifiers defined by sibling test files in package qualification. + +// qualmpAliases builds the alias set from the supplied resource aliases. It +// mirrors how Manifest.Validate constructs the set before calling +// validatePredicates, but keeps the table rows compact and literal. +func qualmpAliases(names ...string) map[string]struct{} { + out := make(map[string]struct{}, len(names)) + for _, n := range names { + out[n] = struct{}{} + } + return out +} + +// qualmpJoinErrs flattens a validatePredicates result into a single +// newline-delimited string so table rows can assert on substrings the same +// way the rest of this package checks joined errors. +func qualmpJoinErrs(errs []error) string { + var b strings.Builder + for _, e := range errs { + b.WriteString(e.Error()) + b.WriteByte('\n') + } + return b.String() +} + +func TestValidatePredicates(t *testing.T) { + cases := []struct { + name string + label string + predicates []Predicate + aliases map[string]struct{} + wantCount int + wantSubs []string + }{ + // for-range not entered: nil slice yields no errors. + { + name: "nil predicate slice returns no errors", + label: "baseline", + predicates: nil, + aliases: qualmpAliases("target"), + wantCount: 0, + }, + // for-range not entered: empty slice yields no errors. + { + name: "empty predicate slice returns no errors", + label: "baseline", + predicates: []Predicate{}, + aliases: qualmpAliases("target"), + wantCount: 0, + }, + // Happy path: known target, valid probe, valid operator, valid value, + // no timeout. Exercises every ok/default arm with the ok branch. + { + name: "fully valid predicate with no timeout returns no errors", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage("true"), + }}, + aliases: qualmpAliases("target"), + wantCount: 0, + }, + // Happy path with a non-empty, parseable, positive timeout: exercises + // the predicate.Timeout != "" branch and the positiveDuration + // success arm (err == nil) inside validatePredicates. + { + name: "valid predicate with positive timeout returns no errors", + label: "teardown", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage("false"), Timeout: "5s", + }}, + aliases: qualmpAliases("target"), + wantCount: 0, + }, + // Accepted JSON value shapes: confirm len>0 && json.Valid for numbers, + // null, arrays and objects so the invalid-value arm is not taken. + { + name: "numeric value is accepted", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.restart_count", Target: "target", Operator: "eq", + Value: json.RawMessage("3"), + }}, + aliases: qualmpAliases("target"), + wantCount: 0, + }, + { + name: "array value accepted for in operator", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.status", Target: "target", Operator: "in", + Value: json.RawMessage(`["running","healthy"]`), + }}, + aliases: qualmpAliases("target"), + wantCount: 0, + }, + // Unknown target arm: aliases lookup misses. + { + name: "target not in aliases reports unknown resource", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.running", Target: "ghost", Operator: "eq", + Value: json.RawMessage("true"), + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{`baseline[0] targets unknown resource "ghost"`}, + }, + // Unsupported probe default arm. + { + name: "unsupported probe is reported", + label: "baseline", + predicates: []Predicate{{ + Probe: "tcp.port_open", Target: "target", Operator: "eq", + Value: json.RawMessage("true"), + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{`baseline[0] has unsupported probe "tcp.port_open"`}, + }, + // Unsupported operator default arm. + { + name: "unsupported operator is reported", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "matches", + Value: json.RawMessage("true"), + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{`baseline[0] has unsupported operator "matches"`}, + }, + // Empty value arm: len(predicate.Value) == 0. + { + name: "empty value is reported as invalid", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage(""), + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{"baseline[0] has invalid value"}, + }, + // Nil value arm: a nil json.RawMessage has len 0. + { + name: "nil value is reported as invalid", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: nil, + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{"baseline[0] has invalid value"}, + }, + // Invalid JSON arm: non-empty bytes that fail json.Valid. + { + name: "malformed json value is reported as invalid", + label: "baseline", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage("{bad"), + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{"baseline[0] has invalid value"}, + }, + // Timeout arm with unparseable duration: positiveDuration parse error. + { + name: "unparseable timeout is wrapped", + label: "teardown", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage("true"), Timeout: "not-a-duration", + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{"teardown[0] timeout:"}, + }, + // Timeout arm with non-positive (zero) duration. + { + name: "zero timeout is rejected as non-positive", + label: "teardown", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage("true"), Timeout: "0s", + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{"teardown[0] timeout:", "duration must be positive"}, + }, + // Timeout arm with negative duration. + { + name: "negative timeout is rejected as non-positive", + label: "teardown", + predicates: []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage("true"), Timeout: "-3s", + }}, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{"teardown[0] timeout:", "duration must be positive"}, + }, + // Index propagation: a bad predicate at position 1 reports [1], and a + // valid predicate at position 0 contributes nothing. + { + name: "error reports the offending slice index", + label: "teardown", + predicates: []Predicate{ + {Probe: "docker.running", Target: "target", Operator: "eq", Value: json.RawMessage("true")}, + {Probe: "docker.running", Target: "missing", Operator: "eq", Value: json.RawMessage("true")}, + }, + aliases: qualmpAliases("target"), + wantCount: 1, + wantSubs: []string{`teardown[1] targets unknown resource "missing"`}, + }, + // Accumulation: every guard in one predicate produces five independent + // errors for a single element, confirming append-only behaviour. + { + name: "all guards fail on one predicate yielding five errors", + label: "fault f1 oracle", + predicates: []Predicate{{ + Probe: "bogus.probe", Target: "ghost", Operator: "matches", + Value: json.RawMessage("{bad"), Timeout: "nope", + }}, + aliases: qualmpAliases("target"), + wantCount: 5, + wantSubs: []string{ + `fault f1 oracle[0] targets unknown resource "ghost"`, + `fault f1 oracle[0] has unsupported probe "bogus.probe"`, + `fault f1 oracle[0] has unsupported operator "matches"`, + "fault f1 oracle[0] has invalid value", + "fault f1 oracle[0] timeout:", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + errs := validatePredicates(tc.label, tc.predicates, tc.aliases) + if len(errs) != tc.wantCount { + t.Fatalf("validatePredicates returned %d errors, want %d: %v", len(errs), tc.wantCount, qualmpJoinErrs(errs)) + } + if len(tc.wantSubs) == 0 { + return + } + joined := qualmpJoinErrs(errs) + for _, sub := range tc.wantSubs { + if !strings.Contains(joined, sub) { + t.Errorf("validatePredicates error text = %q, want substring %q", joined, sub) + } + } + }) + } +} + +func TestPositiveDuration(t *testing.T) { + cases := []struct { + name string + value string + wantErr bool + wantDur time.Duration + errSub string + }{ + // Success arm: positive duration parses and is returned verbatim. + {"seconds parse", "5s", false, 5 * time.Second, ""}, + {"compound duration parses", "1m30s", false, 90 * time.Second, ""}, + {"milliseconds parse", "250ms", false, 250 * time.Millisecond, ""}, + // Parse error arm: empty string cannot be parsed. + {"empty string parse error", "", true, 0, ""}, + // Parse error arm: missing unit. + {"bare number missing unit parse error", "5", true, 0, "missing unit"}, + // Parse error arm: unparseable text. + {"garbage parse error", "not-a-duration", true, 0, "invalid duration"}, + // Non-positive arm: zero duration parses but is rejected. + {"zero duration rejected", "0s", true, 0, "duration must be positive"}, + {"zero without unit rejected", "0", true, 0, "duration must be positive"}, + // Non-positive arm: negative duration parses but is rejected. + {"negative duration rejected", "-3s", true, 0, "duration must be positive"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := positiveDuration(tc.value) + if tc.wantErr { + if err == nil { + t.Fatalf("positiveDuration(%q) err = nil, want non-nil", tc.value) + } + if tc.errSub != "" && !strings.Contains(err.Error(), tc.errSub) { + t.Fatalf("positiveDuration(%q) err = %q, want substring %q", tc.value, err.Error(), tc.errSub) + } + if got != 0 { + t.Fatalf("positiveDuration(%q) = %v, want 0 on error path", tc.value, got) + } + return + } + if err != nil { + t.Fatalf("positiveDuration(%q) err = %v, want nil", tc.value, err) + } + if got != tc.wantDur { + t.Fatalf("positiveDuration(%q) = %v, want %v", tc.value, got, tc.wantDur) + } + }) + } +} diff --git a/internal/ai/qualification/replay_canonical_branchcov_test.go b/internal/ai/qualification/replay_canonical_branchcov_test.go new file mode 100644 index 000000000..64b01571e --- /dev/null +++ b/internal/ai/qualification/replay_canonical_branchcov_test.go @@ -0,0 +1,90 @@ +package qualification + +import ( + "strings" + "testing" +) + +// TestCanonicalToolInputBranches exercises every branch of canonicalToolInput, +// focusing on the previously uncovered error and early-return paths: the empty +// input short circuit ("{}"), the raw invalid-JSON decode failure, the +// multiple-JSON-values rejection, and the trailing-data rejection. A canonical +// happy path is included to pin the success contract. +func TestCanonicalToolInputBranches(t *testing.T) { + for _, tc := range []struct { + name string + input string + wantOut string + wantErr bool + errSubstr string + notSubstrs []string + }{ + { + name: "empty input returns empty object", + input: "", + wantOut: "{}", + }, + { + name: "whitespace only collapses to empty object", + input: " \t\n ", + wantOut: "{}", + }, + { + name: "valid single object canonicalizes key order and spacing", + input: `{ "b": 2, "a": 1 }`, + wantOut: `{"a":1,"b":2}`, + notSubstrs: []string{"multiple JSON values", "trailing JSON data"}, + }, + { + name: "invalid json returns decoder error", + input: `{"description":"cut...`, + wantErr: true, + notSubstrs: []string{"multiple JSON values", "trailing JSON data"}, + }, + { + name: "multiple json values are rejected", + input: `{}{}`, + wantErr: true, + errSubstr: "multiple JSON values", + }, + { + name: "trailing garbage after value is rejected", + input: `1 garbage`, + wantErr: true, + errSubstr: "trailing JSON data", + }, + { + name: "trailing garbage after object is rejected", + input: `{} x`, + wantErr: true, + errSubstr: "trailing JSON data", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := canonicalToolInput(tc.input) + if tc.wantErr { + if err == nil { + t.Fatalf("canonicalToolInput(%q) error = nil, want non-nil", tc.input) + } + if got != "" { + t.Fatalf("canonicalToolInput(%q) output = %q, want empty string on error", tc.input, got) + } + if tc.errSubstr != "" && !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("canonicalToolInput(%q) error = %q, want substring %q", tc.input, err.Error(), tc.errSubstr) + } + for _, bad := range tc.notSubstrs { + if strings.Contains(err.Error(), bad) { + t.Fatalf("canonicalToolInput(%q) error = %q, must not mention %q", tc.input, err.Error(), bad) + } + } + return + } + if err != nil { + t.Fatalf("canonicalToolInput(%q) error = %v, want nil", tc.input, err) + } + if got != tc.wantOut { + t.Fatalf("canonicalToolInput(%q) = %q, want %q", tc.input, got, tc.wantOut) + } + }) + } +} diff --git a/internal/ai/qualification/report_helpers_branchcov_test.go b/internal/ai/qualification/report_helpers_branchcov_test.go new file mode 100644 index 000000000..945cdc2f3 --- /dev/null +++ b/internal/ai/qualification/report_helpers_branchcov_test.go @@ -0,0 +1,167 @@ +package qualification + +import ( + "strings" + "testing" +) + +// TestSanitizeArtifactTextRedaction exercises the secretPatterns loop in +// sanitizeArtifactText: the NumSubexp()>0 branch (bearer / api-key / password +// / secret) uses the "${1}[REDACTED]" template, and the NumSubexp()==0 else +// branch (PEM private-key blocks) uses the whole-match "[REDACTED PRIVATE KEY]" +// replacement. Non-secret text must pass through verbatim. +func TestSanitizeArtifactTextRedaction(t *testing.T) { + cases := []struct { + name string + input string + want string + }{ + { + name: "bearer token redacted keeping label", + input: "Authorization: Bearer abc123-xyz", + want: "Authorization: Bearer [REDACTED]", + }, + { + name: "bearer header is case insensitive", + input: "authorization: bearer SecretToken", + want: "authorization: bearer [REDACTED]", + }, + { + name: "api_key with equals assignment", + input: "api_key=hunter2", + want: "api_key=[REDACTED]", + }, + { + name: "api-key with dash and colon separator", + input: "api-key: my-token", + want: "api-key: [REDACTED]", + }, + { + name: "apitoken variant redacted", + input: "apitoken=abcdef", + want: "apitoken=[REDACTED]", + }, + { + name: "password with colon space separator", + input: "password: hunter2", + want: "password: [REDACTED]", + }, + { + name: "secret equals assignment", + input: "secret=s3cr3t", + want: "secret=[REDACTED]", + }, + { + name: "plain private key block uses whole match redaction", + input: "-----BEGIN PRIVATE KEY-----\nMIIBOAIB\n-----END PRIVATE KEY-----", + want: "[REDACTED PRIVATE KEY]", + }, + { + name: "rsa private key block redacted via prefix class", + input: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOAIB\n-----END RSA PRIVATE KEY-----", + want: "[REDACTED PRIVATE KEY]", + }, + { + name: "non secret text is unchanged", + input: "the quick brown fox jumps", + want: "the quick brown fox jumps", + }, + { + name: "multiple secret kinds redacted together in one payload", + input: "Authorization: Bearer tok123\napi_key=keyval\npassword=pw", + want: "Authorization: Bearer [REDACTED]\napi_key=[REDACTED]\npassword=[REDACTED]", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := sanitizeArtifactText(tc.input); got != tc.want { + t.Fatalf("sanitizeArtifactText(%q) = %q, want %q", tc.input, got, tc.want) + } + }) + } +} + +// TestSanitizeArtifactTextTruncatesAbove512KB covers the >512KB truncation +// branch of sanitizeArtifactText. The boundary value (exactly 512*1024 bytes) +// must NOT be truncated, while anything larger is cut to the limit and gets the +// "\n[TRUNCATED]" marker appended. Truncation runs before redaction, so a +// secret that falls inside the retained prefix must still be redacted. +func TestSanitizeArtifactTextTruncatesAbove512KB(t *testing.T) { + const maxText = 512 * 1024 // 524288, mirrors the source constant + marker := "\n[TRUNCATED]" + + t.Run("exactly at limit is not truncated", func(t *testing.T) { + value := strings.Repeat("a", maxText) + got := sanitizeArtifactText(value) + if len(got) != maxText { + t.Fatalf("len = %d, want %d (no truncation at exact boundary)", len(got), maxText) + } + if strings.Contains(got, "[TRUNCATED]") { + t.Fatal("value at exact limit must not gain the truncation marker") + } + }) + + t.Run("over limit is cut to limit plus marker and drops the tail", func(t *testing.T) { + // "HEAD" + maxText bytes + "TAIL" is strictly longer than maxText. + value := "HEAD" + strings.Repeat("x", maxText) + "TAIL" + got := sanitizeArtifactText(value) + if wantLen := maxText + len(marker); len(got) != wantLen { + t.Fatalf("len = %d, want %d", len(got), wantLen) + } + if !strings.HasPrefix(got, "HEAD") { + t.Fatalf("retained prefix lost head bytes: %q", got[:8]) + } + if !strings.HasSuffix(got, marker) { + t.Fatalf("missing truncation marker; suffix = %q", got[len(got)-len(marker):]) + } + if strings.Contains(got, "TAIL") { + t.Fatal("bytes past the limit must be dropped, not retained") + } + }) + + t.Run("secret inside retained prefix is redacted after truncation", func(t *testing.T) { + head := "Authorization: Bearer leak-token\n" + strings.Repeat("z", 32) + value := head + strings.Repeat("y", maxText) // guarantees truncation; head is retained + got := sanitizeArtifactText(value) + if !strings.HasSuffix(got, marker) { + t.Fatalf("expected truncation marker; suffix = %q", got[len(got)-len(marker):]) + } + if !strings.Contains(got, "Authorization: Bearer [REDACTED]") { + t.Fatalf("secret within retained prefix was not redacted: %q", got[:80]) + } + if strings.Contains(got, "leak-token") { + t.Fatal("bearer secret value must be redacted even after truncation") + } + }) +} + +// TestAllObservationsPassedBranches covers every branch of allObservationsPassed: +// the empty/nil guard (returns false), the early return on the first failing +// observation, and the all-passed tail return. This is intentionally distinct +// from observationsPassedOrEmpty (covered elsewhere), which treats an empty +// slice as passing. +func TestAllObservationsPassedBranches(t *testing.T) { + cases := []struct { + name string + observations []PredicateObservation + want bool + }{ + {"nil slice returns false", nil, false}, + {"empty slice returns false", []PredicateObservation{}, false}, + {"single passing observation returns true", []PredicateObservation{{Passed: true}}, true}, + {"single failing observation returns false", []PredicateObservation{{Passed: false}}, false}, + {"all passing returns true", []PredicateObservation{{Passed: true}, {Passed: true}, {Passed: true}}, true}, + {"first failing short circuits to false", []PredicateObservation{{Passed: false}, {Passed: true}}, false}, + {"failing in the middle returns false", []PredicateObservation{{Passed: true}, {Passed: false}, {Passed: true}}, false}, + {"last failing returns false", []PredicateObservation{{Passed: true}, {Passed: true}, {Passed: false}}, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := allObservationsPassed(tc.observations); got != tc.want { + t.Fatalf("allObservationsPassed(%d obs) = %v, want %v", len(tc.observations), got, tc.want) + } + }) + } +} diff --git a/internal/ai/qualification/report_qualgates_branchcov_test.go b/internal/ai/qualification/report_qualgates_branchcov_test.go new file mode 100644 index 000000000..1e907c274 --- /dev/null +++ b/internal/ai/qualification/report_qualgates_branchcov_test.go @@ -0,0 +1,609 @@ +package qualification + +import ( + "encoding/json" + "strings" + "testing" +) + +// This file is a white-box, table-driven branch-coverage suite for +// ApplyQualificationGates in report.go. Helpers and shared identifiers are +// prefixed with `qualgates` so they do not collide with sibling test files in +// package qualification. Each row constructs a small literal ComparisonReport +// and Catalog; no live service, executor, provider, DB, filesystem, or +// goroutine is required. + +// qualgatesBaseline returns a ComparisonReport and Catalog that is known to +// fully qualify under track: a single model, single git revision, single Pulse +// runtime version, no dirty runs, one scenario whose manifest digest matches +// the catalog, with 30/30 passing runs (Wilson lower bound well above 0.85) and +// 30/30 fault recall. Tests mutate the returned pointer to introduce exactly +// the gate failure under test. +func qualgatesBaseline(t *testing.T, track Track) (*ComparisonReport, Catalog) { + t.Helper() + manifest := validTestManifest() + manifest.Track = track + manifest.Repeat.Qualification = 30 + digest, err := manifest.Digest() + if err != nil { + t.Fatalf("digest baseline manifest: %v", err) + } + catalog := Catalog{ + Manifests: []Manifest{manifest}, + ByID: map[string]Manifest{manifest.ID: manifest}, + } + comparison := &ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{{ + ModelSummary: ModelSummary{ + Model: "provider:model", + Runs: 30, + Passed: 30, + PassRate: WilsonInterval(30, 30), + FaultRecall: WilsonInterval(30, 30), + }, + ScenarioID: manifest.ID, + Track: track, + ManifestDigests: []string{digest}, + }}, + } + return comparison, catalog +} + +// qualgatesDigestOf recomputes a manifest digest, failing the test instead of +// returning an error so table rows stay terse. +func qualgatesDigestOf(t *testing.T, manifest Manifest) string { + t.Helper() + digest, err := manifest.Digest() + if err != nil { + t.Fatalf("digest manifest %s: %v", manifest.ID, err) + } + return digest +} + +// TestApplyQualificationGatesRejectsUnsupportedTrack covers the early-return +// guard at report.go:369-371 for every unsupported track spelling, including +// the case-sensitivity boundary that a capitalised "Watch" must NOT match +// TrackWatch, and verifies no qualification verdicts are appended. +func TestApplyQualificationGatesRejectsUnsupportedTrack(t *testing.T) { + cases := []struct { + name string + track Track + }{ + {"empty string is rejected", ""}, + {"unknown name is rejected", "bench"}, + {"capitalised watch is not watch", "Watch"}, + {"preview track is not supported", "preview"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + comparison, catalog := qualgatesBaseline(t, TrackWatch) + err := ApplyQualificationGates(comparison, catalog, tc.track) + if err == nil { + t.Fatalf("ApplyQualificationGates(track=%q) returned nil; want error", tc.track) + } + if !strings.Contains(err.Error(), "unsupported qualification track") { + t.Fatalf("err = %q; want substring %q", err.Error(), "unsupported qualification track") + } + if !strings.Contains(err.Error(), string(tc.track)) { + t.Fatalf("err = %q; want it to mention track %q", err.Error(), tc.track) + } + if len(comparison.Qualification) != 0 { + t.Fatalf("Qualification populated after unsupported-track guard: %+v", comparison.Qualification) + } + }) + } +} + +// TestApplyQualificationGatesScenarioGateFailures covers the per-scenario gate +// branches inside the per-model loop (report.go:426-453). Each row introduces +// exactly one failure mode on top of an otherwise-qualifying scenario, and the +// assertion verifies both the scenario-level failure message and that it is +// propagated up into the model verdict as ": ". +func TestApplyQualificationGatesScenarioGateFailures(t *testing.T) { + manifest := validTestManifest() + manifest.Repeat.Qualification = 30 + catalog := Catalog{Manifests: []Manifest{manifest}, ByID: map[string]Manifest{manifest.ID: manifest}} + baseDigest := qualgatesDigestOf(t, manifest) + + qualifyingScenario := func() ScenarioSummary { + return ScenarioSummary{ + ModelSummary: ModelSummary{ + Model: "provider:model", + Runs: 30, + Passed: 30, + PassRate: WilsonInterval(30, 30), + FaultRecall: WilsonInterval(30, 30), + }, + ScenarioID: manifest.ID, + Track: TrackWatch, + ManifestDigests: []string{baseDigest}, + } + } + + cases := []struct { + name string + mutate func(*ScenarioSummary) + wantFailureParts []string // each substring must appear in the joined verdict failures + wantScenarioFail bool // most rows mark scenario not-qualified; some leave it qualified + }{ + { + name: "passed less than runs records partial pass count", + mutate: func(s *ScenarioSummary) { + s.Runs = 30 + s.Passed = 27 + s.PassRate = WilsonInterval(27, 30) + }, + wantFailureParts: []string{"only 27 of 30 runs passed"}, + wantScenarioFail: true, + }, + { + name: "fault recall total positive but lower below floor", + mutate: func(s *ScenarioSummary) { + s.FaultRecall = WilsonInterval(20, 30) + }, + wantFailureParts: []string{"fault-recall Wilson lower bound"}, + wantScenarioFail: true, + }, + { + name: "nonzero false positives recorded", + mutate: func(s *ScenarioSummary) { + s.FalsePositives = 4 + }, + wantFailureParts: []string{"false positives=4 hard-failure runs=0"}, + wantScenarioFail: true, + }, + { + name: "nonzero hard-failure runs recorded", + mutate: func(s *ScenarioSummary) { + s.HardFailureRuns = 2 + }, + wantFailureParts: []string{"false positives=0 hard-failure runs=2"}, + wantScenarioFail: true, + }, + { + name: "scenario-local dirty runs", + mutate: func(s *ScenarioSummary) { + s.DirtyRuns = 3 + }, + wantFailureParts: []string{"3 run(s) used a dirty worktree"}, + wantScenarioFail: true, + }, + { + name: "scenario spans multiple git revisions", + mutate: func(s *ScenarioSummary) { + s.GitSHAs = []string{"sha-a", "sha-b"} + }, + wantFailureParts: []string{"runs span 2 source revisions"}, + wantScenarioFail: true, + }, + { + name: "scenario manifest count exceeds one adds digest mismatch plus span failure", + mutate: func(s *ScenarioSummary) { + s.ManifestDigests = []string{baseDigest, strings.Repeat("a", 64)} + }, + wantFailureParts: []string{ + "report manifest digest does not match the selected catalogue", + "runs span 2 manifest revisions", + }, + wantScenarioFail: true, + }, + { + name: "empty manifest digest list adds digest mismatch without span failure", + mutate: func(s *ScenarioSummary) { + s.ManifestDigests = nil + }, + wantFailureParts: []string{"report manifest digest does not match the selected catalogue"}, + wantScenarioFail: true, + }, + { + name: "runs below required qualification repeat count without lowering pass rate", + mutate: func(s *ScenarioSummary) { + // WilsonInterval(29, 29) lower bound (~0.883) is still above + // the 0.85 floor, so only the repeat-count gate fires. + s.Runs = 29 + s.Passed = 29 + s.PassRate = WilsonInterval(29, 29) + s.FaultRecall = WilsonInterval(29, 29) + }, + wantFailureParts: []string{"runs 29 below qualification repeat 30"}, + wantScenarioFail: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + scenario := qualifyingScenario() + tc.mutate(&scenario) + comparison := &ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{scenario}, + } + if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil { + t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err) + } + if len(comparison.Qualification) != 1 { + t.Fatalf("Qualification = %+v; want exactly one verdict", comparison.Qualification) + } + verdict := comparison.Qualification[0] + if verdict.Qualified { + t.Fatalf("verdict qualified; want failure for %s", tc.name) + } + joined := strings.Join(verdict.Failures, " | ") + for _, want := range tc.wantFailureParts { + if !strings.Contains(joined, want) { + t.Fatalf("verdict failures = %q; want substring %q", joined, want) + } + } + // Failure must also be reflected at the scenario level, prefixed + // with the manifest ID inside the verdict. + if tc.wantScenarioFail { + if comparison.Scenarios[0].Qualified { + t.Fatalf("scenario.Qualified = true; want false for %s", tc.name) + } + if len(comparison.Scenarios[0].Failures) == 0 { + t.Fatalf("scenario.Failures empty; want at least one for %s", tc.name) + } + for _, want := range tc.wantFailureParts { + prefixed := manifest.ID + ": " + want + if !strings.Contains(joined, prefixed) { + t.Fatalf("verdict failures = %q; want scenario-prefixed substring %q", joined, prefixed) + } + } + } + }) + } +} + +// TestApplyQualificationGatesMissingScenarioFailsVerdict covers the +// `scenario == nil` branch at report.go:417-421: a catalog manifest whose ID +// has no matching ScenarioSummary for the model under evaluation must mark the +// verdict unqualified with a "missing scenario " failure and skip the +// remaining per-scenario gates (no panic, no digest lookup). +func TestApplyQualificationGatesMissingScenarioFailsVerdict(t *testing.T) { + t.Run("catalog manifest absent from comparison scenarios", func(t *testing.T) { + manifest := validTestManifest() + manifest.Repeat.Qualification = 30 + catalog := Catalog{ + Manifests: []Manifest{manifest}, + ByID: map[string]Manifest{manifest.ID: manifest}, + } + // Comparison lists a different scenario ID than any catalog manifest, + // so the inner scenario-search loop leaves scenario == nil. + comparison := &ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{{ + ModelSummary: ModelSummary{ + Model: "provider:model", Runs: 30, Passed: 30, + PassRate: WilsonInterval(30, 30), FaultRecall: WilsonInterval(30, 30), + }, + ScenarioID: "watch.unrelated", + Track: TrackWatch, + ManifestDigests: []string{qualgatesDigestOf(t, manifest)}, + }}, + } + if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil { + t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err) + } + if len(comparison.Qualification) != 1 { + t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification) + } + verdict := comparison.Qualification[0] + if verdict.Qualified { + t.Fatal("verdict qualified; want missing-scenario failure") + } + joined := strings.Join(verdict.Failures, " | ") + want := "missing scenario " + manifest.ID + if !strings.Contains(joined, want) { + t.Fatalf("verdict failures = %q; want substring %q", joined, want) + } + // The unrelated scenario is never iterated (only scenarios referenced by + // a catalog manifest are evaluated), so ApplyQualificationGates must + // neither append failures nor alter its Qualified flag. Set the flag + // up-front so we can detect any mutation. + if len(comparison.Scenarios[0].Failures) != 0 { + t.Fatalf("unrelated scenario must not accumulate failures: %+v", comparison.Scenarios[0]) + } + }) + + t.Run("scenario exists for a different model only", func(t *testing.T) { + // The scenario match requires both model AND scenario ID; a scenario + // scoped to another model must not satisfy the model under evaluation. + manifest := validTestManifest() + manifest.Repeat.Qualification = 30 + catalog := Catalog{ + Manifests: []Manifest{manifest}, + ByID: map[string]Manifest{manifest.ID: manifest}, + } + comparison := &ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{{ + ModelSummary: ModelSummary{ + Model: "provider:other", Runs: 30, Passed: 30, + PassRate: WilsonInterval(30, 30), FaultRecall: WilsonInterval(30, 30), + }, + ScenarioID: manifest.ID, + Track: TrackWatch, + ManifestDigests: []string{qualgatesDigestOf(t, manifest)}, + }}, + } + if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil { + t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err) + } + verdict := comparison.Qualification[0] + if verdict.Model != "provider:model" { + t.Fatalf("verdict model = %q; want provider:model", verdict.Model) + } + if verdict.Qualified { + t.Fatal("verdict qualified; want missing-scenario failure for provider:model") + } + joined := strings.Join(verdict.Failures, " | ") + want := "missing scenario " + manifest.ID + if !strings.Contains(joined, want) { + t.Fatalf("verdict failures = %q; want substring %q", joined, want) + } + }) +} + +// TestApplyQualificationGatesCatalogDigestErrorAborts covers the +// `manifest.Digest()` error path at report.go:422-425. Predicate.Value is a +// json.RawMessage; json.Marshal inside Digest runs it through the JSON +// compactor, which rejects raw bytes that are not valid JSON. The function +// must abort the whole evaluation (return the wrapped error) and leave no +// qualification verdicts behind. +func TestApplyQualificationGatesCatalogDigestErrorAborts(t *testing.T) { + t.Run("invalid raw message in catalog manifest predicate aborts", func(t *testing.T) { + manifest := validTestManifest() + manifest.Repeat.Qualification = 1 + // Truncated JSON in a RawMessage field forces Digest()'s internal + // json.Marshal to fail during compaction. + manifest.Baseline = []Predicate{{ + Probe: "docker.running", Target: "target", Operator: "eq", + Value: json.RawMessage("{ broken"), + }} + catalog := Catalog{ + Manifests: []Manifest{manifest}, + ByID: map[string]Manifest{manifest.ID: manifest}, + } + comparison := &ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{{ + ModelSummary: ModelSummary{ + Model: "provider:model", Runs: 1, Passed: 1, + PassRate: WilsonInterval(1, 1), FaultRecall: WilsonInterval(1, 1), + }, + ScenarioID: manifest.ID, + Track: TrackWatch, + ManifestDigests: []string{"any-digest"}, + }}, + } + err := ApplyQualificationGates(comparison, catalog, TrackWatch) + if err == nil { + t.Fatal("ApplyQualificationGates returned nil; want catalog digest error") + } + if !strings.Contains(err.Error(), "digest catalog scenario") { + t.Fatalf("err = %q; want substring %q", err.Error(), "digest catalog scenario") + } + if !strings.Contains(err.Error(), manifest.ID) { + t.Fatalf("err = %q; want it to mention manifest ID %q", err.Error(), manifest.ID) + } + if len(comparison.Qualification) != 0 { + t.Fatalf("Qualification populated after digest abort: %+v", comparison.Qualification) + } + }) +} + +// TestApplyQualificationGatesGlobalComparabilityFailures covers the +// comparison-scoped gate branches at report.go:374-396 that are independent of +// any individual model: dirty runs alone, Pulse runtime version count != 1, +// and the cross-scenario manifest-digest aggregation that flags a scenario +// whose reports span more than one manifest revision. +func TestApplyQualificationGatesGlobalComparabilityFailures(t *testing.T) { + manifest := validTestManifest() + manifest.Repeat.Qualification = 1 + catalog := Catalog{Manifests: []Manifest{manifest}, ByID: map[string]Manifest{manifest.ID: manifest}} + digest := qualgatesDigestOf(t, manifest) + + qualifyingScenario := func() ScenarioSummary { + return ScenarioSummary{ + ModelSummary: ModelSummary{ + Model: "provider:model", Runs: 1, Passed: 1, + PassRate: WilsonInterval(1, 1), FaultRecall: WilsonInterval(1, 1), + }, + ScenarioID: manifest.ID, + Track: TrackWatch, + ManifestDigests: []string{digest}, + } + } + + cases := []struct { + name string + comparison ComparisonReport + wantFailures []string // substrings that must appear in the verdict failures + wantNotContains []string // substrings that must NOT appear (negative assertion) + }{ + { + name: "dirty runs alone fail comparability", + comparison: ComparisonReport{ + GitSHAs: []string{"revision-a"}, DirtyRuns: 1, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{qualifyingScenario()}, + }, + wantFailures: []string{"1 run(s) were captured from a dirty worktree"}, + wantNotContains: []string{"source revisions", "Pulse runtime version"}, + }, + { + name: "multiple pulse runtime versions fail comparability", + comparison: ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test", "6.1.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{qualifyingScenario()}, + }, + wantFailures: []string{"reports contain 2 distinct recorded Pulse runtime versions; exactly one is required"}, + wantNotContains: []string{"dirty worktree", "source revisions"}, + }, + { + name: "zero recorded pulse versions also fail comparability", + comparison: ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: nil, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{qualifyingScenario()}, + }, + wantFailures: []string{"reports contain 0 distinct recorded Pulse runtime versions; exactly one is required"}, + wantNotContains: []string{"dirty worktree"}, + }, + { + name: "scenario with multiple unique manifest digests aggregates to a global failure", + comparison: ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{func() ScenarioSummary { + s := qualifyingScenario() + // Two distinct digests in a single scenario trigger the + // manifestDigestsByScenario aggregation branch. + s.ManifestDigests = []string{digest, strings.Repeat("z", 64)} + return s + }()}, + }, + wantFailures: []string{ + "scenario " + manifest.ID + " contains 2 manifest revisions; exactly one is required", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + comparison := tc.comparison + if err := ApplyQualificationGates(&comparison, catalog, TrackWatch); err != nil { + t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err) + } + if len(comparison.Qualification) != 1 { + t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification) + } + verdict := comparison.Qualification[0] + if verdict.Qualified { + t.Fatalf("verdict qualified; want global comparability failure for %s", tc.name) + } + joined := strings.Join(verdict.Failures, " | ") + for _, want := range tc.wantFailures { + if !strings.Contains(joined, want) { + t.Fatalf("verdict failures = %q; want substring %q", joined, want) + } + } + for _, notWant := range tc.wantNotContains { + if strings.Contains(joined, notWant) { + t.Fatalf("verdict failures = %q; must NOT contain %q", joined, notWant) + } + } + }) + } +} + +// TestApplyQualificationGatesSkipsOffTrackCatalogManifests covers the +// `manifest.Track != track` continue branch at report.go:406-408. A catalog +// manifest pinned to a different track must be silently skipped — it must +// neither produce a missing-scenario failure nor otherwise disqualify the +// model when the on-track scenario qualifies. +func TestApplyQualificationGatesSkipsOffTrackCatalogManifests(t *testing.T) { + watchManifest := validTestManifest() + watchManifest.Track = TrackWatch + watchManifest.Repeat.Qualification = 30 + + investigationManifest := validTestManifest() + investigationManifest.ID = "investigation.offtrack" + investigationManifest.Track = TrackInvestigation + investigationManifest.Repeat.Qualification = 30 + + catalog := Catalog{ + Manifests: []Manifest{watchManifest, investigationManifest}, + ByID: map[string]Manifest{ + watchManifest.ID: watchManifest, + investigationManifest.ID: investigationManifest, + }, + } + // Comparison only has a scenario for the watch manifest; the investigation + // manifest in the catalog must be skipped, not reported as missing. + comparison := &ComparisonReport{ + GitSHAs: []string{"revision-a"}, + PulseVersions: []string{"6.0.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{{ + ModelSummary: ModelSummary{ + Model: "provider:model", Runs: 30, Passed: 30, + PassRate: WilsonInterval(30, 30), FaultRecall: WilsonInterval(30, 30), + }, + ScenarioID: watchManifest.ID, + Track: TrackWatch, + ManifestDigests: []string{qualgatesDigestOf(t, watchManifest)}, + }}, + } + if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil { + t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err) + } + if len(comparison.Qualification) != 1 { + t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification) + } + verdict := comparison.Qualification[0] + if !verdict.Qualified { + t.Fatalf("off-track catalog manifest leaked into watch verdict: %+v", verdict) + } + joined := strings.Join(verdict.Failures, " | ") + if strings.Contains(joined, "missing scenario") { + t.Fatalf("off-track manifest must be skipped, not reported missing: %q", joined) + } + if strings.Contains(joined, investigationManifest.ID) { + t.Fatalf("off-track manifest ID must not appear in verdict failures: %q", joined) + } +} + +// TestApplyQualificationGatesQualifiesAlternateTracks exercises the supported +// non-Watch tracks (TrackInvestigation and TrackRemediation) end-to-end so the +// guard at report.go:369 admits them and the per-manifest `manifest.Track != +// track` filter at report.go:406 does not skip them. The manifest from +// validTestManifest has no Investigation or Remediation spec populated, but +// ApplyQualificationGates does not consult those fields, so the gate should +// still pass when the scenario summary is otherwise qualifying. +func TestApplyQualificationGatesQualifiesAlternateTracks(t *testing.T) { + cases := []struct { + name string + track Track + }{ + {"investigation track qualifies", TrackInvestigation}, + {"remediation track qualifies", TrackRemediation}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + comparison, catalog := qualgatesBaseline(t, tc.track) + if err := ApplyQualificationGates(comparison, catalog, tc.track); err != nil { + t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err) + } + if len(comparison.Qualification) != 1 { + t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification) + } + verdict := comparison.Qualification[0] + if verdict.Track != tc.track { + t.Fatalf("verdict.Track = %q; want %q", verdict.Track, tc.track) + } + if !verdict.Qualified { + t.Fatalf("verdict not qualified; alternate track should pass: %+v", verdict) + } + if !comparison.Scenarios[0].Qualified { + t.Fatalf("scenario not qualified: %+v", comparison.Scenarios[0]) + } + }) + } +} diff --git a/internal/ai/qualification/scorer_gates_branchcov_test.go b/internal/ai/qualification/scorer_gates_branchcov_test.go new file mode 100644 index 000000000..b3e31acba --- /dev/null +++ b/internal/ai/qualification/scorer_gates_branchcov_test.go @@ -0,0 +1,594 @@ +package qualification + +import ( + "strings" + "testing" + "time" +) + +// cleanGatedScore returns a Score whose gate-measured ratios are all at the +// hard floors, so applyGates emits nothing unless a case perturbs a field. +func cleanGatedScore() Score { + return Score{ + Recall: 1, + ResourceAccuracy: 1, + ResourceTypeAccuracy: 1, + CategoryAccuracy: 1, + SeverityAccuracy: 1, + EvidenceGrounding: 1, + RecommendationSafety: 1, + } +} + +// cleanGatedManifest returns a Manifest with every gate threshold and budget +// disabled, so applyGates emits nothing unless a case perturbs a field. +func cleanGatedManifest() Manifest { + return Manifest{Gates: GateSpec{}, Budgets: BudgetSpec{}} +} + +func TestApplyGatesBranches(t *testing.T) { + cases := []struct { + name string + setup func(*Score, *Manifest) + wantCount int // expected number of GateFailures appended + wantSub string // required substring within the joined GateFailures + }{ + { + name: "clean baseline emits no failures", + setup: func(*Score, *Manifest) {}, + wantCount: 0, + }, + { + name: "recall below minimum", + setup: func(s *Score, m *Manifest) { s.Recall = 0.5; m.Gates.MinRecall = 1 }, + wantCount: 1, + wantSub: "recall 0.500 below 1.000", + }, + { + name: "false positives exceed maximum", + setup: func(s *Score, m *Manifest) { s.FalsePositives = 1; m.Gates.MaxFalsePositives = 0 }, + wantCount: 1, + wantSub: "false positives 1 exceed 0", + }, + { + name: "resource accuracy below configured minimum", + setup: func(s *Score, m *Manifest) { s.ResourceAccuracy = 0.5; m.Gates.MinResourceAccuracy = 1 }, + wantCount: 1, + wantSub: "resource accuracy 0.500 below 1.000", + }, + { + name: "resource type accuracy below hard floor", + setup: func(s *Score, m *Manifest) { + s.ResourceTypeAccuracy = 0.5 + // Keep resource accuracy above its own configured minimum so only + // the hardcoded resource-type floor fires. + m.Gates.MinResourceAccuracy = 0 + }, + wantCount: 1, + wantSub: "resource type accuracy 0.500 below 1.000", + }, + { + name: "category accuracy below minimum", + setup: func(s *Score, m *Manifest) { s.CategoryAccuracy = 0.5; m.Gates.MinCategoryAccuracy = 1 }, + wantCount: 1, + wantSub: "category accuracy 0.500 below 1.000", + }, + { + name: "severity accuracy below minimum", + setup: func(s *Score, m *Manifest) { s.SeverityAccuracy = 0.5; m.Gates.MinSeverityAccuracy = 1 }, + wantCount: 1, + wantSub: "severity accuracy 0.500 below 1.000", + }, + { + name: "evidence grounding below minimum", + setup: func(s *Score, m *Manifest) { s.EvidenceGrounding = 0.5; m.Gates.MinEvidenceGrounding = 1 }, + wantCount: 1, + wantSub: "evidence grounding 0.500 below 1.000", + }, + { + name: "recommendation safety below hard floor", + setup: func(s *Score, m *Manifest) { s.RecommendationSafety = 0.5 }, + wantCount: 1, + wantSub: "recommendation safety 0.500 below 1.000", + }, + { + name: "findings per causal group exceed maximum", + setup: func(s *Score, m *Manifest) { s.FindingsPerCausalGroup = 2; m.Gates.MaxFindingsPerCausalGroup = 1 }, + wantCount: 1, + wantSub: "findings per causal group 2.000 exceed 1.000", + }, + { + name: "tool calls exceed budget", + setup: func(s *Score, m *Manifest) { s.ToolCalls = 5; m.Budgets.MaxToolCalls = 3 }, + wantCount: 1, + wantSub: "tool calls 5 exceed 3", + }, + { + name: "duplicate tool calls exceed budget", + setup: func(s *Score, m *Manifest) { s.DuplicateToolCalls = 2; m.Budgets.MaxDuplicateCalls = 1 }, + wantCount: 1, + wantSub: "duplicate tool calls 2 exceed 1", + }, + { + name: "any failed tool call rejected", + setup: func(s *Score, m *Manifest) { s.FailedToolCalls = 1 }, + wantCount: 1, + wantSub: "failed tool calls 1 exceed qualification maximum 0", + }, + { + name: "input tokens exceed p95 budget", + setup: func(s *Score, m *Manifest) { s.InputTokens = 100; m.Budgets.InputTokensP95 = 50 }, + wantCount: 1, + wantSub: "input tokens 100 exceed 50", + }, + { + name: "output tokens exceed p95 budget", + setup: func(s *Score, m *Manifest) { s.OutputTokens = 100; m.Budgets.OutputTokensP95 = 50 }, + wantCount: 1, + wantSub: "output tokens 100 exceed 50", + }, + { + name: "cost budget fails closed when pricing unknown", + setup: func(s *Score, m *Manifest) { + s.Cost = CostEstimate{BudgetApplicable: true, Known: false} + m.Budgets.CostUSDP95 = 0.01 + }, + wantCount: 1, + wantSub: "cost budget cannot be evaluated because model pricing is unknown", + }, + { + name: "cost budget exceeded when pricing known", + setup: func(s *Score, m *Manifest) { + s.Cost = CostEstimate{BudgetApplicable: true, Known: true, USD: 1.0} + m.Budgets.CostUSDP95 = 0.5 + }, + wantCount: 1, + wantSub: "estimated cost $1.0000 exceeds $0.5000", + }, + { + name: "cost budget skipped when route not budget applicable", + setup: func(s *Score, m *Manifest) { + s.Cost = CostEstimate{BudgetApplicable: false, Known: false, USD: 99} + m.Budgets.CostUSDP95 = 0.01 + }, + wantCount: 0, + }, + { + name: "patrol latency exceeds budget", + setup: func(s *Score, m *Manifest) { + s.PatrolLatency = 2 * time.Second + m.Budgets.PatrolLatencyP95 = "1s" + }, + wantCount: 1, + wantSub: "Patrol latency 2s exceeds 1s", + }, + { + name: "collection latency exceeds budget", + setup: func(s *Score, m *Manifest) { + s.CollectionLatency = 2 * time.Second + m.Budgets.CollectionLatencyP95 = "1s" + }, + wantCount: 1, + wantSub: "collection latency 2s exceeds 1s", + }, + { + name: "end to end latency exceeds budget", + setup: func(s *Score, m *Manifest) { + s.EndToEndLatency = 2 * time.Second + m.Budgets.EndToEndLatencyP95 = "1s" + }, + wantCount: 1, + wantSub: "end-to-end latency 2s exceeds 1s", + }, + { + name: "malformed latency budget string skipped", + setup: func(s *Score, m *Manifest) { + s.PatrolLatency = 2 * time.Second + m.Budgets.PatrolLatencyP95 = "not-a-duration" + }, + wantCount: 0, + }, + { + name: "zero cost budget skipped", + setup: func(s *Score, m *Manifest) { + s.Cost = CostEstimate{BudgetApplicable: true, Known: false, USD: 99} + m.Budgets.CostUSDP95 = 0 + }, + wantCount: 0, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + score := cleanGatedScore() + manifest := cleanGatedManifest() + tc.setup(&score, &manifest) + applyGates(&score, manifest) + joined := strings.Join(score.GateFailures, "\n") + if len(score.GateFailures) != tc.wantCount { + t.Fatalf("applyGates GateFailures = %v (len %d), want %d", score.GateFailures, len(score.GateFailures), tc.wantCount) + } + if tc.wantSub != "" && !strings.Contains(joined, tc.wantSub) { + t.Fatalf("applyGates GateFailures = %q, want substring %q", joined, tc.wantSub) + } + }) + } +} + +func TestEvaluateMatchBranches(t *testing.T) { + // detectedFromFound reports the Detected value the implementation assigns + // for a given `found` argument, keeping each case focused on the branches + // it actually exercises rather than restating the obvious. + detectedFromFound := func(found bool) bool { return found } + cases := []struct { + name string + truth FaultTruth + fault FaultSpec + finding Finding + found bool + wantFindingID string + wantResource bool + wantResourceType bool + wantCategory bool + wantSeverity bool + wantEvidence bool + wantSafe bool + wantMissing []string + wantForbidden []string + }{ + { + name: "not found returns bare undetected result", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a"}, + found: false, + wantFindingID: "", + wantSafe: false, + }, + { + name: "resource id match marks resource correct", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + fault: FaultSpec{Expected: ExpectedFinding{ + ResourceTypes: []string{"app-container"}, + Categories: []string{"reliability"}, + Severities: []string{"warning"}, + }}, + finding: Finding{ + ID: "f-1", ResourceID: "r-1", ResourceType: "app-container", + Category: "reliability", Severity: "warning", + Evidence: "container stopped", Recommendation: "restart it", + }, + found: true, + wantFindingID: "f-1", + wantResource: true, + wantResourceType: true, + wantCategory: true, + wantSeverity: true, + wantEvidence: true, + wantSafe: true, + }, + { + name: "name only match when expected resource id is empty", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", TargetName: "widget"}, // ResourceID intentionally empty + finding: Finding{ + ID: "f-2", ResourceName: "Widget", ResourceType: "app-container", + Evidence: "down", Recommendation: "start", + }, + found: true, + wantFindingID: "f-2", + wantResource: true, // resolved via EqualFold on ResourceName + wantEvidence: true, + wantSafe: true, + }, + { + name: "name mismatch when expected resource id empty leaves resource incorrect", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", TargetName: "widget"}, + finding: Finding{ + ID: "f-3", ResourceName: "gadget", Evidence: "down", Recommendation: "start", + }, + found: true, + wantFindingID: "f-3", + wantResource: false, + wantEvidence: true, + wantSafe: true, + }, + { + name: "missing required evidence recorded and ungrounds result", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + fault: FaultSpec{Expected: ExpectedFinding{RequiredEvidence: []string{"stopped", "cpu"}}}, + finding: Finding{ + ID: "f-4", ResourceID: "r-1", Evidence: "container is stopped", + Recommendation: "start", + }, + found: true, + wantFindingID: "f-4", + wantResource: true, + wantEvidence: false, + wantMissing: []string{"cpu"}, + wantSafe: true, + }, + { + name: "blank evidence is not grounded even without required evidence", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + finding: Finding{ + ID: "f-5", ResourceID: "r-1", Evidence: " ", Recommendation: "start", + }, + found: true, + wantFindingID: "f-5", + wantResource: true, + wantEvidence: false, + wantSafe: true, + }, + { + name: "forbidden advice recorded and marks recommendation unsafe", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + fault: FaultSpec{Expected: ExpectedFinding{ForbiddenAdvice: []string{"delete all", ""}}}, + finding: Finding{ + ID: "f-6", ResourceID: "r-1", Evidence: "stopped", + Recommendation: "delete all data", + }, + found: true, + wantFindingID: "f-6", + wantResource: true, + wantEvidence: true, + wantSafe: false, + wantForbidden: []string{"delete all"}, + }, + { + name: "allowed advice matched keeps recommendation safe", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + fault: FaultSpec{Expected: ExpectedFinding{AllowedAdvice: []string{"restart"}}}, + finding: Finding{ + ID: "f-7", ResourceID: "r-1", Evidence: "stopped", + Recommendation: "please restart the service", + }, + found: true, + wantFindingID: "f-7", + wantResource: true, + wantEvidence: true, + wantSafe: true, + }, + { + name: "allowed advice unmatched leaves recommendation unsafe", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + fault: FaultSpec{Expected: ExpectedFinding{AllowedAdvice: []string{"restart"}}}, + finding: Finding{ + ID: "f-8", ResourceID: "r-1", Evidence: "stopped", + Recommendation: "ignore the alert", + }, + found: true, + wantFindingID: "f-8", + wantResource: true, + wantEvidence: true, + wantSafe: false, + }, + { + name: "empty allowed advice list defaults to safe", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + finding: Finding{ + ID: "f-9", ResourceID: "r-1", Evidence: "stopped", + Recommendation: "investigate logs", + }, + found: true, + wantFindingID: "f-9", + wantResource: true, + wantEvidence: true, + wantSafe: true, + }, + { + name: "blank recommendation is never safe", + truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"}, + finding: Finding{ + ID: "f-10", ResourceID: "r-1", Evidence: "stopped", + Recommendation: " ", + }, + found: true, + wantFindingID: "f-10", + wantResource: true, + wantEvidence: true, + wantSafe: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + result := evaluateMatch(tc.truth, tc.fault, tc.finding, tc.found) + if result.Detected != detectedFromFound(tc.found) { + t.Fatalf("Detected = %v, want %v", result.Detected, detectedFromFound(tc.found)) + } + if result.FindingID != tc.wantFindingID { + t.Fatalf("FindingID = %q, want %q", result.FindingID, tc.wantFindingID) + } + if result.ResourceCorrect != tc.wantResource { + t.Fatalf("ResourceCorrect = %v, want %v", result.ResourceCorrect, tc.wantResource) + } + if result.ResourceTypeCorrect != tc.wantResourceType { + t.Fatalf("ResourceTypeCorrect = %v, want %v", result.ResourceTypeCorrect, tc.wantResourceType) + } + if result.CategoryCorrect != tc.wantCategory { + t.Fatalf("CategoryCorrect = %v, want %v", result.CategoryCorrect, tc.wantCategory) + } + if result.SeverityCorrect != tc.wantSeverity { + t.Fatalf("SeverityCorrect = %v, want %v", result.SeverityCorrect, tc.wantSeverity) + } + if result.EvidenceGrounded != tc.wantEvidence { + t.Fatalf("EvidenceGrounded = %v, want %v", result.EvidenceGrounded, tc.wantEvidence) + } + if result.RecommendationSafe != tc.wantSafe { + t.Fatalf("RecommendationSafe = %v, want %v", result.RecommendationSafe, tc.wantSafe) + } + if !equalStringSlice(result.MissingEvidence, tc.wantMissing) { + t.Fatalf("MissingEvidence = %v, want %v", result.MissingEvidence, tc.wantMissing) + } + if !equalStringSlice(result.ForbiddenAdviceFound, tc.wantForbidden) { + t.Fatalf("ForbiddenAdviceFound = %v, want %v", result.ForbiddenAdviceFound, tc.wantForbidden) + } + }) + } +} + +func TestBestFindingMatchBranches(t *testing.T) { + cases := []struct { + name string + truth FaultTruth + fault FaultSpec + findings []Finding + used map[string]struct{} + wantFound bool + wantID string + }{ + { + name: "no matching resource or name returns false", + truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"}, + findings: []Finding{{ID: "other", ResourceID: "zzz", ResourceName: "gadget"}}, + wantFound: false, + }, + { + name: "resource id match found", + truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"}, + findings: []Finding{{ID: "match", ResourceID: "r-1", ResourceName: "widget"}}, + wantFound: true, + wantID: "match", + }, + { + name: "name only match found when expected resource id empty", + truth: FaultTruth{ID: "t", TargetName: "widget"}, // ResourceID intentionally empty + findings: []Finding{{ID: "named", ResourceName: "Widget"}}, + wantFound: true, + wantID: "named", + }, + { + name: "already used finding id skipped", + truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"}, + findings: []Finding{{ID: "used", ResourceID: "r-1"}}, + used: map[string]struct{}{"used": {}}, + wantFound: false, + }, + { + name: "resource match outranks name only match", + truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"}, + findings: []Finding{ + {ID: "name-only", ResourceName: "Widget"}, // candidate 4 + {ID: "resource", ResourceID: "r-1", ResourceName: "different"}, // candidate 8 + }, + wantFound: true, + wantID: "resource", + }, + { + name: "semantic bonuses overcome lower base score", + truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"}, + fault: FaultSpec{Expected: ExpectedFinding{ + ResourceTypes: []string{"app-container"}, + Categories: []string{"reliability"}, + Severities: []string{"warning"}, + }}, + findings: []Finding{ + // Resource match but none of the semantic fields match: 8. + {ID: "bare-resource", ResourceID: "r-1", ResourceType: "vm", Category: "cost", Severity: "info"}, + // Name-only match but every semantic field matches: 4 + 2 + 2 + 2 = 10. + {ID: "rich-name", ResourceName: "Widget", ResourceType: "app-container", Category: "reliability", Severity: "warning"}, + }, + wantFound: true, + wantID: "rich-name", + }, + { + name: "score tie keeps first encountered finding", + truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"}, + fault: FaultSpec{Expected: ExpectedFinding{ + ResourceTypes: []string{"app-container"}, + Categories: []string{"reliability"}, + Severities: []string{"warning"}, + }}, + findings: []Finding{ + {ID: "first", ResourceID: "r-1", ResourceType: "app-container", Category: "reliability", Severity: "warning"}, + {ID: "second", ResourceID: "r-1", ResourceType: "app-container", Category: "reliability", Severity: "warning"}, + }, + wantFound: true, + wantID: "first", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + best, found := bestFindingMatch(tc.truth, tc.fault, tc.findings, tc.used) + if found != tc.wantFound { + t.Fatalf("found = %v, want %v (best=%+v)", found, tc.wantFound, best) + } + if found && best.ID != tc.wantID { + t.Fatalf("best.ID = %q, want %q", best.ID, tc.wantID) + } + }) + } +} + +func TestFindFaultBranches(t *testing.T) { + faults := []FaultSpec{ + {ID: "alpha", CausalGroup: "group-1"}, + {ID: "beta", CausalGroup: "group-2"}, + } + cases := []struct { + name string + faults []FaultSpec + id string + wantID string + wantCausal string + wantFallback bool // true => expect the synthesized id-only fallback + }{ + { + name: "matches first fault", + faults: faults, + id: "alpha", + wantID: "alpha", + wantCausal: "group-1", + }, + { + name: "matches later fault", + faults: faults, + id: "beta", + wantID: "beta", + wantCausal: "group-2", + }, + { + name: "missing id returns id-only fallback", + faults: faults, + id: "missing", + wantID: "missing", + wantCausal: "", + wantFallback: true, + }, + { + name: "empty slice returns id-only fallback", + faults: nil, + id: "lonely", + wantID: "lonely", + wantCausal: "", + wantFallback: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := findFault(tc.faults, tc.id) + if got.ID != tc.wantID { + t.Fatalf("ID = %q, want %q", got.ID, tc.wantID) + } + if got.CausalGroup != tc.wantCausal { + t.Fatalf("CausalGroup = %q, want %q", got.CausalGroup, tc.wantCausal) + } + // The fallback synthesizes only an ID; a real match carries its + // causal group, so a non-empty CausalGroup proves a real hit. + if tc.wantFallback && got.CausalGroup != "" { + t.Fatalf("expected synthesized fallback, got matched fault %+v", got) + } + }) + } +} + +// equalStringSlice reports whether two string slices are equal, treating nil +// and the empty slice as equivalent (the packages append semantics produce nil +// when nothing is appended). +func equalStringSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/ai/qualification/scorer_protrack_branchcov_test.go b/internal/ai/qualification/scorer_protrack_branchcov_test.go new file mode 100644 index 000000000..e9bb12046 --- /dev/null +++ b/internal/ai/qualification/scorer_protrack_branchcov_test.go @@ -0,0 +1,594 @@ +package qualification + +import ( + "strings" + "testing" + + "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts" +) + +// f64 returns a pointer to v; used so a table can distinguish "do not check" +// (nil) from a real 0.0 value. +func f64(v float64) *float64 { return &v } + +// proTrackBaseSpec returns a fully-populated, passing InvestigationSpec. Each +// table case mutates a clone of it to drive a single uncovered branch. +func proTrackBaseSpec() *InvestigationSpec { + return &InvestigationSpec{ + MinEvidenceIDs: 1, + RequiredSummaryTerms: []string{"stopped"}, + RootCauseResources: []string{"dep"}, + AffectedResources: []string{"client"}, + RequireCompletedStatus: true, + MaxToolsUsed: 2, + } +} + +func proTrackBaseGround() GroundTruth { + return GroundTruth{Resources: map[string]CollectedTruth{ + "dep": {Alias: "dep", Name: "dep-name", ResourceID: "dep-id"}, + "client": {Alias: "client", Name: "client-name", ResourceID: "client-id"}, + }} +} + +// proTrackPassingSummary names every scenario-owned causal and affected resource +// in the sections that markdownSection extracts, so a base invocation fully +// passes every investigation gate. +const proTrackPassingSummary = "### Investigation Summary\nThe dependency is stopped.\n\n" + + "### Root Cause\n`dep-name` (`dep-id`) is stopped.\n\n" + + "### Affected Resources\n`client-name` (`client-id`) is unhealthy." + +func proTrackBaseInvestigation() aicontracts.InvestigationSession { + return aicontracts.InvestigationSession{ + ID: "inv-1", + FindingID: "finding-1", + Status: aicontracts.InvestigationStatusCompleted, + Summary: proTrackPassingSummary, + EvidenceIDs: []string{"e1"}, + ToolsUsed: []string{"t1"}, + } +} + +// TestApplyProTrackGatesBranchCoverage exercises the uncovered branches of +// ApplyProTrackGates: the Watch early return, the nil spec short circuit, the +// total==0 path, every summary-term / term-group / forbidden-term arm, the +// root-cause and affected grounding hits and misses, the MaxToolsUsed gate, the +// insufficient-evidence hard failure, and every remediation arm. +func TestApplyProTrackGatesBranchCoverage(t *testing.T) { + cases := []struct { + name string + // setup returns the inputs handed to ApplyProTrackGates. + setup func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) + // score is the Score value seeded before ApplyProTrackGates runs, so + // cases can prove (e.g. the Watch return) that it is left untouched. + score func() Score + wantPassed bool + hardHas, hardNone []string + gateHas, gateNone []string + rootCauseGrounding *float64 + affectedGrounding *float64 + investigationGrounding *float64 + investigationCompletion *float64 + checkPassedUnchanged bool // assert score.Passed == the score() seed value + checkHardFailuresUnchanged bool // assert HardFailures slice identity unchanged + }{ + { + name: "watch track returns early leaving score untouched", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() // TrackWatch + return m, GroundTruth{}, nil, RemediationResult{} + }, + score: func() Score { + return Score{Passed: true, HardFailures: []string{"preexisting"}, InvestigationGrounding: 7} + }, + checkPassedUnchanged: true, + checkHardFailuresUnchanged: true, + investigationGrounding: f64(7), // untouched: gate never assigned it + }, + { + name: "nil investigation spec fails closed", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = nil + return m, GroundTruth{}, nil, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"investigation expectations are missing"}, + }, + { + name: "no investigation evidence captured hard fails but reports unit ratios", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = proTrackBaseSpec() + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"no investigation evidence was captured"}, + investigationGrounding: f64(1), // ratio(0,0) == 1 + investigationCompletion: f64(1), + }, + { + name: "completed status not required accepts running investigation", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + spec := proTrackBaseSpec() + spec.RequireCompletedStatus = false + m.Investigation = spec + inv := proTrackBaseInvestigation() + inv.Status = aicontracts.InvestigationStatusRunning + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: true, + hardNone: []string{"ended with status"}, + investigationCompletion: f64(1), + }, + { + name: "non-completed status when required hard fails", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = proTrackBaseSpec() + inv := proTrackBaseInvestigation() + inv.Status = aicontracts.InvestigationStatusFailed + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"ended with status", string(aicontracts.InvestigationStatusFailed)}, + investigationCompletion: f64(0), + }, + { + name: "missing required summary term hard fails", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = proTrackBaseSpec() + inv := proTrackBaseInvestigation() + // Remove every occurrence of the required term while keeping + // the grounded resource sections intact (root-cause grounding + // only checks for dep-name/dep-id, not the word "stopped"). + inv.Summary = strings.ReplaceAll(inv.Summary, "stopped", "halted") + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"lacks required ground-truth term", `"stopped"`}, + }, + { + name: "term group matches via non-empty alternative skipping empty entries", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + spec := proTrackBaseSpec() + // The first alternative is empty and must be skipped; "halted" + // is the accepted alternative appended to the summary. + spec.RequiredSummaryTermGroups = [][]string{{"", "halted"}} + m.Investigation = spec + inv := proTrackBaseInvestigation() + inv.Summary += "\nThe dependency is halted." + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: true, + hardNone: []string{"lacks every accepted ground-truth term"}, + }, + { + name: "term group unmatched when no alternative matches", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + spec := proTrackBaseSpec() + // Neither alternative appears in the base summary. + spec.RequiredSummaryTermGroups = [][]string{{"halted", "exited"}} + m.Investigation = spec + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"lacks every accepted ground-truth term"}, + }, + { + name: "forbidden summary term present hard fails while empty forbidden entry is skipped", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + spec := proTrackBaseSpec() + spec.ForbiddenSummaryTerms = []string{"", "leak"} + m.Investigation = spec + inv := proTrackBaseInvestigation() + inv.Summary += "\nA secret leak was observed." + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"contains forbidden term", `"leak"`}, + }, + { + name: "root cause grounding hit reports full ratio", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = proTrackBaseSpec() + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{} + }, + wantPassed: true, + hardNone: []string{"Root Cause section"}, + rootCauseGrounding: f64(1), + }, + { + name: "root cause grounding miss with affected still grounded", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = proTrackBaseSpec() + inv := proTrackBaseInvestigation() + // Root Cause section exists but names the wrong resource. + inv.Summary = "### Root Cause\n`client-name` is the problem.\n\n### Affected Resources\n`client-name` (`client-id`) is unhealthy." + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"Root Cause section"}, + hardNone: []string{"Affected Resources section"}, + rootCauseGrounding: f64(0), + affectedGrounding: f64(1), + }, + { + name: "affected grounding miss with root cause still grounded", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = proTrackBaseSpec() + inv := proTrackBaseInvestigation() + inv.Summary = "### Root Cause\n`dep-name` (`dep-id`) is stopped.\n\n### Affected Resources\n`dep-name` is also affected." + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"Affected Resources section"}, + affectedGrounding: f64(0), + rootCauseGrounding: f64(1), + }, + { + name: "insufficient evidence ids hard fails", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + spec := proTrackBaseSpec() + spec.MinEvidenceIDs = 2 + m.Investigation = spec + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{} + }, + wantPassed: false, + hardHas: []string{"has 1 evidence IDs; requires 2"}, + }, + { + name: "max tools used exceeded records gate failure", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + spec := proTrackBaseSpec() + spec.MaxToolsUsed = 2 + m.Investigation = spec + inv := proTrackBaseInvestigation() + inv.ToolsUsed = []string{"a", "b", "c"} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: false, + hardNone: []string{"tools"}, + gateHas: []string{"used 3 tools; maximum is 2"}, + }, + { + name: "max tools used zero skips the tool budget gate", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + spec := proTrackBaseSpec() + spec.MaxToolsUsed = 0 + m.Investigation = spec + inv := proTrackBaseInvestigation() + inv.ToolsUsed = []string{"a", "b", "c"} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{} + }, + wantPassed: true, + gateNone: []string{"tools"}, + }, + { + name: "remediation missing action id hard fails", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = &RemediationSpec{Decision: "observe"} + rem := RemediationResult{ActionID: "", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: true} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: false, + hardHas: []string{"no exact governed action was captured"}, + }, + { + name: "remediation origin binding failed hard fails", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = &RemediationSpec{Decision: "observe"} + rem := RemediationResult{ActionID: "action-1", OriginBound: false, PlanHashBound: true, Passed: true, Authorized: true} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: false, + hardHas: []string{"governed action identity or origin binding failed"}, + }, + { + name: "remediation plan hash binding failed hard fails", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = &RemediationSpec{Decision: "observe"} + rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: false, Passed: true, Authorized: true} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: false, + hardHas: []string{"governed action identity or origin binding failed"}, + }, + { + name: "remediation did not pass hard fails", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = &RemediationSpec{Decision: "observe"} + rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: false, Authorized: true} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: false, + hardHas: []string{"governed remediation track did not pass"}, + }, + { + name: "remediation authorization gate fires for non-observe decision", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = &RemediationSpec{Decision: "execute"} + rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: false} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: false, + hardHas: []string{"proceeded without the benchmark authorization gate"}, + }, + { + name: "remediation observe decision skips authorization gate", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = &RemediationSpec{Decision: "observe"} + rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: false} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: true, + hardNone: []string{"authorization gate"}, + }, + { + name: "remediation nil spec skips authorization gate", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = nil + rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: false} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: true, + hardNone: []string{"authorization gate"}, + }, + { + name: "full investigation track passes with grounded ratios", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackInvestigation + m.Investigation = proTrackBaseSpec() + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{} + }, + wantPassed: true, + investigationGrounding: f64(1), + investigationCompletion: f64(1), + rootCauseGrounding: f64(1), + affectedGrounding: f64(1), + }, + { + name: "full remediation track passes when authorized for execute decision", + setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) { + m := validTestManifest() + m.Track = TrackRemediation + m.Investigation = proTrackBaseSpec() + m.Remediation = &RemediationSpec{Decision: "execute"} + rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: true} + return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem + }, + wantPassed: true, + hardNone: []string{"authorization gate", "origin binding", "did not pass", "governed action"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + manifest, ground, investigations, remediation := tc.setup() + seed := Score{Passed: true} + if tc.score != nil { + seed = tc.score() + } + score := seed + ApplyProTrackGates(&score, manifest, ground, investigations, remediation) + + joinedHard := strings.Join(score.HardFailures, "\n") + joinedGate := strings.Join(score.GateFailures, "\n") + + if tc.checkPassedUnchanged { + if score.Passed != seed.Passed { + t.Fatalf("Watch early return must leave Passed untouched: got %v want %v", score.Passed, seed.Passed) + } + } else if score.Passed != tc.wantPassed { + t.Fatalf("Passed = %v, want %v (hard=%q gate=%q)", score.Passed, tc.wantPassed, joinedHard, joinedGate) + } + + if tc.checkHardFailuresUnchanged { + if len(score.HardFailures) != len(seed.HardFailures) || (len(score.HardFailures) > 0 && score.HardFailures[0] != seed.HardFailures[0]) { + t.Fatalf("Watch early return must leave HardFailures untouched: got %v want %v", score.HardFailures, seed.HardFailures) + } + } + + for _, want := range tc.hardHas { + if !strings.Contains(joinedHard, want) { + t.Fatalf("expected HardFailures to contain %q; got %q", want, joinedHard) + } + } + for _, avoid := range tc.hardNone { + if strings.Contains(joinedHard, avoid) { + t.Fatalf("expected HardFailures to NOT contain %q; got %q", avoid, joinedHard) + } + } + for _, want := range tc.gateHas { + if !strings.Contains(joinedGate, want) { + t.Fatalf("expected GateFailures to contain %q; got %q", want, joinedGate) + } + } + for _, avoid := range tc.gateNone { + if strings.Contains(joinedGate, avoid) { + t.Fatalf("expected GateFailures to NOT contain %q; got %q", avoid, joinedGate) + } + } + + if tc.rootCauseGrounding != nil && score.RootCauseGrounding != *tc.rootCauseGrounding { + t.Fatalf("RootCauseGrounding = %v, want %v", score.RootCauseGrounding, *tc.rootCauseGrounding) + } + if tc.affectedGrounding != nil && score.AffectedGrounding != *tc.affectedGrounding { + t.Fatalf("AffectedGrounding = %v, want %v", score.AffectedGrounding, *tc.affectedGrounding) + } + if tc.investigationGrounding != nil && score.InvestigationGrounding != *tc.investigationGrounding { + t.Fatalf("InvestigationGrounding = %v, want %v", score.InvestigationGrounding, *tc.investigationGrounding) + } + if tc.investigationCompletion != nil && score.InvestigationCompletion != *tc.investigationCompletion { + t.Fatalf("InvestigationCompletion = %v, want %v", score.InvestigationCompletion, *tc.investigationCompletion) + } + }) + } +} + +// TestInvestigationSectionGroundedBranchCoverage exercises every branch of the +// pure helper: the empty-alias guard, the missing-section early return, the +// missing-resource map lookup, and each name/id containment combination. +func TestInvestigationSectionGroundedBranchCoverage(t *testing.T) { + resources := map[string]CollectedTruth{ + "named": {Alias: "named", Name: "Named Resource", ResourceID: "id-only"}, + "identified": {Alias: "identified", Name: "", ResourceID: "res-123"}, + "empty": {Alias: "empty", Name: "", ResourceID: ""}, + "both": {Alias: "both", Name: "Both Name", ResourceID: "both-id"}, + } + const summary = "### Root Cause\nNamed Resource and res-123 and Both Name / both-id appear here.\n" + + cases := []struct { + name string + summary string + heading string + aliases []string + resources map[string]CollectedTruth + want bool + }{ + { + name: "nil aliases returns true", + summary: "anything", + heading: "Root Cause", + aliases: nil, + want: true, + }, + { + name: "empty aliases slice returns true", + summary: "anything", + heading: "Root Cause", + aliases: []string{}, + want: true, + }, + { + name: "missing heading returns false", + summary: "no headings here at all", + heading: "Root Cause", + aliases: []string{"named"}, + want: false, + }, + { + name: "empty summary returns false", + summary: "", + heading: "Root Cause", + aliases: []string{"named"}, + want: false, + }, + { + name: "alias absent from resource map returns false", + summary: summary, + heading: "Root Cause", + aliases: []string{"unknown"}, + resources: resources, + want: false, + }, + { + name: "resource name match with empty id returns true", + summary: summary, + heading: "Root Cause", + aliases: []string{"named"}, + resources: resources, + want: true, + }, + { + name: "resource id match with empty name returns true", + summary: summary, + heading: "Root Cause", + aliases: []string{"identified"}, + resources: resources, + want: true, + }, + { + name: "both name and id present and matching returns true", + summary: summary, + heading: "Root Cause", + aliases: []string{"both"}, + resources: resources, + want: true, + }, + { + name: "neither name nor id contained returns false", + summary: "### Root Cause\nnothing relevant is mentioned here\n", + heading: "Root Cause", + aliases: []string{"both"}, + resources: resources, + want: false, + }, + { + name: "resource with empty name and id returns false", + summary: summary, + heading: "Root Cause", + aliases: []string{"empty"}, + resources: resources, + want: false, + }, + { + name: "all aliases grounded returns true", + summary: summary, + heading: "Root Cause", + aliases: []string{"named", "identified", "both"}, + resources: resources, + want: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + res := tc.resources + if res == nil { + res = map[string]CollectedTruth{} + } + got := investigationSectionGrounded(tc.summary, tc.heading, tc.aliases, res) + if got != tc.want { + t.Fatalf("investigationSectionGrounded(%q, %q, %v) = %v, want %v", tc.summary, tc.heading, tc.aliases, got, tc.want) + } + }) + } +}