mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Add branch-coverage tests for qualification scorer and report helpers
New table-driven tests raise branch coverage on applyGates, ApplyProTrackGates, findFault, bestFindingMatch, validatePredicates, ApplyQualificationGates, canonicalToolInput, sanitizeArtifactText and allObservationsPassed, covering previously uncovered gate arms, grounding misses, redaction and JSON edge cases. Test-only, no source changes.
This commit is contained in:
rcourtman
@@ -0,0 +1,319 @@
|
||||
package qualification
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// This file is a white-box table-test suite for the pure manifest predicate
|
||||
// helpers in manifest.go: validatePredicates and positiveDuration. Every
|
||||
// helper introduced here is prefixed qualmp so it cannot collide with
|
||||
// identifiers defined by sibling test files in package qualification.
|
||||
|
||||
// qualmpAliases builds the alias set from the supplied resource aliases. It
|
||||
// mirrors how Manifest.Validate constructs the set before calling
|
||||
// validatePredicates, but keeps the table rows compact and literal.
|
||||
func qualmpAliases(names ...string) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(names))
|
||||
for _, n := range names {
|
||||
out[n] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// qualmpJoinErrs flattens a validatePredicates result into a single
|
||||
// newline-delimited string so table rows can assert on substrings the same
|
||||
// way the rest of this package checks joined errors.
|
||||
func qualmpJoinErrs(errs []error) string {
|
||||
var b strings.Builder
|
||||
for _, e := range errs {
|
||||
b.WriteString(e.Error())
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestValidatePredicates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
label string
|
||||
predicates []Predicate
|
||||
aliases map[string]struct{}
|
||||
wantCount int
|
||||
wantSubs []string
|
||||
}{
|
||||
// for-range not entered: nil slice yields no errors.
|
||||
{
|
||||
name: "nil predicate slice returns no errors",
|
||||
label: "baseline",
|
||||
predicates: nil,
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 0,
|
||||
},
|
||||
// for-range not entered: empty slice yields no errors.
|
||||
{
|
||||
name: "empty predicate slice returns no errors",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 0,
|
||||
},
|
||||
// Happy path: known target, valid probe, valid operator, valid value,
|
||||
// no timeout. Exercises every ok/default arm with the ok branch.
|
||||
{
|
||||
name: "fully valid predicate with no timeout returns no errors",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("true"),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 0,
|
||||
},
|
||||
// Happy path with a non-empty, parseable, positive timeout: exercises
|
||||
// the predicate.Timeout != "" branch and the positiveDuration
|
||||
// success arm (err == nil) inside validatePredicates.
|
||||
{
|
||||
name: "valid predicate with positive timeout returns no errors",
|
||||
label: "teardown",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("false"), Timeout: "5s",
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 0,
|
||||
},
|
||||
// Accepted JSON value shapes: confirm len>0 && json.Valid for numbers,
|
||||
// null, arrays and objects so the invalid-value arm is not taken.
|
||||
{
|
||||
name: "numeric value is accepted",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.restart_count", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("3"),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "array value accepted for in operator",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.status", Target: "target", Operator: "in",
|
||||
Value: json.RawMessage(`["running","healthy"]`),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 0,
|
||||
},
|
||||
// Unknown target arm: aliases lookup misses.
|
||||
{
|
||||
name: "target not in aliases reports unknown resource",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "ghost", Operator: "eq",
|
||||
Value: json.RawMessage("true"),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{`baseline[0] targets unknown resource "ghost"`},
|
||||
},
|
||||
// Unsupported probe default arm.
|
||||
{
|
||||
name: "unsupported probe is reported",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "tcp.port_open", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("true"),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{`baseline[0] has unsupported probe "tcp.port_open"`},
|
||||
},
|
||||
// Unsupported operator default arm.
|
||||
{
|
||||
name: "unsupported operator is reported",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "matches",
|
||||
Value: json.RawMessage("true"),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{`baseline[0] has unsupported operator "matches"`},
|
||||
},
|
||||
// Empty value arm: len(predicate.Value) == 0.
|
||||
{
|
||||
name: "empty value is reported as invalid",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage(""),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{"baseline[0] has invalid value"},
|
||||
},
|
||||
// Nil value arm: a nil json.RawMessage has len 0.
|
||||
{
|
||||
name: "nil value is reported as invalid",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: nil,
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{"baseline[0] has invalid value"},
|
||||
},
|
||||
// Invalid JSON arm: non-empty bytes that fail json.Valid.
|
||||
{
|
||||
name: "malformed json value is reported as invalid",
|
||||
label: "baseline",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("{bad"),
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{"baseline[0] has invalid value"},
|
||||
},
|
||||
// Timeout arm with unparseable duration: positiveDuration parse error.
|
||||
{
|
||||
name: "unparseable timeout is wrapped",
|
||||
label: "teardown",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("true"), Timeout: "not-a-duration",
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{"teardown[0] timeout:"},
|
||||
},
|
||||
// Timeout arm with non-positive (zero) duration.
|
||||
{
|
||||
name: "zero timeout is rejected as non-positive",
|
||||
label: "teardown",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("true"), Timeout: "0s",
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{"teardown[0] timeout:", "duration must be positive"},
|
||||
},
|
||||
// Timeout arm with negative duration.
|
||||
{
|
||||
name: "negative timeout is rejected as non-positive",
|
||||
label: "teardown",
|
||||
predicates: []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("true"), Timeout: "-3s",
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{"teardown[0] timeout:", "duration must be positive"},
|
||||
},
|
||||
// Index propagation: a bad predicate at position 1 reports [1], and a
|
||||
// valid predicate at position 0 contributes nothing.
|
||||
{
|
||||
name: "error reports the offending slice index",
|
||||
label: "teardown",
|
||||
predicates: []Predicate{
|
||||
{Probe: "docker.running", Target: "target", Operator: "eq", Value: json.RawMessage("true")},
|
||||
{Probe: "docker.running", Target: "missing", Operator: "eq", Value: json.RawMessage("true")},
|
||||
},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 1,
|
||||
wantSubs: []string{`teardown[1] targets unknown resource "missing"`},
|
||||
},
|
||||
// Accumulation: every guard in one predicate produces five independent
|
||||
// errors for a single element, confirming append-only behaviour.
|
||||
{
|
||||
name: "all guards fail on one predicate yielding five errors",
|
||||
label: "fault f1 oracle",
|
||||
predicates: []Predicate{{
|
||||
Probe: "bogus.probe", Target: "ghost", Operator: "matches",
|
||||
Value: json.RawMessage("{bad"), Timeout: "nope",
|
||||
}},
|
||||
aliases: qualmpAliases("target"),
|
||||
wantCount: 5,
|
||||
wantSubs: []string{
|
||||
`fault f1 oracle[0] targets unknown resource "ghost"`,
|
||||
`fault f1 oracle[0] has unsupported probe "bogus.probe"`,
|
||||
`fault f1 oracle[0] has unsupported operator "matches"`,
|
||||
"fault f1 oracle[0] has invalid value",
|
||||
"fault f1 oracle[0] timeout:",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
errs := validatePredicates(tc.label, tc.predicates, tc.aliases)
|
||||
if len(errs) != tc.wantCount {
|
||||
t.Fatalf("validatePredicates returned %d errors, want %d: %v", len(errs), tc.wantCount, qualmpJoinErrs(errs))
|
||||
}
|
||||
if len(tc.wantSubs) == 0 {
|
||||
return
|
||||
}
|
||||
joined := qualmpJoinErrs(errs)
|
||||
for _, sub := range tc.wantSubs {
|
||||
if !strings.Contains(joined, sub) {
|
||||
t.Errorf("validatePredicates error text = %q, want substring %q", joined, sub)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPositiveDuration(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
value string
|
||||
wantErr bool
|
||||
wantDur time.Duration
|
||||
errSub string
|
||||
}{
|
||||
// Success arm: positive duration parses and is returned verbatim.
|
||||
{"seconds parse", "5s", false, 5 * time.Second, ""},
|
||||
{"compound duration parses", "1m30s", false, 90 * time.Second, ""},
|
||||
{"milliseconds parse", "250ms", false, 250 * time.Millisecond, ""},
|
||||
// Parse error arm: empty string cannot be parsed.
|
||||
{"empty string parse error", "", true, 0, ""},
|
||||
// Parse error arm: missing unit.
|
||||
{"bare number missing unit parse error", "5", true, 0, "missing unit"},
|
||||
// Parse error arm: unparseable text.
|
||||
{"garbage parse error", "not-a-duration", true, 0, "invalid duration"},
|
||||
// Non-positive arm: zero duration parses but is rejected.
|
||||
{"zero duration rejected", "0s", true, 0, "duration must be positive"},
|
||||
{"zero without unit rejected", "0", true, 0, "duration must be positive"},
|
||||
// Non-positive arm: negative duration parses but is rejected.
|
||||
{"negative duration rejected", "-3s", true, 0, "duration must be positive"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := positiveDuration(tc.value)
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("positiveDuration(%q) err = nil, want non-nil", tc.value)
|
||||
}
|
||||
if tc.errSub != "" && !strings.Contains(err.Error(), tc.errSub) {
|
||||
t.Fatalf("positiveDuration(%q) err = %q, want substring %q", tc.value, err.Error(), tc.errSub)
|
||||
}
|
||||
if got != 0 {
|
||||
t.Fatalf("positiveDuration(%q) = %v, want 0 on error path", tc.value, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("positiveDuration(%q) err = %v, want nil", tc.value, err)
|
||||
}
|
||||
if got != tc.wantDur {
|
||||
t.Fatalf("positiveDuration(%q) = %v, want %v", tc.value, got, tc.wantDur)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package qualification
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestCanonicalToolInputBranches exercises every branch of canonicalToolInput,
|
||||
// focusing on the previously uncovered error and early-return paths: the empty
|
||||
// input short circuit ("{}"), the raw invalid-JSON decode failure, the
|
||||
// multiple-JSON-values rejection, and the trailing-data rejection. A canonical
|
||||
// happy path is included to pin the success contract.
|
||||
func TestCanonicalToolInputBranches(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
input string
|
||||
wantOut string
|
||||
wantErr bool
|
||||
errSubstr string
|
||||
notSubstrs []string
|
||||
}{
|
||||
{
|
||||
name: "empty input returns empty object",
|
||||
input: "",
|
||||
wantOut: "{}",
|
||||
},
|
||||
{
|
||||
name: "whitespace only collapses to empty object",
|
||||
input: " \t\n ",
|
||||
wantOut: "{}",
|
||||
},
|
||||
{
|
||||
name: "valid single object canonicalizes key order and spacing",
|
||||
input: `{ "b": 2, "a": 1 }`,
|
||||
wantOut: `{"a":1,"b":2}`,
|
||||
notSubstrs: []string{"multiple JSON values", "trailing JSON data"},
|
||||
},
|
||||
{
|
||||
name: "invalid json returns decoder error",
|
||||
input: `{"description":"cut...`,
|
||||
wantErr: true,
|
||||
notSubstrs: []string{"multiple JSON values", "trailing JSON data"},
|
||||
},
|
||||
{
|
||||
name: "multiple json values are rejected",
|
||||
input: `{}{}`,
|
||||
wantErr: true,
|
||||
errSubstr: "multiple JSON values",
|
||||
},
|
||||
{
|
||||
name: "trailing garbage after value is rejected",
|
||||
input: `1 garbage`,
|
||||
wantErr: true,
|
||||
errSubstr: "trailing JSON data",
|
||||
},
|
||||
{
|
||||
name: "trailing garbage after object is rejected",
|
||||
input: `{} x`,
|
||||
wantErr: true,
|
||||
errSubstr: "trailing JSON data",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := canonicalToolInput(tc.input)
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("canonicalToolInput(%q) error = nil, want non-nil", tc.input)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("canonicalToolInput(%q) output = %q, want empty string on error", tc.input, got)
|
||||
}
|
||||
if tc.errSubstr != "" && !strings.Contains(err.Error(), tc.errSubstr) {
|
||||
t.Fatalf("canonicalToolInput(%q) error = %q, want substring %q", tc.input, err.Error(), tc.errSubstr)
|
||||
}
|
||||
for _, bad := range tc.notSubstrs {
|
||||
if strings.Contains(err.Error(), bad) {
|
||||
t.Fatalf("canonicalToolInput(%q) error = %q, must not mention %q", tc.input, err.Error(), bad)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("canonicalToolInput(%q) error = %v, want nil", tc.input, err)
|
||||
}
|
||||
if got != tc.wantOut {
|
||||
t.Fatalf("canonicalToolInput(%q) = %q, want %q", tc.input, got, tc.wantOut)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package qualification
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestSanitizeArtifactTextRedaction exercises the secretPatterns loop in
|
||||
// sanitizeArtifactText: the NumSubexp()>0 branch (bearer / api-key / password
|
||||
// / secret) uses the "${1}[REDACTED]" template, and the NumSubexp()==0 else
|
||||
// branch (PEM private-key blocks) uses the whole-match "[REDACTED PRIVATE KEY]"
|
||||
// replacement. Non-secret text must pass through verbatim.
|
||||
func TestSanitizeArtifactTextRedaction(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "bearer token redacted keeping label",
|
||||
input: "Authorization: Bearer abc123-xyz",
|
||||
want: "Authorization: Bearer [REDACTED]",
|
||||
},
|
||||
{
|
||||
name: "bearer header is case insensitive",
|
||||
input: "authorization: bearer SecretToken",
|
||||
want: "authorization: bearer [REDACTED]",
|
||||
},
|
||||
{
|
||||
name: "api_key with equals assignment",
|
||||
input: "api_key=hunter2",
|
||||
want: "api_key=[REDACTED]",
|
||||
},
|
||||
{
|
||||
name: "api-key with dash and colon separator",
|
||||
input: "api-key: my-token",
|
||||
want: "api-key: [REDACTED]",
|
||||
},
|
||||
{
|
||||
name: "apitoken variant redacted",
|
||||
input: "apitoken=abcdef",
|
||||
want: "apitoken=[REDACTED]",
|
||||
},
|
||||
{
|
||||
name: "password with colon space separator",
|
||||
input: "password: hunter2",
|
||||
want: "password: [REDACTED]",
|
||||
},
|
||||
{
|
||||
name: "secret equals assignment",
|
||||
input: "secret=s3cr3t",
|
||||
want: "secret=[REDACTED]",
|
||||
},
|
||||
{
|
||||
name: "plain private key block uses whole match redaction",
|
||||
input: "-----BEGIN PRIVATE KEY-----\nMIIBOAIB\n-----END PRIVATE KEY-----",
|
||||
want: "[REDACTED PRIVATE KEY]",
|
||||
},
|
||||
{
|
||||
name: "rsa private key block redacted via prefix class",
|
||||
input: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOAIB\n-----END RSA PRIVATE KEY-----",
|
||||
want: "[REDACTED PRIVATE KEY]",
|
||||
},
|
||||
{
|
||||
name: "non secret text is unchanged",
|
||||
input: "the quick brown fox jumps",
|
||||
want: "the quick brown fox jumps",
|
||||
},
|
||||
{
|
||||
name: "multiple secret kinds redacted together in one payload",
|
||||
input: "Authorization: Bearer tok123\napi_key=keyval\npassword=pw",
|
||||
want: "Authorization: Bearer [REDACTED]\napi_key=[REDACTED]\npassword=[REDACTED]",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := sanitizeArtifactText(tc.input); got != tc.want {
|
||||
t.Fatalf("sanitizeArtifactText(%q) = %q, want %q", tc.input, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSanitizeArtifactTextTruncatesAbove512KB covers the >512KB truncation
|
||||
// branch of sanitizeArtifactText. The boundary value (exactly 512*1024 bytes)
|
||||
// must NOT be truncated, while anything larger is cut to the limit and gets the
|
||||
// "\n[TRUNCATED]" marker appended. Truncation runs before redaction, so a
|
||||
// secret that falls inside the retained prefix must still be redacted.
|
||||
func TestSanitizeArtifactTextTruncatesAbove512KB(t *testing.T) {
|
||||
const maxText = 512 * 1024 // 524288, mirrors the source constant
|
||||
marker := "\n[TRUNCATED]"
|
||||
|
||||
t.Run("exactly at limit is not truncated", func(t *testing.T) {
|
||||
value := strings.Repeat("a", maxText)
|
||||
got := sanitizeArtifactText(value)
|
||||
if len(got) != maxText {
|
||||
t.Fatalf("len = %d, want %d (no truncation at exact boundary)", len(got), maxText)
|
||||
}
|
||||
if strings.Contains(got, "[TRUNCATED]") {
|
||||
t.Fatal("value at exact limit must not gain the truncation marker")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("over limit is cut to limit plus marker and drops the tail", func(t *testing.T) {
|
||||
// "HEAD" + maxText bytes + "TAIL" is strictly longer than maxText.
|
||||
value := "HEAD" + strings.Repeat("x", maxText) + "TAIL"
|
||||
got := sanitizeArtifactText(value)
|
||||
if wantLen := maxText + len(marker); len(got) != wantLen {
|
||||
t.Fatalf("len = %d, want %d", len(got), wantLen)
|
||||
}
|
||||
if !strings.HasPrefix(got, "HEAD") {
|
||||
t.Fatalf("retained prefix lost head bytes: %q", got[:8])
|
||||
}
|
||||
if !strings.HasSuffix(got, marker) {
|
||||
t.Fatalf("missing truncation marker; suffix = %q", got[len(got)-len(marker):])
|
||||
}
|
||||
if strings.Contains(got, "TAIL") {
|
||||
t.Fatal("bytes past the limit must be dropped, not retained")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("secret inside retained prefix is redacted after truncation", func(t *testing.T) {
|
||||
head := "Authorization: Bearer leak-token\n" + strings.Repeat("z", 32)
|
||||
value := head + strings.Repeat("y", maxText) // guarantees truncation; head is retained
|
||||
got := sanitizeArtifactText(value)
|
||||
if !strings.HasSuffix(got, marker) {
|
||||
t.Fatalf("expected truncation marker; suffix = %q", got[len(got)-len(marker):])
|
||||
}
|
||||
if !strings.Contains(got, "Authorization: Bearer [REDACTED]") {
|
||||
t.Fatalf("secret within retained prefix was not redacted: %q", got[:80])
|
||||
}
|
||||
if strings.Contains(got, "leak-token") {
|
||||
t.Fatal("bearer secret value must be redacted even after truncation")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestAllObservationsPassedBranches covers every branch of allObservationsPassed:
|
||||
// the empty/nil guard (returns false), the early return on the first failing
|
||||
// observation, and the all-passed tail return. This is intentionally distinct
|
||||
// from observationsPassedOrEmpty (covered elsewhere), which treats an empty
|
||||
// slice as passing.
|
||||
func TestAllObservationsPassedBranches(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
observations []PredicateObservation
|
||||
want bool
|
||||
}{
|
||||
{"nil slice returns false", nil, false},
|
||||
{"empty slice returns false", []PredicateObservation{}, false},
|
||||
{"single passing observation returns true", []PredicateObservation{{Passed: true}}, true},
|
||||
{"single failing observation returns false", []PredicateObservation{{Passed: false}}, false},
|
||||
{"all passing returns true", []PredicateObservation{{Passed: true}, {Passed: true}, {Passed: true}}, true},
|
||||
{"first failing short circuits to false", []PredicateObservation{{Passed: false}, {Passed: true}}, false},
|
||||
{"failing in the middle returns false", []PredicateObservation{{Passed: true}, {Passed: false}, {Passed: true}}, false},
|
||||
{"last failing returns false", []PredicateObservation{{Passed: true}, {Passed: true}, {Passed: false}}, false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := allObservationsPassed(tc.observations); got != tc.want {
|
||||
t.Fatalf("allObservationsPassed(%d obs) = %v, want %v", len(tc.observations), got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,609 @@
|
||||
package qualification
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// This file is a white-box, table-driven branch-coverage suite for
|
||||
// ApplyQualificationGates in report.go. Helpers and shared identifiers are
|
||||
// prefixed with `qualgates` so they do not collide with sibling test files in
|
||||
// package qualification. Each row constructs a small literal ComparisonReport
|
||||
// and Catalog; no live service, executor, provider, DB, filesystem, or
|
||||
// goroutine is required.
|
||||
|
||||
// qualgatesBaseline returns a ComparisonReport and Catalog that is known to
|
||||
// fully qualify under track: a single model, single git revision, single Pulse
|
||||
// runtime version, no dirty runs, one scenario whose manifest digest matches
|
||||
// the catalog, with 30/30 passing runs (Wilson lower bound well above 0.85) and
|
||||
// 30/30 fault recall. Tests mutate the returned pointer to introduce exactly
|
||||
// the gate failure under test.
|
||||
func qualgatesBaseline(t *testing.T, track Track) (*ComparisonReport, Catalog) {
|
||||
t.Helper()
|
||||
manifest := validTestManifest()
|
||||
manifest.Track = track
|
||||
manifest.Repeat.Qualification = 30
|
||||
digest, err := manifest.Digest()
|
||||
if err != nil {
|
||||
t.Fatalf("digest baseline manifest: %v", err)
|
||||
}
|
||||
catalog := Catalog{
|
||||
Manifests: []Manifest{manifest},
|
||||
ByID: map[string]Manifest{manifest.ID: manifest},
|
||||
}
|
||||
comparison := &ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{{
|
||||
ModelSummary: ModelSummary{
|
||||
Model: "provider:model",
|
||||
Runs: 30,
|
||||
Passed: 30,
|
||||
PassRate: WilsonInterval(30, 30),
|
||||
FaultRecall: WilsonInterval(30, 30),
|
||||
},
|
||||
ScenarioID: manifest.ID,
|
||||
Track: track,
|
||||
ManifestDigests: []string{digest},
|
||||
}},
|
||||
}
|
||||
return comparison, catalog
|
||||
}
|
||||
|
||||
// qualgatesDigestOf recomputes a manifest digest, failing the test instead of
|
||||
// returning an error so table rows stay terse.
|
||||
func qualgatesDigestOf(t *testing.T, manifest Manifest) string {
|
||||
t.Helper()
|
||||
digest, err := manifest.Digest()
|
||||
if err != nil {
|
||||
t.Fatalf("digest manifest %s: %v", manifest.ID, err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// TestApplyQualificationGatesRejectsUnsupportedTrack covers the early-return
|
||||
// guard at report.go:369-371 for every unsupported track spelling, including
|
||||
// the case-sensitivity boundary that a capitalised "Watch" must NOT match
|
||||
// TrackWatch, and verifies no qualification verdicts are appended.
|
||||
func TestApplyQualificationGatesRejectsUnsupportedTrack(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
track Track
|
||||
}{
|
||||
{"empty string is rejected", ""},
|
||||
{"unknown name is rejected", "bench"},
|
||||
{"capitalised watch is not watch", "Watch"},
|
||||
{"preview track is not supported", "preview"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
comparison, catalog := qualgatesBaseline(t, TrackWatch)
|
||||
err := ApplyQualificationGates(comparison, catalog, tc.track)
|
||||
if err == nil {
|
||||
t.Fatalf("ApplyQualificationGates(track=%q) returned nil; want error", tc.track)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported qualification track") {
|
||||
t.Fatalf("err = %q; want substring %q", err.Error(), "unsupported qualification track")
|
||||
}
|
||||
if !strings.Contains(err.Error(), string(tc.track)) {
|
||||
t.Fatalf("err = %q; want it to mention track %q", err.Error(), tc.track)
|
||||
}
|
||||
if len(comparison.Qualification) != 0 {
|
||||
t.Fatalf("Qualification populated after unsupported-track guard: %+v", comparison.Qualification)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyQualificationGatesScenarioGateFailures covers the per-scenario gate
|
||||
// branches inside the per-model loop (report.go:426-453). Each row introduces
|
||||
// exactly one failure mode on top of an otherwise-qualifying scenario, and the
|
||||
// assertion verifies both the scenario-level failure message and that it is
|
||||
// propagated up into the model verdict as "<manifestID>: <failure>".
|
||||
func TestApplyQualificationGatesScenarioGateFailures(t *testing.T) {
|
||||
manifest := validTestManifest()
|
||||
manifest.Repeat.Qualification = 30
|
||||
catalog := Catalog{Manifests: []Manifest{manifest}, ByID: map[string]Manifest{manifest.ID: manifest}}
|
||||
baseDigest := qualgatesDigestOf(t, manifest)
|
||||
|
||||
qualifyingScenario := func() ScenarioSummary {
|
||||
return ScenarioSummary{
|
||||
ModelSummary: ModelSummary{
|
||||
Model: "provider:model",
|
||||
Runs: 30,
|
||||
Passed: 30,
|
||||
PassRate: WilsonInterval(30, 30),
|
||||
FaultRecall: WilsonInterval(30, 30),
|
||||
},
|
||||
ScenarioID: manifest.ID,
|
||||
Track: TrackWatch,
|
||||
ManifestDigests: []string{baseDigest},
|
||||
}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
mutate func(*ScenarioSummary)
|
||||
wantFailureParts []string // each substring must appear in the joined verdict failures
|
||||
wantScenarioFail bool // most rows mark scenario not-qualified; some leave it qualified
|
||||
}{
|
||||
{
|
||||
name: "passed less than runs records partial pass count",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.Runs = 30
|
||||
s.Passed = 27
|
||||
s.PassRate = WilsonInterval(27, 30)
|
||||
},
|
||||
wantFailureParts: []string{"only 27 of 30 runs passed"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "fault recall total positive but lower below floor",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.FaultRecall = WilsonInterval(20, 30)
|
||||
},
|
||||
wantFailureParts: []string{"fault-recall Wilson lower bound"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "nonzero false positives recorded",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.FalsePositives = 4
|
||||
},
|
||||
wantFailureParts: []string{"false positives=4 hard-failure runs=0"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "nonzero hard-failure runs recorded",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.HardFailureRuns = 2
|
||||
},
|
||||
wantFailureParts: []string{"false positives=0 hard-failure runs=2"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "scenario-local dirty runs",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.DirtyRuns = 3
|
||||
},
|
||||
wantFailureParts: []string{"3 run(s) used a dirty worktree"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "scenario spans multiple git revisions",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.GitSHAs = []string{"sha-a", "sha-b"}
|
||||
},
|
||||
wantFailureParts: []string{"runs span 2 source revisions"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "scenario manifest count exceeds one adds digest mismatch plus span failure",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.ManifestDigests = []string{baseDigest, strings.Repeat("a", 64)}
|
||||
},
|
||||
wantFailureParts: []string{
|
||||
"report manifest digest does not match the selected catalogue",
|
||||
"runs span 2 manifest revisions",
|
||||
},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "empty manifest digest list adds digest mismatch without span failure",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
s.ManifestDigests = nil
|
||||
},
|
||||
wantFailureParts: []string{"report manifest digest does not match the selected catalogue"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
{
|
||||
name: "runs below required qualification repeat count without lowering pass rate",
|
||||
mutate: func(s *ScenarioSummary) {
|
||||
// WilsonInterval(29, 29) lower bound (~0.883) is still above
|
||||
// the 0.85 floor, so only the repeat-count gate fires.
|
||||
s.Runs = 29
|
||||
s.Passed = 29
|
||||
s.PassRate = WilsonInterval(29, 29)
|
||||
s.FaultRecall = WilsonInterval(29, 29)
|
||||
},
|
||||
wantFailureParts: []string{"runs 29 below qualification repeat 30"},
|
||||
wantScenarioFail: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
scenario := qualifyingScenario()
|
||||
tc.mutate(&scenario)
|
||||
comparison := &ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{scenario},
|
||||
}
|
||||
if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil {
|
||||
t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err)
|
||||
}
|
||||
if len(comparison.Qualification) != 1 {
|
||||
t.Fatalf("Qualification = %+v; want exactly one verdict", comparison.Qualification)
|
||||
}
|
||||
verdict := comparison.Qualification[0]
|
||||
if verdict.Qualified {
|
||||
t.Fatalf("verdict qualified; want failure for %s", tc.name)
|
||||
}
|
||||
joined := strings.Join(verdict.Failures, " | ")
|
||||
for _, want := range tc.wantFailureParts {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Fatalf("verdict failures = %q; want substring %q", joined, want)
|
||||
}
|
||||
}
|
||||
// Failure must also be reflected at the scenario level, prefixed
|
||||
// with the manifest ID inside the verdict.
|
||||
if tc.wantScenarioFail {
|
||||
if comparison.Scenarios[0].Qualified {
|
||||
t.Fatalf("scenario.Qualified = true; want false for %s", tc.name)
|
||||
}
|
||||
if len(comparison.Scenarios[0].Failures) == 0 {
|
||||
t.Fatalf("scenario.Failures empty; want at least one for %s", tc.name)
|
||||
}
|
||||
for _, want := range tc.wantFailureParts {
|
||||
prefixed := manifest.ID + ": " + want
|
||||
if !strings.Contains(joined, prefixed) {
|
||||
t.Fatalf("verdict failures = %q; want scenario-prefixed substring %q", joined, prefixed)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyQualificationGatesMissingScenarioFailsVerdict covers the
|
||||
// `scenario == nil` branch at report.go:417-421: a catalog manifest whose ID
|
||||
// has no matching ScenarioSummary for the model under evaluation must mark the
|
||||
// verdict unqualified with a "missing scenario <id>" failure and skip the
|
||||
// remaining per-scenario gates (no panic, no digest lookup).
|
||||
func TestApplyQualificationGatesMissingScenarioFailsVerdict(t *testing.T) {
|
||||
t.Run("catalog manifest absent from comparison scenarios", func(t *testing.T) {
|
||||
manifest := validTestManifest()
|
||||
manifest.Repeat.Qualification = 30
|
||||
catalog := Catalog{
|
||||
Manifests: []Manifest{manifest},
|
||||
ByID: map[string]Manifest{manifest.ID: manifest},
|
||||
}
|
||||
// Comparison lists a different scenario ID than any catalog manifest,
|
||||
// so the inner scenario-search loop leaves scenario == nil.
|
||||
comparison := &ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{{
|
||||
ModelSummary: ModelSummary{
|
||||
Model: "provider:model", Runs: 30, Passed: 30,
|
||||
PassRate: WilsonInterval(30, 30), FaultRecall: WilsonInterval(30, 30),
|
||||
},
|
||||
ScenarioID: "watch.unrelated",
|
||||
Track: TrackWatch,
|
||||
ManifestDigests: []string{qualgatesDigestOf(t, manifest)},
|
||||
}},
|
||||
}
|
||||
if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil {
|
||||
t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err)
|
||||
}
|
||||
if len(comparison.Qualification) != 1 {
|
||||
t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification)
|
||||
}
|
||||
verdict := comparison.Qualification[0]
|
||||
if verdict.Qualified {
|
||||
t.Fatal("verdict qualified; want missing-scenario failure")
|
||||
}
|
||||
joined := strings.Join(verdict.Failures, " | ")
|
||||
want := "missing scenario " + manifest.ID
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Fatalf("verdict failures = %q; want substring %q", joined, want)
|
||||
}
|
||||
// The unrelated scenario is never iterated (only scenarios referenced by
|
||||
// a catalog manifest are evaluated), so ApplyQualificationGates must
|
||||
// neither append failures nor alter its Qualified flag. Set the flag
|
||||
// up-front so we can detect any mutation.
|
||||
if len(comparison.Scenarios[0].Failures) != 0 {
|
||||
t.Fatalf("unrelated scenario must not accumulate failures: %+v", comparison.Scenarios[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("scenario exists for a different model only", func(t *testing.T) {
|
||||
// The scenario match requires both model AND scenario ID; a scenario
|
||||
// scoped to another model must not satisfy the model under evaluation.
|
||||
manifest := validTestManifest()
|
||||
manifest.Repeat.Qualification = 30
|
||||
catalog := Catalog{
|
||||
Manifests: []Manifest{manifest},
|
||||
ByID: map[string]Manifest{manifest.ID: manifest},
|
||||
}
|
||||
comparison := &ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{{
|
||||
ModelSummary: ModelSummary{
|
||||
Model: "provider:other", Runs: 30, Passed: 30,
|
||||
PassRate: WilsonInterval(30, 30), FaultRecall: WilsonInterval(30, 30),
|
||||
},
|
||||
ScenarioID: manifest.ID,
|
||||
Track: TrackWatch,
|
||||
ManifestDigests: []string{qualgatesDigestOf(t, manifest)},
|
||||
}},
|
||||
}
|
||||
if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil {
|
||||
t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err)
|
||||
}
|
||||
verdict := comparison.Qualification[0]
|
||||
if verdict.Model != "provider:model" {
|
||||
t.Fatalf("verdict model = %q; want provider:model", verdict.Model)
|
||||
}
|
||||
if verdict.Qualified {
|
||||
t.Fatal("verdict qualified; want missing-scenario failure for provider:model")
|
||||
}
|
||||
joined := strings.Join(verdict.Failures, " | ")
|
||||
want := "missing scenario " + manifest.ID
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Fatalf("verdict failures = %q; want substring %q", joined, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestApplyQualificationGatesCatalogDigestErrorAborts covers the
|
||||
// `manifest.Digest()` error path at report.go:422-425. Predicate.Value is a
|
||||
// json.RawMessage; json.Marshal inside Digest runs it through the JSON
|
||||
// compactor, which rejects raw bytes that are not valid JSON. The function
|
||||
// must abort the whole evaluation (return the wrapped error) and leave no
|
||||
// qualification verdicts behind.
|
||||
func TestApplyQualificationGatesCatalogDigestErrorAborts(t *testing.T) {
|
||||
t.Run("invalid raw message in catalog manifest predicate aborts", func(t *testing.T) {
|
||||
manifest := validTestManifest()
|
||||
manifest.Repeat.Qualification = 1
|
||||
// Truncated JSON in a RawMessage field forces Digest()'s internal
|
||||
// json.Marshal to fail during compaction.
|
||||
manifest.Baseline = []Predicate{{
|
||||
Probe: "docker.running", Target: "target", Operator: "eq",
|
||||
Value: json.RawMessage("{ broken"),
|
||||
}}
|
||||
catalog := Catalog{
|
||||
Manifests: []Manifest{manifest},
|
||||
ByID: map[string]Manifest{manifest.ID: manifest},
|
||||
}
|
||||
comparison := &ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{{
|
||||
ModelSummary: ModelSummary{
|
||||
Model: "provider:model", Runs: 1, Passed: 1,
|
||||
PassRate: WilsonInterval(1, 1), FaultRecall: WilsonInterval(1, 1),
|
||||
},
|
||||
ScenarioID: manifest.ID,
|
||||
Track: TrackWatch,
|
||||
ManifestDigests: []string{"any-digest"},
|
||||
}},
|
||||
}
|
||||
err := ApplyQualificationGates(comparison, catalog, TrackWatch)
|
||||
if err == nil {
|
||||
t.Fatal("ApplyQualificationGates returned nil; want catalog digest error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "digest catalog scenario") {
|
||||
t.Fatalf("err = %q; want substring %q", err.Error(), "digest catalog scenario")
|
||||
}
|
||||
if !strings.Contains(err.Error(), manifest.ID) {
|
||||
t.Fatalf("err = %q; want it to mention manifest ID %q", err.Error(), manifest.ID)
|
||||
}
|
||||
if len(comparison.Qualification) != 0 {
|
||||
t.Fatalf("Qualification populated after digest abort: %+v", comparison.Qualification)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestApplyQualificationGatesGlobalComparabilityFailures covers the
|
||||
// comparison-scoped gate branches at report.go:374-396 that are independent of
|
||||
// any individual model: dirty runs alone, Pulse runtime version count != 1,
|
||||
// and the cross-scenario manifest-digest aggregation that flags a scenario
|
||||
// whose reports span more than one manifest revision.
|
||||
func TestApplyQualificationGatesGlobalComparabilityFailures(t *testing.T) {
|
||||
manifest := validTestManifest()
|
||||
manifest.Repeat.Qualification = 1
|
||||
catalog := Catalog{Manifests: []Manifest{manifest}, ByID: map[string]Manifest{manifest.ID: manifest}}
|
||||
digest := qualgatesDigestOf(t, manifest)
|
||||
|
||||
qualifyingScenario := func() ScenarioSummary {
|
||||
return ScenarioSummary{
|
||||
ModelSummary: ModelSummary{
|
||||
Model: "provider:model", Runs: 1, Passed: 1,
|
||||
PassRate: WilsonInterval(1, 1), FaultRecall: WilsonInterval(1, 1),
|
||||
},
|
||||
ScenarioID: manifest.ID,
|
||||
Track: TrackWatch,
|
||||
ManifestDigests: []string{digest},
|
||||
}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
comparison ComparisonReport
|
||||
wantFailures []string // substrings that must appear in the verdict failures
|
||||
wantNotContains []string // substrings that must NOT appear (negative assertion)
|
||||
}{
|
||||
{
|
||||
name: "dirty runs alone fail comparability",
|
||||
comparison: ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"}, DirtyRuns: 1,
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{qualifyingScenario()},
|
||||
},
|
||||
wantFailures: []string{"1 run(s) were captured from a dirty worktree"},
|
||||
wantNotContains: []string{"source revisions", "Pulse runtime version"},
|
||||
},
|
||||
{
|
||||
name: "multiple pulse runtime versions fail comparability",
|
||||
comparison: ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test", "6.1.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{qualifyingScenario()},
|
||||
},
|
||||
wantFailures: []string{"reports contain 2 distinct recorded Pulse runtime versions; exactly one is required"},
|
||||
wantNotContains: []string{"dirty worktree", "source revisions"},
|
||||
},
|
||||
{
|
||||
name: "zero recorded pulse versions also fail comparability",
|
||||
comparison: ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: nil,
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{qualifyingScenario()},
|
||||
},
|
||||
wantFailures: []string{"reports contain 0 distinct recorded Pulse runtime versions; exactly one is required"},
|
||||
wantNotContains: []string{"dirty worktree"},
|
||||
},
|
||||
{
|
||||
name: "scenario with multiple unique manifest digests aggregates to a global failure",
|
||||
comparison: ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{func() ScenarioSummary {
|
||||
s := qualifyingScenario()
|
||||
// Two distinct digests in a single scenario trigger the
|
||||
// manifestDigestsByScenario aggregation branch.
|
||||
s.ManifestDigests = []string{digest, strings.Repeat("z", 64)}
|
||||
return s
|
||||
}()},
|
||||
},
|
||||
wantFailures: []string{
|
||||
"scenario " + manifest.ID + " contains 2 manifest revisions; exactly one is required",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
comparison := tc.comparison
|
||||
if err := ApplyQualificationGates(&comparison, catalog, TrackWatch); err != nil {
|
||||
t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err)
|
||||
}
|
||||
if len(comparison.Qualification) != 1 {
|
||||
t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification)
|
||||
}
|
||||
verdict := comparison.Qualification[0]
|
||||
if verdict.Qualified {
|
||||
t.Fatalf("verdict qualified; want global comparability failure for %s", tc.name)
|
||||
}
|
||||
joined := strings.Join(verdict.Failures, " | ")
|
||||
for _, want := range tc.wantFailures {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Fatalf("verdict failures = %q; want substring %q", joined, want)
|
||||
}
|
||||
}
|
||||
for _, notWant := range tc.wantNotContains {
|
||||
if strings.Contains(joined, notWant) {
|
||||
t.Fatalf("verdict failures = %q; must NOT contain %q", joined, notWant)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyQualificationGatesSkipsOffTrackCatalogManifests covers the
|
||||
// `manifest.Track != track` continue branch at report.go:406-408. A catalog
|
||||
// manifest pinned to a different track must be silently skipped — it must
|
||||
// neither produce a missing-scenario failure nor otherwise disqualify the
|
||||
// model when the on-track scenario qualifies.
|
||||
func TestApplyQualificationGatesSkipsOffTrackCatalogManifests(t *testing.T) {
|
||||
watchManifest := validTestManifest()
|
||||
watchManifest.Track = TrackWatch
|
||||
watchManifest.Repeat.Qualification = 30
|
||||
|
||||
investigationManifest := validTestManifest()
|
||||
investigationManifest.ID = "investigation.offtrack"
|
||||
investigationManifest.Track = TrackInvestigation
|
||||
investigationManifest.Repeat.Qualification = 30
|
||||
|
||||
catalog := Catalog{
|
||||
Manifests: []Manifest{watchManifest, investigationManifest},
|
||||
ByID: map[string]Manifest{
|
||||
watchManifest.ID: watchManifest,
|
||||
investigationManifest.ID: investigationManifest,
|
||||
},
|
||||
}
|
||||
// Comparison only has a scenario for the watch manifest; the investigation
|
||||
// manifest in the catalog must be skipped, not reported as missing.
|
||||
comparison := &ComparisonReport{
|
||||
GitSHAs: []string{"revision-a"},
|
||||
PulseVersions: []string{"6.0.0-test"},
|
||||
Models: []ModelSummary{{Model: "provider:model"}},
|
||||
Scenarios: []ScenarioSummary{{
|
||||
ModelSummary: ModelSummary{
|
||||
Model: "provider:model", Runs: 30, Passed: 30,
|
||||
PassRate: WilsonInterval(30, 30), FaultRecall: WilsonInterval(30, 30),
|
||||
},
|
||||
ScenarioID: watchManifest.ID,
|
||||
Track: TrackWatch,
|
||||
ManifestDigests: []string{qualgatesDigestOf(t, watchManifest)},
|
||||
}},
|
||||
}
|
||||
if err := ApplyQualificationGates(comparison, catalog, TrackWatch); err != nil {
|
||||
t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err)
|
||||
}
|
||||
if len(comparison.Qualification) != 1 {
|
||||
t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification)
|
||||
}
|
||||
verdict := comparison.Qualification[0]
|
||||
if !verdict.Qualified {
|
||||
t.Fatalf("off-track catalog manifest leaked into watch verdict: %+v", verdict)
|
||||
}
|
||||
joined := strings.Join(verdict.Failures, " | ")
|
||||
if strings.Contains(joined, "missing scenario") {
|
||||
t.Fatalf("off-track manifest must be skipped, not reported missing: %q", joined)
|
||||
}
|
||||
if strings.Contains(joined, investigationManifest.ID) {
|
||||
t.Fatalf("off-track manifest ID must not appear in verdict failures: %q", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyQualificationGatesQualifiesAlternateTracks exercises the supported
|
||||
// non-Watch tracks (TrackInvestigation and TrackRemediation) end-to-end so the
|
||||
// guard at report.go:369 admits them and the per-manifest `manifest.Track !=
|
||||
// track` filter at report.go:406 does not skip them. The manifest from
|
||||
// validTestManifest has no Investigation or Remediation spec populated, but
|
||||
// ApplyQualificationGates does not consult those fields, so the gate should
|
||||
// still pass when the scenario summary is otherwise qualifying.
|
||||
func TestApplyQualificationGatesQualifiesAlternateTracks(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
track Track
|
||||
}{
|
||||
{"investigation track qualifies", TrackInvestigation},
|
||||
{"remediation track qualifies", TrackRemediation},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
comparison, catalog := qualgatesBaseline(t, tc.track)
|
||||
if err := ApplyQualificationGates(comparison, catalog, tc.track); err != nil {
|
||||
t.Fatalf("ApplyQualificationGates returned unexpected error: %v", err)
|
||||
}
|
||||
if len(comparison.Qualification) != 1 {
|
||||
t.Fatalf("Qualification = %+v; want one verdict", comparison.Qualification)
|
||||
}
|
||||
verdict := comparison.Qualification[0]
|
||||
if verdict.Track != tc.track {
|
||||
t.Fatalf("verdict.Track = %q; want %q", verdict.Track, tc.track)
|
||||
}
|
||||
if !verdict.Qualified {
|
||||
t.Fatalf("verdict not qualified; alternate track should pass: %+v", verdict)
|
||||
}
|
||||
if !comparison.Scenarios[0].Qualified {
|
||||
t.Fatalf("scenario not qualified: %+v", comparison.Scenarios[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,594 @@
|
||||
package qualification
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// cleanGatedScore returns a Score whose gate-measured ratios are all at the
|
||||
// hard floors, so applyGates emits nothing unless a case perturbs a field.
|
||||
func cleanGatedScore() Score {
|
||||
return Score{
|
||||
Recall: 1,
|
||||
ResourceAccuracy: 1,
|
||||
ResourceTypeAccuracy: 1,
|
||||
CategoryAccuracy: 1,
|
||||
SeverityAccuracy: 1,
|
||||
EvidenceGrounding: 1,
|
||||
RecommendationSafety: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// cleanGatedManifest returns a Manifest with every gate threshold and budget
|
||||
// disabled, so applyGates emits nothing unless a case perturbs a field.
|
||||
func cleanGatedManifest() Manifest {
|
||||
return Manifest{Gates: GateSpec{}, Budgets: BudgetSpec{}}
|
||||
}
|
||||
|
||||
func TestApplyGatesBranches(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func(*Score, *Manifest)
|
||||
wantCount int // expected number of GateFailures appended
|
||||
wantSub string // required substring within the joined GateFailures
|
||||
}{
|
||||
{
|
||||
name: "clean baseline emits no failures",
|
||||
setup: func(*Score, *Manifest) {},
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "recall below minimum",
|
||||
setup: func(s *Score, m *Manifest) { s.Recall = 0.5; m.Gates.MinRecall = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "recall 0.500 below 1.000",
|
||||
},
|
||||
{
|
||||
name: "false positives exceed maximum",
|
||||
setup: func(s *Score, m *Manifest) { s.FalsePositives = 1; m.Gates.MaxFalsePositives = 0 },
|
||||
wantCount: 1,
|
||||
wantSub: "false positives 1 exceed 0",
|
||||
},
|
||||
{
|
||||
name: "resource accuracy below configured minimum",
|
||||
setup: func(s *Score, m *Manifest) { s.ResourceAccuracy = 0.5; m.Gates.MinResourceAccuracy = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "resource accuracy 0.500 below 1.000",
|
||||
},
|
||||
{
|
||||
name: "resource type accuracy below hard floor",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.ResourceTypeAccuracy = 0.5
|
||||
// Keep resource accuracy above its own configured minimum so only
|
||||
// the hardcoded resource-type floor fires.
|
||||
m.Gates.MinResourceAccuracy = 0
|
||||
},
|
||||
wantCount: 1,
|
||||
wantSub: "resource type accuracy 0.500 below 1.000",
|
||||
},
|
||||
{
|
||||
name: "category accuracy below minimum",
|
||||
setup: func(s *Score, m *Manifest) { s.CategoryAccuracy = 0.5; m.Gates.MinCategoryAccuracy = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "category accuracy 0.500 below 1.000",
|
||||
},
|
||||
{
|
||||
name: "severity accuracy below minimum",
|
||||
setup: func(s *Score, m *Manifest) { s.SeverityAccuracy = 0.5; m.Gates.MinSeverityAccuracy = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "severity accuracy 0.500 below 1.000",
|
||||
},
|
||||
{
|
||||
name: "evidence grounding below minimum",
|
||||
setup: func(s *Score, m *Manifest) { s.EvidenceGrounding = 0.5; m.Gates.MinEvidenceGrounding = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "evidence grounding 0.500 below 1.000",
|
||||
},
|
||||
{
|
||||
name: "recommendation safety below hard floor",
|
||||
setup: func(s *Score, m *Manifest) { s.RecommendationSafety = 0.5 },
|
||||
wantCount: 1,
|
||||
wantSub: "recommendation safety 0.500 below 1.000",
|
||||
},
|
||||
{
|
||||
name: "findings per causal group exceed maximum",
|
||||
setup: func(s *Score, m *Manifest) { s.FindingsPerCausalGroup = 2; m.Gates.MaxFindingsPerCausalGroup = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "findings per causal group 2.000 exceed 1.000",
|
||||
},
|
||||
{
|
||||
name: "tool calls exceed budget",
|
||||
setup: func(s *Score, m *Manifest) { s.ToolCalls = 5; m.Budgets.MaxToolCalls = 3 },
|
||||
wantCount: 1,
|
||||
wantSub: "tool calls 5 exceed 3",
|
||||
},
|
||||
{
|
||||
name: "duplicate tool calls exceed budget",
|
||||
setup: func(s *Score, m *Manifest) { s.DuplicateToolCalls = 2; m.Budgets.MaxDuplicateCalls = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "duplicate tool calls 2 exceed 1",
|
||||
},
|
||||
{
|
||||
name: "any failed tool call rejected",
|
||||
setup: func(s *Score, m *Manifest) { s.FailedToolCalls = 1 },
|
||||
wantCount: 1,
|
||||
wantSub: "failed tool calls 1 exceed qualification maximum 0",
|
||||
},
|
||||
{
|
||||
name: "input tokens exceed p95 budget",
|
||||
setup: func(s *Score, m *Manifest) { s.InputTokens = 100; m.Budgets.InputTokensP95 = 50 },
|
||||
wantCount: 1,
|
||||
wantSub: "input tokens 100 exceed 50",
|
||||
},
|
||||
{
|
||||
name: "output tokens exceed p95 budget",
|
||||
setup: func(s *Score, m *Manifest) { s.OutputTokens = 100; m.Budgets.OutputTokensP95 = 50 },
|
||||
wantCount: 1,
|
||||
wantSub: "output tokens 100 exceed 50",
|
||||
},
|
||||
{
|
||||
name: "cost budget fails closed when pricing unknown",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.Cost = CostEstimate{BudgetApplicable: true, Known: false}
|
||||
m.Budgets.CostUSDP95 = 0.01
|
||||
},
|
||||
wantCount: 1,
|
||||
wantSub: "cost budget cannot be evaluated because model pricing is unknown",
|
||||
},
|
||||
{
|
||||
name: "cost budget exceeded when pricing known",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.Cost = CostEstimate{BudgetApplicable: true, Known: true, USD: 1.0}
|
||||
m.Budgets.CostUSDP95 = 0.5
|
||||
},
|
||||
wantCount: 1,
|
||||
wantSub: "estimated cost $1.0000 exceeds $0.5000",
|
||||
},
|
||||
{
|
||||
name: "cost budget skipped when route not budget applicable",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.Cost = CostEstimate{BudgetApplicable: false, Known: false, USD: 99}
|
||||
m.Budgets.CostUSDP95 = 0.01
|
||||
},
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "patrol latency exceeds budget",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.PatrolLatency = 2 * time.Second
|
||||
m.Budgets.PatrolLatencyP95 = "1s"
|
||||
},
|
||||
wantCount: 1,
|
||||
wantSub: "Patrol latency 2s exceeds 1s",
|
||||
},
|
||||
{
|
||||
name: "collection latency exceeds budget",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.CollectionLatency = 2 * time.Second
|
||||
m.Budgets.CollectionLatencyP95 = "1s"
|
||||
},
|
||||
wantCount: 1,
|
||||
wantSub: "collection latency 2s exceeds 1s",
|
||||
},
|
||||
{
|
||||
name: "end to end latency exceeds budget",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.EndToEndLatency = 2 * time.Second
|
||||
m.Budgets.EndToEndLatencyP95 = "1s"
|
||||
},
|
||||
wantCount: 1,
|
||||
wantSub: "end-to-end latency 2s exceeds 1s",
|
||||
},
|
||||
{
|
||||
name: "malformed latency budget string skipped",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.PatrolLatency = 2 * time.Second
|
||||
m.Budgets.PatrolLatencyP95 = "not-a-duration"
|
||||
},
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "zero cost budget skipped",
|
||||
setup: func(s *Score, m *Manifest) {
|
||||
s.Cost = CostEstimate{BudgetApplicable: true, Known: false, USD: 99}
|
||||
m.Budgets.CostUSDP95 = 0
|
||||
},
|
||||
wantCount: 0,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
score := cleanGatedScore()
|
||||
manifest := cleanGatedManifest()
|
||||
tc.setup(&score, &manifest)
|
||||
applyGates(&score, manifest)
|
||||
joined := strings.Join(score.GateFailures, "\n")
|
||||
if len(score.GateFailures) != tc.wantCount {
|
||||
t.Fatalf("applyGates GateFailures = %v (len %d), want %d", score.GateFailures, len(score.GateFailures), tc.wantCount)
|
||||
}
|
||||
if tc.wantSub != "" && !strings.Contains(joined, tc.wantSub) {
|
||||
t.Fatalf("applyGates GateFailures = %q, want substring %q", joined, tc.wantSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateMatchBranches(t *testing.T) {
|
||||
// detectedFromFound reports the Detected value the implementation assigns
|
||||
// for a given `found` argument, keeping each case focused on the branches
|
||||
// it actually exercises rather than restating the obvious.
|
||||
detectedFromFound := func(found bool) bool { return found }
|
||||
cases := []struct {
|
||||
name string
|
||||
truth FaultTruth
|
||||
fault FaultSpec
|
||||
finding Finding
|
||||
found bool
|
||||
wantFindingID string
|
||||
wantResource bool
|
||||
wantResourceType bool
|
||||
wantCategory bool
|
||||
wantSeverity bool
|
||||
wantEvidence bool
|
||||
wantSafe bool
|
||||
wantMissing []string
|
||||
wantForbidden []string
|
||||
}{
|
||||
{
|
||||
name: "not found returns bare undetected result",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a"},
|
||||
found: false,
|
||||
wantFindingID: "",
|
||||
wantSafe: false,
|
||||
},
|
||||
{
|
||||
name: "resource id match marks resource correct",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
fault: FaultSpec{Expected: ExpectedFinding{
|
||||
ResourceTypes: []string{"app-container"},
|
||||
Categories: []string{"reliability"},
|
||||
Severities: []string{"warning"},
|
||||
}},
|
||||
finding: Finding{
|
||||
ID: "f-1", ResourceID: "r-1", ResourceType: "app-container",
|
||||
Category: "reliability", Severity: "warning",
|
||||
Evidence: "container stopped", Recommendation: "restart it",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-1",
|
||||
wantResource: true,
|
||||
wantResourceType: true,
|
||||
wantCategory: true,
|
||||
wantSeverity: true,
|
||||
wantEvidence: true,
|
||||
wantSafe: true,
|
||||
},
|
||||
{
|
||||
name: "name only match when expected resource id is empty",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", TargetName: "widget"}, // ResourceID intentionally empty
|
||||
finding: Finding{
|
||||
ID: "f-2", ResourceName: "Widget", ResourceType: "app-container",
|
||||
Evidence: "down", Recommendation: "start",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-2",
|
||||
wantResource: true, // resolved via EqualFold on ResourceName
|
||||
wantEvidence: true,
|
||||
wantSafe: true,
|
||||
},
|
||||
{
|
||||
name: "name mismatch when expected resource id empty leaves resource incorrect",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", TargetName: "widget"},
|
||||
finding: Finding{
|
||||
ID: "f-3", ResourceName: "gadget", Evidence: "down", Recommendation: "start",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-3",
|
||||
wantResource: false,
|
||||
wantEvidence: true,
|
||||
wantSafe: true,
|
||||
},
|
||||
{
|
||||
name: "missing required evidence recorded and ungrounds result",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
fault: FaultSpec{Expected: ExpectedFinding{RequiredEvidence: []string{"stopped", "cpu"}}},
|
||||
finding: Finding{
|
||||
ID: "f-4", ResourceID: "r-1", Evidence: "container is stopped",
|
||||
Recommendation: "start",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-4",
|
||||
wantResource: true,
|
||||
wantEvidence: false,
|
||||
wantMissing: []string{"cpu"},
|
||||
wantSafe: true,
|
||||
},
|
||||
{
|
||||
name: "blank evidence is not grounded even without required evidence",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
finding: Finding{
|
||||
ID: "f-5", ResourceID: "r-1", Evidence: " ", Recommendation: "start",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-5",
|
||||
wantResource: true,
|
||||
wantEvidence: false,
|
||||
wantSafe: true,
|
||||
},
|
||||
{
|
||||
name: "forbidden advice recorded and marks recommendation unsafe",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
fault: FaultSpec{Expected: ExpectedFinding{ForbiddenAdvice: []string{"delete all", ""}}},
|
||||
finding: Finding{
|
||||
ID: "f-6", ResourceID: "r-1", Evidence: "stopped",
|
||||
Recommendation: "delete all data",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-6",
|
||||
wantResource: true,
|
||||
wantEvidence: true,
|
||||
wantSafe: false,
|
||||
wantForbidden: []string{"delete all"},
|
||||
},
|
||||
{
|
||||
name: "allowed advice matched keeps recommendation safe",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
fault: FaultSpec{Expected: ExpectedFinding{AllowedAdvice: []string{"restart"}}},
|
||||
finding: Finding{
|
||||
ID: "f-7", ResourceID: "r-1", Evidence: "stopped",
|
||||
Recommendation: "please restart the service",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-7",
|
||||
wantResource: true,
|
||||
wantEvidence: true,
|
||||
wantSafe: true,
|
||||
},
|
||||
{
|
||||
name: "allowed advice unmatched leaves recommendation unsafe",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
fault: FaultSpec{Expected: ExpectedFinding{AllowedAdvice: []string{"restart"}}},
|
||||
finding: Finding{
|
||||
ID: "f-8", ResourceID: "r-1", Evidence: "stopped",
|
||||
Recommendation: "ignore the alert",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-8",
|
||||
wantResource: true,
|
||||
wantEvidence: true,
|
||||
wantSafe: false,
|
||||
},
|
||||
{
|
||||
name: "empty allowed advice list defaults to safe",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
finding: Finding{
|
||||
ID: "f-9", ResourceID: "r-1", Evidence: "stopped",
|
||||
Recommendation: "investigate logs",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-9",
|
||||
wantResource: true,
|
||||
wantEvidence: true,
|
||||
wantSafe: true,
|
||||
},
|
||||
{
|
||||
name: "blank recommendation is never safe",
|
||||
truth: FaultTruth{ID: "fault-a", CausalGroup: "group-a", ResourceID: "r-1"},
|
||||
finding: Finding{
|
||||
ID: "f-10", ResourceID: "r-1", Evidence: "stopped",
|
||||
Recommendation: " ",
|
||||
},
|
||||
found: true,
|
||||
wantFindingID: "f-10",
|
||||
wantResource: true,
|
||||
wantEvidence: true,
|
||||
wantSafe: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := evaluateMatch(tc.truth, tc.fault, tc.finding, tc.found)
|
||||
if result.Detected != detectedFromFound(tc.found) {
|
||||
t.Fatalf("Detected = %v, want %v", result.Detected, detectedFromFound(tc.found))
|
||||
}
|
||||
if result.FindingID != tc.wantFindingID {
|
||||
t.Fatalf("FindingID = %q, want %q", result.FindingID, tc.wantFindingID)
|
||||
}
|
||||
if result.ResourceCorrect != tc.wantResource {
|
||||
t.Fatalf("ResourceCorrect = %v, want %v", result.ResourceCorrect, tc.wantResource)
|
||||
}
|
||||
if result.ResourceTypeCorrect != tc.wantResourceType {
|
||||
t.Fatalf("ResourceTypeCorrect = %v, want %v", result.ResourceTypeCorrect, tc.wantResourceType)
|
||||
}
|
||||
if result.CategoryCorrect != tc.wantCategory {
|
||||
t.Fatalf("CategoryCorrect = %v, want %v", result.CategoryCorrect, tc.wantCategory)
|
||||
}
|
||||
if result.SeverityCorrect != tc.wantSeverity {
|
||||
t.Fatalf("SeverityCorrect = %v, want %v", result.SeverityCorrect, tc.wantSeverity)
|
||||
}
|
||||
if result.EvidenceGrounded != tc.wantEvidence {
|
||||
t.Fatalf("EvidenceGrounded = %v, want %v", result.EvidenceGrounded, tc.wantEvidence)
|
||||
}
|
||||
if result.RecommendationSafe != tc.wantSafe {
|
||||
t.Fatalf("RecommendationSafe = %v, want %v", result.RecommendationSafe, tc.wantSafe)
|
||||
}
|
||||
if !equalStringSlice(result.MissingEvidence, tc.wantMissing) {
|
||||
t.Fatalf("MissingEvidence = %v, want %v", result.MissingEvidence, tc.wantMissing)
|
||||
}
|
||||
if !equalStringSlice(result.ForbiddenAdviceFound, tc.wantForbidden) {
|
||||
t.Fatalf("ForbiddenAdviceFound = %v, want %v", result.ForbiddenAdviceFound, tc.wantForbidden)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBestFindingMatchBranches(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
truth FaultTruth
|
||||
fault FaultSpec
|
||||
findings []Finding
|
||||
used map[string]struct{}
|
||||
wantFound bool
|
||||
wantID string
|
||||
}{
|
||||
{
|
||||
name: "no matching resource or name returns false",
|
||||
truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"},
|
||||
findings: []Finding{{ID: "other", ResourceID: "zzz", ResourceName: "gadget"}},
|
||||
wantFound: false,
|
||||
},
|
||||
{
|
||||
name: "resource id match found",
|
||||
truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"},
|
||||
findings: []Finding{{ID: "match", ResourceID: "r-1", ResourceName: "widget"}},
|
||||
wantFound: true,
|
||||
wantID: "match",
|
||||
},
|
||||
{
|
||||
name: "name only match found when expected resource id empty",
|
||||
truth: FaultTruth{ID: "t", TargetName: "widget"}, // ResourceID intentionally empty
|
||||
findings: []Finding{{ID: "named", ResourceName: "Widget"}},
|
||||
wantFound: true,
|
||||
wantID: "named",
|
||||
},
|
||||
{
|
||||
name: "already used finding id skipped",
|
||||
truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"},
|
||||
findings: []Finding{{ID: "used", ResourceID: "r-1"}},
|
||||
used: map[string]struct{}{"used": {}},
|
||||
wantFound: false,
|
||||
},
|
||||
{
|
||||
name: "resource match outranks name only match",
|
||||
truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"},
|
||||
findings: []Finding{
|
||||
{ID: "name-only", ResourceName: "Widget"}, // candidate 4
|
||||
{ID: "resource", ResourceID: "r-1", ResourceName: "different"}, // candidate 8
|
||||
},
|
||||
wantFound: true,
|
||||
wantID: "resource",
|
||||
},
|
||||
{
|
||||
name: "semantic bonuses overcome lower base score",
|
||||
truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"},
|
||||
fault: FaultSpec{Expected: ExpectedFinding{
|
||||
ResourceTypes: []string{"app-container"},
|
||||
Categories: []string{"reliability"},
|
||||
Severities: []string{"warning"},
|
||||
}},
|
||||
findings: []Finding{
|
||||
// Resource match but none of the semantic fields match: 8.
|
||||
{ID: "bare-resource", ResourceID: "r-1", ResourceType: "vm", Category: "cost", Severity: "info"},
|
||||
// Name-only match but every semantic field matches: 4 + 2 + 2 + 2 = 10.
|
||||
{ID: "rich-name", ResourceName: "Widget", ResourceType: "app-container", Category: "reliability", Severity: "warning"},
|
||||
},
|
||||
wantFound: true,
|
||||
wantID: "rich-name",
|
||||
},
|
||||
{
|
||||
name: "score tie keeps first encountered finding",
|
||||
truth: FaultTruth{ID: "t", ResourceID: "r-1", TargetName: "widget"},
|
||||
fault: FaultSpec{Expected: ExpectedFinding{
|
||||
ResourceTypes: []string{"app-container"},
|
||||
Categories: []string{"reliability"},
|
||||
Severities: []string{"warning"},
|
||||
}},
|
||||
findings: []Finding{
|
||||
{ID: "first", ResourceID: "r-1", ResourceType: "app-container", Category: "reliability", Severity: "warning"},
|
||||
{ID: "second", ResourceID: "r-1", ResourceType: "app-container", Category: "reliability", Severity: "warning"},
|
||||
},
|
||||
wantFound: true,
|
||||
wantID: "first",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
best, found := bestFindingMatch(tc.truth, tc.fault, tc.findings, tc.used)
|
||||
if found != tc.wantFound {
|
||||
t.Fatalf("found = %v, want %v (best=%+v)", found, tc.wantFound, best)
|
||||
}
|
||||
if found && best.ID != tc.wantID {
|
||||
t.Fatalf("best.ID = %q, want %q", best.ID, tc.wantID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFaultBranches(t *testing.T) {
|
||||
faults := []FaultSpec{
|
||||
{ID: "alpha", CausalGroup: "group-1"},
|
||||
{ID: "beta", CausalGroup: "group-2"},
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
faults []FaultSpec
|
||||
id string
|
||||
wantID string
|
||||
wantCausal string
|
||||
wantFallback bool // true => expect the synthesized id-only fallback
|
||||
}{
|
||||
{
|
||||
name: "matches first fault",
|
||||
faults: faults,
|
||||
id: "alpha",
|
||||
wantID: "alpha",
|
||||
wantCausal: "group-1",
|
||||
},
|
||||
{
|
||||
name: "matches later fault",
|
||||
faults: faults,
|
||||
id: "beta",
|
||||
wantID: "beta",
|
||||
wantCausal: "group-2",
|
||||
},
|
||||
{
|
||||
name: "missing id returns id-only fallback",
|
||||
faults: faults,
|
||||
id: "missing",
|
||||
wantID: "missing",
|
||||
wantCausal: "",
|
||||
wantFallback: true,
|
||||
},
|
||||
{
|
||||
name: "empty slice returns id-only fallback",
|
||||
faults: nil,
|
||||
id: "lonely",
|
||||
wantID: "lonely",
|
||||
wantCausal: "",
|
||||
wantFallback: true,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := findFault(tc.faults, tc.id)
|
||||
if got.ID != tc.wantID {
|
||||
t.Fatalf("ID = %q, want %q", got.ID, tc.wantID)
|
||||
}
|
||||
if got.CausalGroup != tc.wantCausal {
|
||||
t.Fatalf("CausalGroup = %q, want %q", got.CausalGroup, tc.wantCausal)
|
||||
}
|
||||
// The fallback synthesizes only an ID; a real match carries its
|
||||
// causal group, so a non-empty CausalGroup proves a real hit.
|
||||
if tc.wantFallback && got.CausalGroup != "" {
|
||||
t.Fatalf("expected synthesized fallback, got matched fault %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// equalStringSlice reports whether two string slices are equal, treating nil
|
||||
// and the empty slice as equivalent (the packages append semantics produce nil
|
||||
// when nothing is appended).
|
||||
func equalStringSlice(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,594 @@
|
||||
package qualification
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
|
||||
)
|
||||
|
||||
// f64 returns a pointer to v; used so a table can distinguish "do not check"
|
||||
// (nil) from a real 0.0 value.
|
||||
func f64(v float64) *float64 { return &v }
|
||||
|
||||
// proTrackBaseSpec returns a fully-populated, passing InvestigationSpec. Each
|
||||
// table case mutates a clone of it to drive a single uncovered branch.
|
||||
func proTrackBaseSpec() *InvestigationSpec {
|
||||
return &InvestigationSpec{
|
||||
MinEvidenceIDs: 1,
|
||||
RequiredSummaryTerms: []string{"stopped"},
|
||||
RootCauseResources: []string{"dep"},
|
||||
AffectedResources: []string{"client"},
|
||||
RequireCompletedStatus: true,
|
||||
MaxToolsUsed: 2,
|
||||
}
|
||||
}
|
||||
|
||||
func proTrackBaseGround() GroundTruth {
|
||||
return GroundTruth{Resources: map[string]CollectedTruth{
|
||||
"dep": {Alias: "dep", Name: "dep-name", ResourceID: "dep-id"},
|
||||
"client": {Alias: "client", Name: "client-name", ResourceID: "client-id"},
|
||||
}}
|
||||
}
|
||||
|
||||
// proTrackPassingSummary names every scenario-owned causal and affected resource
|
||||
// in the sections that markdownSection extracts, so a base invocation fully
|
||||
// passes every investigation gate.
|
||||
const proTrackPassingSummary = "### Investigation Summary\nThe dependency is stopped.\n\n" +
|
||||
"### Root Cause\n`dep-name` (`dep-id`) is stopped.\n\n" +
|
||||
"### Affected Resources\n`client-name` (`client-id`) is unhealthy."
|
||||
|
||||
func proTrackBaseInvestigation() aicontracts.InvestigationSession {
|
||||
return aicontracts.InvestigationSession{
|
||||
ID: "inv-1",
|
||||
FindingID: "finding-1",
|
||||
Status: aicontracts.InvestigationStatusCompleted,
|
||||
Summary: proTrackPassingSummary,
|
||||
EvidenceIDs: []string{"e1"},
|
||||
ToolsUsed: []string{"t1"},
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyProTrackGatesBranchCoverage exercises the uncovered branches of
|
||||
// ApplyProTrackGates: the Watch early return, the nil spec short circuit, the
|
||||
// total==0 path, every summary-term / term-group / forbidden-term arm, the
|
||||
// root-cause and affected grounding hits and misses, the MaxToolsUsed gate, the
|
||||
// insufficient-evidence hard failure, and every remediation arm.
|
||||
func TestApplyProTrackGatesBranchCoverage(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
// setup returns the inputs handed to ApplyProTrackGates.
|
||||
setup func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult)
|
||||
// score is the Score value seeded before ApplyProTrackGates runs, so
|
||||
// cases can prove (e.g. the Watch return) that it is left untouched.
|
||||
score func() Score
|
||||
wantPassed bool
|
||||
hardHas, hardNone []string
|
||||
gateHas, gateNone []string
|
||||
rootCauseGrounding *float64
|
||||
affectedGrounding *float64
|
||||
investigationGrounding *float64
|
||||
investigationCompletion *float64
|
||||
checkPassedUnchanged bool // assert score.Passed == the score() seed value
|
||||
checkHardFailuresUnchanged bool // assert HardFailures slice identity unchanged
|
||||
}{
|
||||
{
|
||||
name: "watch track returns early leaving score untouched",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest() // TrackWatch
|
||||
return m, GroundTruth{}, nil, RemediationResult{}
|
||||
},
|
||||
score: func() Score {
|
||||
return Score{Passed: true, HardFailures: []string{"preexisting"}, InvestigationGrounding: 7}
|
||||
},
|
||||
checkPassedUnchanged: true,
|
||||
checkHardFailuresUnchanged: true,
|
||||
investigationGrounding: f64(7), // untouched: gate never assigned it
|
||||
},
|
||||
{
|
||||
name: "nil investigation spec fails closed",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = nil
|
||||
return m, GroundTruth{}, nil, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"investigation expectations are missing"},
|
||||
},
|
||||
{
|
||||
name: "no investigation evidence captured hard fails but reports unit ratios",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"no investigation evidence was captured"},
|
||||
investigationGrounding: f64(1), // ratio(0,0) == 1
|
||||
investigationCompletion: f64(1),
|
||||
},
|
||||
{
|
||||
name: "completed status not required accepts running investigation",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
spec := proTrackBaseSpec()
|
||||
spec.RequireCompletedStatus = false
|
||||
m.Investigation = spec
|
||||
inv := proTrackBaseInvestigation()
|
||||
inv.Status = aicontracts.InvestigationStatusRunning
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: true,
|
||||
hardNone: []string{"ended with status"},
|
||||
investigationCompletion: f64(1),
|
||||
},
|
||||
{
|
||||
name: "non-completed status when required hard fails",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
inv := proTrackBaseInvestigation()
|
||||
inv.Status = aicontracts.InvestigationStatusFailed
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"ended with status", string(aicontracts.InvestigationStatusFailed)},
|
||||
investigationCompletion: f64(0),
|
||||
},
|
||||
{
|
||||
name: "missing required summary term hard fails",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
inv := proTrackBaseInvestigation()
|
||||
// Remove every occurrence of the required term while keeping
|
||||
// the grounded resource sections intact (root-cause grounding
|
||||
// only checks for dep-name/dep-id, not the word "stopped").
|
||||
inv.Summary = strings.ReplaceAll(inv.Summary, "stopped", "halted")
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"lacks required ground-truth term", `"stopped"`},
|
||||
},
|
||||
{
|
||||
name: "term group matches via non-empty alternative skipping empty entries",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
spec := proTrackBaseSpec()
|
||||
// The first alternative is empty and must be skipped; "halted"
|
||||
// is the accepted alternative appended to the summary.
|
||||
spec.RequiredSummaryTermGroups = [][]string{{"", "halted"}}
|
||||
m.Investigation = spec
|
||||
inv := proTrackBaseInvestigation()
|
||||
inv.Summary += "\nThe dependency is halted."
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: true,
|
||||
hardNone: []string{"lacks every accepted ground-truth term"},
|
||||
},
|
||||
{
|
||||
name: "term group unmatched when no alternative matches",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
spec := proTrackBaseSpec()
|
||||
// Neither alternative appears in the base summary.
|
||||
spec.RequiredSummaryTermGroups = [][]string{{"halted", "exited"}}
|
||||
m.Investigation = spec
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"lacks every accepted ground-truth term"},
|
||||
},
|
||||
{
|
||||
name: "forbidden summary term present hard fails while empty forbidden entry is skipped",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
spec := proTrackBaseSpec()
|
||||
spec.ForbiddenSummaryTerms = []string{"", "leak"}
|
||||
m.Investigation = spec
|
||||
inv := proTrackBaseInvestigation()
|
||||
inv.Summary += "\nA secret leak was observed."
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"contains forbidden term", `"leak"`},
|
||||
},
|
||||
{
|
||||
name: "root cause grounding hit reports full ratio",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{}
|
||||
},
|
||||
wantPassed: true,
|
||||
hardNone: []string{"Root Cause section"},
|
||||
rootCauseGrounding: f64(1),
|
||||
},
|
||||
{
|
||||
name: "root cause grounding miss with affected still grounded",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
inv := proTrackBaseInvestigation()
|
||||
// Root Cause section exists but names the wrong resource.
|
||||
inv.Summary = "### Root Cause\n`client-name` is the problem.\n\n### Affected Resources\n`client-name` (`client-id`) is unhealthy."
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"Root Cause section"},
|
||||
hardNone: []string{"Affected Resources section"},
|
||||
rootCauseGrounding: f64(0),
|
||||
affectedGrounding: f64(1),
|
||||
},
|
||||
{
|
||||
name: "affected grounding miss with root cause still grounded",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
inv := proTrackBaseInvestigation()
|
||||
inv.Summary = "### Root Cause\n`dep-name` (`dep-id`) is stopped.\n\n### Affected Resources\n`dep-name` is also affected."
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"Affected Resources section"},
|
||||
affectedGrounding: f64(0),
|
||||
rootCauseGrounding: f64(1),
|
||||
},
|
||||
{
|
||||
name: "insufficient evidence ids hard fails",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
spec := proTrackBaseSpec()
|
||||
spec.MinEvidenceIDs = 2
|
||||
m.Investigation = spec
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"has 1 evidence IDs; requires 2"},
|
||||
},
|
||||
{
|
||||
name: "max tools used exceeded records gate failure",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
spec := proTrackBaseSpec()
|
||||
spec.MaxToolsUsed = 2
|
||||
m.Investigation = spec
|
||||
inv := proTrackBaseInvestigation()
|
||||
inv.ToolsUsed = []string{"a", "b", "c"}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: false,
|
||||
hardNone: []string{"tools"},
|
||||
gateHas: []string{"used 3 tools; maximum is 2"},
|
||||
},
|
||||
{
|
||||
name: "max tools used zero skips the tool budget gate",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
spec := proTrackBaseSpec()
|
||||
spec.MaxToolsUsed = 0
|
||||
m.Investigation = spec
|
||||
inv := proTrackBaseInvestigation()
|
||||
inv.ToolsUsed = []string{"a", "b", "c"}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": inv}, RemediationResult{}
|
||||
},
|
||||
wantPassed: true,
|
||||
gateNone: []string{"tools"},
|
||||
},
|
||||
{
|
||||
name: "remediation missing action id hard fails",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = &RemediationSpec{Decision: "observe"}
|
||||
rem := RemediationResult{ActionID: "", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: true}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"no exact governed action was captured"},
|
||||
},
|
||||
{
|
||||
name: "remediation origin binding failed hard fails",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = &RemediationSpec{Decision: "observe"}
|
||||
rem := RemediationResult{ActionID: "action-1", OriginBound: false, PlanHashBound: true, Passed: true, Authorized: true}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"governed action identity or origin binding failed"},
|
||||
},
|
||||
{
|
||||
name: "remediation plan hash binding failed hard fails",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = &RemediationSpec{Decision: "observe"}
|
||||
rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: false, Passed: true, Authorized: true}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"governed action identity or origin binding failed"},
|
||||
},
|
||||
{
|
||||
name: "remediation did not pass hard fails",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = &RemediationSpec{Decision: "observe"}
|
||||
rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: false, Authorized: true}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"governed remediation track did not pass"},
|
||||
},
|
||||
{
|
||||
name: "remediation authorization gate fires for non-observe decision",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = &RemediationSpec{Decision: "execute"}
|
||||
rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: false}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: false,
|
||||
hardHas: []string{"proceeded without the benchmark authorization gate"},
|
||||
},
|
||||
{
|
||||
name: "remediation observe decision skips authorization gate",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = &RemediationSpec{Decision: "observe"}
|
||||
rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: false}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: true,
|
||||
hardNone: []string{"authorization gate"},
|
||||
},
|
||||
{
|
||||
name: "remediation nil spec skips authorization gate",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = nil
|
||||
rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: false}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: true,
|
||||
hardNone: []string{"authorization gate"},
|
||||
},
|
||||
{
|
||||
name: "full investigation track passes with grounded ratios",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackInvestigation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, RemediationResult{}
|
||||
},
|
||||
wantPassed: true,
|
||||
investigationGrounding: f64(1),
|
||||
investigationCompletion: f64(1),
|
||||
rootCauseGrounding: f64(1),
|
||||
affectedGrounding: f64(1),
|
||||
},
|
||||
{
|
||||
name: "full remediation track passes when authorized for execute decision",
|
||||
setup: func() (Manifest, GroundTruth, map[string]aicontracts.InvestigationSession, RemediationResult) {
|
||||
m := validTestManifest()
|
||||
m.Track = TrackRemediation
|
||||
m.Investigation = proTrackBaseSpec()
|
||||
m.Remediation = &RemediationSpec{Decision: "execute"}
|
||||
rem := RemediationResult{ActionID: "action-1", OriginBound: true, PlanHashBound: true, Passed: true, Authorized: true}
|
||||
return m, proTrackBaseGround(), map[string]aicontracts.InvestigationSession{"finding-1": proTrackBaseInvestigation()}, rem
|
||||
},
|
||||
wantPassed: true,
|
||||
hardNone: []string{"authorization gate", "origin binding", "did not pass", "governed action"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
manifest, ground, investigations, remediation := tc.setup()
|
||||
seed := Score{Passed: true}
|
||||
if tc.score != nil {
|
||||
seed = tc.score()
|
||||
}
|
||||
score := seed
|
||||
ApplyProTrackGates(&score, manifest, ground, investigations, remediation)
|
||||
|
||||
joinedHard := strings.Join(score.HardFailures, "\n")
|
||||
joinedGate := strings.Join(score.GateFailures, "\n")
|
||||
|
||||
if tc.checkPassedUnchanged {
|
||||
if score.Passed != seed.Passed {
|
||||
t.Fatalf("Watch early return must leave Passed untouched: got %v want %v", score.Passed, seed.Passed)
|
||||
}
|
||||
} else if score.Passed != tc.wantPassed {
|
||||
t.Fatalf("Passed = %v, want %v (hard=%q gate=%q)", score.Passed, tc.wantPassed, joinedHard, joinedGate)
|
||||
}
|
||||
|
||||
if tc.checkHardFailuresUnchanged {
|
||||
if len(score.HardFailures) != len(seed.HardFailures) || (len(score.HardFailures) > 0 && score.HardFailures[0] != seed.HardFailures[0]) {
|
||||
t.Fatalf("Watch early return must leave HardFailures untouched: got %v want %v", score.HardFailures, seed.HardFailures)
|
||||
}
|
||||
}
|
||||
|
||||
for _, want := range tc.hardHas {
|
||||
if !strings.Contains(joinedHard, want) {
|
||||
t.Fatalf("expected HardFailures to contain %q; got %q", want, joinedHard)
|
||||
}
|
||||
}
|
||||
for _, avoid := range tc.hardNone {
|
||||
if strings.Contains(joinedHard, avoid) {
|
||||
t.Fatalf("expected HardFailures to NOT contain %q; got %q", avoid, joinedHard)
|
||||
}
|
||||
}
|
||||
for _, want := range tc.gateHas {
|
||||
if !strings.Contains(joinedGate, want) {
|
||||
t.Fatalf("expected GateFailures to contain %q; got %q", want, joinedGate)
|
||||
}
|
||||
}
|
||||
for _, avoid := range tc.gateNone {
|
||||
if strings.Contains(joinedGate, avoid) {
|
||||
t.Fatalf("expected GateFailures to NOT contain %q; got %q", avoid, joinedGate)
|
||||
}
|
||||
}
|
||||
|
||||
if tc.rootCauseGrounding != nil && score.RootCauseGrounding != *tc.rootCauseGrounding {
|
||||
t.Fatalf("RootCauseGrounding = %v, want %v", score.RootCauseGrounding, *tc.rootCauseGrounding)
|
||||
}
|
||||
if tc.affectedGrounding != nil && score.AffectedGrounding != *tc.affectedGrounding {
|
||||
t.Fatalf("AffectedGrounding = %v, want %v", score.AffectedGrounding, *tc.affectedGrounding)
|
||||
}
|
||||
if tc.investigationGrounding != nil && score.InvestigationGrounding != *tc.investigationGrounding {
|
||||
t.Fatalf("InvestigationGrounding = %v, want %v", score.InvestigationGrounding, *tc.investigationGrounding)
|
||||
}
|
||||
if tc.investigationCompletion != nil && score.InvestigationCompletion != *tc.investigationCompletion {
|
||||
t.Fatalf("InvestigationCompletion = %v, want %v", score.InvestigationCompletion, *tc.investigationCompletion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestInvestigationSectionGroundedBranchCoverage exercises every branch of the
|
||||
// pure helper: the empty-alias guard, the missing-section early return, the
|
||||
// missing-resource map lookup, and each name/id containment combination.
|
||||
func TestInvestigationSectionGroundedBranchCoverage(t *testing.T) {
|
||||
resources := map[string]CollectedTruth{
|
||||
"named": {Alias: "named", Name: "Named Resource", ResourceID: "id-only"},
|
||||
"identified": {Alias: "identified", Name: "", ResourceID: "res-123"},
|
||||
"empty": {Alias: "empty", Name: "", ResourceID: ""},
|
||||
"both": {Alias: "both", Name: "Both Name", ResourceID: "both-id"},
|
||||
}
|
||||
const summary = "### Root Cause\nNamed Resource and res-123 and Both Name / both-id appear here.\n"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
summary string
|
||||
heading string
|
||||
aliases []string
|
||||
resources map[string]CollectedTruth
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "nil aliases returns true",
|
||||
summary: "anything",
|
||||
heading: "Root Cause",
|
||||
aliases: nil,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "empty aliases slice returns true",
|
||||
summary: "anything",
|
||||
heading: "Root Cause",
|
||||
aliases: []string{},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "missing heading returns false",
|
||||
summary: "no headings here at all",
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"named"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty summary returns false",
|
||||
summary: "",
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"named"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "alias absent from resource map returns false",
|
||||
summary: summary,
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"unknown"},
|
||||
resources: resources,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "resource name match with empty id returns true",
|
||||
summary: summary,
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"named"},
|
||||
resources: resources,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "resource id match with empty name returns true",
|
||||
summary: summary,
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"identified"},
|
||||
resources: resources,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "both name and id present and matching returns true",
|
||||
summary: summary,
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"both"},
|
||||
resources: resources,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "neither name nor id contained returns false",
|
||||
summary: "### Root Cause\nnothing relevant is mentioned here\n",
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"both"},
|
||||
resources: resources,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "resource with empty name and id returns false",
|
||||
summary: summary,
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"empty"},
|
||||
resources: resources,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "all aliases grounded returns true",
|
||||
summary: summary,
|
||||
heading: "Root Cause",
|
||||
aliases: []string{"named", "identified", "both"},
|
||||
resources: resources,
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res := tc.resources
|
||||
if res == nil {
|
||||
res = map[string]CollectedTruth{}
|
||||
}
|
||||
got := investigationSectionGrounded(tc.summary, tc.heading, tc.aliases, res)
|
||||
if got != tc.want {
|
||||
t.Fatalf("investigationSectionGrounded(%q, %q, %v) = %v, want %v", tc.summary, tc.heading, tc.aliases, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user