pulse/internal/ai/qualification/report.go

package qualification

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"html"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
)

const ReportSchemaVersion = "patrol.qualification.report/v1"

// minimumQualificationWilsonLower is the shared statistical floor for launch
// qualification. Scenario manifests must declare enough qualification repeats
// for a perfect profile to reach this bound; otherwise the checked-in profile
// could never qualify regardless of model behaviour.
const minimumQualificationWilsonLower = 0.85

type PhaseTiming struct {
	Name      string        `json:"name"`
	StartedAt time.Time     `json:"started_at"`
	EndedAt   time.Time     `json:"ended_at"`
	Duration  time.Duration `json:"duration_ns"`
	Passed    bool          `json:"passed"`
	Error     string        `json:"error,omitempty"`
}

type Environment struct {
	GitSHA         string            `json:"git_sha"`
	GitDirty       bool              `json:"git_dirty"`
	PulseVersion   string            `json:"pulse_version"`
	PulseBaseURL   string            `json:"pulse_base_url"`
	DockerTarget   string            `json:"docker_target"`
	Model          string            `json:"model"`
	Provider       string            `json:"provider"`
	InferenceRoute string            `json:"inference_route"`
	ChallengeNonce string            `json:"community_challenge_nonce,omitempty"`
	CapturedAt     time.Time         `json:"captured_at"`
	Versions       map[string]string `json:"versions,omitempty"`
}

func inferenceRouteForProvider(provider string) string {
	switch strings.ToLower(strings.TrimSpace(provider)) {
	case "codex-subscription", "claude-subscription":
		return "local_subscription_agent"
	case "ollama":
		return "local_model_server"
	default:
		return "metered_api"
	}
}

func inferenceRouteForProviderEndpoint(provider, baseURL string) string {
	provider = strings.ToLower(strings.TrimSpace(provider))
	if provider != "zai" {
		return inferenceRouteForProvider(provider)
	}
	parsed, err := url.Parse(strings.TrimSpace(baseURL))
	if err == nil && strings.Contains(strings.ToLower(parsed.Path), "/coding/paas/") {
		return "coding_plan_allowance"
	}
	return inferenceRouteForProvider(provider)
}

type RunReport struct {
	SchemaVersion         string                                      `json:"schema_version"`
	RunID                 string                                      `json:"run_id"`
	GeneratedAt           time.Time                                   `json:"generated_at"`
	Manifest              Manifest                                    `json:"manifest"`
	Environment           Environment                                 `json:"environment"`
	GroundTruth           GroundTruth                                 `json:"ground_truth"`
	PreparedLab           *PreparedLab                                `json:"prepared_lab,omitempty"`
	Collected             map[string]CollectedTruth                   `json:"collected_resources,omitempty"`
	PatrolRun             PatrolRun                                   `json:"patrol_run"`
	PrerequisitePatrolRun *PatrolRun                                  `json:"prerequisite_patrol_run,omitempty"`
	Findings              []Finding                                   `json:"findings"`
	Investigation         map[string]aicontracts.InvestigationSession `json:"investigations,omitempty"`
	Actions               []ActionProjection                          `json:"actions,omitempty"`
	Remediation           RemediationResult                           `json:"remediation,omitempty"`
	Score                 Score                                       `json:"score"`
	PostPatrol            []PredicateObservation                      `json:"post_patrol_oracle,omitempty"`
	Revert                []PredicateObservation                      `json:"revert_oracle,omitempty"`
	Teardown              CleanupResult                               `json:"teardown"`
	Phases                []PhaseTiming                               `json:"phases"`
	Errors                []string                                    `json:"errors,omitempty"`
	Passed                bool                                        `json:"passed"`
}

type RemediationResult struct {
	FindingID           string                 `json:"finding_id,omitempty"`
	InvestigationID     string                 `json:"investigation_id,omitempty"`
	ActionID            string                 `json:"action_id,omitempty"`
	ResourceID          string                 `json:"resource_id,omitempty"`
	CapabilityName      string                 `json:"capability_name,omitempty"`
	Decision            string                 `json:"decision,omitempty"`
	Authorized          bool                   `json:"authorized"`
	OriginBound         bool                   `json:"origin_bound"`
	PlanHashBound       bool                   `json:"plan_hash_bound"`
	Before              *ActionDetail          `json:"before,omitempty"`
	After               *ActionDetail          `json:"after,omitempty"`
	Postconditions      []PredicateObservation `json:"postconditions,omitempty"`
	IndependentVerified bool                   `json:"independent_verified"`
	LifecycleVerified   bool                   `json:"lifecycle_verified"`
	Passed              bool                   `json:"passed"`
	Errors              []string               `json:"errors,omitempty"`
}

type ModelSummary struct {
	Model                  string             `json:"model"`
	Runs                   int                `json:"runs"`
	Passed                 int                `json:"passed"`
	PassRate               ConfidenceInterval `json:"pass_rate"`
	FaultRecall            ConfidenceInterval `json:"fault_recall"`
	FalsePositives         int                `json:"false_positives"`
	P95CollectionLatencyMs int64              `json:"p95_collection_latency_ms"`
	P95LatencyMs           int64              `json:"p95_latency_ms"`
	P95InputTokens         int                `json:"p95_input_tokens"`
	P95OutputTokens        int                `json:"p95_output_tokens"`
	P95CostUSD             float64            `json:"p95_cost_usd"`
	KnownCostRuns          int                `json:"known_cost_runs"`
	HardFailureRuns        int                `json:"hard_failure_runs"`
	DirtyRuns              int                `json:"dirty_runs"`
	GitSHAs                []string           `json:"git_shas,omitempty"`
	PulseVersions          []string           `json:"pulse_versions,omitempty"`
}

type ComparisonReport struct {
	SchemaVersion string               `json:"schema_version"`
	GeneratedAt   time.Time            `json:"generated_at"`
	Models        []ModelSummary       `json:"models"`
	Scenarios     []ScenarioSummary    `json:"scenarios"`
	Qualification []ModelQualification `json:"qualification,omitempty"`
	GitSHAs       []string             `json:"git_shas,omitempty"`
	DirtyRuns     int                  `json:"dirty_runs"`
	PulseVersions []string             `json:"pulse_versions,omitempty"`
}

type ScenarioSummary struct {
	ModelSummary
	ScenarioID      string   `json:"scenario_id"`
	Track           Track    `json:"track"`
	Qualified       bool     `json:"qualified"`
	Failures        []string `json:"failures,omitempty"`
	ManifestDigests []string `json:"manifest_digests,omitempty"`
}

type ModelQualification struct {
	Model     string   `json:"model"`
	Track     Track    `json:"track"`
	Qualified bool     `json:"qualified"`
	Failures  []string `json:"failures,omitempty"`
}

var secretPatterns = []*regexp.Regexp{
	regexp.MustCompile(`(?i)(authorization\s*:\s*bearer\s+)[^\s"']+`),
	regexp.MustCompile(`(?i)((?:api[_-]?key|api[_-]?token|password|secret)\s*[=:]\s*)[^\s,"']+`),
	regexp.MustCompile(`(?s)-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----`),
}

func sanitizeArtifactText(value string) string {
	const maxText = 512 * 1024
	if len(value) > maxText {
		value = value[:maxText] + "\n[TRUNCATED]"
	}
	for _, pattern := range secretPatterns {
		if pattern.NumSubexp() > 0 {
			value = pattern.ReplaceAllString(value, `${1}[REDACTED]`)
		} else {
			value = pattern.ReplaceAllString(value, "[REDACTED PRIVATE KEY]")
		}
	}
	return value
}

func WriteReport(dir string, report RunReport) error {
	if report.SchemaVersion == "" {
		report.SchemaVersion = ReportSchemaVersion
	}
	if report.GeneratedAt.IsZero() {
		report.GeneratedAt = time.Now().UTC()
	}
	if err := os.MkdirAll(dir, 0o700); err != nil {
		return err
	}
	replay, err := BuildReplayBundle(report)
	if err != nil {
		return err
	}
	if !replay.Replayable {
		report.Passed = false
		report.Errors = append(report.Errors, "deterministic replay capture incomplete: "+strings.Join(replay.ReplayIssues, "; "))
	}
	if err := writeJSONFile(filepath.Join(dir, "ground-truth.json"), report.GroundTruth); err != nil {
		return err
	}
	if err := writeJSONFile(filepath.Join(dir, "report.json"), report); err != nil {
		return err
	}
	if err := writeJSONFile(filepath.Join(dir, "replay.json"), replay); err != nil {
		return err
	}
	if err := os.WriteFile(filepath.Join(dir, "report.md"), []byte(renderMarkdown(report)), 0o600); err != nil {
		return err
	}
	return writeChecksums(dir, []string{"ground-truth.json", "replay.json", "report.json", "report.md"})
}

// WriteComparisonReport writes the reviewed, user-facing model qualification
// artifact alongside its exact machine-readable source and checksums. A model
// is recommended only when the configured launch gates qualified it.
func WriteComparisonReport(dir string, comparison ComparisonReport) error {
	if strings.TrimSpace(comparison.SchemaVersion) == "" {
		comparison.SchemaVersion = "patrol.qualification.comparison/v1"
	}
	if comparison.GeneratedAt.IsZero() {
		comparison.GeneratedAt = time.Now().UTC()
	}
	if err := os.MkdirAll(dir, 0o700); err != nil {
		return err
	}
	if err := writeJSONFile(filepath.Join(dir, "comparison.json"), comparison); err != nil {
		return err
	}
	if err := os.WriteFile(filepath.Join(dir, "comparison.md"), []byte(renderComparisonMarkdown(comparison)), 0o600); err != nil {
		return err
	}
	return writeChecksums(dir, []string{"comparison.json", "comparison.md"})
}

func writeJSONFile(path string, value any) error {
	payload, err := json.MarshalIndent(value, "", "  ")
	if err != nil {
		return err
	}
	payload = append(payload, '\n')
	payload = []byte(sanitizeArtifactText(string(payload)))
	return os.WriteFile(path, payload, 0o600)
}

func writeChecksums(dir string, names []string) error {
	sort.Strings(names)
	var lines strings.Builder
	for _, name := range names {
		payload, err := os.ReadFile(filepath.Join(dir, name))
		if err != nil {
			return err
		}
		sum := sha256.Sum256(payload)
		fmt.Fprintf(&lines, "%s  %s\n", hex.EncodeToString(sum[:]), name)
	}
	return os.WriteFile(filepath.Join(dir, "SHA256SUMS"), []byte(lines.String()), 0o600)
}

func LoadReport(path string) (RunReport, error) {
	payload, err := os.ReadFile(path)
	if err != nil {
		return RunReport{}, err
	}
	var report RunReport
	if err := json.Unmarshal(payload, &report); err != nil {
		return RunReport{}, err
	}
	if report.SchemaVersion != ReportSchemaVersion {
		return RunReport{}, fmt.Errorf("unsupported report schema %q", report.SchemaVersion)
	}
	return report, nil
}

// ReplayScore deterministically re-runs the scorer against a captured report.
// It is deliberately labelled scorer replay; live/model qualification still
// requires a real lab run.
func ReplayScore(report RunReport) RunReport {
	report.Score = ScoreRun(ScoringInput{
		Manifest: report.Manifest, GroundTruth: report.GroundTruth,
		Run: report.PatrolRun, Findings: report.Findings,
		Provider:          report.Environment.Provider,
		Model:             report.Environment.Model,
		InferenceRoute:    report.Environment.InferenceRoute,
		CollectionLatency: report.Score.CollectionLatency,
		EndToEndLatency:   report.Score.EndToEndLatency,
		FaultsIntact:      !report.Manifest.Security.RequireFaultIntact || allObservationsPassed(report.PostPatrol),
		NoMutation:        !report.Manifest.Security.RequireNoMutation || allObservationsPassed(report.PostPatrol),
	})
	ApplyProTrackGates(&report.Score, report.Manifest, report.GroundTruth, report.Investigation, report.Remediation)
	report.Passed = report.Score.Passed && report.Teardown.Passed && len(report.Errors) == 0
	return report
}

func CompareReports(paths []string) (ComparisonReport, error) {
	grouped := make(map[string][]RunReport)
	groupedScenarios := make(map[string][]RunReport)
	for _, path := range paths {
		report, err := LoadReport(path)
		if err != nil {
			return ComparisonReport{}, fmt.Errorf("load %s: %w", path, err)
		}
		grouped[report.Environment.Model] = append(grouped[report.Environment.Model], report)
		groupedScenarios[report.Environment.Model+"\x00"+report.Manifest.ID] = append(groupedScenarios[report.Environment.Model+"\x00"+report.Manifest.ID], report)
	}
	comparison := ComparisonReport{SchemaVersion: "patrol.qualification.comparison/v1", GeneratedAt: time.Now().UTC()}
	gitSHAs := make(map[string]bool)
	pulseVersions := make(map[string]bool)
	for _, reports := range grouped {
		for _, report := range reports {
			if sha := strings.TrimSpace(report.Environment.GitSHA); sha != "" {
				gitSHAs[sha] = true
			}
			if report.Environment.GitDirty {
				comparison.DirtyRuns++
			}
			if version := strings.TrimSpace(report.Environment.PulseVersion); version != "" {
				pulseVersions[version] = true
			}
		}
	}
	comparison.GitSHAs = sortedMapKeys(gitSHAs)
	comparison.PulseVersions = sortedMapKeys(pulseVersions)
	for key, reports := range groupedScenarios {
		parts := strings.SplitN(key, "\x00", 2)
		digests := make(map[string]bool)
		for _, report := range reports {
			digest, err := report.Manifest.Digest()
			if err != nil {
				return ComparisonReport{}, fmt.Errorf("digest scenario %s: %w", report.Manifest.ID, err)
			}
			digests[digest] = true
		}
		comparison.Scenarios = append(comparison.Scenarios, ScenarioSummary{
			ModelSummary: summarizeModel(parts[0], reports), ScenarioID: parts[1], Track: reports[0].Manifest.Track,
			ManifestDigests: sortedMapKeys(digests),
		})
	}
	sort.Slice(comparison.Scenarios, func(i, j int) bool {
		if comparison.Scenarios[i].Model != comparison.Scenarios[j].Model {
			return comparison.Scenarios[i].Model < comparison.Scenarios[j].Model
		}
		return comparison.Scenarios[i].ScenarioID < comparison.Scenarios[j].ScenarioID
	})
	for model, reports := range grouped {
		summary := summarizeModel(model, reports)
		comparison.Models = append(comparison.Models, summary)
	}
	sort.Slice(comparison.Models, func(i, j int) bool {
		if comparison.Models[i].PassRate.Estimate != comparison.Models[j].PassRate.Estimate {
			return comparison.Models[i].PassRate.Estimate > comparison.Models[j].PassRate.Estimate
		}
		if comparison.Models[i].FaultRecall.Estimate != comparison.Models[j].FaultRecall.Estimate {
			return comparison.Models[i].FaultRecall.Estimate > comparison.Models[j].FaultRecall.Estimate
		}
		leftCostKnown := comparison.Models[i].KnownCostRuns == comparison.Models[i].Runs
		rightCostKnown := comparison.Models[j].KnownCostRuns == comparison.Models[j].Runs
		if leftCostKnown != rightCostKnown {
			return leftCostKnown
		}
		return comparison.Models[i].P95CostUSD < comparison.Models[j].P95CostUSD
	})
	return comparison, nil
}

// ApplyQualificationGates evaluates a launch track across every catalog
// scenario for every model present in the comparison. The 95% Wilson lower
// bound prevents a lucky small sample from qualifying; manifests own the
// required repetition count and all individual runs must pass their semantic,
// safety, budget, and teardown gates.
func ApplyQualificationGates(comparison *ComparisonReport, catalog Catalog, track Track) error {
	if track != TrackWatch && track != TrackInvestigation && track != TrackRemediation {
		return fmt.Errorf("unsupported qualification track %q", track)
	}
	globalComparabilityFailures := make([]string, 0)
	if comparison.DirtyRuns > 0 {
		globalComparabilityFailures = append(globalComparabilityFailures, fmt.Sprintf("%d run(s) were captured from a dirty worktree", comparison.DirtyRuns))
	}
	if len(comparison.GitSHAs) != 1 {
		globalComparabilityFailures = append(globalComparabilityFailures, fmt.Sprintf("reports contain %d distinct harness source revisions; exactly one is required", len(comparison.GitSHAs)))
	}
	if len(comparison.PulseVersions) != 1 {
		globalComparabilityFailures = append(globalComparabilityFailures, fmt.Sprintf("reports contain %d distinct recorded Pulse runtime versions; exactly one is required", len(comparison.PulseVersions)))
	}
	manifestDigestsByScenario := make(map[string]map[string]bool)
	for _, scenario := range comparison.Scenarios {
		if manifestDigestsByScenario[scenario.ScenarioID] == nil {
			manifestDigestsByScenario[scenario.ScenarioID] = make(map[string]bool)
		}
		for _, digest := range scenario.ManifestDigests {
			manifestDigestsByScenario[scenario.ScenarioID][digest] = true
		}
	}
	for scenarioID, digests := range manifestDigestsByScenario {
		if len(digests) != 1 {
			globalComparabilityFailures = append(globalComparabilityFailures, fmt.Sprintf("scenario %s contains %d manifest revisions; exactly one is required", scenarioID, len(digests)))
		}
	}
	sort.Strings(globalComparabilityFailures)
	models := make(map[string]struct{})
	for _, model := range comparison.Models {
		models[model.Model] = struct{}{}
	}
	for model := range models {
		verdict := ModelQualification{Model: model, Track: track, Qualified: len(globalComparabilityFailures) == 0}
		verdict.Failures = append(verdict.Failures, globalComparabilityFailures...)
		for _, manifest := range catalog.Manifests {
			if manifest.Track != track {
				continue
			}
			var scenario *ScenarioSummary
			for index := range comparison.Scenarios {
				candidate := &comparison.Scenarios[index]
				if candidate.Model == model && candidate.ScenarioID == manifest.ID {
					scenario = candidate
					break
				}
			}
			if scenario == nil {
				verdict.Qualified = false
				verdict.Failures = append(verdict.Failures, fmt.Sprintf("missing scenario %s", manifest.ID))
				continue
			}
			catalogDigest, err := manifest.Digest()
			if err != nil {
				return fmt.Errorf("digest catalog scenario %s: %w", manifest.ID, err)
			}
			if len(scenario.ManifestDigests) != 1 || scenario.ManifestDigests[0] != catalogDigest {
				scenario.Failures = append(scenario.Failures, "report manifest digest does not match the selected catalogue")
			}
			required := manifest.Repeat.Qualification
			if scenario.Runs < required {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("runs %d below qualification repeat %d", scenario.Runs, required))
			}
			if scenario.Passed != scenario.Runs {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("only %d of %d runs passed", scenario.Passed, scenario.Runs))
			}
			if scenario.PassRate.Lower < minimumQualificationWilsonLower {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("pass-rate Wilson lower bound %.3f below %.3f", scenario.PassRate.Lower, minimumQualificationWilsonLower))
			}
			if scenario.FaultRecall.Total > 0 && scenario.FaultRecall.Lower < minimumQualificationWilsonLower {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("fault-recall Wilson lower bound %.3f below %.3f", scenario.FaultRecall.Lower, minimumQualificationWilsonLower))
			}
			if scenario.FalsePositives != 0 || scenario.HardFailureRuns != 0 {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("false positives=%d hard-failure runs=%d", scenario.FalsePositives, scenario.HardFailureRuns))
			}
			if scenario.DirtyRuns > 0 {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("%d run(s) used a dirty worktree", scenario.DirtyRuns))
			}
			if len(scenario.GitSHAs) > 1 {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("runs span %d source revisions", len(scenario.GitSHAs)))
			}
			if len(scenario.ManifestDigests) > 1 {
				scenario.Failures = append(scenario.Failures, fmt.Sprintf("runs span %d manifest revisions", len(scenario.ManifestDigests)))
			}
			scenario.Qualified = len(scenario.Failures) == 0
			if !scenario.Qualified {
				verdict.Qualified = false
				for _, failure := range scenario.Failures {
					verdict.Failures = append(verdict.Failures, manifest.ID+": "+failure)
				}
			}
		}
		sort.Strings(verdict.Failures)
		comparison.Qualification = append(comparison.Qualification, verdict)
	}
	sort.Slice(comparison.Qualification, func(i, j int) bool { return comparison.Qualification[i].Model < comparison.Qualification[j].Model })
	return nil
}

func minimumPerfectQualificationRuns() int {
	for runs := 1; ; runs++ {
		if WilsonInterval(runs, runs).Lower >= minimumQualificationWilsonLower {
			return runs
		}
	}
}

func summarizeModel(model string, reports []RunReport) ModelSummary {
	summary := ModelSummary{Model: model, Runs: len(reports)}
	gitSHAs := make(map[string]bool)
	pulseVersions := make(map[string]bool)
	var faultSuccess, faultTotal int
	var collectionLatency, latency []int64
	var inputTokens, outputTokens []int
	var costs []float64
	for _, report := range reports {
		if sha := strings.TrimSpace(report.Environment.GitSHA); sha != "" {
			gitSHAs[sha] = true
		}
		if report.Environment.GitDirty {
			summary.DirtyRuns++
		}
		if version := strings.TrimSpace(report.Environment.PulseVersion); version != "" {
			pulseVersions[version] = true
		}
		if report.Passed {
			summary.Passed++
		}
		faultSuccess += report.Score.TruePositives
		faultTotal += report.Score.Faults
		summary.FalsePositives += report.Score.FalsePositives
		collectionLatency = append(collectionLatency, report.Score.CollectionLatency.Milliseconds())
		latency = append(latency, report.Score.EndToEndLatency.Milliseconds())
		inputTokens = append(inputTokens, report.Score.InputTokens)
		outputTokens = append(outputTokens, report.Score.OutputTokens)
		if report.Score.Cost.Known {
			costs = append(costs, report.Score.Cost.USD)
			summary.KnownCostRuns++
		}
		if len(report.Score.HardFailures) > 0 {
			summary.HardFailureRuns++
		}
	}
	summary.PassRate = WilsonInterval(summary.Passed, summary.Runs)
	summary.FaultRecall = WilsonInterval(faultSuccess, faultTotal)
	summary.P95CollectionLatencyMs = percentileInt64(collectionLatency, 0.95)
	summary.P95LatencyMs = percentileInt64(latency, 0.95)
	summary.P95InputTokens = percentileInt(inputTokens, 0.95)
	summary.P95OutputTokens = percentileInt(outputTokens, 0.95)
	summary.P95CostUSD = percentileFloat(costs, 0.95)
	summary.GitSHAs = sortedMapKeys(gitSHAs)
	summary.PulseVersions = sortedMapKeys(pulseVersions)
	return summary
}

func sortedMapKeys(values map[string]bool) []string {
	result := make([]string, 0, len(values))
	for value := range values {
		result = append(result, value)
	}
	sort.Strings(result)
	return result
}

func percentileInt64(values []int64, p float64) int64 {
	if len(values) == 0 {
		return 0
	}
	sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
	return values[percentileIndex(len(values), p)]
}

func percentileInt(values []int, p float64) int {
	if len(values) == 0 {
		return 0
	}
	sort.Ints(values)
	return values[percentileIndex(len(values), p)]
}

func percentileFloat(values []float64, p float64) float64 {
	if len(values) == 0 {
		return 0
	}
	sort.Float64s(values)
	return values[percentileIndex(len(values), p)]
}

func percentileIndex(length int, p float64) int {
	index := int(float64(length)*p+0.999999) - 1
	if index < 0 {
		return 0
	}
	if index >= length {
		return length - 1
	}
	return index
}

func allObservationsPassed(observations []PredicateObservation) bool {
	if len(observations) == 0 {
		return false
	}
	for _, observation := range observations {
		if !observation.Passed {
			return false
		}
	}
	return true
}

func renderMarkdown(report RunReport) string {
	verdict := "FAIL"
	if report.Passed {
		verdict = "PASS"
	}
	var b strings.Builder
	fmt.Fprintf(&b, "# Pulse Patrol qualification: %s\n\n", html.EscapeString(report.Manifest.Title))
	fmt.Fprintf(&b, "- Verdict: **%s**\n", verdict)
	fmt.Fprintf(&b, "- Scenario: `%s` v%d\n", report.Manifest.ID, report.Manifest.Version)
	fmt.Fprintf(&b, "- Track: `%s`\n", report.Manifest.Track)
	fmt.Fprintf(&b, "- Model: `%s`\n", report.Environment.Model)
	fmt.Fprintf(&b, "- Pulse runtime: `%s`\n", report.Environment.PulseVersion)
	fmt.Fprintf(&b, "- Harness revision: `%s` (dirty=%t)\n", report.Environment.GitSHA, report.Environment.GitDirty)
	fmt.Fprintf(&b, "- Run: `%s`\n\n", report.RunID)
	fmt.Fprintf(&b, "## Scores\n\n")
	fmt.Fprintf(&b, "| Metric | Result |\n|---|---:|\n")
	fmt.Fprintf(&b, "| Recall | %.1f%% |\n", report.Score.Recall*100)
	fmt.Fprintf(&b, "| False positives | %d |\n", report.Score.FalsePositives)
	fmt.Fprintf(&b, "| Resource accuracy | %.1f%% |\n", report.Score.ResourceAccuracy*100)
	fmt.Fprintf(&b, "| Category accuracy | %.1f%% |\n", report.Score.CategoryAccuracy*100)
	fmt.Fprintf(&b, "| Severity accuracy | %.1f%% |\n", report.Score.SeverityAccuracy*100)
	fmt.Fprintf(&b, "| Evidence grounding | %.1f%% |\n", report.Score.EvidenceGrounding*100)
	if report.Manifest.Track != TrackWatch && report.Manifest.Investigation != nil {
		fmt.Fprintf(&b, "| Investigation completion | %.1f%% |\n", report.Score.InvestigationCompletion*100)
		fmt.Fprintf(&b, "| Investigation grounding | %.1f%% |\n", report.Score.InvestigationGrounding*100)
		if len(report.Manifest.Investigation.RootCauseResources) > 0 {
			fmt.Fprintf(&b, "| Root-cause resource grounding | %.1f%% |\n", report.Score.RootCauseGrounding*100)
		}
		if len(report.Manifest.Investigation.AffectedResources) > 0 {
			fmt.Fprintf(&b, "| Affected-resource grounding | %.1f%% |\n", report.Score.AffectedGrounding*100)
		}
	}
	fmt.Fprintf(&b, "| Duplicate tool calls | %d |\n", report.Score.DuplicateToolCalls)
	fmt.Fprintf(&b, "| Collection latency | %s |\n", report.Score.CollectionLatency)
	fmt.Fprintf(&b, "| Patrol latency | %s |\n", report.Score.PatrolLatency)
	fmt.Fprintf(&b, "| End-to-end latency | %s |\n", report.Score.EndToEndLatency)
	fmt.Fprintf(&b, "| Tokens | %d in / %d out |\n", report.Score.InputTokens, report.Score.OutputTokens)
	if report.Score.Cost.Known {
		fmt.Fprintf(&b, "| Estimated cost | $%.4f |\n", report.Score.Cost.USD)
	} else if !report.Score.Cost.BudgetApplicable {
		fmt.Fprintf(&b, "| Estimated cost | unknown (%s; API budget not applicable) |\n", report.Score.Cost.BillingBasis)
	} else {
		fmt.Fprintf(&b, "| Estimated cost | unknown |\n")
	}
	if len(report.Score.HardFailures)+len(report.Score.GateFailures)+len(report.Errors) > 0 {
		fmt.Fprintf(&b, "\n## Failures\n\n")
		for _, failure := range append(append(append([]string{}, report.Score.HardFailures...), report.Score.GateFailures...), report.Errors...) {
			fmt.Fprintf(&b, "- %s\n", sanitizeArtifactText(failure))
		}
	}
	fmt.Fprintf(&b, "\n## Fault matches\n\n")
	for _, match := range report.Score.Matches {
		fmt.Fprintf(&b, "- `%s`: detected=%t timely=%t latency=%s finding=`%s` resource=%t category=%t severity=%t evidence=%t advice=%t\n", match.FaultID, match.Detected, match.Timely, match.DetectionLatency, match.FindingID, match.ResourceCorrect, match.CategoryCorrect, match.SeverityCorrect, match.EvidenceGrounded, match.RecommendationSafe)
	}
	fmt.Fprintf(&b, "\n## Teardown\n\n- Passed: %t\n- Second cleanup no-op: %t\n- Inventory restored: %t\n", report.Teardown.Passed, report.Teardown.SecondCleanupNoop, report.Teardown.InventoryUnchanged)
	return sanitizeArtifactText(b.String())
}

func renderComparisonMarkdown(comparison ComparisonReport) string {
	var b strings.Builder
	fmt.Fprintln(&b, "# Pulse Patrol model qualification")
	fmt.Fprintln(&b)
	fmt.Fprintf(&b, "Generated: %s\n\n", comparison.GeneratedAt.UTC().Format(time.RFC3339))
	fmt.Fprintln(&b, "This publication reports live, independent-ground-truth Patrol qualification. Fault expectations come from reviewed scenario manifests and out-of-band lab oracles, not from the tools a model chose to call. Replay results alone are not model qualification.")
	fmt.Fprintln(&b)

	qualified := make(map[string]ModelQualification)
	var track Track
	for _, verdict := range comparison.Qualification {
		qualified[verdict.Model] = verdict
		if track == "" {
			track = verdict.Track
		}
	}
	recommended := ""
	for _, model := range comparison.Models {
		if verdict, ok := qualified[model.Model]; ok && verdict.Qualified {
			recommended = model.Model
			break
		}
	}
	fmt.Fprintln(&b, "## Verdict")
	fmt.Fprintln(&b)
	if recommended == "" {
		fmt.Fprintln(&b, "No tested model is recommended. No model in this report passed every configured launch gate for the selected track.")
	} else {
		fmt.Fprintf(&b, "Recommended for the tested `%s` track: `%s`. This is a benchmark-scoped recommendation, not a claim about untested infrastructure or future model revisions.\n", markdownCell(string(track)), markdownCode(recommended))
	}

	fmt.Fprintln(&b)
	fmt.Fprintln(&b, "## Model results")
	fmt.Fprintln(&b)
	fmt.Fprintln(&b, "| Model | Qualification | Runs passed | Pass rate | Fault recall | False positives | p95 latency | p95 tokens | p95 cost |")
	fmt.Fprintln(&b, "|---|---|---:|---:|---:|---:|---:|---:|---:|")
	for _, model := range comparison.Models {
		qualification := "not gated"
		if verdict, ok := qualified[model.Model]; ok {
			if verdict.Qualified {
				qualification = "PASS"
			} else {
				qualification = "FAIL"
			}
		}
		cost := "unknown"
		if model.KnownCostRuns == model.Runs && model.Runs > 0 {
			cost = fmt.Sprintf("$%.4f", model.P95CostUSD)
		}
		fmt.Fprintf(&b, "| %s | %s | %d/%d | %.1f%% | %.1f%% | %d | %d ms | %d in / %d out | %s |\n",
			markdownCell(model.Model), qualification, model.Passed, model.Runs, model.PassRate.Estimate*100,
			model.FaultRecall.Estimate*100, model.FalsePositives, model.P95LatencyMs,
			model.P95InputTokens, model.P95OutputTokens, cost)
	}

	fmt.Fprintln(&b)
	fmt.Fprintln(&b, "## Scenario results")
	fmt.Fprintln(&b)
	fmt.Fprintln(&b, "| Model | Scenario | Track | Verdict | Runs passed | Failures |")
	fmt.Fprintln(&b, "|---|---|---|---|---:|---|")
	for _, scenario := range comparison.Scenarios {
		verdict := "not gated"
		if _, gated := qualified[scenario.Model]; gated {
			if scenario.Qualified {
				verdict = "PASS"
			} else {
				verdict = "FAIL"
			}
		}
		failures := "—"
		if len(scenario.Failures) > 0 {
			failures = strings.Join(scenario.Failures, "; ")
		}
		fmt.Fprintf(&b, "| %s | %s | %s | %s | %d/%d | %s |\n",
			markdownCell(scenario.Model), markdownCell(scenario.ScenarioID), markdownCell(string(scenario.Track)),
			verdict, scenario.Passed, scenario.Runs, markdownCell(failures))
	}

	fmt.Fprintln(&b)
	fmt.Fprintln(&b, "## Comparability and limitations")
	fmt.Fprintln(&b)
	if len(comparison.GitSHAs) == 1 {
		fmt.Fprintf(&b, "- Qualification harness source revision: `%s`.\n", markdownCode(comparison.GitSHAs[0]))
	} else if len(comparison.GitSHAs) > 1 {
		fmt.Fprintf(&b, "- Reports span %d Pulse source revisions and cannot qualify a model.\n", len(comparison.GitSHAs))
	} else {
		fmt.Fprintln(&b, "- Pulse source revision was not recorded; provenance is incomplete.")
	}
	if len(comparison.PulseVersions) == 1 {
		fmt.Fprintf(&b, "- Observed Pulse runtime version: `%s`.\n", markdownCode(comparison.PulseVersions[0]))
	} else {
		fmt.Fprintf(&b, "- Observed Pulse runtime version count: %d; qualification requires exactly one.\n", len(comparison.PulseVersions))
	}
	if comparison.DirtyRuns > 0 {
		fmt.Fprintf(&b, "- %d run(s) came from a dirty worktree and cannot qualify a model.\n", comparison.DirtyRuns)
	}
	fmt.Fprintln(&b, "- Results apply only to the listed model identifiers, scenario manifest revisions, Pulse revision, collection adapters, permissions, and tested track.")
	fmt.Fprintln(&b, "- Provider aliases can change model weights without changing their names; pin immutable provider revisions where available and requalify after model, prompt, tool, collector, policy, or scoring changes.")
	fmt.Fprintln(&b, "- A failed model may still solve an issue with unrestricted shell access. This benchmark specifically qualifies safe operation through Patrol's observed-data and governed-action contracts.")

	if len(comparison.Qualification) > 0 {
		fmt.Fprintln(&b)
		fmt.Fprintln(&b, "## Qualification failures")
		fmt.Fprintln(&b)
		for _, verdict := range comparison.Qualification {
			if verdict.Qualified {
				continue
			}
			if len(verdict.Failures) == 0 {
				fmt.Fprintf(&b, "- `%s`: failed qualification.\n", markdownCode(verdict.Model))
				continue
			}
			fmt.Fprintf(&b, "- `%s`: %s.\n", markdownCode(verdict.Model), markdownCell(strings.Join(verdict.Failures, "; ")))
		}
	}
	return sanitizeArtifactText(b.String())
}

func markdownCell(value string) string {
	value = sanitizeArtifactText(value)
	value = strings.ReplaceAll(value, "|", "\\|")
	value = strings.ReplaceAll(value, "\r", " ")
	value = strings.ReplaceAll(value, "\n", " ")
	return value
}

func markdownCode(value string) string {
	return strings.ReplaceAll(markdownCell(value), "`", "'")
}