pulse/scripts/release_control/ai_runtime_docs_policy_test.py

#!/usr/bin/env python3
"""Guard public Pulse Intelligence docs against retired AI shorthand."""

from __future__ import annotations

import unittest

from repo_file_io import read_repo_text

PUBLIC_AI_DOC_PATHS = (
    "docs/AI.md",
    "docs/AI_AUTONOMY.md",
    "docs/API.md",
    "docs/FAQ.md",
    "docs/PULSE_PRO.md",
    "docs/README.md",
)


class AIRuntimeDocsPolicyTest(unittest.TestCase):
    def test_public_ai_overview_uses_productized_context_language(self) -> None:
        content = read_repo_text("docs/AI.md")
        normalized_content = " ".join(content.split())

        self.assertIn("# Pulse Intelligence", content)
        self.assertIn(
            "Uses prior alert context, Patrol run history, and resource timelines",
            content,
        )
        self.assertIn("Investigation Context", content)
        self.assertIn("the configured LLM owns diagnosis", content)
        self.assertIn(
            "Pulse supplies context, capabilities, safety gates, approval state, and audit trails",
            content,
        )
        self.assertIn("shared **Pulse Intelligence Core**", content)
        self.assertIn("<!-- pulse-intelligence-overview:start -->", content)
        self.assertIn("<!-- pulse-intelligence-overview:end -->", content)
        self.assertIn(
            "Canonical context, governed actions, safety gates, approval state, action audit, and verification",
            normalized_content,
        )
        self.assertIn("Patrol as the primary built-in operator", normalized_content)
        self.assertIn(
            "Pulse Patrol**: Patrol is the first-party operations surface: it checks infrastructure, investigates issues, follows the chosen Patrol mode before acting, verifies outcomes, and records what happened.",
            normalized_content,
        )
        self.assertIn(
            "Pulse Assistant**: The contextual explanation, approval, and handoff surface for Patrol findings, governed actions, verification, and operator questions. Affordances: tools and interactive questions.",
            normalized_content,
        )
        self.assertIn(
            "Pulse MCP**: The external-agent adapter that projects canonical Pulse Intelligence capabilities as MCP tools. Affordances: tools, resources, prompts, and capability metadata.",
            normalized_content,
        )
        self.assertIn("Pulse MCP", content)
        self.assertIn("AI_PATROL_QUALIFICATION.md", content)
        self.assertIn("full-track local suite", normalized_content)
        self.assertIn("privacy-allowlisted community evidence export", normalized_content)
        self.assertIn("Every active finding shown or returned to a Patrol run", content)
        self.assertIn("Silence is not an all-clear signal", normalized_content)
        self.assertIn("### The Patrol attention queue", content)
        self.assertIn("### Operational objectives and model-authored observers", content)
        self.assertIn("Patrol objectives retain an operator's desired outcome", normalized_content)
        self.assertIn("`patrol_propose_observer`", content)
        self.assertIn("The proposal boundary is intentionally not an execution boundary", normalized_content)
        self.assertIn("Coverage remains `uncovered` until a core-owned validator", normalized_content)
        self.assertIn(
            "it cannot create lifecycle truth, invent action authority, or hide uncertainty",
            normalized_content,
        )
        self.assertIn(
            "Suppression is temporary and requires an operator reason plus a bounded expiry",
            normalized_content,
        )
        self.assertIn(
            "A successful command does not resolve the attention item",
            normalized_content,
        )
        self.assertIn(
            "The Assistant model matrix below proves Assistant orchestration only",
            normalized_content,
        )
        self.assertIn("### Tool Inventory", content)
        self.assertIn("The Assistant tool list is registry-owned at runtime", content)
        self.assertIn(
            "Each turn receives an available-tool manifest generated from Pulse's governed tool registry",
            normalized_content,
        )
        self.assertIn(
            "tool-result handling, approval boundaries, and Patrol-only tool filtering",
            normalized_content,
        )
        self.assertIn(
            "use the live `/api/agent/capabilities` manifest or `pulse-mcp` `tools/list`",
            normalized_content,
        )
        self.assertIn(
            "The same manifest also carries reusable `workflowPrompts` metadata",
            normalized_content,
        )
        self.assertIn(
            "Pulse Assistant-compatible starters and MCP `prompts/list` clients discover the same fleet triage, resource investigation, and Patrol finding review workflows",
            normalized_content,
        )
        self.assertIn(
            "Pulse Assistant and `pulse-mcp` are sibling surfaces over Pulse Intelligence",
            normalized_content,
        )
        self.assertIn("Patrol mode and governed Pulse Intelligence operations", normalized_content)
        self.assertIn("not competing implementations, and neither replaces the other", normalized_content)
        self.assertIn(
            "New operational capabilities should be added to the canonical API manifest first",
            normalized_content,
        )
        self.assertIn(
            "MCP-only actions and Assistant-only copies of the same business logic are drift",
            normalized_content,
        )
        self.assertNotIn("two supported operator-facing surfaces", normalized_content)
        self.assertNotIn("Pulse Assistant should be replaced by", normalized_content)
        self.assertNotIn("MCP replaces Pulse Assistant", normalized_content)
        self.assertNotIn("Assistant can be removed", normalized_content)
        # The Patrol summary card and its supporting-context section are gone,
        # so the docs point at where check history actually lives.
        self.assertIn("Activity > Finding options and history", normalized_content)
        self.assertNotIn("legacy operational score remain", normalized_content)
        self.assertNotIn("collapsed supporting-context section", normalized_content)
        self.assertIn("- **Anthropic** (API key)", content)
        self.assertIn("Anthropic OAuth is not a supported runtime authentication method", normalized_content)
        self.assertIn("does not make Anthropic configured", normalized_content)
        self.assertIn("go run ./cmd/eval -scenario resource-context", content)
        self.assertIn("EVAL_RESOURCE_CONTEXT_FORBIDDEN", content)
        self.assertIn("Pulse does not convert them into Pulse-authored findings", content)
        self.assertIn(
            "critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors",
            normalized_content,
        )
        self.assertIn(
            "Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors",
            normalized_content,
        )
        self.assertNotIn("Anthropic** (API key or OAuth)", content)
        self.assertNotIn("Anthropic API key or OAuth", content)
        self.assertNotIn("successful remediations (incident memory)", content)
        self.assertNotIn("**Incident memory**", content)
        self.assertNotIn("**Incident Memory**", content)
        self.assertNotIn("incident memory, investigation orchestration", content)
        self.assertNotIn("learns what's normal", content)
        self.assertNotIn("multi-layered intelligence platform", content)
        self.assertNotIn("capacity predictions", content)
        self.assertNotIn("Deterministic Signal Detection", content)
        self.assertNotIn("Pulse AI is built", content)
        self.assertNotIn("counts and feature flags only", normalized_content)
        self.assertNotIn("### Available Tools", content)
        self.assertNotIn("| Tool | Classification | Purpose |", content)
        self.assertNotIn("| `pulse_query`, `pulse_discovery` |", content)
        self.assertNotIn("| `patrol_report_finding` | Patrol |", content)
        self.assertNotIn("active_alert", content)
        self.assertNotIn("auto-recovery", content)
        self.assertNotIn('EVAL_RESOURCE_CONTEXT_FORBIDDEN="/mnt/pve/finance-db,/var/lib/homeassistant,secret"', content)
        self.assertNotRegex(content, r"(?i)understands resources before you ask")

    def test_public_ai_docs_define_local_subscription_agent_safety_boundary(self) -> None:
        content = read_repo_text("docs/AI.md")
        normalized_content = " ".join(content.split())

        self.assertIn("codex-subscription:gpt-5.6-luna", content)
        self.assertIn("claude-subscription:sonnet", content)
        self.assertIn("claude-subscription:opus", content)
        self.assertIn("local_subscription_agent", content)
        self.assertIn("API-key environment variables", normalized_content)
        self.assertIn("dedicated, least-privilege OS account", normalized_content)
        self.assertIn("Pulse retains tool execution and policy enforcement", normalized_content)
        self.assertIn("Pulse audits Claude's buffered event stream", normalized_content)
        self.assertIn("accepts only the first call whose name was explicitly offered", normalized_content)
        self.assertIn("undeclared local tool attempts fail closed", normalized_content)
        self.assertIn("does not grant Claude Code direct access to the infrastructure", normalized_content)
        self.assertIn("not represented as a zero-dollar API price", normalized_content)
        self.assertIn("per-run metered-API budget as not applicable", normalized_content)
        self.assertIn("bounded output allowance", normalized_content)
        self.assertIn("low reasoning-effort hint", normalized_content)
        self.assertIn("does not reduce the tools available to Watch", normalized_content)
        self.assertIn("Pro investigation reasoning", normalized_content)
        self.assertIn("normal reasoning depth", normalized_content)
        self.assertIn("inference_route=coding_plan_allowance", content)
        self.assertIn("per-run monetary cost unknown", normalized_content)
        self.assertIn("standard Z.ai `/api/paas/` endpoint remains a `metered_api` route", normalized_content)
        self.assertIn("Pulse runs as the `pulse` account with `HOME=/opt/pulse`", normalized_content)
        self.assertIn("a CLI installed or authenticated only as `root` is intentionally unavailable", normalized_content)
        self.assertIn("claude auth status --json", content)
        self.assertIn("PULSE_CLAUDE_CLI_PATH", content)
        self.assertIn("PULSE_CODEX_CLI_PATH", content)

    def test_public_ai_privacy_copy_discloses_outbound_usage_telemetry(self) -> None:
        content = read_repo_text("docs/AI.md")
        normalized_content = " ".join(content.split())

        self.assertIn("## Privacy", content)
        self.assertIn("Outbound usage telemetry", content)
        self.assertIn("rotating pseudonymous install ID", normalized_content)
        self.assertIn(
            "no hostnames, credentials, prompts, chat messages, command text, action output, token values, IP addresses, or resource identifiers in the payload",
            normalized_content,
        )
        self.assertIn("enabled by default and can be disabled any time", normalized_content)
        self.assertNotIn("anonymous telemetry", normalized_content.lower())

    def test_patrol_qualification_docs_keep_community_evidence_local_and_bounded(self) -> None:
        content = read_repo_text("docs/AI_PATROL_QUALIFICATION.md")
        normalized_content = " ".join(content.split())

        self.assertIn("-mode live-suite", content)
        self.assertIn("-mode export-contribution", content)
        self.assertIn("The export command performs no network request", normalized_content)
        self.assertIn("explicit allowlist", normalized_content)
        self.assertIn("It never copies raw findings", normalized_content)
        self.assertIn("Community tested", content)
        self.assertIn("Community validated", content)
        self.assertIn("Pulse certified", content)
        self.assertIn(
            "Only Pulse-certified models may become the default hosted route",
            normalized_content,
        )
        self.assertIn("`max_evidence_calls`", content)
        self.assertIn("actual evidence-call volume", normalized_content)
        self.assertIn("completed model responses separately", normalized_content)
        self.assertIn("terminal typed proposal does not count as evidence", normalized_content)

    def test_public_patrol_docs_separate_evidence_and_model_turn_budgets(self) -> None:
        overview = read_repo_text("docs/AI.md")
        control_doc = read_repo_text("docs/AI_AUTONOMY.md")
        normalized_control = " ".join(control_doc.split())

        self.assertIn("| `MaxEvidenceCalls` | 15 |", overview)
        self.assertIn("| `MaxTurns` | 17 |", overview)
        self.assertIn("terminal typed proposal is reserved separately", overview)
        self.assertIn("Maximum evidence-tool calls per investigation", normalized_control)
        self.assertIn("separate model-response safety ceiling", normalized_control)

    def test_public_ai_overview_describes_dimensioned_patrol_model_readiness(self) -> None:
        content = read_repo_text("docs/AI.md")
        normalized_content = " ".join(content.split())

        self.assertIn("### Patrol model readiness advisor", content)
        self.assertIn("synthetic data only and never calls infrastructure tools", normalized_content)
        self.assertIn("provider connectivity, exact typed streaming tool calls", normalized_content)
        self.assertIn("Patrol-shaped context fixtures", normalized_content)
        self.assertIn("projected loop latency", normalized_content)
        self.assertIn("never certifies **Safe auto-fix** or **Autopilot**", normalized_content)
        self.assertIn("without replacing the last completed evidence", normalized_content)
        self.assertIn("`patrol_readiness` usage category", normalized_content)
        self.assertNotIn("Patrol model ready", content)

    def test_public_ai_overview_defines_local_compatible_provider_lifecycle(self) -> None:
        content = read_repo_text("docs/AI.md")
        normalized_content = " ".join(content.split())

        self.assertIn("llama.cpp, LocalAI, LM Studio", normalized_content)
        self.assertIn(
            "API key is optional when the custom endpoint is intentionally keyless",
            normalized_content,
        )
        self.assertIn("Custom OpenAI-compatible model catalogs are authoritative", normalized_content)
        self.assertIn("Pulse lists every non-empty model ID returned by the endpoint", normalized_content)
        self.assertIn("Pulse omits empty Authorization headers", normalized_content)
        self.assertIn("Blank is the default", normalized_content)
        self.assertIn("the Ollama server's own policy applies", normalized_content)
        self.assertIn("Provider transport health is reported independently from Patrol capability", normalized_content)
        self.assertIn("does not weaken Patrol's fail-closed tool/action admission", normalized_content)

    def test_public_ai_docs_use_current_surface_naming(self) -> None:
        for doc_path in PUBLIC_AI_DOC_PATHS:
            with self.subTest(doc_path=doc_path):
                content = read_repo_text(doc_path)
                self.assertNotIn("Pulse AI", content)
                self.assertNotIn("Settings → System → AI Assistant", content)
                self.assertNotIn("Settings > System > AI Assistant", content)
                self.assertNotIn("Settings > Pulse Assistant", content)

    def test_public_patrol_docs_use_mode_language(self) -> None:
        overview = read_repo_text("docs/AI.md")
        control_doc = read_repo_text("docs/AI_AUTONOMY.md")

        self.assertIn("Pro adds hands-on Patrol modes, issue investigation, governed fixes, verified outcomes, and 90-day history", overview)
        self.assertIn("## Patrol Modes", overview)
        self.assertIn("four modes", overview)
        self.assertIn("**Watch only**", overview)
        self.assertIn("**Ask before changes**", overview)
        self.assertIn("**Auto-fix safe issues**", overview)
        self.assertIn("**Policy autopilot**", overview)
        self.assertIn("When a finding is created in a Pro Patrol mode", overview)
        self.assertIn("**Patrol modes (Pro and above)**", overview)
        self.assertIn("**Governed fixes (Pro and above)**", overview)
        self.assertIn("**Issue investigation (Pro and above)**", overview)
        self.assertIn("Patrol mode and governed Pulse Intelligence operations", overview)
        self.assertNotIn("## Patrol Control Levels", overview)
        self.assertNotIn("four control levels", overview)
        self.assertNotIn("When a finding is created in a Pro control level", overview)
        self.assertNotIn("**Patrol control (Pro and above)**", overview)
        self.assertNotIn("**Alert investigation (Pro and above)**", overview)
        self.assertNotIn("Patrol control and governed Pulse Intelligence operations", overview)
        self.assertNotIn("## Autonomy Levels", overview)
        self.assertNotIn("autonomy modes", overview)

        self.assertIn("# Pulse Intelligence Modes and Safety Configuration", control_doc)
        self.assertIn("**Patrol Mode**", control_doc)
        self.assertIn("Patrol mode is configured on the **Patrol** page.", control_doc)
        self.assertIn("## Patrol Modes", control_doc)
        self.assertIn("Patrol mode sets how far Pulse can go", control_doc)
        self.assertIn("**UI:** Patrol → Patrol mode", control_doc)
        self.assertIn("The API retains `autonomy_level` for compatibility", control_doc)
        self.assertNotIn("Patrol Control Level", control_doc)
        self.assertNotIn("## Patrol Control Levels", control_doc)
        self.assertNotIn("Patrol control is configured on the **Patrol** page.", control_doc)
        self.assertNotIn("Patrol control sets how far Pulse can go", control_doc)
        self.assertNotIn("**UI:** Patrol → Patrol control", control_doc)
        self.assertNotIn("Patrol Autonomy Levels", control_doc)
        self.assertNotIn("Patrol autonomy controls", control_doc)
        self.assertNotIn("Patrol Autonomy Level", control_doc)


if __name__ == "__main__":
    unittest.main()