mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-26 05:29:45 +00:00
rcourtman
a32356ae94
With the web chat window and its backend routes retired (#3126, #3137), the pulse_assistant surface only serves Pulse Mobile's chat. Its manifest description still called it the explanation and handoff surface for Patrol findings, and that text is public: the docs/AI.md overview, the MCP README surface block, and the External agents settings panel all show it. It now names the Pulse Mobile role and says the web chat is retired. The browser's fallback affordances for an Assistant surface with none declared still offered interactive questions. They now match the backend default of tools only, so no client path advertises a question tool nobody can answer.
341 lines
19 KiB
Python
341 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Guard public Pulse Intelligence docs against retired AI shorthand."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from repo_file_io import read_repo_text
|
|
|
|
PUBLIC_AI_DOC_PATHS = (
|
|
"docs/AI.md",
|
|
"docs/AI_AUTONOMY.md",
|
|
"docs/API.md",
|
|
"docs/FAQ.md",
|
|
"docs/PULSE_PRO.md",
|
|
"docs/README.md",
|
|
)
|
|
|
|
|
|
class AIRuntimeDocsPolicyTest(unittest.TestCase):
|
|
def test_public_ai_overview_uses_productized_context_language(self) -> None:
|
|
content = read_repo_text("docs/AI.md")
|
|
normalized_content = " ".join(content.split())
|
|
|
|
self.assertIn("# Pulse Intelligence", content)
|
|
self.assertIn(
|
|
"Uses prior alert context, Patrol run history, and resource timelines",
|
|
content,
|
|
)
|
|
self.assertIn("Investigation Context", content)
|
|
self.assertIn("the configured LLM owns diagnosis", content)
|
|
self.assertIn(
|
|
"Pulse supplies context, capabilities, safety gates, approval state, and audit trails",
|
|
content,
|
|
)
|
|
self.assertIn("shared **Pulse Intelligence Core**", content)
|
|
self.assertIn("<!-- pulse-intelligence-overview:start -->", content)
|
|
self.assertIn("<!-- pulse-intelligence-overview:end -->", content)
|
|
self.assertIn(
|
|
"Canonical context, governed actions, safety gates, approval state, action audit, and verification",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("Patrol as the primary built-in operator", normalized_content)
|
|
self.assertIn(
|
|
"Pulse Patrol**: Patrol is the first-party operations surface: it checks infrastructure, investigates issues, follows the chosen Patrol mode before acting, verifies outcomes, and records what happened.",
|
|
normalized_content,
|
|
)
|
|
self.assertIn(
|
|
"Pulse Assistant**: The chat surface Pulse Mobile uses for operator questions over the same governed tools, approvals, and verification; the in-app web chat window is retired. Affordances: tools.",
|
|
normalized_content,
|
|
)
|
|
self.assertIn(
|
|
"Pulse MCP**: The external-agent adapter that projects canonical Pulse Intelligence capabilities as MCP tools. Affordances: tools, resources, prompts, and capability metadata.",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("Pulse MCP", content)
|
|
self.assertIn("AI_PATROL_QUALIFICATION.md", content)
|
|
self.assertIn("full-track local suite", normalized_content)
|
|
self.assertIn("privacy-allowlisted community evidence export", normalized_content)
|
|
self.assertIn("Every active finding shown or returned to a Patrol run", content)
|
|
self.assertIn("Silence is not an all-clear signal", normalized_content)
|
|
self.assertIn("### The Patrol attention queue", content)
|
|
self.assertIn("### Operational objectives and model-authored observers", content)
|
|
self.assertIn("Patrol objectives retain an operator's desired outcome", normalized_content)
|
|
self.assertIn("`patrol_propose_observer`", content)
|
|
self.assertIn("The proposal boundary is intentionally not an execution boundary", normalized_content)
|
|
self.assertIn("Coverage remains `uncovered` until a core-owned validator", normalized_content)
|
|
self.assertIn(
|
|
"Suppression is temporary and requires an operator reason plus a bounded expiry",
|
|
normalized_content,
|
|
)
|
|
self.assertIn(
|
|
"A successful command does not resolve the attention item",
|
|
normalized_content,
|
|
)
|
|
self.assertIn(
|
|
"The Assistant model matrix below proves Assistant orchestration only",
|
|
normalized_content,
|
|
)
|
|
# The in-app Assistant chat is retired: external agents use pulse-mcp
|
|
# over the same canonical manifest, and Pulse Mobile keeps its chat tab
|
|
# until the app retires.
|
|
self.assertIn("### Using Pulse from Claude Code, Codex, and other agents", content)
|
|
self.assertIn("Pulse has no in-app chat window", normalized_content)
|
|
self.assertIn("Settings → Pulse Intelligence → External agents", normalized_content)
|
|
self.assertIn("The tool list is registry-owned at runtime", content)
|
|
self.assertIn(
|
|
"the live `/api/agent/capabilities` manifest or `pulse-mcp` `tools/list`",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("The same manifest carries reusable `workflowPrompts` metadata", normalized_content)
|
|
self.assertIn(
|
|
"MCP `prompts/list` clients discover the fleet triage, resource investigation, and Patrol finding review workflows",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("Patrol mode and governed Pulse Intelligence operations", normalized_content)
|
|
self.assertIn(
|
|
"New operational capabilities are added to the canonical API manifest first",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("MCP-only copies of the same business logic are drift", normalized_content)
|
|
self.assertIn("## Pulse Assistant chat (retired)", content)
|
|
self.assertIn(
|
|
"Pulse Mobile keeps its chat tab until the app retires on 31 March 2027",
|
|
normalized_content,
|
|
)
|
|
self.assertNotIn("Explain with Assistant", content)
|
|
self.assertNotIn("Open in Assistant", content)
|
|
self.assertNotIn("sibling surfaces", normalized_content)
|
|
# The Patrol summary card and its supporting-context section are gone,
|
|
# so the docs point at where check history actually lives.
|
|
self.assertIn("Activity > Finding options and history", normalized_content)
|
|
self.assertNotIn("legacy operational score remain", normalized_content)
|
|
self.assertNotIn("collapsed supporting-context section", normalized_content)
|
|
self.assertIn("- **Anthropic** (API key)", content)
|
|
self.assertIn("Anthropic OAuth is not a supported runtime authentication method", normalized_content)
|
|
self.assertIn("does not make Anthropic configured", normalized_content)
|
|
self.assertIn("go run ./cmd/eval -scenario resource-context", content)
|
|
self.assertIn("EVAL_RESOURCE_CONTEXT_FORBIDDEN", content)
|
|
self.assertIn("Pulse does not convert them into Pulse-authored findings", content)
|
|
self.assertIn(
|
|
"critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors",
|
|
normalized_content,
|
|
)
|
|
self.assertIn(
|
|
"Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors",
|
|
normalized_content,
|
|
)
|
|
self.assertNotIn("Anthropic** (API key or OAuth)", content)
|
|
self.assertNotIn("Anthropic API key or OAuth", content)
|
|
self.assertNotIn("successful remediations (incident memory)", content)
|
|
self.assertNotIn("**Incident memory**", content)
|
|
self.assertNotIn("**Incident Memory**", content)
|
|
self.assertNotIn("incident memory, investigation orchestration", content)
|
|
self.assertNotIn("learns what's normal", content)
|
|
self.assertNotIn("multi-layered intelligence platform", content)
|
|
self.assertNotIn("capacity predictions", content)
|
|
self.assertNotIn("Deterministic Signal Detection", content)
|
|
self.assertNotIn("Pulse AI is built", content)
|
|
self.assertNotIn("counts and feature flags only", normalized_content)
|
|
self.assertNotIn("### Available Tools", content)
|
|
self.assertNotIn("| Tool | Classification | Purpose |", content)
|
|
self.assertNotIn("| `pulse_query`, `pulse_discovery` |", content)
|
|
# AI Discovery and its pulse_discovery tool were removed on 2026-10-11.
|
|
self.assertNotIn("pulse_discovery", content)
|
|
self.assertNotIn("Discovery Before Action", content)
|
|
self.assertNotIn("| `patrol_report_finding` | Patrol |", content)
|
|
self.assertNotIn("active_alert", content)
|
|
self.assertNotIn("auto-recovery", content)
|
|
self.assertNotIn('EVAL_RESOURCE_CONTEXT_FORBIDDEN="/mnt/pve/finance-db,/var/lib/homeassistant,secret"', content)
|
|
self.assertNotRegex(content, r"(?i)understands resources before you ask")
|
|
|
|
def test_public_ai_docs_define_local_subscription_agent_safety_boundary(self) -> None:
|
|
content = read_repo_text("docs/AI.md")
|
|
normalized_content = " ".join(content.split())
|
|
|
|
self.assertIn("codex-subscription:gpt-5.6-luna", content)
|
|
self.assertIn("claude-subscription:sonnet", content)
|
|
self.assertIn("claude-subscription:opus", content)
|
|
self.assertIn("local_subscription_agent", content)
|
|
self.assertIn("API-key environment variables", normalized_content)
|
|
self.assertIn("dedicated, least-privilege OS account", normalized_content)
|
|
self.assertIn("Pulse retains tool execution and policy enforcement", normalized_content)
|
|
self.assertIn("Pulse audits Claude's buffered event stream", normalized_content)
|
|
self.assertIn("accepts only the first call whose name was explicitly offered", normalized_content)
|
|
self.assertIn("undeclared local tool attempts fail closed", normalized_content)
|
|
self.assertIn("does not grant Claude Code direct access to the infrastructure", normalized_content)
|
|
self.assertIn("not represented as a zero-dollar API price", normalized_content)
|
|
self.assertIn("per-run metered-API budget as not applicable", normalized_content)
|
|
self.assertIn("bounded output allowance", normalized_content)
|
|
self.assertIn("low reasoning-effort hint", normalized_content)
|
|
self.assertIn("does not reduce the tools available to Watch", normalized_content)
|
|
self.assertIn("Pro investigation reasoning", normalized_content)
|
|
self.assertIn("normal reasoning depth", normalized_content)
|
|
self.assertIn("inference_route=coding_plan_allowance", content)
|
|
self.assertIn("per-run monetary cost unknown", normalized_content)
|
|
self.assertIn("standard Z.ai `/api/paas/` endpoint remains a `metered_api` route", normalized_content)
|
|
self.assertIn("Pulse runs as the `pulse` account with `HOME=/opt/pulse`", normalized_content)
|
|
self.assertIn("a CLI installed or authenticated only as `root` is intentionally unavailable", normalized_content)
|
|
self.assertIn("claude auth status --json", content)
|
|
self.assertIn("PULSE_CLAUDE_CLI_PATH", content)
|
|
self.assertIn("PULSE_CODEX_CLI_PATH", content)
|
|
|
|
def test_public_ai_privacy_copy_discloses_outbound_usage_telemetry(self) -> None:
|
|
content = read_repo_text("docs/AI.md")
|
|
normalized_content = " ".join(content.split())
|
|
|
|
self.assertIn("## Privacy", content)
|
|
self.assertIn("Outbound usage telemetry", content)
|
|
self.assertIn("rotating pseudonymous install ID", normalized_content)
|
|
self.assertIn(
|
|
"no hostnames, credentials, prompts, chat messages, command text, action output, token values, IP addresses, or resource identifiers in the payload",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("enabled by default and can be disabled any time", normalized_content)
|
|
self.assertNotIn("anonymous telemetry", normalized_content.lower())
|
|
|
|
def test_patrol_qualification_docs_keep_community_evidence_local_and_bounded(self) -> None:
|
|
content = read_repo_text("docs/AI_PATROL_QUALIFICATION.md")
|
|
normalized_content = " ".join(content.split())
|
|
|
|
self.assertIn("-mode live-suite", content)
|
|
self.assertIn("-mode export-contribution", content)
|
|
self.assertIn("The export command performs no network request", normalized_content)
|
|
self.assertIn("explicit allowlist", normalized_content)
|
|
self.assertIn("It never copies raw findings", normalized_content)
|
|
self.assertIn("Community tested", content)
|
|
self.assertIn("Community validated", content)
|
|
self.assertIn("Pulse certified", content)
|
|
self.assertIn(
|
|
"Only Pulse-certified models may become the default hosted route",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("`max_evidence_calls`", content)
|
|
self.assertIn("actual evidence-call volume", normalized_content)
|
|
self.assertIn("completed model responses separately", normalized_content)
|
|
self.assertIn("terminal typed proposal does not count as evidence", normalized_content)
|
|
|
|
def test_public_patrol_docs_separate_evidence_and_model_turn_budgets(self) -> None:
|
|
overview = read_repo_text("docs/AI.md")
|
|
control_doc = read_repo_text("docs/AI_AUTONOMY.md")
|
|
normalized_control = " ".join(control_doc.split())
|
|
|
|
self.assertIn("| `MaxEvidenceCalls` | 10 |", overview)
|
|
self.assertIn("| `MaxTurns` | 12 |", overview)
|
|
self.assertIn("terminal typed proposal is reserved separately", overview)
|
|
self.assertIn("Maximum evidence-tool calls per investigation", normalized_control)
|
|
self.assertIn("separate model-response safety ceiling", normalized_control)
|
|
|
|
def test_public_ai_overview_describes_dimensioned_patrol_model_readiness(self) -> None:
|
|
content = read_repo_text("docs/AI.md")
|
|
normalized_content = " ".join(content.split())
|
|
|
|
self.assertIn("### Patrol model readiness advisor", content)
|
|
self.assertIn("synthetic data only and never calls infrastructure tools", normalized_content)
|
|
self.assertIn("provider connectivity, exact typed streaming tool calls", normalized_content)
|
|
self.assertIn("Patrol-shaped context fixtures", normalized_content)
|
|
self.assertIn("projected loop latency", normalized_content)
|
|
self.assertIn("never certifies **Safe auto-fix** or **Autopilot**", normalized_content)
|
|
self.assertIn("without replacing the last completed evidence", normalized_content)
|
|
self.assertIn("`patrol_readiness` usage category", normalized_content)
|
|
self.assertNotIn("Patrol model ready", content)
|
|
|
|
def test_public_ai_overview_defines_local_compatible_provider_lifecycle(self) -> None:
|
|
content = read_repo_text("docs/AI.md")
|
|
normalized_content = " ".join(content.split())
|
|
|
|
self.assertIn("llama.cpp, LocalAI, LM Studio", normalized_content)
|
|
self.assertIn(
|
|
"API key is optional when the custom endpoint is intentionally keyless",
|
|
normalized_content,
|
|
)
|
|
self.assertIn("Custom OpenAI-compatible model catalogs are authoritative", normalized_content)
|
|
self.assertIn("Pulse lists every non-empty model ID returned by the endpoint", normalized_content)
|
|
self.assertIn("Pulse omits empty Authorization headers", normalized_content)
|
|
self.assertIn("Blank is the default", normalized_content)
|
|
self.assertIn("the Ollama server's own policy applies", normalized_content)
|
|
self.assertIn("Provider transport health is reported independently from Patrol capability", normalized_content)
|
|
self.assertIn("does not weaken Patrol's fail-closed tool/action admission", normalized_content)
|
|
|
|
def test_public_ai_docs_use_current_surface_naming(self) -> None:
|
|
for doc_path in PUBLIC_AI_DOC_PATHS:
|
|
with self.subTest(doc_path=doc_path):
|
|
content = read_repo_text(doc_path)
|
|
self.assertNotIn("Pulse AI", content)
|
|
self.assertNotIn("Settings → System → AI Assistant", content)
|
|
self.assertNotIn("Settings > System > AI Assistant", content)
|
|
self.assertNotIn("Settings > Pulse Assistant", content)
|
|
|
|
def test_public_patrol_docs_use_mode_language(self) -> None:
|
|
overview = read_repo_text("docs/AI.md")
|
|
control_doc = read_repo_text("docs/AI_AUTONOMY.md")
|
|
|
|
self.assertIn("Pro adds hands-on Patrol modes, issue investigation, governed fixes, verified outcomes, and 90-day history", overview)
|
|
self.assertIn("## Patrol Modes", overview)
|
|
self.assertIn("four modes", overview)
|
|
self.assertIn("**Watch only**", overview)
|
|
self.assertIn("**Ask first**", overview)
|
|
self.assertIn("**Safe auto-fix**", overview)
|
|
self.assertIn("**Autopilot**", overview)
|
|
self.assertIn("When a finding is created in a Pro Patrol mode", overview)
|
|
self.assertIn("**Patrol modes (Pro and above)**", overview)
|
|
self.assertIn("**Governed fixes (Pro and above)**", overview)
|
|
self.assertIn("**Issue investigation (Pro and above)**", overview)
|
|
self.assertIn("Patrol mode and governed Pulse Intelligence operations", overview)
|
|
self.assertNotIn("## Patrol Control Levels", overview)
|
|
self.assertNotIn("four control levels", overview)
|
|
self.assertNotIn("When a finding is created in a Pro control level", overview)
|
|
self.assertNotIn("**Patrol control (Pro and above)**", overview)
|
|
self.assertNotIn("**Alert investigation (Pro and above)**", overview)
|
|
self.assertNotIn("Patrol control and governed Pulse Intelligence operations", overview)
|
|
self.assertNotIn("## Autonomy Levels", overview)
|
|
self.assertNotIn("autonomy modes", overview)
|
|
|
|
self.assertIn("# Pulse Intelligence Modes and Safety Configuration", control_doc)
|
|
self.assertIn("**Patrol Mode**", control_doc)
|
|
self.assertIn("Patrol mode is configured on the **Patrol** page.", control_doc)
|
|
self.assertIn("## Patrol Modes", control_doc)
|
|
self.assertIn("Patrol mode sets how far Pulse can go", control_doc)
|
|
self.assertIn("**UI:** Patrol → Patrol mode", control_doc)
|
|
self.assertIn(
|
|
"The API retains `autonomy_level` for compatibility:",
|
|
" ".join(control_doc.split()),
|
|
)
|
|
self.assertNotIn("Patrol Control Level", control_doc)
|
|
self.assertNotIn("## Patrol Control Levels", control_doc)
|
|
self.assertNotIn("Patrol control is configured on the **Patrol** page.", control_doc)
|
|
self.assertNotIn("Patrol control sets how far Pulse can go", control_doc)
|
|
self.assertNotIn("**UI:** Patrol → Patrol control", control_doc)
|
|
self.assertNotIn("Patrol Autonomy Levels", control_doc)
|
|
self.assertNotIn("Patrol autonomy controls", control_doc)
|
|
self.assertNotIn("Patrol Autonomy Level", control_doc)
|
|
|
|
|
|
def test_existing_help_matches_planning_boundary(self):
|
|
sections = {
|
|
"AI.md": (
|
|
"## Pulse Assistant chat (retired)",
|
|
("pulse-mcp", "Have Patrol investigate", "Pulse Mobile", "31 March 2027", "Never auto-remediate"),
|
|
),
|
|
"AI_AUTONOMY.md": (
|
|
"## Pulse Mobile chat control level",
|
|
("Read-only", "Ask first", "Actions", "does not execute", "stored", "entitlement", "Patrol", "verification", "Never auto-remediate", "31 March 2027"),
|
|
),
|
|
}
|
|
for name, (heading, retained_terms) in sections.items():
|
|
with self.subTest(name=name):
|
|
text = read_repo_text("docs/" + name)
|
|
self.assertEqual(text, read_repo_text("frontend-modern/public/docs/" + name))
|
|
start = text.index(heading)
|
|
end = text.index("\n---", start)
|
|
section = " ".join(text[start:end].split())
|
|
for retained in retained_terms:
|
|
self.assertIn(retained, section)
|
|
for retired in ("executes commands without prompting", "AI executes actions without prompting", "APPROVAL_REQUIRED", "Approve** to execute", "Chat action mode"):
|
|
self.assertNotIn(retired, section)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|