Files
pulse/internal/api/route_inventory_test.go
T
Richard Courtman ccea5ba761 Fail actions stranded without agent completion evidence and add force-fail (#1649)
An action that reached executing had exactly one way out: a terminal
operation receipt from the agent. Expiry skips executing rows on purpose,
and reconciliation bailed for every non-terminal query answer, so a Docker
update whose agent restarted mid-pull sat in executing forever with no
operator route out of it.

The agent-side receipt store rewrites accepted and started receipts to
interrupted on every Open(), and nothing can move an interrupted or
tombstoned receipt back to terminal. An identity-correlated answer of
either kind is therefore proof the operation will never report, and the
three executors now settle the action immediately as inconclusive with a
message telling the operator Pulse cannot confirm the effect and the
resource needs checking by hand.

A not_found answer, or a receipt still merely accepted or started, may
still be completed, so those keep waiting and only settle once the
dispatch attempt is older than one hour, the same threshold the
pulse-intelligence telemetry already uses to call an executing action
stuck. Every typed operation timeout is far shorter than that, so an
in-flight mutation is never cut short. A transport error answering the
query is still not evidence and preserves receipt_pending unchanged.

RecoverExecutingActions drives all of this on the existing two-minute
recovery loop, so rows already wedged before the upgrade heal themselves
without anyone touching them.

For the residue that reconciliation cannot reach, an agent that was
reinstalled or a legacy executing row with no dispatch attempt, POST
/api/actions/{id}/force-fail writes the same inconclusive terminal truth
under an operator attribution. It never touches the transport, refuses
anything already terminal, and is gated on admin plus settings:write on
top of the execute capability check.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-29 12:14:28 +01:00

702 lines
19 KiB
Go

package api
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"testing"
)
func TestRouterRouteInventory(t *testing.T) {
literalRoutes, dynamicRoutes, bareLiteralRoutes := parseRouterRoutes(t)
expectedAll := sliceToSet(t, allRouteAllowlist, "all route allowlist")
expectedPublic := sliceToSet(t, publicRouteAllowlist, "public route allowlist")
expectedDynamic := sliceToSet(t, dynamicRouteAllowlist, "dynamic route allowlist")
expectedBare := sliceToSet(t, bareRouteAllowlist, "bare route allowlist")
actualAll := sliceToSet(t, literalRoutes, "router routes")
actualDynamic := sliceToSet(t, dynamicRoutes, "router dynamic routes")
actualBare := sliceToSet(t, bareLiteralRoutes, "router bare routes")
if missing := setDifference(actualAll, expectedAll); len(missing) > 0 {
t.Fatalf("routes missing from allowlist: %s", strings.Join(sortedKeys(missing), ", "))
}
if stale := setDifference(expectedAll, actualAll); len(stale) > 0 {
t.Fatalf("allowlist contains routes not in router.go: %s", strings.Join(sortedKeys(stale), ", "))
}
if missing := setDifference(actualDynamic, expectedDynamic); len(missing) > 0 {
t.Fatalf("dynamic routes missing from allowlist: %s", strings.Join(sortedKeys(missing), ", "))
}
if stale := setDifference(expectedDynamic, actualDynamic); len(stale) > 0 {
t.Fatalf("dynamic allowlist contains routes not in router.go: %s", strings.Join(sortedKeys(stale), ", "))
}
if missing := setDifference(expectedPublic, expectedAll); len(missing) > 0 {
t.Fatalf("public routes missing from full allowlist: %s", strings.Join(sortedKeys(missing), ", "))
}
if missing := setDifference(actualBare, expectedBare); len(missing) > 0 {
t.Fatalf("bare routes missing from allowlist: %s", strings.Join(sortedKeys(missing), ", "))
}
if stale := setDifference(expectedBare, actualBare); len(stale) > 0 {
t.Fatalf("bare allowlist contains routes not registered bare in router.go: %s", strings.Join(sortedKeys(stale), ", "))
}
if missing := setDifference(expectedPublic, expectedBare); len(missing) > 0 {
t.Fatalf("public routes must be registered bare: %s", strings.Join(sortedKeys(missing), ", "))
}
}
func TestRouterRouteInventory_VMwarePhase1ExclusionIntegrity(t *testing.T) {
literalRoutes, _, _ := parseRouterRoutes(t)
actual := sliceToSet(t, literalRoutes, "router routes")
forbidden := []string{
"/api/vmware/hosts",
"/api/vmware/vms",
"/api/vmware/datastores",
"/api/vmware/events",
"/api/vmware/tasks",
"/api/vmware/alarms",
}
for _, route := range forbidden {
if _, ok := actual[route]; ok {
t.Fatalf("vmware phase-1 exclusion failed: unexpected route %q is registered", route)
}
}
var vmwareRoutes []string
for route := range actual {
if strings.HasPrefix(route, "/api/vmware/") {
vmwareRoutes = append(vmwareRoutes, route)
}
}
sort.Strings(vmwareRoutes)
expected := []string{
"/api/vmware/connections",
"/api/vmware/connections/",
"/api/vmware/connections/preview",
"/api/vmware/connections/test",
}
if strings.Join(vmwareRoutes, ",") != strings.Join(expected, ",") {
t.Fatalf("vmware phase-1 route family drifted: got %v, want %v", vmwareRoutes, expected)
}
}
func parseRouterRoutes(t *testing.T) ([]string, []string, []string) {
t.Helper()
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatalf("failed to locate test file path")
}
routerFiles := []string{
"router.go",
"router_routes_registration.go",
"router_routes_ai_relay.go",
"router_routes_monitoring.go",
"router_routes_auth_security.go",
"router_routes_licensing.go",
"router_routes_cloud.go",
"router_routes_debug.go",
}
fset := token.NewFileSet()
var literalRoutes []string
var dynamicRoutes []string
var bareLiteralRoutes []string
var found bool
for _, routerFile := range routerFiles {
routerPath := filepath.Join(filepath.Dir(file), routerFile)
fileAST, err := parser.ParseFile(fset, routerPath, nil, 0)
if err != nil {
t.Fatalf("parse %s: %v", routerFile, err)
}
ast.Inspect(fileAST, func(node ast.Node) bool {
call, ok := node.(*ast.CallExpr)
if !ok {
return true
}
selector, ok := call.Fun.(*ast.SelectorExpr)
if !ok || selector.Sel == nil {
return true
}
if selector.Sel.Name != "Handle" && selector.Sel.Name != "HandleFunc" {
return true
}
if len(call.Args) < 2 {
return true
}
route, isDynamic := routeLiteral(call.Args[0])
if route == "" {
return true
}
found = true
if isDynamic {
dynamicRoutes = append(dynamicRoutes, route)
} else {
literalRoutes = append(literalRoutes, route)
if !isProtectedHandler(call.Args[1]) {
bareLiteralRoutes = append(bareLiteralRoutes, route)
}
}
return true
})
}
if !found {
t.Fatalf("no routes found in parsed router files")
}
return literalRoutes, dynamicRoutes, bareLiteralRoutes
}
func routeLiteral(expr ast.Expr) (string, bool) {
switch v := expr.(type) {
case *ast.BasicLit:
if v.Kind != token.STRING {
return "", true
}
unquoted, err := strconv.Unquote(v.Value)
if err != nil {
unquoted = strings.Trim(v.Value, "`\"'")
}
return unquoted, false
case *ast.Ident:
return v.Name, true
case *ast.SelectorExpr:
return fmt.Sprintf("%s.%s", selectorName(v.X), v.Sel.Name), true
default:
return fmt.Sprintf("%T", expr), true
}
}
func selectorName(expr ast.Expr) string {
switch v := expr.(type) {
case *ast.Ident:
return v.Name
case *ast.SelectorExpr:
return selectorName(v.X) + "." + v.Sel.Name
default:
return ""
}
}
func isProtectedHandler(expr ast.Expr) bool {
call, ok := expr.(*ast.CallExpr)
if !ok {
return false
}
switch fn := call.Fun.(type) {
case *ast.Ident:
if isAuthWrapper(fn.Name) {
return true
}
case *ast.SelectorExpr:
if isAuthWrapper(fn.Sel.Name) {
return true
}
}
for _, arg := range call.Args {
if isProtectedHandler(arg) {
return true
}
}
return false
}
func isAuthWrapper(name string) bool {
switch name {
case "RequireAuth", "RequireAdmin", "RequirePermission", "RequireOrgOwnerOrPlatformAdmin", "RequirePlatformAdmin":
return true
default:
return false
}
}
func sliceToSet(t *testing.T, items []string, name string) map[string]struct{} {
t.Helper()
set := make(map[string]struct{}, len(items))
for _, item := range items {
if _, exists := set[item]; exists {
t.Fatalf("duplicate entry %q in %s", item, name)
}
set[item] = struct{}{}
}
return set
}
func setDifference(a, b map[string]struct{}) map[string]struct{} {
diff := make(map[string]struct{})
for key := range a {
if _, ok := b[key]; !ok {
diff[key] = struct{}{}
}
}
return diff
}
func sortedKeys(set map[string]struct{}) []string {
keys := make([]string, 0, len(set))
for key := range set {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
var dynamicRouteAllowlist = []string{
"agentcapabilities.OperationsLoopStatusCompatibilityPath",
"agentcapabilities.PatrolControlStatusCapabilityPath",
}
var publicRouteAllowlist = []string{
"/api/health",
"/api/version",
"/api/agent/capabilities",
"/api/agent/version",
"/api/agent/ws",
"/api/server/info",
"/api/security/status",
"/api/security/validate-bootstrap-token",
"/api/security/quick-setup",
"/api/security/recovery",
"/api/login",
"/api/public/signup",
"/api/public/magic-link/request",
"/api/public/magic-link/verify",
"/api/cloud/handoff/exchange",
"/api/ai/oauth/callback",
"/api/webhooks/stripe",
"/api/setup-script",
"/api/config/export",
"/api/config/import",
"/api/system/verify-temperature-ssh",
"/api/system/ssh-config",
"/api/auto-register",
"/api/auto-unregister",
"/auth/cloud-handoff",
licensePurchaseActivationPath,
"/install.sh",
"/install.ps1",
"/download/pulse-agent",
}
var bareRouteAllowlist = []string{
"/api/agent-deploy/jobs/",
"/api/agent-deploy/preflights/",
"/api/agent/capabilities",
"/api/agent/version",
"/api/agent/ws",
"/api/ai/oauth/callback",
"/api/auto-register",
"/api/auto-unregister",
"/api/webhooks/stripe",
"/api/config/export",
"/api/config/import",
"/api/config/nodes",
"/api/config/nodes/",
"/api/config/nodes/test-config",
"/api/config/nodes/test-connection",
"/api/config/system",
"/api/connections",
"/api/connections/probe",
"/api/availability-targets",
"/api/availability-targets/test",
"/api/availability-targets/",
"/api/truenas/connections",
"/api/truenas/connections/preview",
"/api/truenas/connections/test",
"/api/truenas/connections/",
"/api/vmware/connections",
"/api/vmware/connections/preview",
"/api/vmware/connections/test",
"/api/vmware/connections/",
"/api/health",
"/api/login",
"/api/logout",
"/api/logs/level",
"/api/oidc/",
"/api/security/apply-restart",
"/api/security/change-password",
"/api/security/dev/reset-first-run",
"/api/security/quick-setup",
"/api/security/recovery",
"/api/security/regenerate-token",
"/api/security/reset-lockout",
"/api/security/status",
"/api/security/validate-bootstrap-token",
"/api/security/validate-token",
"/api/server/info",
"/api/setup-script",
"/api/state",
"/api/state/summary",
"/api/system/mock-mode",
"/api/system/ssh-config",
"/api/system/verify-temperature-ssh",
"/api/version",
"/download/pulse-agent",
"/install.ps1",
"/install.sh",
"/api/public/signup",
"/api/public/magic-link/request",
"/api/public/magic-link/verify",
"/api/cloud/handoff/exchange",
"/api/clusters/",
"/auth/cloud-handoff",
licensePurchaseActivationPath,
"/ws",
"/api/saml/",
}
var allRouteAllowlist = []string{
"/debug/pprof",
"/debug/pprof/",
"/debug/pprof/cmdline",
"/debug/pprof/profile",
"/debug/pprof/symbol",
"/debug/pprof/trace",
"/api/health",
"/api/monitoring/scheduler/health",
"/api/state",
"/api/state/summary",
"/api/logs/stream",
"/api/logs/download",
"/api/logs/level",
"/api/agents/docker/report",
"/api/agents/kubernetes/report",
"/api/agents/agent/report",
"/api/agents/host/report",
"/api/agents/agent/lookup",
"/api/agents/host/lookup",
"/api/agents/agent/uninstall",
"/api/agents/host/uninstall",
"/api/agents/agent/unlink",
"/api/agents/host/unlink",
"/api/agents/agent/link",
"/api/agents/host/link",
"/api/agents/diagnostics",
"/api/agents/agent/",
"/api/agents/host/",
"/api/agents/docker/commands/",
"/api/agents/docker/runtimes/",
"/api/agents/docker/containers/update",
"/api/agents/kubernetes/clusters/",
"/api/version",
"/api/storage/",
"/api/storage-charts",
"/api/charts",
"/api/charts/workloads",
"/api/charts/infrastructure",
"/api/charts/storage-summary",
"/api/charts/workloads-summary",
"/api/metrics-store/stats",
"/api/metrics-store/history",
"/api/diagnostics",
"/api/diagnostics/docker/prepare-token",
"/api/config",
"/api/recovery/points",
"/api/recovery/series",
"/api/recovery/facets",
"/api/recovery/rollups",
"/api/recovery/postures",
"/api/resources",
"/api/resources/storage-incidents",
"/api/resources/storage-summary",
"/api/resources/k8s/namespaces",
"/api/resources/stats",
"/api/resources/timeline",
"/api/resources/",
"/api/resources/{id}/facets",
"/api/resources/{id}/timeline",
"/api/resources/{id}/operator-state",
"/api/resources/{id}/maintenance-verifications",
"POST /api/resources/{id}/maintenance-verifications/rerun",
"/api/maintenance-verifications/{reportId}",
"POST /api/maintenance-verifications/{reportId}/review",
"/api/agent/resource-context/{id}",
"/api/agent/resource-capabilities/{id}",
"/api/agent/fleet-context",
"/api/agent/capabilities",
"/api/agent/events",
"POST /api/actions/plan",
"GET /api/actions/pending",
"GET /api/actions",
"GET /api/actions/{id}",
"POST /api/actions/{id}/decision",
"POST /api/actions/{id}/execute",
"POST /api/actions/{id}/force-fail",
"/api/guests/metadata",
"/api/guests/metadata/",
"/api/docker/metadata",
"/api/docker/metadata/",
"/api/docker/runtimes/metadata",
"/api/docker/runtimes/metadata/",
"/api/agents/metadata",
"/api/agents/metadata/",
"GET /api/settings/relay",
"GET /api/settings/relay/status",
"PUT /api/settings/relay",
"GET /api/onboarding/qr",
"POST /api/onboarding/validate",
"GET /api/onboarding/deep-link",
"/api/updates/check",
"/api/updates/apply",
"/api/updates/rollback",
"/api/updates/status",
"/api/updates/release-notes",
"/api/updates/stream",
"/api/updates/plan",
"/api/updates/history",
"/api/updates/history/entry",
"/api/infra-updates",
"/api/infra-updates/summary",
"/api/infra-updates/check",
"/api/infra-updates/agent/",
"/api/infra-updates/",
"/api/config/nodes",
"/api/security/validate-bootstrap-token",
"/api/config/nodes/test-config",
"/api/config/nodes/test-connection",
"/api/config/nodes/",
"/api/connections",
"/api/connections/probe",
"/api/availability-targets",
"/api/availability-targets/test",
"/api/availability-targets/",
"/api/truenas/connections",
"/api/truenas/connections/preview",
"/api/truenas/connections/test",
"/api/truenas/connections/",
"/api/vmware/connections",
"/api/vmware/connections/preview",
"/api/vmware/connections/test",
"/api/vmware/connections/",
"/api/admin/profiles/",
"/api/config/system",
"/api/system/settings/telemetry-preview",
"/api/system/settings/telemetry-reset-id",
"/api/system/mock-mode",
"/api/license/status",
"/api/webhooks/stripe",
"/api/license/features",
"/api/license/activate",
"/api/license/clear",
"GET /api/license/runtime-capabilities",
"GET /api/license/commercial-posture",
"GET /api/license/entitlements",
"GET /api/license/monitored-system-ledger",
"POST /api/license/monitored-system-ledger/explain",
"POST /api/license/monitored-system-ledger/preview",
"GET /auth/license-purchase-start",
"GET /api/orgs",
"POST /api/orgs",
"GET /api/orgs/{id}",
"PUT /api/orgs/{id}",
"DELETE /api/orgs/{id}",
"GET /api/orgs/{id}/members",
"POST /api/orgs/{id}/members",
"DELETE /api/orgs/{id}/members/{userId}",
"GET /api/orgs/{id}/invitations",
"DELETE /api/orgs/{id}/invitations/{userId}",
"GET /api/org-invitations",
"POST /api/org-invitations/{id}/accept",
"DELETE /api/org-invitations/{id}",
"GET /api/orgs/{id}/shares",
"GET /api/orgs/{id}/shares/incoming",
"POST /api/orgs/{id}/shares/incoming/{shareId}/accept",
"POST /api/orgs/{id}/shares",
"DELETE /api/orgs/{id}/shares/incoming/{shareId}",
"DELETE /api/orgs/{id}/shares/{shareId}",
"GET /api/audit",
"GET /api/audit/",
"GET /api/audit/{id}/verify",
"GET /api/audit/export",
"GET /api/audit/summary",
"GET /api/audit/actions",
"GET /api/audit/actions/{id}/events",
"GET /api/audit/exports",
"GET /api/admin/orgs/{id}/billing-state",
"PUT /api/admin/orgs/{id}/billing-state",
"POST /api/admin/orgs/{id}/suspend",
"POST /api/admin/orgs/{id}/unsuspend",
"POST /api/admin/orgs/{id}/soft-delete",
"GET /api/hosted/organizations",
"POST /api/admin/orgs/{id}/agent-install-command",
"/api/public/signup",
"/api/public/magic-link/request",
"/api/public/magic-link/verify",
"/api/cloud/handoff/exchange",
"/auth/cloud-handoff",
licensePurchaseActivationPath,
"/api/saml/",
"/api/admin/roles",
"/api/admin/roles/",
"/api/admin/users",
"/api/admin/users/",
"GET /api/admin/rbac/integrity",
"POST /api/admin/rbac/reset-admin",
"/api/admin/reports/generate",
"/api/admin/reports/generate-multi",
"/api/admin/reports/catalog",
"/api/admin/reports/inventory/vms/export",
"GET /api/admin/reports/schedules",
"POST /api/admin/reports/schedules",
"PUT /api/admin/reports/schedules/{id}",
"DELETE /api/admin/reports/schedules/{id}",
"POST /api/admin/reports/schedules/{id}/run",
"/api/admin/webhooks/audit",
"/api/security/change-password",
"/api/security/dev/reset-first-run",
"/api/logout",
"/api/login",
"/api/security/reset-lockout",
"/api/oidc/",
"/api/security/sso/providers/test",
"/api/security/sso/providers/metadata/preview",
"/api/security/sso/providers",
"/api/security/sso/providers/",
"/api/security/tokens/relay-mobile",
"/api/security/tokens",
"/api/security/tokens/",
"/api/security/status",
"/api/security/quick-setup",
"/api/security/regenerate-token",
"/api/security/validate-token",
"/api/security/apply-restart",
"/api/security/recovery",
"/api/config/export",
"/api/config/import",
"/api/setup-script",
"/api/setup-script-url",
"/api/agent-install-command",
"/api/auto-register",
"/api/auto-unregister",
"/api/discover",
"/api/test-notification",
"/api/alerts/",
"/api/notifications/",
"/api/notifications/dlq",
"/api/notifications/queue/stats",
"/api/notifications/dlq/retry",
"/api/notifications/dlq/delete",
"/api/system/settings",
"/api/system/settings/update",
"/api/system/ssh-config",
"/api/system/verify-temperature-ssh",
"/api/settings/ai",
"/api/settings/ai/update",
"/api/ai/test",
"/api/ai/test/{provider}",
"/api/ai/models",
"/api/ai/execute",
"/api/ai/execute/stream",
"/api/ai/kubernetes/analyze",
"/api/ai/investigate-alert",
"/api/ai/run-command",
"/api/ai/knowledge",
"/api/ai/knowledge/save",
"/api/ai/knowledge/delete",
"/api/ai/knowledge/export",
"/api/ai/knowledge/import",
"/api/ai/knowledge/clear",
"/api/ai/debug/context",
"/api/ai/agents",
"/api/ai/cost/summary",
"/api/ai/cost/reset",
"/api/ai/cost/export",
"/api/ai/oauth/start",
"/api/ai/oauth/exchange",
"/api/ai/oauth/callback",
"/api/ai/oauth/disconnect",
"/api/ai/patrol/status",
"/api/ai/patrol/stream",
"/api/ai/patrol/findings",
"/api/ai/patrol/attention",
"/api/ai/patrol/attention/",
"/api/ai/patrol/history",
"/api/ai/patrol/run",
"/api/ai/patrol/preflight",
"/api/ai/patrol/readiness",
"/api/ai/patrol/acknowledge",
"/api/ai/patrol/dismiss",
"/api/ai/patrol/findings/note",
"/api/ai/patrol/suppress",
"/api/ai/patrol/snooze",
"/api/ai/patrol/resolve",
"/api/ai/patrol/runs",
"/api/ai/patrol/runs/",
"/api/ai/patrol/suppressions",
"/api/ai/patrol/suppressions/",
"/api/ai/patrol/dismissed",
"/api/ai/patrol/autonomy",
"/api/ai/patrol/autonomy/acknowledgements",
"/api/ai/patrol/autonomy/acknowledgements/",
"/api/ai/findings/",
"/api/ai/intelligence",
"/api/ai/intelligence/patterns",
"/api/ai/intelligence/predictions",
"/api/ai/intelligence/correlations",
"/api/ai/intelligence/changes",
"/api/ai/intelligence/baselines",
"/api/ai/intelligence/remediations",
"/api/ai/intelligence/anomalies",
"/api/ai/intelligence/learning",
"/api/ai/unified/findings",
"/api/ai/forecast",
"/api/ai/forecasts/overview",
"/api/ai/learning/preferences",
"/api/ai/proxmox/events",
"/api/ai/proxmox/correlations",
"/api/ai/remediation/plans",
"/api/ai/remediation/plan",
"/api/ai/remediation/approve",
"/api/ai/remediation/execute",
"/api/ai/remediation/rollback",
"/api/ai/circuit/status",
"/api/ai/incidents",
"/api/ai/incidents/",
"/api/ai/status",
"/api/ai/assistant/surface-tools",
"/api/ai/workflow-prompts/render",
"/api/ai/workflow-prompts/activity",
"/api/ai/chat",
"/api/ai/sessions",
"/api/ai/sessions/",
"/api/ai/approvals",
"/api/ai/approvals/",
"/api/ai/question/",
"/api/discovery",
"/api/discovery/status",
"/api/discovery/run",
"/api/discovery/settings",
"/api/discovery/info/",
"/api/discovery/type/",
"/api/discovery/agent/",
"/api/discovery/",
"/api/agent/ws",
"POST /api/agent/workflow-prompt-activity",
"/api/agents/agent/enroll",
"/api/agents/host/enroll",
"/api/clusters/",
"/api/agent-deploy/preflights/",
"/api/agent-deploy/jobs/",
"/install.sh",
"/install.ps1",
"/download/pulse-agent",
"/api/agent/version",
"/api/server/info",
"/ws",
"/simple-stats",
}