From a5629d57014f58252ddbdfe7ec515ba113f38b0d Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 12 May 2026 23:00:42 +0100 Subject: [PATCH] add capacity-forecast action template registry Deterministic remediation proposals for capacity findings, keyed by (resourceType, metric). Templates ship Allowed=false, RequiresApproval=true, and emit a preflight-only ActionPlan until a Pulse write capability is wired for the resource type. Registers PBS datastore prune+GC, ZFS pool snapshot prune, and VM/CT disk expand variants. CapacityActionPlanSource = "capacity_forecast" is the wire-side marker the FindingsPanel approval card variant keys off. --- .../ai/forecast/capacity_action_templates.go | 293 ++++++++++++++++++ .../capacity_action_templates_test.go | 217 +++++++++++++ 2 files changed, 510 insertions(+) create mode 100644 internal/ai/forecast/capacity_action_templates.go create mode 100644 internal/ai/forecast/capacity_action_templates_test.go diff --git a/internal/ai/forecast/capacity_action_templates.go b/internal/ai/forecast/capacity_action_templates.go new file mode 100644 index 000000000..7da04dc76 --- /dev/null +++ b/internal/ai/forecast/capacity_action_templates.go @@ -0,0 +1,293 @@ +// capacity_action_templates.go: deterministic remediation proposals for +// capacity findings. +// +// When a capacity finding crosses (or is projected to cross) a threshold, +// patrol_findings.go projects the finding into a CapacityFindingInput and +// asks this registry for a deterministic ActionPlan proposal. The proposal +// is attached to the finding's RemediationPlan and surfaced in +// FindingsPanel.tsx as an explicit approve/reject card. +// +// Every template here MUST set RequiresApproval=true. When no Pulse write +// capability is wired for the proposed remediation, the template sets +// Allowed=false and ships preflight-only intent so the operator can +// approve and act manually until a capability is added. We intentionally +// do NOT invent capabilities in pulse_control / agentexec; making a +// proposal that references an unimplemented capability would be a lie +// at the audit layer. +package forecast + +import ( + "fmt" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// CapacityActionPlanSource identifies forecast-driven proposals on the wire. +// FindingsPanel uses this to render the capacity-forecast approval card +// variant rather than the generic remediation plan card. +const CapacityActionPlanSource = "capacity_forecast" + +// proposalTTL controls how long a proposed plan remains presentable before +// it is considered stale. Operator approval still flows through the action +// engine (which has its own freshness check), so this is just the +// presentation-side staleness for findings that go un-actioned. +const proposalTTL = 30 * time.Minute + +// CapacityFindingInput is the minimum context the registry needs to build a +// deterministic capacity-remediation proposal. +// +// patrol_findings.go projects a *Finding plus any available forecast +// snapshot into this struct so the forecast package can stay free of an +// import on the broader internal/ai package. +type CapacityFindingInput struct { + FindingID string + ResourceID string + ResourceName string + ResourceType string + Node string + Metric string + CurrentValue float64 + PredictedValue float64 + ThresholdValue float64 + TimeToThreshold *time.Duration + Now time.Time +} + +type templateKey struct { + ResourceType string + Metric string +} + +type capacityActionTemplate func(input CapacityFindingInput) *unifiedresources.ActionPlan + +// capacityActionTemplates registers the deterministic templates. +// +// Lookups are normalized via lower-case + trim on both fields. The wire-in +// in patrol_findings.go funnels VM/CT and PBS findings through the +// canonical aliases below so callers don't have to remember the +// resource_type variants used across signal detectors and unified +// resources. +var capacityActionTemplates = map[templateKey]capacityActionTemplate{ + {ResourceType: "pbs-datastore", Metric: "usage_percent"}: pbsDatastorePruneAndGCTemplate, + {ResourceType: "pbs", Metric: "usage_percent"}: pbsDatastorePruneAndGCTemplate, + {ResourceType: "storage", Metric: "usage_percent"}: zfsPoolSnapshotPruneTemplate, + {ResourceType: "qemu", Metric: "disk_usage_percent"}: vmDiskExpandTemplate, + {ResourceType: "vm", Metric: "disk_usage_percent"}: vmDiskExpandTemplate, + {ResourceType: "lxc", Metric: "disk_usage_percent"}: vmDiskExpandTemplate, + {ResourceType: "system-container", Metric: "disk_usage_percent"}: vmDiskExpandTemplate, +} + +// BuildActionPlanForFinding returns a deterministic ActionPlan proposal for +// the given (resourceType, metric) pair, or nil if no template is +// registered. +// +// The returned plan is guaranteed to have RequiresApproval=true. Templates +// can choose Allowed=false to indicate "no write capability wired yet — +// preflight-only proposal." +func BuildActionPlanForFinding(input CapacityFindingInput) *unifiedresources.ActionPlan { + key := templateKey{ + ResourceType: strings.ToLower(strings.TrimSpace(input.ResourceType)), + Metric: strings.ToLower(strings.TrimSpace(input.Metric)), + } + fn, ok := capacityActionTemplates[key] + if !ok { + return nil + } + plan := fn(input) + if plan == nil { + return nil + } + plan.RequiresApproval = true + return plan +} + +// HasCapacityActionTemplate reports whether a template exists for the +// (resourceType, metric) pair without constructing a plan. +func HasCapacityActionTemplate(resourceType, metric string) bool { + _, ok := capacityActionTemplates[templateKey{ + ResourceType: strings.ToLower(strings.TrimSpace(resourceType)), + Metric: strings.ToLower(strings.TrimSpace(metric)), + }] + return ok +} + +// CapacityActionTemplateKey is a registered (resourceType, metric) pair. +type CapacityActionTemplateKey struct { + ResourceType string + Metric string +} + +// CapacityActionTemplateKeys returns the registered (resourceType, metric) +// pairs. Useful for tests and for surfacing the catalog in observability +// tooling. Order is not stable across calls. +func CapacityActionTemplateKeys() []CapacityActionTemplateKey { + out := make([]CapacityActionTemplateKey, 0, len(capacityActionTemplates)) + for k := range capacityActionTemplates { + out = append(out, CapacityActionTemplateKey{ResourceType: k.ResourceType, Metric: k.Metric}) + } + return out +} + +func resolveNow(in CapacityFindingInput) time.Time { + if !in.Now.IsZero() { + return in.Now.UTC() + } + return time.Now().UTC() +} + +func displayName(in CapacityFindingInput) string { + name := strings.TrimSpace(in.ResourceName) + if name == "" { + name = strings.TrimSpace(in.ResourceID) + } + if name == "" { + name = "(unknown)" + } + return name +} + +func currentStateString(in CapacityFindingInput) string { + parts := []string{fmt.Sprintf("%s=%.1f%%", in.Metric, in.CurrentValue)} + if in.PredictedValue > 0 && in.PredictedValue != in.CurrentValue { + parts = append(parts, fmt.Sprintf("projected=%.1f%%", in.PredictedValue)) + } + if in.ThresholdValue > 0 { + parts = append(parts, fmt.Sprintf("threshold=%.1f%%", in.ThresholdValue)) + } + if in.TimeToThreshold != nil { + parts = append(parts, fmt.Sprintf("ttb=%s", roundDuration(*in.TimeToThreshold))) + } + return strings.Join(parts, ", ") +} + +func roundDuration(d time.Duration) time.Duration { + if d >= 24*time.Hour { + return d.Round(time.Hour) + } + if d >= time.Hour { + return d.Round(time.Minute) + } + return d.Round(time.Second) +} + +// --- Templates --- + +func pbsDatastorePruneAndGCTemplate(in CapacityFindingInput) *unifiedresources.ActionPlan { + now := resolveNow(in) + name := displayName(in) + + msg := fmt.Sprintf( + "PBS datastore %q is at %.1f%% usage (projected %.1f%%). Propose: prune snapshots against the configured retention policy, then run garbage-collect to reclaim chunk-store space. No PBS prune/GC capability is wired into Pulse yet, so this proposal is preflight-only — approve to record intent and run the remediation manually until the capability lands.", + name, in.CurrentValue, in.PredictedValue, + ) + + return &unifiedresources.ActionPlan{ + ActionID: "capacity-forecast-" + uuid.NewString(), + Allowed: false, + RequiresApproval: true, + ApprovalPolicy: unifiedresources.ApprovalAdmin, + Message: msg, + PlannedAt: now, + ExpiresAt: now.Add(proposalTTL), + Preflight: &unifiedresources.ActionPreflight{ + Target: fmt.Sprintf("pbs-datastore/%s", name), + CurrentState: currentStateString(in), + IntendedChange: "Prune backups against retention policy, then run garbage-collect.", + DryRunAvailable: false, + DryRunSummary: "No PBS prune/GC capability is wired into Pulse yet; this proposal records intent and surfaces the operator-facing remediation.", + SafetyChecks: []string{ + "Operator must explicitly approve before any execution path is wired.", + "This proposal ships with Allowed=false; the action broker will refuse execution.", + "Verify the configured retention policy matches your recovery objectives before approving.", + }, + VerificationSteps: []string{ + "After manual prune+GC, re-check the datastore usage on the next Patrol pass.", + "Confirm chunk-store free space increased and that recent backups remain restorable.", + }, + GeneratedAt: now, + }, + } +} + +func zfsPoolSnapshotPruneTemplate(in CapacityFindingInput) *unifiedresources.ActionPlan { + now := resolveNow(in) + name := displayName(in) + + msg := fmt.Sprintf( + "Storage pool %q is at %.1f%% usage (projected %.1f%%). Propose: prune oldest auto-snapshots and surface the largest reclaimable datasets. No snapshot-prune capability is wired into Pulse yet, so this proposal is preflight-only — approve to record intent and run the remediation manually until the capability lands.", + name, in.CurrentValue, in.PredictedValue, + ) + + return &unifiedresources.ActionPlan{ + ActionID: "capacity-forecast-" + uuid.NewString(), + Allowed: false, + RequiresApproval: true, + ApprovalPolicy: unifiedresources.ApprovalAdmin, + Message: msg, + PlannedAt: now, + ExpiresAt: now.Add(proposalTTL), + Preflight: &unifiedresources.ActionPreflight{ + Target: fmt.Sprintf("storage/%s", name), + CurrentState: currentStateString(in), + IntendedChange: "Prune oldest auto-snapshots, then list largest reclaimable datasets for review.", + DryRunAvailable: false, + DryRunSummary: "No snapshot-prune capability is wired into Pulse yet; this proposal records intent and surfaces the operator-facing remediation.", + SafetyChecks: []string{ + "Operator must explicitly approve before any execution path is wired.", + "This proposal ships with Allowed=false; the action broker will refuse execution.", + "Confirm snapshot retention is sufficient before approving — this proposal targets oldest auto-snapshots only.", + }, + VerificationSteps: []string{ + "After manual snapshot prune, re-check pool usage on the next Patrol pass.", + "Confirm that critical snapshots required for rollback or replication are still present.", + }, + GeneratedAt: now, + }, + } +} + +func vmDiskExpandTemplate(in CapacityFindingInput) *unifiedresources.ActionPlan { + now := resolveNow(in) + name := displayName(in) + + rt := strings.ToUpper(strings.TrimSpace(in.ResourceType)) + if rt == "" { + rt = "GUEST" + } + + msg := fmt.Sprintf( + "%s %q is at %.1f%% disk usage (projected %.1f%%). Propose: expand the guest disk by the next sensible increment, or compact the qcow2 image if the underlying allocation has grown beyond the in-guest footprint. No guest-disk write capability is wired into Pulse yet, so this proposal is preflight-only — approve to record intent and run the remediation manually until the capability lands.", + rt, name, in.CurrentValue, in.PredictedValue, + ) + + return &unifiedresources.ActionPlan{ + ActionID: "capacity-forecast-" + uuid.NewString(), + Allowed: false, + RequiresApproval: true, + ApprovalPolicy: unifiedresources.ApprovalAdmin, + Message: msg, + PlannedAt: now, + ExpiresAt: now.Add(proposalTTL), + Preflight: &unifiedresources.ActionPreflight{ + Target: fmt.Sprintf("%s/%s", strings.ToLower(strings.TrimSpace(in.ResourceType)), name), + CurrentState: currentStateString(in), + IntendedChange: "Expand guest disk by the next sensible increment, or compact the qcow2 image.", + DryRunAvailable: false, + DryRunSummary: "No guest-disk capability is wired into Pulse yet; this proposal records intent and surfaces the operator-facing remediation.", + SafetyChecks: []string{ + "Operator must explicitly approve before any execution path is wired.", + "This proposal ships with Allowed=false; the action broker will refuse execution.", + "Take a snapshot before expanding or compacting — disk operations risk filesystem damage if interrupted.", + }, + VerificationSteps: []string{ + "After manual expand/compact, re-check disk usage on the next Patrol pass.", + "Inside the guest, confirm the partition and filesystem were resized to use the new capacity.", + }, + GeneratedAt: now, + }, + } +} diff --git a/internal/ai/forecast/capacity_action_templates_test.go b/internal/ai/forecast/capacity_action_templates_test.go new file mode 100644 index 000000000..201c79dee --- /dev/null +++ b/internal/ai/forecast/capacity_action_templates_test.go @@ -0,0 +1,217 @@ +package forecast + +import ( + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func TestBuildActionPlanForFinding_ReturnsNilForUnknownPair(t *testing.T) { + plan := BuildActionPlanForFinding(CapacityFindingInput{ + FindingID: "f-1", + ResourceType: "wireguard-tunnel", + Metric: "handshakes_per_minute", + }) + if plan != nil { + t.Fatalf("expected nil plan for unknown (resourceType, metric); got %+v", plan) + } +} + +func TestBuildActionPlanForFinding_RequiresApprovalInvariantHoldsForEveryTemplate(t *testing.T) { + for _, key := range CapacityActionTemplateKeys() { + t.Run(key.ResourceType+"/"+key.Metric, func(t *testing.T) { + plan := BuildActionPlanForFinding(CapacityFindingInput{ + FindingID: "f-" + key.ResourceType, + ResourceID: "res/" + key.ResourceType, + ResourceName: "test-" + key.ResourceType, + ResourceType: key.ResourceType, + Metric: key.Metric, + CurrentValue: 88.0, + PredictedValue: 95.0, + ThresholdValue: 90.0, + }) + if plan == nil { + t.Fatal("expected plan, got nil") + } + if !plan.RequiresApproval { + t.Errorf("RequiresApproval = false; templates MUST set RequiresApproval=true") + } + if plan.Allowed { + t.Errorf("Allowed = true; lane currently ships preflight-only proposals (no write capability wired). " + + "If a capability lands later, update this assertion alongside the template.") + } + if plan.ApprovalPolicy != unifiedresources.ApprovalAdmin { + t.Errorf("ApprovalPolicy = %q; want %q", plan.ApprovalPolicy, unifiedresources.ApprovalAdmin) + } + if plan.Message == "" { + t.Error("Message must be non-empty operator-facing description") + } + if plan.ActionID == "" { + t.Error("ActionID must be set") + } + if !strings.HasPrefix(plan.ActionID, "capacity-forecast-") { + t.Errorf("ActionID = %q; want capacity-forecast- prefix so audit can identify forecast-driven actions", plan.ActionID) + } + if plan.Preflight == nil { + t.Fatal("Preflight is required so operators can see the proposed change before approving") + } + if plan.Preflight.IntendedChange == "" { + t.Error("Preflight.IntendedChange must describe the proposed remediation") + } + if plan.Preflight.DryRunAvailable { + t.Error("Preflight.DryRunAvailable = true; no provider dry-run exists yet for these templates") + } + if len(plan.Preflight.SafetyChecks) == 0 { + t.Error("Preflight.SafetyChecks must be non-empty so the audit trail records the gate") + } + if len(plan.Preflight.VerificationSteps) == 0 { + t.Error("Preflight.VerificationSteps must be non-empty so post-action verification is documented") + } + if plan.PlannedAt.IsZero() { + t.Error("PlannedAt must be set") + } + if !plan.ExpiresAt.After(plan.PlannedAt) { + t.Errorf("ExpiresAt (%v) must be after PlannedAt (%v)", plan.ExpiresAt, plan.PlannedAt) + } + }) + } +} + +func TestBuildActionPlanForFinding_PBSDatastoreTemplateMessageMentionsPruneAndGC(t *testing.T) { + plan := BuildActionPlanForFinding(CapacityFindingInput{ + FindingID: "f-pbs-1", + ResourceID: "pbs-1/datastore/main", + ResourceName: "main", + ResourceType: "pbs-datastore", + Metric: "usage_percent", + CurrentValue: 91.4, + PredictedValue: 96.0, + ThresholdValue: 90.0, + }) + if plan == nil { + t.Fatal("expected plan, got nil") + } + msg := strings.ToLower(plan.Message) + if !strings.Contains(msg, "prune") { + t.Errorf("PBS template message missing prune verb: %q", plan.Message) + } + if !strings.Contains(msg, "garbage-collect") && !strings.Contains(msg, "garbage collect") && !strings.Contains(msg, "gc") { + t.Errorf("PBS template message missing garbage-collect verb: %q", plan.Message) + } + if !strings.Contains(plan.Message, "91.4") { + t.Errorf("PBS template message missing current value 91.4: %q", plan.Message) + } + if !strings.Contains(plan.Message, "96.0") { + t.Errorf("PBS template message missing projected value 96.0: %q", plan.Message) + } +} + +func TestBuildActionPlanForFinding_StoragePoolTemplateMessageMentionsSnapshotPrune(t *testing.T) { + plan := BuildActionPlanForFinding(CapacityFindingInput{ + FindingID: "f-zfs-1", + ResourceID: "node-a/storage/tank", + ResourceName: "tank", + ResourceType: "storage", + Metric: "usage_percent", + CurrentValue: 87.2, + PredictedValue: 93.5, + ThresholdValue: 90.0, + }) + if plan == nil { + t.Fatal("expected plan, got nil") + } + if !strings.Contains(strings.ToLower(plan.Message), "snapshot") { + t.Errorf("storage template message missing snapshot reference: %q", plan.Message) + } +} + +func TestBuildActionPlanForFinding_VMDiskTemplateMessageMentionsExpand(t *testing.T) { + plan := BuildActionPlanForFinding(CapacityFindingInput{ + FindingID: "f-vm-1", + ResourceID: "node-a/qemu/101", + ResourceName: "appserver", + ResourceType: "qemu", + Metric: "disk_usage_percent", + CurrentValue: 89.1, + PredictedValue: 95.0, + ThresholdValue: 90.0, + }) + if plan == nil { + t.Fatal("expected plan, got nil") + } + if !strings.Contains(strings.ToLower(plan.Message), "expand") { + t.Errorf("vm disk template message missing expand verb: %q", plan.Message) + } +} + +func TestBuildActionPlanForFinding_NormalizesCaseAndWhitespace(t *testing.T) { + plan := BuildActionPlanForFinding(CapacityFindingInput{ + FindingID: "f-norm", + ResourceType: " STORAGE ", + Metric: " Usage_Percent ", + CurrentValue: 80, + }) + if plan == nil { + t.Fatal("expected plan after case/whitespace normalization, got nil") + } +} + +func TestBuildActionPlanForFinding_RespectsExplicitNow(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + plan := BuildActionPlanForFinding(CapacityFindingInput{ + FindingID: "f-now", + ResourceType: "storage", + Metric: "usage_percent", + CurrentValue: 80, + Now: now, + }) + if plan == nil { + t.Fatal("expected plan, got nil") + } + if !plan.PlannedAt.Equal(now) { + t.Errorf("PlannedAt = %v; want %v (caller's Now must be honored for deterministic tests)", plan.PlannedAt, now) + } + wantExpiry := now.Add(proposalTTL) + if !plan.ExpiresAt.Equal(wantExpiry) { + t.Errorf("ExpiresAt = %v; want %v", plan.ExpiresAt, wantExpiry) + } +} + +func TestHasCapacityActionTemplate(t *testing.T) { + cases := []struct { + name string + resourceType string + metric string + want bool + }{ + {"pbs-datastore-usage", "pbs-datastore", "usage_percent", true}, + {"storage-usage", "storage", "usage_percent", true}, + {"qemu-disk", "qemu", "disk_usage_percent", true}, + {"vm-disk-alias", "vm", "disk_usage_percent", true}, + {"lxc-disk", "lxc", "disk_usage_percent", true}, + {"system-container-disk-alias", "system-container", "disk_usage_percent", true}, + {"unknown-pair", "node", "uptime_seconds", false}, + {"capacity-cpu-not-registered", "qemu", "cpu_percent", false}, + {"empty-strings", "", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := HasCapacityActionTemplate(tc.resourceType, tc.metric) + if got != tc.want { + t.Errorf("HasCapacityActionTemplate(%q, %q) = %v; want %v", tc.resourceType, tc.metric, got, tc.want) + } + }) + } +} + +func TestCapacityActionPlanSourceIsStable(t *testing.T) { + // The frontend FindingsPanel.tsx looks for this exact string to + // distinguish the capacity-forecast approval card from the generic + // remediation plan card. If you change this constant, update the + // frontend at the same time. + if CapacityActionPlanSource != "capacity_forecast" { + t.Fatalf("CapacityActionPlanSource = %q; frontend depends on the literal %q", CapacityActionPlanSource, "capacity_forecast") + } +}