From 6c491257f051686fa1d067458dc04f26e64a5654 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 27 Aug 2026 20:13:09 +0100 Subject: [PATCH 1/3] feat(patrol): verify Docker actions independently --- .../v6/internal/subsystems/agent-lifecycle.md | 10 + .../v6/internal/subsystems/ai-runtime.md | 11 + .../v6/internal/subsystems/api-contracts.md | 15 ++ .../subsystems/patrol-intelligence.md | 10 + .../agentexec/docker_observation_codec.go | 137 +++++++++++++ .../docker_observation_codec_test.go | 58 ++++++ internal/agentexec/server.go | 194 +++++++++++++----- internal/agentexec/server_websocket_test.go | 77 +++++++ internal/agentexec/types.go | 63 ++++-- .../api/docker_container_action_executor.go | 59 +++++- .../docker_container_action_executor_test.go | 46 +++++ .../docker_restart_colima_real_lab_test.go | 2 +- internal/hostagent/commands.go | 74 +++++-- .../hostagent/commands_registration_test.go | 3 + internal/hostagent/docker_lifecycle.go | 4 + internal/hostagent/docker_lifecycle_test.go | 18 ++ .../runtime_surface_audit_test.go | 2 + 17 files changed, 691 insertions(+), 92 deletions(-) create mode 100644 internal/agentexec/docker_observation_codec.go create mode 100644 internal/agentexec/docker_observation_codec_test.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index ff1fbca17..fb01c3810 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -1671,6 +1671,16 @@ the intentionally sparse public response. It may render only the strict redacted projection produced by the matching managed-runtime run and must reject missing or mismatched proposal, action, attempt, receipt, finding, and evidence identities. + Unified Agent registration separately advertises the versioned, read-only + Docker observation protocol. `docker_container_observe` is not an action, + preflight grant, or durable operation: it accepts only an action-bound + digest, runtime enum, and immutable container ID, invokes the fixed local + Docker/Podman lifecycle inspector, and returns either one bounded snapshot + or a typed inconclusive reason. It must never enter the operation receipt + store, execute a mutation, accept command text, or infer action truth. The + server validates freshness and correlation and owns all evidence-class and + verification projection. Older agents advertise version zero and therefore + fail closed without receiving an unsupported request. RG-06 has a separate mutation harness at `scripts/intelligence_lab/patrol_autonomy_colima.py` and `internal/api/patrol_autonomy_colima_real_lab_test.go`. It must run from a diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 07388d1d9..aa45a7eb1 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -7258,6 +7258,17 @@ and terminal push copy stays explicitly inconclusive. This keeps legacy single-outcome consumers conservative while `ActionDisposition.ActionResultV2` retains both truth axes and the evidence source. +The production Docker/Podman Patrol action path now supplies the independent +postcondition that this rule requires. After a successful typed lifecycle or +update receipt, the API executor performs a separate action-bound daemon +observation through the versioned read-only Unified Agent protocol. Only a +fresh, identity-matched daemon snapshot from the distinct daemon observer +trust domain can project to `fix_verified`; unsupported older agents, +inspection failure, stale facts, or mismatched identities remain +`fix_verification_unknown`. The mutation receipt's own same-agent readback +continues to project only `agent_attested` evidence, so adding the production +observer does not loosen the verification rule. + ### Proxmox guest lifecycle Patrol detector floor `internal/ai/findings_proxmox_lifecycle.go` owns the deterministic production diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 5a3f85ad6..15de29c53 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -8935,6 +8935,21 @@ the action, subject, observation and receipt times, before/after facts, and canonical evidence digest. It is independent operational evidence, not a cryptographic attestation. +Production Docker and Podman lifecycle execution must install that observer, +not leave it as a real-lab-only seam. After the typed mutation receipt returns, +the API executor issues one separate, bounded `docker_container_observe` +request through the authenticated Unified Agent session. The request contains +only an action ID, immutable container ID, runtime enum, protocol version and +canonical digest; it carries no operation verb, command, approval, dispatch +lease, or mutation authority. The agent transports a fresh fixed daemon +inspection but does not author action truth. The API validates the action and +subject binding, freshness, protocol version and digest, assigns the daemon +observer trust domain, and alone projects the observation into +`ActionResultV2`. Missing support, inspection failure, disconnect, timeout, +stale evidence, or identity drift retains the original agent-attested or +inconclusive result. It must never downgrade execution success or fabricate +independence. + Proxmox VM and LXC lifecycle execution now consumes that same two-axis truth contract in production. The node agent remains the executor, while the API composition root injects the tenant-scoped monitoring client as a direct diff --git a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md index 974b5885b..42ef1efc1 100644 --- a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md +++ b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md @@ -2400,6 +2400,16 @@ not-attempted or inconclusive verification remains verification unknown. A successful execution never overwrites contradictory verification. Task 11 still owns browser wording and proof for the distinct terminal states. +Docker/Podman Patrol remediation must have a production post-action observer, +not only a qualification-lab injection. The canonical action executor issues a +separate versioned, action-bound daemon observation after the typed mutation +receipt. Only fresh independent daemon evidence may advance the originating +investigation and finding to `fix_verified`; older agents, observation errors, +stale snapshots, digest or subject mismatch, and same-agent receipt evidence +remain verification unknown. This closes the prior structural dead end where +an unhealthy-container restart could execute in production but the executor +had no observer capable of satisfying Patrol's verified-outcome contract. + Patrol now consumes the server-derived effective Autopilot mode. Requested `full` is admitted only with a current persisted human acknowledgement and exact activation for the same actor credential and organization; legacy diff --git a/internal/agentexec/docker_observation_codec.go b/internal/agentexec/docker_observation_codec.go new file mode 100644 index 000000000..bd73ff79b --- /dev/null +++ b/internal/agentexec/docker_observation_codec.go @@ -0,0 +1,137 @@ +package agentexec + +import ( + "fmt" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt" +) + +const DockerContainerObservationProtocolVersion = 1 + +const dockerContainerObservationMaxClockSkew = 5 * time.Minute + +func DecodeDockerContainerObservationPayload(data []byte) (DockerContainerObservationPayload, error) { + var payload DockerContainerObservationPayload + if err := decodeStrictDockerLifecycle(data, &payload); err != nil { + return DockerContainerObservationPayload{}, err + } + if err := ValidateDockerContainerObservationPayload(&payload); err != nil { + return DockerContainerObservationPayload{}, err + } + return payload, nil +} + +func DecodeDockerContainerObservationResultPayload(data []byte) (DockerContainerObservationResultPayload, error) { + var result DockerContainerObservationResultPayload + if err := decodeStrictDockerLifecycle(data, &result); err != nil { + return DockerContainerObservationResultPayload{}, err + } + if err := ValidateDockerContainerObservationResultPayload(&result); err != nil { + return DockerContainerObservationResultPayload{}, err + } + return result, nil +} + +func BindDockerContainerObservationPayload(payload *DockerContainerObservationPayload) error { + if payload == nil { + return fmt.Errorf("docker container observation payload is required") + } + payload.ProtocolVersion = DockerContainerObservationProtocolVersion + digest, err := dockerContainerObservationRequestDigest(*payload) + if err != nil { + return err + } + payload.RequestDigest = digest + return nil +} + +func dockerContainerObservationRequestDigest(payload DockerContainerObservationPayload) (string, error) { + return operationreceipt.DigestCanonicalJSON(struct { + ActionID string `json:"action_id"` + ProtocolVersion int `json:"protocol_version"` + Runtime string `json:"runtime"` + ContainerID string `json:"container_id"` + }{strings.TrimSpace(payload.ActionID), payload.ProtocolVersion, strings.ToLower(strings.TrimSpace(payload.Runtime)), strings.ToLower(strings.TrimSpace(payload.ContainerID))}) +} + +func ValidateDockerContainerObservationPayload(payload *DockerContainerObservationPayload) error { + if payload == nil { + return fmt.Errorf("docker container observation payload is required") + } + payload.RequestID = strings.TrimSpace(payload.RequestID) + payload.ActionID = strings.TrimSpace(payload.ActionID) + payload.RequestDigest = strings.TrimSpace(payload.RequestDigest) + payload.Runtime = strings.ToLower(strings.TrimSpace(payload.Runtime)) + payload.ContainerID = strings.ToLower(strings.TrimSpace(payload.ContainerID)) + if payload.RequestID == "" || len(payload.RequestID) > maxRequestIDLength || payload.ActionID == "" || len(payload.ActionID) > maxRequestIDLength { + return fmt.Errorf("invalid docker observation request identity") + } + if payload.ProtocolVersion != DockerContainerObservationProtocolVersion { + return fmt.Errorf("unsupported docker observation protocol version %d", payload.ProtocolVersion) + } + if payload.Runtime != "docker" && payload.Runtime != "podman" { + return fmt.Errorf("unsupported container runtime %q", payload.Runtime) + } + if !dockerContainerIDPattern.MatchString(payload.ContainerID) { + return fmt.Errorf("container id must be an immutable hexadecimal id") + } + digest, err := dockerContainerObservationRequestDigest(*payload) + if err != nil { + return err + } + if payload.RequestDigest != digest { + return fmt.Errorf("docker observation request digest mismatch") + } + return nil +} + +func ValidateDockerContainerObservationResultPayload(result *DockerContainerObservationResultPayload) error { + if result == nil { + return fmt.Errorf("docker container observation result is required") + } + result.RequestID = strings.TrimSpace(result.RequestID) + result.ActionID = strings.TrimSpace(result.ActionID) + result.RequestDigest = strings.TrimSpace(result.RequestDigest) + result.ReasonCode = strings.TrimSpace(result.ReasonCode) + result.Snapshot.ContainerID = strings.ToLower(strings.TrimSpace(result.Snapshot.ContainerID)) + result.Snapshot.State = strings.ToLower(strings.TrimSpace(result.Snapshot.State)) + result.Snapshot.ObservedAt = result.Snapshot.ObservedAt.UTC() + result.Snapshot.StartedAt = result.Snapshot.StartedAt.UTC() + if result.RequestID == "" || len(result.RequestID) > maxRequestIDLength || result.ActionID == "" || len(result.ActionID) > maxRequestIDLength { + return fmt.Errorf("invalid docker observation result identity") + } + if result.ProtocolVersion != DockerContainerObservationProtocolVersion || !hostUpdateInventoryHashPattern.MatchString(result.RequestDigest) { + return fmt.Errorf("invalid docker observation result binding") + } + if !result.Observed { + if !IsActionRefusalReasonCode(result.ReasonCode) || result.Snapshot.ContainerID != "" || result.Snapshot.State != "" || result.Snapshot.Running || !result.Snapshot.StartedAt.IsZero() || result.Snapshot.RestartCount != 0 || !result.Snapshot.ObservedAt.IsZero() { + return fmt.Errorf("inconclusive docker observation requires a bounded reason and no snapshot") + } + return nil + } + if result.ReasonCode != "" || !dockerContainerIDPattern.MatchString(result.Snapshot.ContainerID) || result.Snapshot.State == "" || len(result.Snapshot.State) > 32 || result.Snapshot.RestartCount < 0 || result.Snapshot.ObservedAt.IsZero() { + return fmt.Errorf("invalid docker observation snapshot") + } + return nil +} + +func ValidateDockerContainerObservationResultForRequest(req DockerContainerObservationPayload, result DockerContainerObservationResultPayload, receivedAt time.Time) error { + if err := ValidateDockerContainerObservationPayload(&req); err != nil { + return err + } + if err := ValidateDockerContainerObservationResultPayload(&result); err != nil { + return err + } + if result.RequestID != req.RequestID || result.ActionID != req.ActionID || result.ProtocolVersion != req.ProtocolVersion || result.RequestDigest != req.RequestDigest || (result.Observed && result.Snapshot.ContainerID != req.ContainerID) { + return fmt.Errorf("docker observation result identity mismatch") + } + if !result.Observed { + return nil + } + if receivedAt.IsZero() || result.Snapshot.ObservedAt.After(receivedAt.UTC().Add(dockerContainerObservationMaxClockSkew)) || result.Snapshot.ObservedAt.Before(receivedAt.UTC().Add(-15*time.Minute)) { + return fmt.Errorf("docker observation result is stale or clock-skewed") + } + return nil +} diff --git a/internal/agentexec/docker_observation_codec_test.go b/internal/agentexec/docker_observation_codec_test.go new file mode 100644 index 000000000..0d79315f0 --- /dev/null +++ b/internal/agentexec/docker_observation_codec_test.go @@ -0,0 +1,58 @@ +package agentexec + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestDockerContainerObservationContractBindsRequestAndFreshResult(t *testing.T) { + now := time.Now().UTC() + req := DockerContainerObservationPayload{ + RequestID: "observe-1", ActionID: "action-1", Runtime: "docker", ContainerID: strings.Repeat("a", 64), + } + if err := BindDockerContainerObservationPayload(&req); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + decoded, err := DecodeDockerContainerObservationPayload(encoded) + if err != nil { + t.Fatal(err) + } + result := DockerContainerObservationResultPayload{ + RequestID: decoded.RequestID, ActionID: decoded.ActionID, ProtocolVersion: decoded.ProtocolVersion, RequestDigest: decoded.RequestDigest, + Observed: true, + Snapshot: DockerContainerLifecycleSnapshot{ContainerID: decoded.ContainerID, State: "running", Running: true, ObservedAt: now}, + } + if err := ValidateDockerContainerObservationResultForRequest(decoded, result, now); err != nil { + t.Fatalf("fresh bound result rejected: %v", err) + } + + tampered := result + tampered.ActionID = "different-action" + if err := ValidateDockerContainerObservationResultForRequest(decoded, tampered, now); err == nil { + t.Fatal("cross-action observation result was accepted") + } + stale := result + stale.Snapshot.ObservedAt = now.Add(-16 * time.Minute) + if err := ValidateDockerContainerObservationResultForRequest(decoded, stale, now); err == nil { + t.Fatal("stale observation result was accepted") + } +} + +func TestDockerContainerObservationWireCarriesFactsNotActionTruth(t *testing.T) { + payload, err := json.Marshal(DockerContainerObservationResultPayload{}) + if err != nil { + t.Fatal(err) + } + encoded := string(payload) + for _, forbidden := range []string{`"success"`, `"verification"`, `"evidence"`, `"independent"`, `"command"`, `"operation"`, `"approval"`} { + if strings.Contains(encoded, forbidden) { + t.Fatalf("observation wire contains forbidden action-truth or authority field %s: %s", forbidden, encoded) + } + } +} diff --git a/internal/agentexec/server.go b/internal/agentexec/server.go index ad9e30c52..87c37c4ce 100644 --- a/internal/agentexec/server.go +++ b/internal/agentexec/server.go @@ -61,29 +61,30 @@ var hostStorageCleanupFingerprintPattern = regexp.MustCompile(`^sha256:[a-f0-9]{ // Server manages WebSocket connections from agents type Server struct { - mu sync.RWMutex - agents map[string]*agentConn // organizationID + agentID -> connection - pendingReqs map[string]chan CommandResultPayload // scoped request key -> response channel - pendingHostStorageCleanups map[string]chan HostStorageCleanupResultPayload // scoped request key -> typed storage-cleanup response - pendingHostUpdates map[string]chan HostUpdateResultPayload // scoped request key -> typed host-update response - pendingDockerContainerLifecycles map[string]chan DockerContainerLifecycleResultPayload - pendingDockerContainerUpdates map[string]chan DockerContainerUpdateResultPayload - pendingActionPreflights map[string]chan ActionPreflightResultPayload - pendingHostOperations map[string]pendingHostOperation // scoped request key -> exact typed APT operation/query identity - pendingOperationQueries map[string]pendingOperationQuery - deploySubs map[string]chan DeployProgressPayload // deploySubKey(agentID, jobID) -> progress subscriber - admitToken AgentRegistrationValidator - validateSession AgentSessionValidator - commandPolicy *CommandPolicy - ipConnCounts map[string]int - maxConnsPerIP int - shutdown chan struct{} - shutdownOnce sync.Once - pingInterval time.Duration - commandAuthorizationVerifier func(CommandAuthorizationRequest) error - newCommandApprovalGrant func([]byte, string, ExecuteCommandPayload, time.Time, time.Duration) (*CommandApprovalGrant, error) - now func() time.Time - agentRegisteredNotifier func(AgentAdmission) + mu sync.RWMutex + agents map[string]*agentConn // organizationID + agentID -> connection + pendingReqs map[string]chan CommandResultPayload // scoped request key -> response channel + pendingHostStorageCleanups map[string]chan HostStorageCleanupResultPayload // scoped request key -> typed storage-cleanup response + pendingHostUpdates map[string]chan HostUpdateResultPayload // scoped request key -> typed host-update response + pendingDockerContainerLifecycles map[string]chan DockerContainerLifecycleResultPayload + pendingDockerContainerUpdates map[string]chan DockerContainerUpdateResultPayload + pendingDockerContainerObservations map[string]chan DockerContainerObservationResultPayload + pendingActionPreflights map[string]chan ActionPreflightResultPayload + pendingHostOperations map[string]pendingHostOperation // scoped request key -> exact typed APT operation/query identity + pendingOperationQueries map[string]pendingOperationQuery + deploySubs map[string]chan DeployProgressPayload // deploySubKey(agentID, jobID) -> progress subscriber + admitToken AgentRegistrationValidator + validateSession AgentSessionValidator + commandPolicy *CommandPolicy + ipConnCounts map[string]int + maxConnsPerIP int + shutdown chan struct{} + shutdownOnce sync.Once + pingInterval time.Duration + commandAuthorizationVerifier func(CommandAuthorizationRequest) error + newCommandApprovalGrant func([]byte, string, ExecuteCommandPayload, time.Time, time.Duration) (*CommandApprovalGrant, error) + now func() time.Time + agentRegisteredNotifier func(AgentAdmission) } const defaultOrganizationID = "default" @@ -190,25 +191,26 @@ func NewServerWithAdmissionValidator(admit AgentRegistrationValidator, validateS } return &Server{ - agents: make(map[string]*agentConn), - pendingReqs: make(map[string]chan CommandResultPayload), - pendingHostStorageCleanups: make(map[string]chan HostStorageCleanupResultPayload), - pendingHostUpdates: make(map[string]chan HostUpdateResultPayload), - pendingDockerContainerLifecycles: make(map[string]chan DockerContainerLifecycleResultPayload), - pendingDockerContainerUpdates: make(map[string]chan DockerContainerUpdateResultPayload), - pendingActionPreflights: make(map[string]chan ActionPreflightResultPayload), - pendingHostOperations: make(map[string]pendingHostOperation), - pendingOperationQueries: make(map[string]pendingOperationQuery), - deploySubs: make(map[string]chan DeployProgressPayload), - admitToken: admit, - validateSession: validateSession, - commandPolicy: DefaultPolicy(), - ipConnCounts: make(map[string]int), - maxConnsPerIP: defaultMaxWebSocketConnectionsPerIP, - shutdown: make(chan struct{}), - pingInterval: defaultPingInterval, - newCommandApprovalGrant: NewCommandApprovalGrant, - now: time.Now, + agents: make(map[string]*agentConn), + pendingReqs: make(map[string]chan CommandResultPayload), + pendingHostStorageCleanups: make(map[string]chan HostStorageCleanupResultPayload), + pendingHostUpdates: make(map[string]chan HostUpdateResultPayload), + pendingDockerContainerLifecycles: make(map[string]chan DockerContainerLifecycleResultPayload), + pendingDockerContainerUpdates: make(map[string]chan DockerContainerUpdateResultPayload), + pendingDockerContainerObservations: make(map[string]chan DockerContainerObservationResultPayload), + pendingActionPreflights: make(map[string]chan ActionPreflightResultPayload), + pendingHostOperations: make(map[string]pendingHostOperation), + pendingOperationQueries: make(map[string]pendingOperationQuery), + deploySubs: make(map[string]chan DeployProgressPayload), + admitToken: admit, + validateSession: validateSession, + commandPolicy: DefaultPolicy(), + ipConnCounts: make(map[string]int), + maxConnsPerIP: defaultMaxWebSocketConnectionsPerIP, + shutdown: make(chan struct{}), + pingInterval: defaultPingInterval, + newCommandApprovalGrant: NewCommandApprovalGrant, + now: time.Now, } } @@ -967,16 +969,17 @@ func (s *Server) HandleWebSocket(w http.ResponseWriter, r *http.Request) { ac := &agentConn{ conn: conn, agent: ConnectedAgent{ - OrganizationID: admission.OrganizationID, - TokenID: admission.TokenID, - AgentID: admission.AgentID, - Hostname: admission.Hostname, - Version: reg.Version, - Platform: reg.Platform, - Tags: reg.Tags, - ConnectedAt: time.Now(), - OperationReceiptVersion: reg.OperationReceiptVersion, - ActionPreflightVersion: reg.ActionPreflightVersion, + OrganizationID: admission.OrganizationID, + TokenID: admission.TokenID, + AgentID: admission.AgentID, + Hostname: admission.Hostname, + Version: reg.Version, + Platform: reg.Platform, + Tags: reg.Tags, + ConnectedAt: time.Now(), + OperationReceiptVersion: reg.OperationReceiptVersion, + ActionPreflightVersion: reg.ActionPreflightVersion, + DockerObservationVersion: reg.DockerObservationVersion, }, admission: admission, sessionKey: agentSessionKey(admission.OrganizationID, admission.AgentID), @@ -1264,6 +1267,23 @@ func (s *Server) readLoop(ac *agentConn) { } } + case MsgTypeDockerContainerObserveResult: + result, decodeErr := DecodeDockerContainerObservationResultPayload(msg.Payload) + if decodeErr != nil { + log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid docker container observation result") + continue + } + s.mu.RLock() + ch, ok := s.pendingDockerContainerObservations[pendingRequestKey(connectionSessionKey(ac), result.RequestID)] + s.mu.RUnlock() + if ok { + select { + case ch <- result: + default: + log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Docker observation result channel full, dropping") + } + } + case MsgTypeHostStorageCleanupResult: result, decodeErr := DecodeHostStorageCleanupResultPayload(msg.Payload) if decodeErr != nil { @@ -1893,6 +1913,76 @@ func (s *Server) PreflightAction(ctx context.Context, agentID string, req Action } } +// ObserveDockerContainer asks the current Unified Agent for a fresh read-only +// Docker/Podman daemon observation. It is a separate request from the mutation +// receipt and carries no dispatch authority. +func (s *Server) ObserveDockerContainer(ctx context.Context, agentID string, req DockerContainerObservationPayload) (*DockerContainerObservationResultPayload, error) { + if s == nil { + return nil, fmt.Errorf("agent execution server is unavailable") + } + agentID = strings.TrimSpace(agentID) + if agentID == "" { + return nil, fmt.Errorf("agent id is required") + } + if strings.TrimSpace(req.RequestID) == "" { + req.RequestID = uuid.NewString() + } + if err := BindDockerContainerObservationPayload(&req); err != nil { + return nil, err + } + if err := ValidateDockerContainerObservationPayload(&req); err != nil { + return nil, err + } + ac, ok := s.connectionForContext(ctx, agentID) + if !ok { + return nil, fmt.Errorf("agent %s not connected", agentID) + } + if ac.agent.DockerObservationVersion != DockerContainerObservationProtocolVersion { + return nil, fmt.Errorf("agent does not support docker observation protocol") + } + ch := make(chan DockerContainerObservationResultPayload, 1) + key := pendingRequestKey(connectionSessionKey(ac), req.RequestID) + s.mu.Lock() + if _, exists := s.pendingDockerContainerObservations[key]; exists { + s.mu.Unlock() + return nil, fmt.Errorf("docker observation request %q is already pending", req.RequestID) + } + s.pendingDockerContainerObservations[key] = ch + s.mu.Unlock() + defer func() { + s.mu.Lock() + delete(s.pendingDockerContainerObservations, key) + s.mu.Unlock() + }() + msg, err := NewMessage(MsgTypeDockerContainerObserve, req.RequestID, req) + if err != nil { + return nil, err + } + ac.writeMu.Lock() + err = s.sendMessage(ac.conn, msg) + ac.writeMu.Unlock() + if err != nil { + return nil, fmt.Errorf("failed to send docker observation request: %w", err) + } + timer := time.NewTimer(20 * time.Second) + defer timer.Stop() + select { + case result := <-ch: + if err := ValidateDockerContainerObservationResultForRequest(req, result, s.currentTime()); err != nil { + return nil, fmt.Errorf("docker observation result validation failed: %w", err) + } + return &result, nil + case <-timer.C: + return nil, fmt.Errorf("docker observation timed out") + case <-ctx.Done(): + return nil, ctx.Err() + case <-ac.done: + return nil, fmt.Errorf("agent %s disconnected before docker observation result", agentID) + case <-s.shutdown: + return nil, errServerShuttingDown + } +} + // ExecuteHostStorageCleanup dispatches the closed package-cache cleanup // operation. No command text, path, package selector, or removal policy crosses // the server/agent boundary. diff --git a/internal/agentexec/server_websocket_test.go b/internal/agentexec/server_websocket_test.go index 92907e890..c4fba9f0e 100644 --- a/internal/agentexec/server_websocket_test.go +++ b/internal/agentexec/server_websocket_test.go @@ -749,6 +749,83 @@ func TestActionPreflightRoundTripIsReadOnlyAndDigestBound(t *testing.T) { } } +func TestDockerContainerObservationRoundTripIsReadOnlyAndActionBound(t *testing.T) { + s := NewServer(allowAllTestTokens) + ts := newWSServer(t, s) + defer ts.Close() + conn, _, err := dialAgentExecWebSocket(t, ts.URL) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + wsWriteMessage(t, conn, mustNewMessage(t, MsgTypeAgentRegister, "", AgentRegisterPayload{ + AgentID: "docker-observer", Hostname: "host1", Version: "6", Platform: "linux", Token: "any", + DockerObservationVersion: DockerContainerObservationProtocolVersion, + })) + _ = wsReadRegisteredPayload(t, conn) + containerID := strings.Repeat("a", 64) + agentErr := make(chan error, 1) + go func() { + msg, readErr := wsReadRawMessageWithTimeout(conn, 2*time.Second) + if readErr != nil { + agentErr <- readErr + return + } + if msg.Type != MsgTypeDockerContainerObserve || bytes.Contains(*msg.Payload, []byte(`"command"`)) || bytes.Contains(*msg.Payload, []byte(`"operation"`)) { + agentErr <- fmt.Errorf("unexpected docker observation envelope: %#v", msg) + return + } + payload, decodeErr := DecodeDockerContainerObservationPayload(*msg.Payload) + if decodeErr != nil { + agentErr <- decodeErr + return + } + result := DockerContainerObservationResultPayload{ + RequestID: payload.RequestID, ActionID: payload.ActionID, ProtocolVersion: payload.ProtocolVersion, RequestDigest: payload.RequestDigest, + Observed: true, + Snapshot: DockerContainerLifecycleSnapshot{ContainerID: payload.ContainerID, State: "running", Running: true, ObservedAt: time.Now().UTC()}, + } + agentErr <- conn.WriteJSON(mustNewMessage(t, MsgTypeDockerContainerObserveResult, payload.RequestID, result)) + }() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result, err := s.ObserveDockerContainer(ctx, "docker-observer", DockerContainerObservationPayload{ActionID: "action-1", Runtime: "docker", ContainerID: containerID}) + if err != nil { + t.Fatal(err) + } + if result.ActionID != "action-1" || result.Snapshot.ContainerID != containerID || !result.Snapshot.Running { + t.Fatalf("result=%#v", result) + } + if err := <-agentErr; err != nil { + t.Fatal(err) + } +} + +func TestDockerContainerObservationRejectsUnsupportedAgentBeforeDispatch(t *testing.T) { + s := NewServer(allowAllTestTokens) + ts := newWSServer(t, s) + defer ts.Close() + conn, _, err := dialAgentExecWebSocket(t, ts.URL) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + wsWriteMessage(t, conn, mustNewMessage(t, MsgTypeAgentRegister, "", AgentRegisterPayload{ + AgentID: "older-agent", Hostname: "host1", Version: "6.3", Platform: "linux", Token: "any", + })) + _ = wsReadRegisteredPayload(t, conn) + started := time.Now() + _, err = s.ObserveDockerContainer(context.Background(), "older-agent", DockerContainerObservationPayload{ + ActionID: "action-1", Runtime: "docker", ContainerID: strings.Repeat("a", 64), + }) + if err == nil || !strings.Contains(err.Error(), "does not support docker observation protocol") { + t.Fatalf("error=%v", err) + } + if time.Since(started) > time.Second { + t.Fatalf("unsupported agent rejection was not immediate: %s", time.Since(started)) + } +} + func TestValidateHostUpdatePayloadRejectsOpenEndedAuthority(t *testing.T) { for _, req := range []HostUpdatePayload{ {RequestID: "r1", ActionID: "a1", Operation: "run_command", ExpectedInventoryHash: "sha256:" + strings.Repeat("a", 64)}, diff --git a/internal/agentexec/types.go b/internal/agentexec/types.go index b951d760e..014dcf9ab 100644 --- a/internal/agentexec/types.go +++ b/internal/agentexec/types.go @@ -19,6 +19,7 @@ const ( MsgTypeHostUpdateResult MessageType = "host_update_result" MsgTypeDockerContainerLifecycleResult MessageType = "docker_container_lifecycle_result" MsgTypeDockerContainerUpdateResult MessageType = "docker_container_update_result" + MsgTypeDockerContainerObserveResult MessageType = "docker_container_observe_result" MsgTypeActionPreflightResult MessageType = "action_preflight_result" MsgTypeOperationQueryResult MessageType = "agent_operation_query_result" @@ -31,6 +32,7 @@ const ( MsgTypeHostUpdate MessageType = "host_update" MsgTypeDockerContainerLifecycle MessageType = "docker_container_lifecycle" MsgTypeDockerContainerUpdate MessageType = "docker_container_update" + MsgTypeDockerContainerObserve MessageType = "docker_container_observe" MsgTypeActionPreflight MessageType = "action_preflight" MsgTypeOperationQuery MessageType = "agent_operation_query" MsgTypeDeployPreflight MessageType = "deploy_preflight" @@ -88,14 +90,15 @@ func (m Message) DecodePayload(target any) error { // AgentRegisterPayload is sent by agent on connection type AgentRegisterPayload struct { - AgentID string `json:"agent_id"` - Hostname string `json:"hostname"` - Version string `json:"version"` - Platform string `json:"platform"` // "linux", "windows", "darwin" - Tags []string `json:"tags,omitempty"` - Token string `json:"token"` // API token for authentication - OperationReceiptVersion int `json:"operation_receipt_version,omitempty"` - ActionPreflightVersion int `json:"action_preflight_version,omitempty"` + AgentID string `json:"agent_id"` + Hostname string `json:"hostname"` + Version string `json:"version"` + Platform string `json:"platform"` // "linux", "windows", "darwin" + Tags []string `json:"tags,omitempty"` + Token string `json:"token"` // API token for authentication + OperationReceiptVersion int `json:"operation_receipt_version,omitempty"` + ActionPreflightVersion int `json:"action_preflight_version,omitempty"` + DockerObservationVersion int `json:"docker_observation_version,omitempty"` } // RegisteredPayload is sent by server after successful registration @@ -205,6 +208,29 @@ type DockerContainerLifecycleSnapshot struct { ObservedAt time.Time `json:"observed_at"` } +// DockerContainerObservationPayload is a server-initiated, read-only daemon +// observation made after a typed mutation has returned. It is deliberately a +// separate protocol from the mutation receipt so verification is not authored +// by the executor result. +type DockerContainerObservationPayload struct { + RequestID string `json:"request_id"` + ActionID string `json:"action_id"` + ProtocolVersion int `json:"protocol_version"` + RequestDigest string `json:"request_digest"` + Runtime string `json:"runtime"` + ContainerID string `json:"container_id"` +} + +type DockerContainerObservationResultPayload struct { + RequestID string `json:"request_id"` + ActionID string `json:"action_id"` + ProtocolVersion int `json:"protocol_version"` + RequestDigest string `json:"request_digest"` + Observed bool `json:"observed"` + ReasonCode string `json:"reason_code,omitempty"` + Snapshot DockerContainerLifecycleSnapshot `json:"snapshot"` +} + type DockerContainerLifecycleResultPayload struct { RequestID string `json:"request_id"` ActionID string `json:"action_id"` @@ -480,16 +506,17 @@ func IsActionRefusalReasonCode(code string) bool { // ConnectedAgent represents an agent connected via WebSocket type ConnectedAgent struct { - OrganizationID string - TokenID string - AgentID string - Hostname string - Version string - Platform string - Tags []string - ConnectedAt time.Time - OperationReceiptVersion int - ActionPreflightVersion int + OrganizationID string + TokenID string + AgentID string + Hostname string + Version string + Platform string + Tags []string + ConnectedAt time.Time + OperationReceiptVersion int + ActionPreflightVersion int + DockerObservationVersion int } // --- Deploy protocol payloads --- diff --git a/internal/api/docker_container_action_executor.go b/internal/api/docker_container_action_executor.go index 8b1750e5d..edd415a94 100644 --- a/internal/api/docker_container_action_executor.go +++ b/internal/api/docker_container_action_executor.go @@ -34,7 +34,34 @@ type dockerContainerPostconditionObservation struct { } type dockerContainerPostconditionObserver interface { - ObserveDockerContainer(context.Context, string, string) (dockerContainerPostconditionObservation, error) + ObserveDockerContainer(context.Context, string, string, string, string) (dockerContainerPostconditionObservation, error) +} + +type dockerContainerObservationCommander interface { + ObserveDockerContainer(context.Context, string, agentexec.DockerContainerObservationPayload) (*agentexec.DockerContainerObservationResultPayload, error) +} + +type agentDockerContainerPostconditionObserver struct { + commander dockerContainerObservationCommander +} + +func (o agentDockerContainerPostconditionObserver) ObserveDockerContainer(ctx context.Context, actionID, agentID, runtime, containerID string) (dockerContainerPostconditionObservation, error) { + result, err := o.commander.ObserveDockerContainer(ctx, agentID, agentexec.DockerContainerObservationPayload{ + ActionID: actionID, Runtime: runtime, ContainerID: containerID, + }) + if err != nil { + return dockerContainerPostconditionObservation{}, err + } + if result == nil { + return dockerContainerPostconditionObservation{}, fmt.Errorf("docker daemon observation unavailable") + } + if !result.Observed { + return dockerContainerPostconditionObservation{}, fmt.Errorf("docker daemon observation unavailable: %s", strings.TrimSpace(result.ReasonCode)) + } + return dockerContainerPostconditionObservation{ + ObserverID: "docker-daemon:" + agentID, TrustDomain: "docker-daemon:" + agentID, + Method: "typed_docker_daemon_observation", Snapshot: result.Snapshot, ReceivedAt: time.Now().UTC(), + }, nil } type dockerContainerLifecycleAgentCommander interface { @@ -106,7 +133,11 @@ func newDockerContainerActionExecutor(resources *ResourceHandlers, agents action if resources == nil || agents == nil { return nil } - return dockerContainerActionExecutor{resources: resources, agents: agents} + executor := dockerContainerActionExecutor{resources: resources, agents: agents} + if commander, ok := agents.(dockerContainerObservationCommander); ok { + executor.observer = agentDockerContainerPostconditionObserver{commander: commander} + } + return executor } func (e dockerContainerActionExecutor) ActionHandlerNames() []string { @@ -183,7 +214,7 @@ func (e dockerContainerActionExecutor) ExecuteAction(ctx context.Context, record } var independent *dockerContainerPostconditionObservation if e.observer != nil { - if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, req.ContainerID); observeErr == nil { + if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, agentID, runtime, req.ContainerID); observeErr == nil { independent = &observation } } @@ -231,7 +262,7 @@ func (e dockerContainerActionExecutor) executeDockerContainerUpdate(ctx context. } var independent *dockerContainerPostconditionObservation if e.observer != nil && result.NewContainerID != "" { - if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, result.NewContainerID); observeErr == nil { + if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, agentID, runtime, result.NewContainerID); observeErr == nil { independent = &observation } } @@ -287,7 +318,7 @@ func (e dockerContainerActionExecutor) ReconcileActionDispatch(ctx context.Conte } var independent *dockerContainerPostconditionObservation if e.observer != nil && result.NewContainerID != "" { - if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, result.NewContainerID); observeErr == nil { + if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, attempt.AgentID, e.dockerObservationRuntime(ctx, record), result.NewContainerID); observeErr == nil { independent = &observation } } @@ -305,7 +336,7 @@ func (e dockerContainerActionExecutor) ReconcileActionDispatch(ctx context.Conte req := agentexec.DockerContainerLifecyclePayload{RequestID: attempt.ID, ActionID: record.ID, Operation: attempt.OperationKind, OperationVersion: attempt.OperationVersion, RequestDigest: attempt.RequestDigest, Runtime: "docker", ContainerID: result.ContainerID, ExpectedState: result.Before.State, ExpectedStartedAt: result.Before.StartedAt} var independent *dockerContainerPostconditionObservation if e.observer != nil { - if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, result.ContainerID); observeErr == nil { + if observation, observeErr := e.observer.ObserveDockerContainer(ctx, record.ID, attempt.AgentID, e.dockerObservationRuntime(ctx, record), result.ContainerID); observeErr == nil { independent = &observation } } @@ -317,6 +348,22 @@ func (e dockerContainerActionExecutor) ReconcileActionDispatch(ctx context.Conte return execution, receipt, true, nil } +func (e dockerContainerActionExecutor) dockerObservationRuntime(ctx context.Context, record unified.ActionAuditRecord) string { + if e.resources != nil { + if registry, err := e.resources.buildRegistry(GetOrgID(ctx)); err == nil { + if resource, ok := registry.Get(record.Request.ResourceID); ok && resource != nil { + if runtime, runtimeErr := dockerContainerRuntime(*resource); runtimeErr == nil { + return runtime + } + } + } + } + // Older durable attempts did not retain runtime separately. Docker is the + // compatibility default; a Podman target with no current resource fails + // the observation safely and retains agent-attested truth. + return "docker" +} + func (e dockerContainerActionExecutor) CheckActionAvailable(ctx context.Context, req unified.ActionRequest, resource unified.Resource) unified.ResourceActionReadiness { operation := strings.TrimSpace(req.CapabilityName) capability, ok := findDockerLifecycleCapability(resource.Capabilities, operation) diff --git a/internal/api/docker_container_action_executor_test.go b/internal/api/docker_container_action_executor_test.go index 3b0c01890..e2040e519 100644 --- a/internal/api/docker_container_action_executor_test.go +++ b/internal/api/docker_container_action_executor_test.go @@ -37,6 +37,23 @@ type scopedFakeDockerActionAgentCommander struct { lastOrg string } +type observingDockerActionAgentCommander struct { + *fakeDockerActionAgentCommander + observations []agentexec.DockerContainerObservationPayload +} + +func (f *observingDockerActionAgentCommander) ObserveDockerContainer(_ context.Context, _ string, req agentexec.DockerContainerObservationPayload) (*agentexec.DockerContainerObservationResultPayload, error) { + if err := agentexec.BindDockerContainerObservationPayload(&req); err != nil { + return nil, err + } + f.observations = append(f.observations, req) + return &agentexec.DockerContainerObservationResultPayload{ + RequestID: req.RequestID, ActionID: req.ActionID, ProtocolVersion: req.ProtocolVersion, RequestDigest: req.RequestDigest, + Observed: true, + Snapshot: agentexec.DockerContainerLifecycleSnapshot{ContainerID: req.ContainerID, State: "running", Running: true, StartedAt: time.Now().UTC(), ObservedAt: time.Now().UTC()}, + }, nil +} + func (f *scopedFakeDockerActionAgentCommander) IsAgentConnectedForOrganization(organizationID, agentID string) bool { f.lastOrg = organizationID return f.IsAgentConnected(agentID) @@ -221,6 +238,35 @@ func TestDockerContainerActionExecutorDispatchesPodmanRestartAndVerification(t * } } +func TestDockerContainerActionExecutorUsesProductionDaemonObservationForIndependentTruth(t *testing.T) { + now := time.Now().UTC() + h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()}) + h.SetStateProvider(resourceUnifiedSeedProvider{ + snapshot: models.StateSnapshot{LastUpdate: now}, + resources: []unified.Resource{dockerContainerActionResource("app-container:api", "docker", "running", now)}, + }) + agents := &observingDockerActionAgentCommander{fakeDockerActionAgentCommander: &fakeDockerActionAgentCommander{}} + executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor) + if executor.observer == nil { + t.Fatal("production constructor did not wire the typed daemon observer") + } + record := dockerContainerActionRecord("act_container", "app-container:api", "restart") + result, err := executor.ExecuteAction(dockerActionDispatchContext(t, executor, record), record) + if err != nil { + t.Fatalf("ExecuteAction: %v", err) + } + if len(agents.observations) != 1 || agents.observations[0].ActionID != record.ID { + t.Fatalf("observations=%#v", agents.observations) + } + truth := result.ActionResultV2 + if truth == nil || truth.Execution.Status != unified.ActionExecutionSucceeded || truth.Verification.Status != unified.ActionVerificationConfirmed || truth.Verification.EvidenceClass != unified.ActionEvidenceIndependent { + t.Fatalf("canonical truth=%#v", truth) + } + if len(truth.Verification.Evidence) != 1 || truth.Verification.Evidence[0].ObserverTrustDomain == truth.Verification.Evidence[0].ExecutorTrustDomain { + t.Fatalf("verification evidence=%#v", truth.Verification.Evidence) + } +} + func TestDockerContainerActionExecutorResolvesCommandAgentByDockerHostname(t *testing.T) { now := time.Now().UTC() resource := dockerContainerActionResource("app-container:api", "docker", "running", now) diff --git a/internal/api/docker_restart_colima_real_lab_test.go b/internal/api/docker_restart_colima_real_lab_test.go index e55be1813..4da42dba7 100644 --- a/internal/api/docker_restart_colima_real_lab_test.go +++ b/internal/api/docker_restart_colima_real_lab_test.go @@ -246,7 +246,7 @@ type dockerRestartLabNotification struct { type colimaDirectObserver struct{} -func (colimaDirectObserver) ObserveDockerContainer(ctx context.Context, actionID, containerID string) (dockerContainerPostconditionObservation, error) { +func (colimaDirectObserver) ObserveDockerContainer(ctx context.Context, actionID, _, _, containerID string) (dockerContainerPostconditionObservation, error) { snapshot, err := observeColimaContainer(ctx, containerID) return dockerContainerPostconditionObservation{ObserverID: "task06-colima-direct", TrustDomain: "daemon:colima-direct", Method: "docker_context_colima_inspect", Snapshot: snapshot, ReceivedAt: time.Now().UTC()}, err } diff --git a/internal/hostagent/commands.go b/internal/hostagent/commands.go index 0e4f9191a..fae70a110 100644 --- a/internal/hostagent/commands.go +++ b/internal/hostagent/commands.go @@ -205,6 +205,8 @@ const ( msgTypeDockerContainerLifecycleResult messageType = "docker_container_lifecycle_result" msgTypeDockerContainerUpdate messageType = "docker_container_update" msgTypeDockerContainerUpdateResult messageType = "docker_container_update_result" + msgTypeDockerContainerObserve messageType = "docker_container_observe" + msgTypeDockerContainerObserveResult messageType = "docker_container_observe_result" msgTypeActionPreflight messageType = "action_preflight" msgTypeActionPreflightResult messageType = "action_preflight_result" msgTypeOperationQuery messageType = "agent_operation_query" @@ -224,14 +226,15 @@ type wsMessage struct { } type registerPayload struct { - AgentID string `json:"agent_id"` - Hostname string `json:"hostname"` - Version string `json:"version"` - Platform string `json:"platform"` - Tags []string `json:"tags,omitempty"` - Token string `json:"token"` - OperationReceiptVersion int `json:"operation_receipt_version,omitempty"` - ActionPreflightVersion int `json:"action_preflight_version,omitempty"` + AgentID string `json:"agent_id"` + Hostname string `json:"hostname"` + Version string `json:"version"` + Platform string `json:"platform"` + Tags []string `json:"tags,omitempty"` + Token string `json:"token"` + OperationReceiptVersion int `json:"operation_receipt_version,omitempty"` + ActionPreflightVersion int `json:"action_preflight_version,omitempty"` + DockerObservationVersion int `json:"docker_observation_version,omitempty"` } type registeredPayload struct { @@ -457,13 +460,14 @@ func (c *CommandClient) buildWebSocketOrigin() (string, error) { func (c *CommandClient) sendRegistration(conn *websocket.Conn) error { payload, err := json.Marshal(registerPayload{ - AgentID: c.agentID, - Hostname: c.hostname, - Version: c.version, - Platform: c.platform, - Token: c.apiToken, - OperationReceiptVersion: c.operationReceiptVersion(), - ActionPreflightVersion: agentexec.ActionPreflightProtocolVersion, + AgentID: c.agentID, + Hostname: c.hostname, + Version: c.version, + Platform: c.platform, + Token: c.apiToken, + OperationReceiptVersion: c.operationReceiptVersion(), + ActionPreflightVersion: agentexec.ActionPreflightProtocolVersion, + DockerObservationVersion: agentexec.DockerContainerObservationProtocolVersion, }) if err != nil { return fmt.Errorf("marshal registration payload: %w", err) @@ -601,6 +605,14 @@ func (c *CommandClient) handleMessages(ctx context.Context, conn *websocket.Conn } go c.handleActionPreflight(ctx, conn, payload) + case msgTypeDockerContainerObserve: + payload, err := agentexec.DecodeDockerContainerObservationPayload(msg.Payload) + if err != nil { + c.logger.Warn().Err(err).Msg("Dropping invalid docker container observation request") + continue + } + go c.handleDockerContainerObservation(ctx, conn, payload) + case msgTypeHostStorageCleanup: payload, err := agentexec.DecodeHostStorageCleanupPayload(msg.Payload) if err != nil { @@ -726,6 +738,38 @@ func (c *CommandClient) handleActionPreflight(ctx context.Context, conn *websock } } +func (c *CommandClient) handleDockerContainerObservation(ctx context.Context, conn *websocket.Conn, payload agentexec.DockerContainerObservationPayload) { + observeCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + observer, ok := c.dockerLifecycle.(interface { + Observe(context.Context, string, string) (agentexec.DockerContainerLifecycleSnapshot, error) + }) + result := agentexec.DockerContainerObservationResultPayload{ + RequestID: payload.RequestID, ActionID: payload.ActionID, ProtocolVersion: payload.ProtocolVersion, + RequestDigest: payload.RequestDigest, + } + if !ok || observer == nil { + result.ReasonCode = agentexec.ActionRefusalCapabilityUnavailable + } else if snapshot, err := observer.Observe(observeCtx, payload.Runtime, payload.ContainerID); err != nil { + result.ReasonCode = agentexec.ActionRefusalTargetInspectionUnavailable + c.logger.Debug().Err(err).Str("request_id", payload.RequestID).Msg("Docker container observation failed") + } else { + result.Observed = true + result.Snapshot = snapshot + } + encoded, err := json.Marshal(result) + if err != nil { + return + } + msg := wsMessage{Type: msgTypeDockerContainerObserveResult, ID: result.RequestID, Timestamp: time.Now(), Payload: encoded} + c.connMu.Lock() + err = conn.WriteJSON(msg) + c.connMu.Unlock() + if err != nil { + c.logger.Debug().Err(err).Str("request_id", result.RequestID).Msg("Failed to send docker container observation result") + } +} + func (c *CommandClient) preflightHostUpdate(ctx context.Context, payload agentexec.HostUpdatePayload) (bool, string) { if c.packageUpdates == nil { return false, agentexec.ActionRefusalCapabilityUnavailable diff --git a/internal/hostagent/commands_registration_test.go b/internal/hostagent/commands_registration_test.go index eadd7b92b..4a0445381 100644 --- a/internal/hostagent/commands_registration_test.go +++ b/internal/hostagent/commands_registration_test.go @@ -105,6 +105,9 @@ func TestCommandClient_sendRegistration_WritesExpectedPayload(t *testing.T) { if payload.Token != "token-1" { t.Fatalf("payload.Token = %q, want %q", payload.Token, "token-1") } + if payload.DockerObservationVersion != 1 { + t.Fatalf("payload.DockerObservationVersion = %d, want 1", payload.DockerObservationVersion) + } } func TestCommandClient_waitForRegistration_AcceptsSuccess(t *testing.T) { diff --git a/internal/hostagent/docker_lifecycle.go b/internal/hostagent/docker_lifecycle.go index ffb25b4dc..b3439f329 100644 --- a/internal/hostagent/docker_lifecycle.go +++ b/internal/hostagent/docker_lifecycle.go @@ -21,6 +21,10 @@ type dockerLifecycleManager interface { Preflight(context.Context, agentexec.DockerContainerLifecyclePayload) (bool, string) } +func (m *localDockerLifecycleManager) Observe(ctx context.Context, runtime, containerID string) (agentexec.DockerContainerLifecycleSnapshot, error) { + return m.inspect(ctx, runtime, containerID) +} + // DockerContainerLifecycleOperator is the narrow bridge from the host command // channel to the unified agent's connected Docker / Podman module. It keeps // lifecycle execution on the daemon API and avoids requiring a second runtime diff --git a/internal/hostagent/docker_lifecycle_test.go b/internal/hostagent/docker_lifecycle_test.go index f38aab4fd..399052dcc 100644 --- a/internal/hostagent/docker_lifecycle_test.go +++ b/internal/hostagent/docker_lifecycle_test.go @@ -58,6 +58,24 @@ func TestDockerLifecycleManagerUsesConnectedModuleWithoutExternalCLI(t *testing. } } +func TestDockerLifecycleManagerObservationIsReadOnly(t *testing.T) { + now := time.Now().UTC() + operator := &stubDockerLifecycleOperator{snapshots: []agentexec.DockerContainerLifecycleSnapshot{{ + ContainerID: dockerLifecycleTestContainerID, State: "running", Running: true, ObservedAt: now, + }}} + manager := newLocalDockerLifecycleManager(operator) + manager.run = func(context.Context, string, ...string) ([]byte, error) { + return nil, fmt.Errorf("external runtime CLI must not be called") + } + snapshot, err := manager.Observe(context.Background(), "docker", dockerLifecycleTestContainerID) + if err != nil { + t.Fatal(err) + } + if snapshot.ContainerID != dockerLifecycleTestContainerID || !snapshot.Running || len(operator.mutations) != 0 { + t.Fatalf("snapshot=%#v mutations=%v", snapshot, operator.mutations) + } +} + func TestDockerLifecycleManagerRestartPerformsOneMutationAndBoundedReadback(t *testing.T) { t.Setenv("DOCKER_CONTEXT", "") before := time.Now().UTC().Add(-time.Minute).Truncate(time.Nanosecond) diff --git a/internal/mutationregistry/runtime_surface_audit_test.go b/internal/mutationregistry/runtime_surface_audit_test.go index f36904aff..d814c0530 100644 --- a/internal/mutationregistry/runtime_surface_audit_test.go +++ b/internal/mutationregistry/runtime_surface_audit_test.go @@ -164,6 +164,7 @@ func TestTransportCommandCatalogsResolveToRegistry(t *testing.T) { "MsgTypeHostStorageCleanupResult": {Role: TransportRoleOperationResult}, "MsgTypeHostUpdateResult": {Role: TransportRoleOperationResult}, "MsgTypeDockerContainerLifecycleResult": {Role: TransportRoleOperationResult}, "MsgTypeDockerContainerUpdateResult": {Role: TransportRoleOperationResult}, + "MsgTypeDockerContainerObserveResult": {Role: TransportRoleOperationResult}, "MsgTypeOperationQueryResult": {Role: TransportRoleOperationReceipt}, "MsgTypeRegistered": {Role: TransportRoleProtocol}, "MsgTypePong": {Role: TransportRoleProtocol}, "MsgTypeExecuteCmd": {Role: TransportRoleMutationRequest, MutationID: "transport.agent.raw-command", DurableAuthorityID: "assistant.resource-action"}, @@ -173,6 +174,7 @@ func TestTransportCommandCatalogsResolveToRegistry(t *testing.T) { "MsgTypeHostUpdate": {Role: TransportRoleMutationRequest, MutationID: "transport.agent.host-package-update", DurableAuthorityID: "resource.host.package-update"}, "MsgTypeDockerContainerLifecycle": {Role: TransportRoleMutationRequest, MutationID: "transport.agent.docker-container-lifecycle", DurableAuthorityID: "resource.docker.container-lifecycle"}, "MsgTypeDockerContainerUpdate": {Role: TransportRoleMutationRequest, MutationID: "transport.agent.docker-container-update", DurableAuthorityID: "resource.docker.container-update"}, + "MsgTypeDockerContainerObserve": {Role: TransportRoleOperationQuery}, "MsgTypeOperationQuery": {Role: TransportRoleOperationQuery}, "MsgTypeActionPreflight": {Role: TransportRoleOperationQuery}, "MsgTypeActionPreflightResult": {Role: TransportRoleOperationResult}, From 81b09d870fb39112cbcef70becf09d6f30d77393 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 27 Aug 2026 20:29:17 +0100 Subject: [PATCH 2/3] Add predictive storage capacity alerts --- .../v6/internal/subsystems/alerts.md | 17 + .../v6/internal/subsystems/monitoring.md | 13 + internal/alerts/alerts_test.go | 24 + internal/alerts/canonical_lifecycle.go | 50 ++- internal/alerts/canonical_metric.go | 5 + internal/alerts/capacity_forecast.go | 424 ++++++++++++++++++ internal/alerts/capacity_forecast_test.go | 227 ++++++++++ internal/alerts/metric_runtime.go | 10 +- internal/alerts/storage.go | 15 +- internal/alerts/unified_eval.go | 22 +- internal/alerts/unified_eval_test.go | 22 + .../monitoring/canonical_guardrails_test.go | 2 +- internal/monitoring/ceph.go | 2 +- internal/monitoring/metrics_history.go | 3 + internal/monitoring/metrics_history_test.go | 26 ++ internal/monitoring/monitor_alert_sync.go | 3 +- internal/monitoring/monitor_alerts.go | 2 +- .../monitoring/monitor_polling_storage.go | 2 +- .../monitoring/storage_capacity_forecast.go | 115 +++++ .../storage_capacity_forecast_test.go | 53 +++ 20 files changed, 1002 insertions(+), 35 deletions(-) create mode 100644 internal/alerts/capacity_forecast.go create mode 100644 internal/alerts/capacity_forecast_test.go create mode 100644 internal/monitoring/storage_capacity_forecast.go create mode 100644 internal/monitoring/storage_capacity_forecast_test.go diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 88d78dcdd..a441879e4 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -193,6 +193,22 @@ lifecycle when the agent heartbeat is unhealthy, and resolves only when a fresh result from the currently assigned agent arrives. Notification delivery, acknowledgement, history, and recovery reuse the normal alert pipeline. +Storage capacity forecasting is an alert-grade evidence boundary, not an AI +prose feature. It requires at least 24 hours of valid history, hourly median +normalization, a fresh terminal sample, agreement between full-window and +recent positive slopes, and confidence of at least 0.80 before it can fire. +Raw poll count alone must never manufacture confidence. Forecast risk opens +only when projected exhaustion is within seven days (critical within one day) +and recovers only after the projection moves beyond fourteen days or trusted +evidence proves growth stopped. Missing, shallow, stale, or low-confidence +history is unknown rather than recovery for an already-active forecast. +Predictive and static usage policy share `metric-threshold:usage` as one +canonical occurrence: a forecast that later crosses the percentage threshold +keeps its start time, acknowledgement, history, and timeline instead of +emitting a forecast recovery plus a second capacity alert. The same contract +applies to Proxmox, Ceph, TrueNAS pools/datasets, and vSphere datastores through +their existing platform thresholds and disable policy. + Active-alert restore is opt-out at construction. `NewManagerWithDataDir` accepts `ManagerOption` values, and `WithoutPersistedAlertRestore` starts the manager with an empty active-alert set instead of reading `active-alerts.json`. Mock @@ -252,6 +268,7 @@ default construction path still restores. 45. `internal/alerts/docker.go` 46. `internal/alerts/pbs.go` 47. `internal/alerts/storage.go` +47a. `internal/alerts/capacity_forecast.go` 48. `internal/alerts/node.go` 49. `internal/alerts/host.go` 50. `internal/alerts/backup_snapshot.go` diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 4a644be5d..b94276350 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -420,6 +420,18 @@ lifecycle, and treats a fresh result from that same assignment as recovery. Assignment trackers are removed with their targets and reset when agent identity changes. +Storage capacity forecasting consumes monitoring-owned percentage history +through `internal/monitoring/storage_capacity_forecast.go`. The bridge combines +the durable SQLite series with the in-memory tail and current observation, then +caches the alert-owned trend result for a bounded interval. Durable history is +required in the read path so restart cannot erase a previously earned +confidence floor. Canonical storage resources must resolve their metrics target +before trend evaluation, keeping TrueNAS and vSphere forecast identity aligned +with the series written by `syncUnifiedStorageMetrics`; Proxmox and Ceph retain +their source-native storage history IDs. Monitoring supplies evidence only and +must not choose forecast horizons, severity, hysteresis, notification routing, +or lifecycle identity. + Monitoring ingest keeps mock mode hermetic. The unified read path already substitutes the mock snapshot wholesale, so anything that runs after that substitution has to be suppressed explicitly rather than assumed hidden. Server @@ -510,6 +522,7 @@ cleanup so readers cannot retain orphaned runtime or alert projections. 50. `internal/models/ceph_cluster_identity.go` 51. `internal/truenas/types.go` 52. `internal/monitoring/monitor_alert_sync.go` +52a. `internal/monitoring/storage_capacity_forecast.go` 53. `internal/monitoring/platform_poller_shared.go` 54. `internal/monitoring/monitor_backups.go` 55. `internal/monitoring/resource_stale_thresholds.go` diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index b50a2aac8..60f8ec2f5 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -20878,3 +20878,27 @@ func TestCleanupRemovesOnlyStaleSMARTCounterSnapshots(t *testing.T) { t.Fatal("current SMART counter snapshot was removed") } } + +func TestCheckStorageForecastUsesCanonicalUsageAlertIdentity(t *testing.T) { + m := newTestManager(t) + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70} + m.mu.Unlock() + + storage := models.Storage{ID: "forecast-proof", Name: "archive", Status: "active", Usage: 72} + trend := CapacityTrendObservation{ + Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 5, + Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour, + } + m.CheckStorageWithCapacityTrend(storage, trend) + m.CheckStorageWithCapacityTrend(storage, trend) + + alert := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) + if alert.CanonicalSpecID != canonicalMetricSpecID(storage.ID, "usage") { + t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID) + } + if got := alert.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginForecast { + t.Fatalf("capacity origin = %v, want forecast", got) + } +} diff --git a/internal/alerts/canonical_lifecycle.go b/internal/alerts/canonical_lifecycle.go index 73d35c45d..b978dd0f2 100644 --- a/internal/alerts/canonical_lifecycle.go +++ b/internal/alerts/canonical_lifecycle.go @@ -13,23 +13,27 @@ import ( ) type canonicalLifecycleAlertParams struct { - Spec alertspecs.ResourceAlertSpec - Evidence alertspecs.AlertEvidence - IntentSignal string - PolicyDisabledNoLock func() bool - AlertID string - AlertType string - ResourceID string - ResourceName string - Node string - Instance string - Message string - Metadata map[string]interface{} - AddToRecent bool - AddToHistory bool - RateLimit bool - DispatchAsync bool - IntentBackup BackupIntentContext + Spec alertspecs.ResourceAlertSpec + Evidence alertspecs.AlertEvidence + IntentSignal string + PolicyDisabledNoLock func() bool + AlertID string + AlertType string + ResourceID string + ResourceName string + Node string + Instance string + Message string + Value float64 + Threshold float64 + Metadata map[string]interface{} + AddToRecent bool + AddToHistory bool + RateLimit bool + DispatchAsync bool + NotifyOnSeverityChange bool + AddToHistoryOnSeverityChange bool + IntentBackup BackupIntentContext } type canonicalStatefulAlertParams struct { @@ -460,8 +464,8 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert Node: params.Node, Instance: params.Instance, Message: params.Message, - Value: 0, - Threshold: 0, + Value: params.Value, + Threshold: params.Threshold, StartTime: incident.StartedAt, LastSeen: params.Evidence.ObservedAt, Metadata: cloneMetadata(params.Metadata), @@ -501,6 +505,14 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert if existing != nil { if primary == reducer.EventSeverityChanged { result.Transition = transition(alertspecs.EvaluationTransitionSeverityChanged, alertspecs.AlertStateFiring, alertspecs.AlertStateFiring) + if params.AddToHistoryOnSeverityChange { + m.historyManager.AddAlertTransition(*alert) + } + if params.NotifyOnSeverityChange { + if !params.RateLimit || m.checkRateLimit(trackingKey) { + m.dispatchAlert(alert, params.DispatchAsync) + } + } } return result, true } diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index a91161469..e8422355c 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -327,6 +327,11 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec if existingAlert.Metadata == nil { existingAlert.Metadata = map[string]interface{}{} } + if opts != nil { + for _, key := range opts.RemoveMetadata { + delete(existingAlert.Metadata, key) + } + } for k, v := range alertMetadata { existingAlert.Metadata[k] = v } diff --git a/internal/alerts/capacity_forecast.go b/internal/alerts/capacity_forecast.go new file mode 100644 index 000000000..b435829b3 --- /dev/null +++ b/internal/alerts/capacity_forecast.go @@ -0,0 +1,424 @@ +package alerts + +import ( + "fmt" + "math" + "sort" + "strconv" + "time" + + alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rs/zerolog/log" +) + +const ( + capacityForecastLookback = 7 * 24 * time.Hour + capacityForecastBucketWidth = time.Hour + capacityForecastMinimumSpan = 24 * time.Hour + capacityForecastFreshness = 2 * time.Hour + capacityForecastMinimumBuckets = 12 + capacityForecastMinimumRate = 0.1 + capacityForecastMinConfidence = 0.80 + capacityForecastWarningHorizon = 7 * 24 * time.Hour + capacityForecastCriticalWindow = 24 * time.Hour + capacityForecastRecoveryWindow = 14 * 24 * time.Hour +) + +// CapacityMetricPoint is one percentage-utilization observation used to +// estimate when a capacity-backed resource will fill. Callers may provide raw +// samples at any cadence; the estimator normalizes them into hourly medians so +// a fast poller cannot manufacture confidence by repeating nearly identical +// observations. +type CapacityMetricPoint struct { + Timestamp time.Time + Value float64 +} + +// CapacityTrendObservation is detector evidence, not an alert decision. +// Ready means the evidence coverage is sufficient to say whether a trend is +// actionable. A non-ready observation must not be interpreted as recovery. +type CapacityTrendObservation struct { + Ready bool + Reason string + ObservedAt time.Time + DailyChange float64 + Confidence float64 + SampleCount int + BucketCount int + CoverageSpan time.Duration +} + +type capacityBucketPoint struct { + timestamp time.Time + value float64 +} + +// EstimateCapacityTrend produces a conservative, time-aware capacity trend. +// Alert policy (warning/critical horizons, threshold coexistence, and +// hysteresis) remains in Manager; this function only establishes trustworthy +// trend evidence. +func EstimateCapacityTrend(points []CapacityMetricPoint, now time.Time) CapacityTrendObservation { + if now.IsZero() { + now = time.Now() + } + result := CapacityTrendObservation{Reason: "insufficient-history"} + if len(points) == 0 { + return result + } + + cutoff := now.Add(-capacityForecastLookback) + buckets := make(map[int64][]float64) + validSamples := 0 + latest := time.Time{} + for _, point := range points { + if point.Timestamp.IsZero() || point.Timestamp.Before(cutoff) || point.Timestamp.After(now.Add(5*time.Minute)) { + continue + } + if math.IsNaN(point.Value) || math.IsInf(point.Value, 0) || point.Value < 0 || point.Value > 100 { + continue + } + bucket := point.Timestamp.Unix() / int64(capacityForecastBucketWidth/time.Second) + buckets[bucket] = append(buckets[bucket], point.Value) + validSamples++ + if point.Timestamp.After(latest) { + latest = point.Timestamp + } + } + result.SampleCount = validSamples + if validSamples == 0 || latest.IsZero() { + result.Reason = "no-valid-samples" + return result + } + result.ObservedAt = latest + if now.Sub(latest) > capacityForecastFreshness { + result.Reason = "stale-history" + return result + } + + normalized := make([]capacityBucketPoint, 0, len(buckets)) + for bucket, values := range buckets { + sort.Float64s(values) + median := values[len(values)/2] + if len(values)%2 == 0 { + median = (values[len(values)/2-1] + values[len(values)/2]) / 2 + } + normalized = append(normalized, capacityBucketPoint{ + timestamp: time.Unix(bucket*int64(capacityForecastBucketWidth/time.Second), 0), + value: median, + }) + } + sort.Slice(normalized, func(i, j int) bool { + return normalized[i].timestamp.Before(normalized[j].timestamp) + }) + result.BucketCount = len(normalized) + if len(normalized) < capacityForecastMinimumBuckets { + result.Reason = "insufficient-hourly-coverage" + return result + } + result.CoverageSpan = normalized[len(normalized)-1].timestamp.Sub(normalized[0].timestamp) + if result.CoverageSpan < capacityForecastMinimumSpan { + result.Reason = "history-window-too-short" + return result + } + + overallSlope, overallR2 := capacityLinearRegression(normalized) + recentStart := len(normalized) / 2 + if len(normalized)-recentStart < 8 { + recentStart = len(normalized) - 8 + } + recentSlope, _ := capacityLinearRegression(normalized[recentStart:]) + overallDaily := overallSlope * 24 + recentDaily := recentSlope * 24 + result.DailyChange = overallDaily + + // A historic rise that has flattened or reversed is not an impending + // exhaustion signal. Requiring both windows to rise also rejects one-off + // capacity jumps caused by a resize or a telemetry discontinuity. + if overallDaily <= capacityForecastMinimumRate || recentDaily <= capacityForecastMinimumRate { + result.Ready = true + result.Reason = "not-increasing" + return result + } + + spanFactor := math.Min(1, result.CoverageSpan.Hours()/48) + agreement := math.Min(overallDaily, recentDaily) / math.Max(overallDaily, recentDaily) + result.Confidence = clampCapacityConfidence(overallR2 * spanFactor * agreement) + result.Ready = true + if result.Confidence < capacityForecastMinConfidence { + result.Reason = "low-confidence" + return result + } + result.Reason = "increasing" + return result +} + +func capacityLinearRegression(points []capacityBucketPoint) (slopePerHour, rSquared float64) { + if len(points) < 2 { + return 0, 0 + } + start := points[0].timestamp + n := float64(len(points)) + var sumX, sumY, sumXY, sumX2 float64 + for _, point := range points { + x := point.timestamp.Sub(start).Hours() + sumX += x + sumY += point.value + sumXY += x * point.value + sumX2 += x * x + } + denominator := n*sumX2 - sumX*sumX + if denominator == 0 { + return 0, 0 + } + slopePerHour = (n*sumXY - sumX*sumY) / denominator + intercept := (sumY - slopePerHour*sumX) / n + meanY := sumY / n + var residual, total float64 + for _, point := range points { + x := point.timestamp.Sub(start).Hours() + predicted := intercept + slopePerHour*x + residual += math.Pow(point.value-predicted, 2) + total += math.Pow(point.value-meanY, 2) + } + if total == 0 { + return slopePerHour, 0 + } + return slopePerHour, clampCapacityConfidence(1 - residual/total) +} + +func clampCapacityConfidence(value float64) float64 { + if value < 0 { + return 0 + } + if value > 1 { + return 1 + } + return value +} + +const ( + capacityAlertOriginKey = "capacityAlertOrigin" + capacityAlertOriginThreshold = "threshold" + capacityAlertOriginForecast = "forecast" +) + +var capacityForecastMetadataKeys = []string{ + "forecastConfidence", + "forecastDailyChangePct", + "forecastDaysToFull", + "forecastObservedAt", + "forecastSampleCount", + "forecastBucketCount", + "forecastCoverageSeconds", +} + +func (m *Manager) evaluateStorageCapacity(storage models.Storage, thresholds ThresholdConfig, trend CapacityTrendObservation) { + input := &UnifiedResourceInput{ + ID: storage.ID, + Type: "storage", + Name: storage.Name, + Node: storage.Node, + Instance: storage.Instance, + Disk: &UnifiedResourceMetric{Percent: storage.Usage}, + } + m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool { + if !m.config.Enabled { + return true + } + allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("storage") + return allDisabled || m.resolveStorageThresholdsNoLock(storage).Disabled + }) +} + +func (m *Manager) evaluateUnifiedCapacity(input *UnifiedResourceInput, thresholds ThresholdConfig, trend CapacityTrendObservation, policyDisabledNoLock func() bool) { + if input == nil || input.Disk == nil { + return + } + threshold := thresholds.Usage + if threshold == nil || threshold.Trigger <= 0 { + m.evaluateUnifiedMetrics(input, thresholds, nil) + return + } + + current := input.DiskValue() + active, origin := m.activeStorageCapacityOrigin(input.ID) + staticTriggered := current >= threshold.Trigger + staticLatched := active && origin == capacityAlertOriginThreshold && threshold.Clear > 0 && current >= threshold.Clear + if staticTriggered || staticLatched { + m.evaluateUnifiedMetrics(input, thresholds, &metricOptions{ + Metadata: map[string]interface{}{ + capacityAlertOriginKey: capacityAlertOriginThreshold, + }, + RemoveMetadata: capacityForecastMetadataKeys, + }) + return + } + + // Insufficient or low-confidence history is unknown, not recovery. Keep an + // already-firing forecast occurrence untouched until trustworthy evidence + // says the risk has receded; normal stale-alert cleanup remains the final + // bound if telemetry never becomes usable again. + recoveryFloor := threshold.Clear + if recoveryFloor <= 0 { + recoveryFloor = threshold.Trigger + } + if active && origin == capacityAlertOriginForecast && current >= recoveryFloor && (!trend.Ready || trend.Reason == "low-confidence") { + return + } + + eta, hasETA := capacityTimeToFull(current, trend.DailyChange) + forecastTrusted := trend.Ready && trend.Confidence >= capacityForecastMinConfidence && hasETA + forecastTriggered := forecastTrusted && eta <= capacityForecastWarningHorizon + forecastLatched := active && origin == capacityAlertOriginForecast && forecastTrusted && eta <= capacityForecastRecoveryWindow + if forecastTriggered || forecastLatched { + m.evaluateCapacityForecast(input, thresholds, trend, eta, policyDisabledNoLock) + return + } + + // No predictive risk remains. Run the ordinary metric evaluator so a + // forecast recovery and every existing static hysteresis rule retain the + // same canonical state, history, and notification behavior. + m.evaluateUnifiedMetrics(input, thresholds, &metricOptions{ + Metadata: map[string]interface{}{ + capacityAlertOriginKey: capacityAlertOriginThreshold, + }, + RemoveMetadata: capacityForecastMetadataKeys, + }) +} + +func (m *Manager) activeStorageCapacityOrigin(resourceID string) (bool, string) { + trackingKey := canonicalMetricStateID(resourceID, "usage") + m.mu.RLock() + defer m.mu.RUnlock() + alert, exists := m.getActiveAlertNoLock(trackingKey) + if !exists || alert == nil { + return false, "" + } + if alert.Metadata == nil { + return true, capacityAlertOriginThreshold + } + origin, _ := alert.Metadata[capacityAlertOriginKey].(string) + if origin == "" { + origin = capacityAlertOriginThreshold + } + return true, origin +} + +func capacityTimeToFull(current, dailyChange float64) (time.Duration, bool) { + if current < 0 || current >= 100 || dailyChange <= capacityForecastMinimumRate || math.IsNaN(dailyChange) || math.IsInf(dailyChange, 0) { + return 0, false + } + days := (100 - current) / dailyChange + if days <= 0 || math.IsNaN(days) || math.IsInf(days, 0) { + return 0, false + } + return time.Duration(days * float64(24*time.Hour)), true +} + +func (m *Manager) evaluateCapacityForecast(input *UnifiedResourceInput, thresholds ThresholdConfig, trend CapacityTrendObservation, eta time.Duration, policyDisabledNoLock func() bool) { + resourceID := input.ID + specID := canonicalMetricSpecID(resourceID, "usage") + resourceType, ok := unifiedMetricResourceType(input.Type) + if !ok { + return + } + spec, err := buildCanonicalSeverityThresholdSpec( + specID, + resourceID, + input.Name, + resourceType, + "capacity-risk", + 1, + 2, + false, + ) + if err != nil { + log.Warn().Err(err).Str("resourceID", input.ID).Msg("Skipping invalid storage capacity forecast spec") + return + } + spec.ConfirmationsRequired = 2 + if err := spec.Validate(); err != nil { + log.Warn().Err(err).Str("resourceID", input.ID).Msg("Skipping invalid confirmed storage capacity forecast spec") + return + } + + riskScore := 1.0 + if eta <= capacityForecastCriticalWindow { + riskScore = 2 + } + observedAt := m.policyNow() + daysToFull := eta.Hours() / 24 + message := fmt.Sprintf( + "%s projected to fill in %s (%.1f%% used, +%.2f%%/day)", + unifiedAlertType(input.Type), + formatCapacityETA(eta), + input.DiskValue(), + trend.DailyChange, + ) + attributes := map[string]string{ + "capacity_alert_origin": capacityAlertOriginForecast, + "confidence": strconv.FormatFloat(trend.Confidence, 'f', 3, 64), + "current_usage_percent": strconv.FormatFloat(input.DiskValue(), 'f', 2, 64), + "daily_change_percent": strconv.FormatFloat(trend.DailyChange, 'f', 3, 64), + "forecast_days_to_full": strconv.FormatFloat(daysToFull, 'f', 2, 64), + "history_bucket_count": strconv.Itoa(trend.BucketCount), + "history_coverage_seconds": strconv.FormatInt(int64(trend.CoverageSpan/time.Second), 10), + } + metadata := map[string]interface{}{ + "resourceType": input.Type, + "clearThreshold": thresholds.Usage.Clear, + capacityAlertOriginKey: capacityAlertOriginForecast, + "forecastConfidence": trend.Confidence, + "forecastDailyChangePct": trend.DailyChange, + "forecastDaysToFull": daysToFull, + "forecastObservedAt": trend.ObservedAt, + "forecastSampleCount": trend.SampleCount, + "forecastBucketCount": trend.BucketCount, + "forecastCoverageSeconds": int64(trend.CoverageSpan / time.Second), + } + _, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{ + Spec: spec, + Evidence: alertspecs.AlertEvidence{ + ObservedAt: observedAt, + Summary: message, + Attributes: attributes, + SeverityThreshold: &alertspecs.SeverityThresholdEvidence{ + Metric: "capacity-risk", + Direction: alertspecs.ThresholdDirectionAbove, + Observed: riskScore, + }, + }, + IntentSignal: MetricAlertIntentSignal("usage"), + PolicyDisabledNoLock: policyDisabledNoLock, + AlertID: canonicalMetricStateID(resourceID, "usage"), + AlertType: "usage", + ResourceID: resourceID, + ResourceName: input.Name, + Node: input.Node, + Instance: input.Instance, + Message: message, + Value: input.DiskValue(), + Threshold: 100, + Metadata: metadata, + AddToRecent: true, + AddToHistory: true, + RateLimit: true, + NotifyOnSeverityChange: true, + AddToHistoryOnSeverityChange: true, + }) +} + +func formatCapacityETA(eta time.Duration) string { + if eta < 2*time.Hour { + return "about 1 hour" + } + if eta < 24*time.Hour { + return fmt.Sprintf("about %.0f hours", math.Ceil(eta.Hours())) + } + days := int(math.Ceil(eta.Hours() / 24)) + if days == 1 { + return "about 1 day" + } + return fmt.Sprintf("about %d days", days) +} diff --git a/internal/alerts/capacity_forecast_test.go b/internal/alerts/capacity_forecast_test.go new file mode 100644 index 000000000..234b0f7bd --- /dev/null +++ b/internal/alerts/capacity_forecast_test.go @@ -0,0 +1,227 @@ +package alerts + +import ( + "math" + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +func TestEstimateCapacityTrendRequiresCoverageAndRecentAgreement(t *testing.T) { + now := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + + t.Run("trusts a clean multi-day fill trend", func(t *testing.T) { + points := make([]CapacityMetricPoint, 0, 73) + for hour := 72; hour >= 0; hour-- { + age := time.Duration(hour) * time.Hour + points = append(points, CapacityMetricPoint{ + Timestamp: now.Add(-age), + Value: 55 + float64(72-hour)*0.08, + }) + } + trend := EstimateCapacityTrend(points, now) + if !trend.Ready || trend.Reason != "increasing" { + t.Fatalf("trend = %+v, want trusted increasing evidence", trend) + } + if math.Abs(trend.DailyChange-1.92) > 0.02 { + t.Fatalf("DailyChange = %.3f, want about 1.92", trend.DailyChange) + } + if trend.Confidence < capacityForecastMinConfidence { + t.Fatalf("Confidence = %.3f, want >= %.2f", trend.Confidence, capacityForecastMinConfidence) + } + }) + + t.Run("rejects dense samples without a full-day span", func(t *testing.T) { + points := make([]CapacityMetricPoint, 0, 300) + for i := 0; i < 300; i++ { + points = append(points, CapacityMetricPoint{ + Timestamp: now.Add(-time.Duration(300-i) * time.Minute), + Value: 50 + float64(i)*0.02, + }) + } + trend := EstimateCapacityTrend(points, now) + if trend.Ready || trend.Reason != "insufficient-hourly-coverage" { + t.Fatalf("trend = %+v, want insufficient hourly coverage", trend) + } + }) + + t.Run("does not extrapolate a historic jump after growth stops", func(t *testing.T) { + points := make([]CapacityMetricPoint, 0, 73) + for hour := 72; hour >= 0; hour-- { + elapsed := 72 - hour + value := 50.0 + if elapsed >= 24 { + value = 70 + } + points = append(points, CapacityMetricPoint{Timestamp: now.Add(-time.Duration(hour) * time.Hour), Value: value}) + } + trend := EstimateCapacityTrend(points, now) + if !trend.Ready || trend.Reason != "not-increasing" { + t.Fatalf("trend = %+v, want a ready non-increasing decision", trend) + } + }) +} + +func TestStorageForecastSharesLifecycleWithStaticUsageAlert(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.config.SuppressionWindow = 0 + m.config.MinimumDelta = 0 + m.config.ActivationState = ActivationActive + m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70} + m.mu.Unlock() + + deliveries := 0 + resolved := 0 + m.SetAlertCallback(func(*Alert) { deliveries++ }) + m.SetResolvedCallback(func(string) { resolved++ }) + + storage := models.Storage{ + ID: "storage-forecast-1", + Name: "archive", + Node: "pve1", + Instance: "lab", + Status: "active", + Usage: 72, + } + trend := CapacityTrendObservation{ + Ready: true, + Reason: "increasing", + ObservedAt: time.Now(), + DailyChange: 5, + Confidence: 0.98, + SampleCount: 400, + BucketCount: 72, + CoverageSpan: 72 * time.Hour, + } + + // Forecast alerts require two independent evaluation cycles. + m.CheckStorageWithCapacityTrend(storage, trend) + if testHasActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) { + t.Fatal("forecast activated before its confirmation floor") + } + m.CheckStorageWithCapacityTrend(storage, trend) + alert := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) + if alert.Type != "usage" || alert.Level != AlertLevelWarning { + t.Fatalf("forecast alert = type %q level %q, want usage warning", alert.Type, alert.Level) + } + if got := alert.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginForecast { + t.Fatalf("capacity origin = %v, want forecast", got) + } + if !strings.Contains(alert.Message, "projected to fill") { + t.Fatalf("message = %q, want predictive explanation", alert.Message) + } + if deliveries != 1 { + t.Fatalf("deliveries = %d, want one forecast activation", deliveries) + } + start := alert.StartTime + if err := m.AcknowledgeAlert(alert.ID, "operator"); err != nil { + t.Fatalf("acknowledge forecast: %v", err) + } + + // A real threshold breach upgrades the same occurrence. It must not emit a + // forecast recovery or a second unacknowledged activation. + storage.Usage = 91 + m.CheckStorageWithCapacityTrend(storage, trend) + upgraded := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) + if upgraded.StartTime != start { + t.Fatalf("occurrence start changed from %s to %s", start, upgraded.StartTime) + } + if !upgraded.Acknowledged || upgraded.AckUser != "operator" { + t.Fatalf("acknowledgement was not preserved: %+v", upgraded) + } + if upgraded.Level != AlertLevelCritical { + t.Fatalf("upgraded level = %q, want critical", upgraded.Level) + } + if got := upgraded.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginThreshold { + t.Fatalf("capacity origin = %v, want threshold", got) + } + if _, exists := upgraded.Metadata["forecastDaysToFull"]; exists { + t.Fatal("stale forecast metadata survived the static threshold transition") + } + if deliveries != 1 { + t.Fatalf("deliveries = %d, want acknowledged transition not to page again", deliveries) + } + if resolved != 0 { + t.Fatalf("resolved callbacks = %d, forecast-to-threshold transition must stay one incident", resolved) + } + if got := len(m.GetActiveAlerts()); got != 1 { + t.Fatalf("active alerts = %d, want one canonical capacity incident", got) + } +} + +func TestStorageForecastTreatsUncertainEvidenceAsUnknown(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70} + m.mu.Unlock() + + storage := models.Storage{ID: "storage-forecast-unknown", Name: "media", Status: "active", Usage: 70} + trusted := CapacityTrendObservation{ + Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 6, + Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour, + } + m.CheckStorageWithCapacityTrend(storage, trusted) + m.CheckStorageWithCapacityTrend(storage, trusted) + before := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) + + uncertain := trusted + uncertain.Confidence = 0.3 + uncertain.Reason = "low-confidence" + m.CheckStorageWithCapacityTrend(storage, uncertain) + after := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) + if after.StartTime != before.StartTime { + t.Fatal("uncertain evidence replaced the active forecast occurrence") + } + + recovered := trusted + recovered.DailyChange = 0 + recovered.Confidence = 0 + recovered.Reason = "not-increasing" + m.CheckStorageWithCapacityTrend(storage, recovered) + if testHasActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) { + t.Fatal("positive non-increasing evidence did not recover the forecast") + } +} + +func TestUnifiedStorageForecastUsesPlatformPolicyAndCanonicalIdentity(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + usage := &HysteresisThreshold{Trigger: 85, Clear: 75} + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.config.TrueNASDefaults.Usage = usage + m.mu.Unlock() + + input := &UnifiedResourceInput{ + ID: "truenas:atlas/pool:tank", + Type: "truenas-pool", + Name: "tank", + Node: "atlas", + Instance: "TrueNAS", + Disk: &UnifiedResourceMetric{Percent: 70}, + } + trend := CapacityTrendObservation{ + Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 7, + Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour, + } + m.CheckUnifiedResourceWithCapacityTrend(input, trend) + m.CheckUnifiedResourceWithCapacityTrend(input, trend) + + alert := testRequireActiveAlert(t, m, canonicalMetricStateID(input.ID, "usage")) + if alert.ResourceID != input.ID || alert.Type != "usage" { + t.Fatalf("alert identity = resource %q type %q", alert.ResourceID, alert.Type) + } + if got := alert.Metadata["resourceType"]; got != "truenas-pool" { + t.Fatalf("resourceType metadata = %v, want truenas-pool", got) + } + if got := alert.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginForecast { + t.Fatalf("capacity origin = %v, want forecast", got) + } +} diff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.go index fb5412c6c..e941f354f 100644 --- a/internal/alerts/metric_runtime.go +++ b/internal/alerts/metric_runtime.go @@ -166,8 +166,9 @@ func (m *Manager) getGlobalMetricTimeThreshold(metricType string) (int, bool) { // checkMetric checks a single metric against its threshold with hysteresis. type metricOptions struct { - Metadata map[string]interface{} - Message string + Metadata map[string]interface{} + RemoveMetadata []string + Message string // MonitorOnly suppresses external notifications while still tracking the alert. MonitorOnly bool } @@ -333,6 +334,11 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource if existingAlert.Metadata == nil { existingAlert.Metadata = map[string]interface{}{} } + if opts != nil { + for _, key := range opts.RemoveMetadata { + delete(existingAlert.Metadata, key) + } + } existingAlert.Metadata["resourceType"] = resourceType existingAlert.Metadata["clearThreshold"] = threshold.Clear existingAlert.Metadata["monitorOnly"] = monitorOnly diff --git a/internal/alerts/storage.go b/internal/alerts/storage.go index 93618ca21..20e4e70ba 100644 --- a/internal/alerts/storage.go +++ b/internal/alerts/storage.go @@ -15,6 +15,12 @@ import ( // CheckStorage checks storage against thresholds func (m *Manager) CheckStorage(storage models.Storage) { + m.CheckStorageWithCapacityTrend(storage, CapacityTrendObservation{}) +} + +// CheckStorageWithCapacityTrend evaluates static storage policy and optional +// predictive evidence as one canonical capacity lifecycle. +func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend CapacityTrendObservation) { m.mu.RLock() if !m.config.Enabled { m.mu.RUnlock() @@ -97,14 +103,7 @@ func (m *Manager) CheckStorage(storage models.Storage) { // Check usage if storage is online - checkMetric will skip if threshold is nil or <= 0 if storage.Status != "offline" && storage.Status != "unavailable" && storage.Usage > 0 { - m.evaluateUnifiedMetrics(&UnifiedResourceInput{ - ID: storage.ID, - Type: "storage", - Name: storage.Name, - Node: storage.Node, - Instance: storage.Instance, - Disk: &UnifiedResourceMetric{Percent: storage.Usage}, - }, thresholds, nil) + m.evaluateStorageCapacity(storage, thresholds, trend) } // Check ZFS pool status if this is ZFS storage diff --git a/internal/alerts/unified_eval.go b/internal/alerts/unified_eval.go index 681e1f13c..7e065cb9b 100644 --- a/internal/alerts/unified_eval.go +++ b/internal/alerts/unified_eval.go @@ -462,6 +462,12 @@ func unifiedMetricResourceType(typeKey string) (unifiedresources.ResourceType, b } func (m *Manager) CheckUnifiedResourceMetrics(resources []unifiedresources.Resource) { + m.CheckUnifiedResourceMetricsWithCapacityTrends(resources, nil) +} + +// CheckUnifiedResourceMetricsWithCapacityTrends evaluates canonical resources +// with optional alert-grade capacity evidence keyed by canonical resource ID. +func (m *Manager) CheckUnifiedResourceMetricsWithCapacityTrends(resources []unifiedresources.Resource, capacityTrends map[string]CapacityTrendObservation) { if m == nil { return } @@ -470,7 +476,7 @@ func (m *Manager) CheckUnifiedResourceMetrics(resources []unifiedresources.Resou if !ok { continue } - m.CheckUnifiedResource(input) + m.CheckUnifiedResourceWithCapacityTrend(input, capacityTrends[input.ID]) } } @@ -730,6 +736,13 @@ func (i *UnifiedResourceInput) TemperatureValue() float64 { // available metric. Discrete event alerts (offline, RAID, backup age, etc.) // are NOT evaluated here — they remain in the typed Check* methods. func (m *Manager) CheckUnifiedResource(input *UnifiedResourceInput) { + m.CheckUnifiedResourceWithCapacityTrend(input, CapacityTrendObservation{}) +} + +// CheckUnifiedResourceWithCapacityTrend keeps predictive and static capacity +// evaluation on the same canonical lifecycle for every admitted storage +// platform. +func (m *Manager) CheckUnifiedResourceWithCapacityTrend(input *UnifiedResourceInput, trend CapacityTrendObservation) { if input == nil { return } @@ -757,5 +770,12 @@ func (m *Manager) CheckUnifiedResource(input *UnifiedResourceInput) { Str("resourceType", unifiedAlertType(input.Type)). Msg("Evaluating unified resource metrics") + if unifiedStorageUsageResourceType(input.Type) && input.Disk != nil { + m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool { + return !m.config.Enabled || m.unifiedPlatformAlertsDisabledNoLock(input.Type) || m.resolveResourceThresholds(input.Type, input.ID).Disabled + }) + return + } + m.evaluateUnifiedMetrics(input, thresholds, nil) } diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go index 4a808e7e5..9584df70d 100644 --- a/internal/alerts/unified_eval_test.go +++ b/internal/alerts/unified_eval_test.go @@ -1537,3 +1537,25 @@ func TestUnifiedResourceInputExcludesDockerAppContainers(t *testing.T) { t.Fatalf("truenas app containers must stay unified-eval owned, got %+v ok=%v", input, ok) } } + +func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testing.T) { + m := newTestManager(t) + configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig()) + input := &UnifiedResourceInput{ + ID: "vmware:lab/datastore:archive", + Type: "vmware-datastore", + Name: "archive", + Disk: &UnifiedResourceMetric{Percent: 70}, + } + trend := CapacityTrendObservation{ + Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 7, + Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour, + } + m.CheckUnifiedResourceWithCapacityTrend(input, trend) + m.CheckUnifiedResourceWithCapacityTrend(input, trend) + + alert := testRequireActiveAlert(t, m, canonicalMetricStateID(input.ID, "usage")) + if alert.CanonicalSpecID != canonicalMetricSpecID(input.ID, "usage") { + t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID) + } +} diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go index 118a5ab9a..7ca105988 100644 --- a/internal/monitoring/canonical_guardrails_test.go +++ b/internal/monitoring/canonical_guardrails_test.go @@ -494,7 +494,7 @@ func TestUnifiedResourceAlertSyncEvaluatesMetricsBeforeIncidents(t *testing.T) { } source := string(data) - metricIndex := strings.Index(source, "CheckUnifiedResourceMetrics(resources)") + metricIndex := strings.Index(source, "CheckUnifiedResourceMetricsWithCapacityTrends(resources") incidentIndex := strings.Index(source, "SyncUnifiedResourceIncidents(resources)") if metricIndex < 0 { t.Fatalf("monitor alert sync must run unified resource metric evaluation") diff --git a/internal/monitoring/ceph.go b/internal/monitoring/ceph.go index 354cce2d2..6d609721e 100644 --- a/internal/monitoring/ceph.go +++ b/internal/monitoring/ceph.go @@ -93,7 +93,7 @@ func (m *Monitor) checkCephPoolStorage(cluster models.CephCluster) { } } if m.alertManager != nil { - m.alertManager.CheckStorage(storage) + m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, timestamp)) } } } diff --git a/internal/monitoring/metrics_history.go b/internal/monitoring/metrics_history.go index 3fee8361b..ea7accbb8 100644 --- a/internal/monitoring/metrics_history.go +++ b/internal/monitoring/metrics_history.go @@ -53,6 +53,9 @@ type MetricsHistory struct { diskMetrics map[string]*DiskMetrics // key: disk metrics resource ID maxDataPoints int retentionTime time.Duration + + capacityForecastMu sync.Mutex + capacityForecastCache map[string]storageCapacityForecastCacheEntry } // NewMetricsHistory creates a new metrics history tracker diff --git a/internal/monitoring/metrics_history_test.go b/internal/monitoring/metrics_history_test.go index 3954ceaa9..153e949a9 100644 --- a/internal/monitoring/metrics_history_test.go +++ b/internal/monitoring/metrics_history_test.go @@ -3,6 +3,8 @@ package monitoring import ( "testing" "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" ) func TestNewMetricsHistory(t *testing.T) { @@ -1395,3 +1397,27 @@ func TestCleanupMetricsReturnsNilForExpiredData(t *testing.T) { t.Errorf("cleanupMetrics should return nil for fully expired data, got slice with len=%d cap=%d", len(result), cap(result)) } } + +func TestMetricsHistorySuppliesAlertGradeStorageCapacityTrend(t *testing.T) { + now := time.Now().UTC().Truncate(time.Hour) + history := NewMetricsHistory(1000, 7*24*time.Hour) + for hour := 72; hour >= 0; hour-- { + history.AddStorageMetric( + "forecast-history-proof", + "usage", + 60+float64(72-hour)*0.08, + now.Add(-time.Duration(hour)*time.Hour), + ) + } + + monitor := &Monitor{metricsHistory: history} + trend := monitor.storageCapacityTrend(models.Storage{ + ID: "forecast-history-proof", Name: "archive", Status: "active", Usage: 65.76, + }, now) + if !trend.Ready || trend.Reason != "increasing" { + t.Fatalf("trend = %+v, want trusted increasing evidence", trend) + } + if trend.Confidence < 0.8 { + t.Fatalf("confidence = %.3f, want alert-grade evidence", trend.Confidence) + } +} diff --git a/internal/monitoring/monitor_alert_sync.go b/internal/monitoring/monitor_alert_sync.go index 98a037e41..c8fe2bc3c 100644 --- a/internal/monitoring/monitor_alert_sync.go +++ b/internal/monitoring/monitor_alert_sync.go @@ -2,6 +2,7 @@ package monitoring import ( "strings" + "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/logging" @@ -132,7 +133,7 @@ func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources. m.migrateAvailabilityLinksToCanonicalIDs(resources) - m.alertManager.CheckUnifiedResourceMetrics(resources) + m.alertManager.CheckUnifiedResourceMetricsWithCapacityTrends(resources, m.unifiedStorageCapacityTrends(resources, time.Now())) m.alertManager.SyncUnifiedResourceIncidents(resources) m.syncAlertsToState() } diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go index d2dc6c13e..2000eea3e 100644 --- a/internal/monitoring/monitor_alerts.go +++ b/internal/monitoring/monitor_alerts.go @@ -549,7 +549,7 @@ func (m *Monitor) checkMockAlerts() { Str("name", storage.Name). Float64("usage", storage.Usage). Msg("Checking storage for alerts") - m.alertManager.CheckStorage(storage) + m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now())) } // Check alerts for PBS instances diff --git a/internal/monitoring/monitor_polling_storage.go b/internal/monitoring/monitor_polling_storage.go index c2adaeb87..718a89dbe 100644 --- a/internal/monitoring/monitor_polling_storage.go +++ b/internal/monitoring/monitor_polling_storage.go @@ -710,7 +710,7 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string, } if m.alertManager != nil { - m.alertManager.CheckStorage(storage) + m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now())) } } diff --git a/internal/monitoring/storage_capacity_forecast.go b/internal/monitoring/storage_capacity_forecast.go new file mode 100644 index 000000000..d1ff8c5f9 --- /dev/null +++ b/internal/monitoring/storage_capacity_forecast.go @@ -0,0 +1,115 @@ +package monitoring + +import ( + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rs/zerolog/log" +) + +const ( + storageCapacityForecastLookback = 7 * 24 * time.Hour + storageCapacityForecastRefresh = 15 * time.Minute +) + +type storageCapacityForecastCacheEntry struct { + trend alerts.CapacityTrendObservation + expiresAt time.Time +} + +// storageCapacityTrend bridges monitoring-owned history into alert-owned +// predictive policy. SQLite history is included so a process restart does not +// erase confidence; the in-memory tail and current observation cover buffered +// writes and installations where durable metrics are unavailable. +func (m *Monitor) storageCapacityTrend(storage models.Storage, now time.Time) alerts.CapacityTrendObservation { + if m == nil || storage.ID == "" || storage.Usage <= 0 { + return alerts.CapacityTrendObservation{Reason: "invalid-current-usage"} + } + return m.storageCapacityTrendFor(storage.ID, storage.ID, storage.Usage, now) +} + +func (m *Monitor) storageCapacityTrendFor(cacheID, historyID string, currentUsage float64, now time.Time) alerts.CapacityTrendObservation { + if m == nil || cacheID == "" || historyID == "" || currentUsage <= 0 { + return alerts.CapacityTrendObservation{Reason: "invalid-current-usage"} + } + if now.IsZero() { + now = time.Now() + } + + if m.metricsHistory != nil { + m.metricsHistory.capacityForecastMu.Lock() + if cached, ok := m.metricsHistory.capacityForecastCache[cacheID]; ok && now.Before(cached.expiresAt) { + m.metricsHistory.capacityForecastMu.Unlock() + return cached.trend + } + m.metricsHistory.capacityForecastMu.Unlock() + } + + points := make([]alerts.CapacityMetricPoint, 0, 256) + if m.metricsStore != nil { + stored, err := m.metricsStore.Query( + "storage", + historyID, + "usage", + now.Add(-storageCapacityForecastLookback), + now, + int64(time.Hour/time.Second), + ) + if err != nil { + log.Debug().Err(err).Str("storage", cacheID).Msg("Persistent capacity history unavailable; using in-memory tail") + } else { + for _, point := range stored { + points = append(points, alerts.CapacityMetricPoint{Timestamp: point.Timestamp, Value: point.Value}) + } + } + } + if m.metricsHistory != nil { + for _, point := range m.metricsHistory.GetAllStorageMetrics(historyID, storageCapacityForecastLookback)["usage"] { + points = append(points, alerts.CapacityMetricPoint{Timestamp: point.Timestamp, Value: point.Value}) + } + } + points = append(points, alerts.CapacityMetricPoint{Timestamp: now, Value: currentUsage}) + + trend := alerts.EstimateCapacityTrend(points, now) + if m.metricsHistory != nil { + m.metricsHistory.capacityForecastMu.Lock() + if m.metricsHistory.capacityForecastCache == nil { + m.metricsHistory.capacityForecastCache = make(map[string]storageCapacityForecastCacheEntry) + } + m.metricsHistory.capacityForecastCache[cacheID] = storageCapacityForecastCacheEntry{ + trend: trend, + expiresAt: now.Add(storageCapacityForecastRefresh), + } + m.metricsHistory.capacityForecastMu.Unlock() + } + return trend +} + +func (m *Monitor) unifiedStorageCapacityTrends(resources []unifiedresources.Resource, now time.Time) map[string]alerts.CapacityTrendObservation { + trends := make(map[string]alerts.CapacityTrendObservation) + var resolver MetricsTargetResourceStore + if candidate, ok := m.resourceStore.(MetricsTargetResourceStore); ok { + resolver = candidate + } + for _, resource := range resources { + input, ok := alerts.UnifiedResourceInputFromResource(resource) + if !ok || input.Disk == nil { + continue + } + switch input.Type { + case "truenas-pool", "truenas-dataset", "vmware-datastore": + default: + continue + } + historyID := input.ID + if resolver != nil { + if target := resolver.MetricsTargetForResource(resource.ID); target != nil && target.ResourceType == "storage" && target.ResourceID != "" { + historyID = target.ResourceID + } + } + trends[input.ID] = m.storageCapacityTrendFor(input.ID, historyID, input.DiskValue(), now) + } + return trends +} diff --git a/internal/monitoring/storage_capacity_forecast_test.go b/internal/monitoring/storage_capacity_forecast_test.go new file mode 100644 index 000000000..08a1c3f3a --- /dev/null +++ b/internal/monitoring/storage_capacity_forecast_test.go @@ -0,0 +1,53 @@ +package monitoring + +import ( + "path/filepath" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" +) + +func TestStorageCapacityTrendUsesDurableHistoryAfterRestart(t *testing.T) { + now := time.Now().UTC().Truncate(time.Hour) + config := metrics.DefaultConfig(t.TempDir()) + config.DBPath = filepath.Join(t.TempDir(), "capacity-metrics.db") + store, err := metrics.NewStore(config) + if err != nil { + t.Fatalf("new metrics store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + writes := make([]metrics.WriteMetric, 0, 72) + for hour := 72; hour >= 1; hour-- { + writes = append(writes, metrics.WriteMetric{ + ResourceType: "storage", + ResourceID: "durable-storage", + MetricType: "usage", + Value: 60 + float64(72-hour)*0.08, + Timestamp: now.Add(-time.Duration(hour) * time.Hour), + Tier: metrics.TierHourly, + }) + } + store.WriteBatchSync(writes) + + // Deliberately omit MetricsHistory: this models the process immediately + // after restart, when the in-memory ring is empty but SQLite remains. + monitor := &Monitor{metricsStore: store} + trend := monitor.storageCapacityTrend(models.Storage{ + ID: "durable-storage", + Name: "archive", + Status: "active", + Usage: 65.76, + }, now) + if !trend.Ready || trend.Reason != "increasing" { + t.Fatalf("trend = %+v, want trusted increasing evidence from durable history", trend) + } + if trend.CoverageSpan < 48*time.Hour { + t.Fatalf("coverage = %s, want at least 48h", trend.CoverageSpan) + } + if trend.Confidence < 0.8 { + t.Fatalf("confidence = %.3f, want alert-grade evidence", trend.Confidence) + } +} From d3c45bea58c970258a8b901c673a46cd826cc775 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 27 Aug 2026 20:36:44 +0100 Subject: [PATCH 3/3] Converge remaining drawer detail rows --- .../subsystems/frontend-primitives.md | 11 + .../v6/internal/subsystems/registry.json | 18 +- .../internal/subsystems/unified-resources.md | 5 + frontend-modern/browser-verification.json | 47 ++-- .../scripts/shared-template-registry.json | 1 + .../AvailabilityProbeStatusCard.tsx | 191 +++++++-------- .../ResourceDetailDrawerOverviewTab.tsx | 230 ++++++++---------- .../ResourceDetailDrawer.history.test.tsx | 6 + ...esourceDetailDrawer.service-cards.test.tsx | 17 ++ .../AvailabilityProbeSuggestionCard.tsx | 37 ++- .../AvailabilityProbeStatusCard.test.tsx | 6 + .../SharedPrimitives.guardrails.test.ts | 14 ++ .../docker/DockerHostDrawerOverview.tsx | 31 +-- 13 files changed, 308 insertions(+), 306 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 41a3ccf07..d7203b68f 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -215,6 +215,7 @@ overflow. 1a. `frontend-modern/src/components/Infrastructure/resourceDetailDrawerMetricsHistoryModel.ts` 1b. `frontend-modern/src/components/Workloads/nodeDrawerModel.ts` 1c. `frontend-modern/src/features/docker/dockerHostDrawerModel.ts` + 1d. `frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx` 2. `frontend-modern/src/components/Settings/Settings.tsx` 3. `frontend-modern/src/components/Settings/SettingsDialogs.tsx` 4. `frontend-modern/src/components/Settings/SettingsPageShell.tsx` @@ -1226,6 +1227,12 @@ not a replacement status card, CTA band, or page-local nested card. otherwise the shared `lg` transition is canonical. Feature surfaces own the labels, values, wrapping, and selection behavior, but must not restore drawer-width `justify-between` rows on desktop. + This contract also covers secondary drawer facts in availability status and + suggestion cards, resource change-history entries, Docker/PBS/PMG service + support panels, and Docker container-update management cards. Headers, + status summaries, actions, disk capacity summaries, and RAID state pairs may + retain intentional endpoint alignment; ordinary label/value facts in those + surfaces may not. Read-only metadata chips belong to `MetadataBadge` and domain wrappers over it. Organization role and share-status chips must use `OrganizationRoleBadge` and `OrganizationShareStatusBadge`, so role/status @@ -6185,6 +6192,10 @@ Known platform resources use that same compact presentation when availability is attached. `AvailabilityProbeStatusCard` is the shared detail primitive for Workloads and Docker host drawers; it renders the complete target, protocol, latest result, latency when relevant, evidence freshness, and last observation. +Its fact rows, and the matching service/probe/target rows in +`AvailabilityProbeSuggestionCard`, compose `InfoCardKeyValueRow` so phone +layouts remain condensed while wide cards keep each value adjacent to its +label. Plural attached checks render as repeated bounded cards from `availabilityChecks`, while the row keeps one compatibility summary. Expired successful evidence must render an amber `Stale` state with no green diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index d90c300ae..b6bf1b1e1 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4899,6 +4899,7 @@ "frontend-modern/src/components/SetupWizard/SetupCompletionPreview.tsx", "frontend-modern/src/components/SetupWizard/SetupWizard.tsx", "frontend-modern/src/components/Toast/Toast.tsx", + "frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx", "frontend-modern/src/components/Workloads/nodeDrawerModel.ts", "frontend-modern/src/features/docker/dockerHostDrawerModel.ts", "frontend-modern/src/features/platformPage/platformEstateOverviewModel.ts", @@ -5185,8 +5186,21 @@ "frontend-modern/src/utils/__tests__/reportingResourceTypes.test.ts" ] }, - { - "id": "workload-presentation-helpers", + { + "id": "compact-info-card-consumers", + "label": "compact information card consumer proof", + "match_prefixes": [], + "match_files": [ + "frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx" + ], + "allow_same_subsystem_tests": false, + "test_prefixes": [], + "exact_files": [ + "frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts" + ] + }, + { + "id": "workload-presentation-helpers", "label": "workload presentation helper proof", "match_prefixes": [], "match_files": [ diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 00cff987b..c83bf6ee5 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1329,6 +1329,11 @@ and temperature facts must also compose the frontend-primitives `InfoCardKeyValueRow`. Mobile rows retain their condensed endpoint layout; desktop rows use the shared fixed label track so labels and values remain visually adjacent instead of spanning the full drawer width. +The same boundary applies to availability facts, resource change-history +metadata, Docker/PBS/PMG service facts, nested PMG queue/mail breakdowns, and +Docker container-update management facts. Their headers, actions, and compact +status summaries remain feature-owned, while ordinary label/value rows compose +`InfoCardKeyValueRow` rather than restoring a drawer-width flex split. Curated technical inventory follows the shared compact-row contract instead of the secondary-card contract. Docker-host drawers must project system, runtime, memory, storage, and telemetry facts through diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index a79b4887d..c1766e9ba 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,43 +1,36 @@ { "version": 1, - "base_sha": "e0baecc418d15a6638ef80ed1e2b39d08aa9a5ad", - "verified_at": "2026-08-27T18:57:57Z", + "base_sha": "81b09d870fb39112cbcef70becf09d6f30d77393", + "verified_at": "2026-08-27T19:35:10Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/components/Storage/StorageDetailKeyValueRow.tsx", - "frontend-modern/src/components/shared/InfoCardFrame.tsx", - "frontend-modern/src/components/shared/cards/DisksCard.tsx", - "frontend-modern/src/components/shared/cards/HardwareCard.tsx", - "frontend-modern/src/components/shared/cards/RootDiskCard.tsx", - "frontend-modern/src/components/shared/cards/SystemInfoCard.tsx", - "frontend-modern/src/components/shared/cards/TemperaturesCard.tsx", - "frontend-modern/src/features/storageBackups/detailPresentation.ts" + "frontend-modern/src/components/Infrastructure/AvailabilityProbeStatusCard.tsx", + "frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx", + "frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx", + "frontend-modern/src/features/docker/DockerHostDrawerOverview.tsx" ], "content_sha256": { - "frontend-modern/src/components/Storage/StorageDetailKeyValueRow.tsx": "db827a709c0f018c24dd0821ae9845022a2ba15c7052b6471926957aafdf6270", - "frontend-modern/src/components/shared/InfoCardFrame.tsx": "0819bce063e2caf72ffbbd3beb398fd4776a7fe089da7eac5c899bd92fffe66c", - "frontend-modern/src/components/shared/cards/DisksCard.tsx": "f9721f90d6b1c1a7030bbc1767c90a4d353d8c4a82843e47ad37aae8f687c46d", - "frontend-modern/src/components/shared/cards/HardwareCard.tsx": "66df4609cc019cf5a7d0928abb82a2ad88f99683cd277ee1f0e38b9b48d281dc", - "frontend-modern/src/components/shared/cards/RootDiskCard.tsx": "48a1698a5781639a2fcb027837edc6c7b580e392b37041c89911f23c71d80fc6", - "frontend-modern/src/components/shared/cards/SystemInfoCard.tsx": "723fab36dbd6eeaf882f6eb89d2d0c7c276f5b930c17bfcbc44413a13f531880", - "frontend-modern/src/components/shared/cards/TemperaturesCard.tsx": "f91cecf1de23abec0b7b571b02516643206ba2b27f7cb37cd8b941f2588229dd", - "frontend-modern/src/features/storageBackups/detailPresentation.ts": "6dd2da6052aa37e6cfe2488356a296c578c26f7aed3ee8e49f6a5362135b34e7" + "frontend-modern/src/components/Infrastructure/AvailabilityProbeStatusCard.tsx": "2d991718c178866ba5bd7d1b89e7a1a9bec6fcd30133bc2fd549aa2fd3ccf4e5", + "frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx": "ac68e2e7f5d954ca5bd23b44f347817ce9a39414ab493d22ad52340ceb3296e2", + "frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx": "a8c549b1284befee652c9ecc79c3c449602d1a102f3867acb81f4a5c9f5fd042", + "frontend-modern/src/features/docker/DockerHostDrawerOverview.tsx": "ab63fba8e1a06cb5951e0d3dc637d92ef371f2b5db5da2dfb80408cd477c0ea5" }, - "routes": ["/standalone/machines", "/proxmox/storage"], + "routes": ["/standalone/availability", "/proxmox/backups/date", "/docker/overview"], "viewports": [ { "width": 1920, "height": 800 }, { "width": 390, "height": 844 } ], "states": [ - "Apollo-114 expanded with Platform details and Machine details visible", - "west-f-service-pool expanded with Configuration and ZFS Pool cards visible" + "Solar inverter web panel expanded with availability facts visible", + "backup-vault expanded with Platform details and PBS service facts visible", + "auth-service-01 expanded on Manage with Container updates visible" ], "interactions": [ - "expanded the Apollo-114 row, opened Platform details, and revealed the nested Machine details", - "confirmed System, Hardware, Disks, and temperature facts use an adjacent fixed label track at 1920px", - "confirmed machine facts retain condensed right-aligned values and clean wrapping at 390px", - "expanded west-f-service-pool and confirmed Configuration values sit adjacent to their labels at 1920px", - "confirmed storage Configuration values retain condensed right alignment and clean wrapping at 390px", - "reloaded the phone machine route and repeated the complete disclosure interaction" + "expanded the Solar inverter web panel and confirmed desktop availability values use an adjacent fixed label track", + "confirmed availability facts retain condensed right-aligned values and clean wrapping at 390px", + "expanded backup-vault, opened Platform details, and revealed the PBS service section", + "confirmed PBS service facts use adjacent desktop values and retain compact mobile alignment", + "expanded auth-service-01, selected Manage, and confirmed Container updates facts use adjacent desktop values", + "confirmed Docker management facts retain condensed right alignment at 390px" ] } diff --git a/frontend-modern/scripts/shared-template-registry.json b/frontend-modern/scripts/shared-template-registry.json index a697ac9c0..13a0f3426 100644 --- a/frontend-modern/scripts/shared-template-registry.json +++ b/frontend-modern/scripts/shared-template-registry.json @@ -466,6 +466,7 @@ { "path": "src/components/shared/cards/RootDiskCard.tsx" }, { "path": "src/components/shared/cards/SystemInfoCard.tsx" }, { "path": "src/components/shared/cards/TemperaturesCard.tsx" }, + { "path": "src/components/Workloads/AvailabilityProbeSuggestionCard.tsx" }, { "path": "src/components/Workloads/DrawerDiskListCard.tsx" }, { "path": "src/features/docker/DockerHostDrawerOverview.tsx" }, { "path": "src/features/storageBackups/detailPresentation.ts" } diff --git a/frontend-modern/src/components/Infrastructure/AvailabilityProbeStatusCard.tsx b/frontend-modern/src/components/Infrastructure/AvailabilityProbeStatusCard.tsx index e4144e020..f16bffbfe 100644 --- a/frontend-modern/src/components/Infrastructure/AvailabilityProbeStatusCard.tsx +++ b/frontend-modern/src/components/Infrastructure/AvailabilityProbeStatusCard.tsx @@ -2,7 +2,7 @@ import { For, Show, createMemo } from 'solid-js'; import { Activity, AlertCircle, Check } from 'lucide-solid'; import type { ResourceAvailabilityMeta } from '@/types/resource'; -import { InfoCardFrame } from '@/components/shared/InfoCardFrame'; +import { InfoCardFrame, InfoCardKeyValueRow } from '@/components/shared/InfoCardFrame'; import { getAvailabilityProbeMethodLabel, getAvailabilityProbeEndpointLabel, @@ -129,130 +129,111 @@ export function AvailabilityProbeStatusCard(props: AvailabilityProbeStatusCardPr
-
- Latency - —} - > - —} > - {latency()} - - -
-
- Method - - {method()} - -
-
- Target - - {targetAddr()} - -
+ + {latency()} + + + } + /> + + -
- Checked - {lastChecked()} -
+ -
- Freshness - - {presentation()?.freshnessLabel ?? 'freshness unknown'} - -
+ {(label) => ( -
- Resource - {label()} -
+ )} {(cert) => (
-
- Certificate - - {certificateTrust().label} - -
+ -
- Subject - - {cert().subject} - -
+ {(expiry) => ( -
- Expires - - {expiry()} - -
+ )} -
- Hostname - - {cert().hostnameValid ? 'Matches' : 'Mismatch'} - -
+ -
- Issuer - - {cert().issuer} - -
+ {(fingerprint) => ( -
- SHA-256 - - {fingerprint().slice(0, 16)}… - -
+ )}
diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx index 814e3f640..9f84d2b00 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx @@ -18,7 +18,7 @@ import { RaidCard } from '@/components/shared/cards/RaidCard'; import { DiscoveryTab } from '@/components/Discovery/DiscoveryTab'; import { DiscoveryLoadingFallback } from '@/components/shared/DiscoveryLoadingFallback'; import { FormSelect } from '@/components/shared/FormSelect'; -import { InfoCardFrame } from '@/components/shared/InfoCardFrame'; +import { InfoCardFrame, InfoCardKeyValueRow } from '@/components/shared/InfoCardFrame'; import { WebInterfaceUrlField } from '@/components/shared/WebInterfaceUrlField'; import { WEB_INTERFACE_LINK_COLOR_CLASS, @@ -575,31 +575,27 @@ export const ResourceDetailDrawerOverviewTab: Component{sourceTypePresentation.label}
-
- Confidence - - {formatConfidenceLabel(change.confidence)} - -
-
- Adapter - - {sourceAdapterPresentation?.label || '—'} - -
+ + -
- Actor - {change.actor} -
+ -
- Transition - - {change.from || '—'} → {change.to || '—'} - -
+
@@ -726,27 +722,24 @@ export const ResourceDetailDrawerOverviewTab: Component
-
- Containers - - {formatInteger(drawer.dockerContainerCount())} - -
-
- Updates - 0 ? 'text-sky-700 dark:text-sky-300' : 'text-base-content'}`} - > - {formatInteger(drawer.dockerUpdatesAvailable())} - -
+ + 0 + ? 'text-sky-700 dark:text-sky-300' + : '' + } + /> -
- Checked - - {drawer.dockerUpdatesCheckedRelative()} - -
+ @@ -758,24 +751,24 @@ export const ResourceDetailDrawerOverviewTab: Component
-
- Action - - {formatIdentifierLabel(drawer.dockerHostCommand()?.type, { - fallback: 'command', - })} - -
-
- State - - {formatIdentifierLabel(drawer.dockerHostCommand()?.status, { - fallback: 'unknown', - })} - -
+ +
-
- State - - {connection.label} - -
+ -
- Version - {pbs().version} -
+ -
- Uptime - - {formatUptime(pbs().uptimeSeconds ?? resource.uptime ?? 0)} - -
+ 0}> -
- Active tasks - - {formatInteger(drawer.pbsActiveTaskCount())} - -
+
@@ -1141,25 +1127,19 @@ export const ResourceDetailDrawerOverviewTab: Component
-
- State - - {connection.label} - -
+ -
- Version - {pmg().version} -
+ -
- Uptime - - {formatUptime(pmg().uptimeSeconds ?? resource.uptime ?? 0)} - -
+
@@ -1187,22 +1167,21 @@ export const ResourceDetailDrawerOverviewTab: Component -
- Nodes - - {formatInteger(pmg().nodeCount)} - -
+ -
- Updated - - {drawer.pmgUpdatedRelative()} - -
+
@@ -1213,14 +1192,13 @@ export const ResourceDetailDrawerOverviewTab: Component {(entry) => ( -
- {entry.label} - - {formatInteger(entry.value)} - -
+ )}
@@ -1232,12 +1210,10 @@ export const ResourceDetailDrawerOverviewTab: Component {(entry) => ( -
- {entry.label} - - {formatInteger(entry.value)} - -
+ )}
diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx index 7ee1fd8ca..5753f8901 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx @@ -802,6 +802,12 @@ describe('ResourceDetailDrawer change history section', () => { expect(panel.getByText('Routine restart requested')).toBeInTheDocument(); expect(panel.getByText('Confidence')).toBeInTheDocument(); expect(panel.getByText('Adapter')).toBeInTheDocument(); + expect(panel.getByText('Confidence').parentElement).toHaveClass( + 'justify-between', + 'lg:grid', + 'lg:grid-cols-[7rem_minmax(0,1fr)]', + ); + expect(panel.getByText('High')).toHaveClass('text-right', 'lg:text-left'); expect(panel.getByText('Metadata')).toBeInTheDocument(); expect(panel.getByText(/"ticket": "INC-1234"/)).toBeInTheDocument(); expect(panel.queryByText('Capabilities')).toBeNull(); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.service-cards.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.service-cards.test.tsx index e702469b1..5ef3e9ff4 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.service-cards.test.tsx +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.service-cards.test.tsx @@ -113,6 +113,12 @@ describe('ResourceDetailDrawer service cards', () => { expect(serviceDetails.queryByText('Connection')).toBeNull(); expect(serviceDetails.getAllByText('State').length).toBeGreaterThan(0); expect(serviceDetails.getAllByText('pbs-main.local').length).toBeGreaterThan(0); + expect(serviceDetails.getByText('State').parentElement).toHaveClass( + 'justify-between', + 'lg:grid', + 'lg:grid-cols-[7rem_minmax(0,1fr)]', + ); + expect(serviceDetails.getByText('Healthy')).toHaveClass('text-right', 'lg:text-left'); expect(queryByText('Backup summary')).toBeNull(); expect(queryByText('Job breakdown')).toBeNull(); expect(queryByText('Types')).toBeNull(); @@ -307,6 +313,12 @@ describe('ResourceDetailDrawer service cards', () => { expect(serviceDetails.queryByText('Connection')).toBeNull(); expect(serviceDetails.getAllByText('State').length).toBeGreaterThan(0); expect(serviceDetails.getAllByText('pmg-main.local').length).toBeGreaterThan(0); + expect(serviceDetails.getByText('State').parentElement).toHaveClass( + 'justify-between', + 'lg:grid', + 'lg:grid-cols-[7rem_minmax(0,1fr)]', + ); + expect(serviceDetails.getByText('Healthy')).toHaveClass('text-right', 'lg:text-left'); expect(queryByText('Mail flow summary')).toBeNull(); expect(queryByText('Queue breakdown')).toBeNull(); expect(queryByText('Mail processing')).toBeNull(); @@ -384,6 +396,11 @@ describe('ResourceDetailDrawer service cards', () => { expect(getByText('18 containers · 4 updates')).toBeInTheDocument(); fireEvent.click(getByRole('button', { name: 'Show service' })); expect(getByText('Docker runtime')).toBeInTheDocument(); + expect(getByText('Containers').parentElement).toHaveClass( + 'justify-between', + 'lg:grid', + 'lg:grid-cols-[7rem_minmax(0,1fr)]', + ); expect(queryByText('Container Updates')).toBeNull(); expect(queryByText('Check now')).toBeNull(); expect(queryByText('Show update controls')).toBeNull(); diff --git a/frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx b/frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx index c8d549c16..f2e3dedcb 100644 --- a/frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx +++ b/frontend-modern/src/components/Workloads/AvailabilityProbeSuggestionCard.tsx @@ -7,7 +7,7 @@ import { type AvailabilityTarget, } from '@/api/availabilityTargets'; import type { AvailabilityProbeSuggestion } from '@/types/discovery'; -import { InfoCardFrame } from '@/components/shared/InfoCardFrame'; +import { InfoCardFrame, InfoCardKeyValueRow } from '@/components/shared/InfoCardFrame'; interface AvailabilityProbeSuggestionCardProps { suggestion: AvailabilityProbeSuggestion; @@ -64,28 +64,19 @@ export function AvailabilityProbeSuggestionCard(props: AvailabilityProbeSuggesti
-
- Service - - {props.suggestion.service_name} - -
-
- Probe - {protocolLabel()} -
-
- Target - - {props.suggestion.address} - -
+ + +
{error()}
 diff --git a/frontend-modern/src/components/Workloads/__tests__/AvailabilityProbeStatusCard.test.tsx b/frontend-modern/src/components/Workloads/__tests__/AvailabilityProbeStatusCard.test.tsx index e9797623d..89c591e98 100644 --- a/frontend-modern/src/components/Workloads/__tests__/AvailabilityProbeStatusCard.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/AvailabilityProbeStatusCard.test.tsx @@ -21,6 +21,12 @@ describe('AvailabilityProbeStatusCard', () => { expect(screen.getByText('Not checked')).toBeInTheDocument(); expect(screen.queryByText('Down')).not.toBeInTheDocument(); expect(screen.getByText('freshness unknown')).toBeInTheDocument(); + expect(screen.getByText('Latency').parentElement).toHaveClass( + 'justify-between', + 'lg:grid', + 'lg:grid-cols-[7rem_minmax(0,1fr)]', + ); + expect(screen.getByText('freshness unknown')).toHaveClass('text-right', 'lg:text-left'); }); it('shows stale evidence and an unresolved canonical resource link', () => { diff --git a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts index a74750013..6364fec79 100644 --- a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts +++ b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts @@ -1171,6 +1171,7 @@ describe('shared primitive guardrails', () => { 'src/components/shared/cards/RootDiskCard.tsx', 'src/components/shared/cards/SystemInfoCard.tsx', 'src/components/shared/cards/TemperaturesCard.tsx', + 'src/components/Workloads/AvailabilityProbeSuggestionCard.tsx', 'src/components/Workloads/DrawerDiskListCard.tsx', 'src/features/docker/DockerHostDrawerOverview.tsx', 'src/features/storageBackups/detailPresentation.ts', @@ -1233,6 +1234,19 @@ describe('shared primitive guardrails', () => { expect(source).toContain('InfoCardKeyValueRow'); expect(source).not.toContain('flex items-center justify-between'); } + + for (const consumerPath of [ + 'src/components/Infrastructure/AvailabilityProbeStatusCard.tsx', + 'src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx', + 'src/components/Workloads/AvailabilityProbeSuggestionCard.tsx', + 'src/features/docker/DockerHostDrawerOverview.tsx', + ]) { + const source = readFrontendSource(consumerPath); + expect(source).toContain('InfoCardKeyValueRow'); + expect(source).not.toMatch( + /class="flex items-(?:center|start) justify-between gap-2">\s*/, + ); + } }); it('keeps shared subtabs as one primitive and leaves shell styling to owning surfaces', () => { diff --git a/frontend-modern/src/features/docker/DockerHostDrawerOverview.tsx b/frontend-modern/src/features/docker/DockerHostDrawerOverview.tsx index 9458063fb..44ec32a69 100644 --- a/frontend-modern/src/features/docker/DockerHostDrawerOverview.tsx +++ b/frontend-modern/src/features/docker/DockerHostDrawerOverview.tsx @@ -5,7 +5,7 @@ import { type DrawerDiskListItem, } from '@/components/Workloads/DrawerDiskListCard'; import { AvailabilityProbeStatusCards } from '@/components/Infrastructure/AvailabilityProbeStatusCard'; -import { InfoCardFrame } from '@/components/shared/InfoCardFrame'; +import { InfoCardFrame, InfoCardKeyValueRow } from '@/components/shared/InfoCardFrame'; import { TechnicalDetailsSection } from '@/components/shared/TechnicalDetailsDisclosure'; import { DrawerAttentionSection } from '@/components/shared/DrawerAttentionSection'; import { @@ -99,32 +99,19 @@ export function DockerHostDrawerManagement(props: DockerHostDrawerOverviewProps) Container updates
-
- Available - {updatesAvailable()} -
+ {(timestamp) => ( -
- Last checked - {formatRelativeTime(timestamp)} -
+ )} -
- - {titleCase(cleanText(hostCommand()?.type).replace(/_/g, ' ') || 'Command')} - - - {titleCase(cleanText(hostCommand()?.status).replace(/_/g, ' ') || 'unknown')} - -
+