From 8098b9782562eb477c9d65fbc3de7fbecc167436 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:42:33 +0100 Subject: [PATCH] Audit agent config fetches only when they carry new information Agents poll /api/agents/agent/{id}/config every minute, and every successful poll was written as a signed agent_config_fetch audit event. On a five-agent homelab that was 483,000 of 483,101 audit rows (250 MB) over 75 days, burying the 64 logins and 25 CSRF failures the log exists to show; at the default 90-day retention it is roughly 60 MB per agent. Every failed fetch is still audited. A successful fetch is audited when it is new audit information: the agent's first delivery after Pulse starts, a different token or desiredConfig hash, or an unchanged delivery last recorded 24 hours earlier. Recorded successes now carry the delivered config hash and the reason. The tracker is per handler, keyed by org and agent, and bounded to 4,096 entries. Existing rows are left to age out under retention rather than deleted, since removing entries from a security log would look like tampering. AUDIT_LOGGING.md and the API and agent-lifecycle contracts describe the new rule. --- docs/AUDIT_LOGGING.md | 2 +- .../v6/internal/subsystems/agent-lifecycle.md | 5 ++ .../v6/internal/subsystems/api-contracts.md | 10 +++ .../internal/subsystems/storage-recovery.md | 11 +++ frontend-modern/public/docs/AUDIT_LOGGING.md | 2 +- internal/api/agent_config_fetch_audit.go | 80 +++++++++++++++++++ internal/api/agent_ingest.go | 21 ++++- ...gent_removal_lifecycle_integration_test.go | 71 ++++++++++++++++ internal/api/unified_agent_handlers_test.go | 33 ++++++++ 9 files changed, 230 insertions(+), 5 deletions(-) create mode 100644 internal/api/agent_config_fetch_audit.go diff --git a/docs/AUDIT_LOGGING.md b/docs/AUDIT_LOGGING.md index 15ff2bdd32..b6d397f85f 100644 --- a/docs/AUDIT_LOGGING.md +++ b/docs/AUDIT_LOGGING.md @@ -31,7 +31,7 @@ Pulse automatically captures the following events: | `agent_profile_assigned` | Agent profile assignments | Profile `production` assigned to agent | | `agent_profile_unassigned` | Agent profile removals | Profile removed from agent | | `user_roles_updated` | RBAC role assignments changed | Updated roles for user jane: [operator] | -| `agent_config_fetch` | Agent configuration retrieval attempts | Agent config fetched successfully | +| `agent_config_fetch` | Every failed agent configuration fetch. A successful fetch is recorded on the agent's first delivery after Pulse starts, when its token or delivered configuration changes, and otherwise once a day; agents poll every minute, so repeat polls are not recorded | `agent_id=… token_id=… config=sha256:… reason=config_changed` | Each event includes: - **Timestamp** (UTC) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 2dc2cee1e4..3e8d5f9ddc 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -5226,6 +5226,11 @@ agent truth, effective enforcement, and bounded reason separately so lifecycle surfaces can show desired-disabled/applied-enabled and desired-enabled/applied-disabled as drift or attention, and no-report cases as pending or unknown rather than in-sync. + +Routine config polling is not a per-poll audit event: +`/api/agents/agent/{id}/config` audits every failed fetch, and a successful one +only on the agent's first delivery after start, a token or desired-config +change, or once a day (see the API contract). That same canonical /api/auto-register path must also complete the live post-registration contract after persistence: it must trigger discovery refresh and emit the canonical `node_auto_registered` WebSocket payload instead of diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e79dbd405f..bfc559794f 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -8694,6 +8694,16 @@ payload shape: `agentId`, `commandsEnabled`, `settings`, `issuedAt`, and signed command decision and the signed settings payload, restricted to the agent-applied settings key schema, rather than treating `desiredConfig` itself as a direct member of that signature payload. + +Its `agent_config_fetch` audit events record every failed fetch but a +successful fetch only when it is new audit information: the agent's first +delivery after Pulse starts, a different token or `desiredConfig` hash, or an +unchanged delivery last recorded 24 hours earlier. Agents poll every minute, +and auditing each poll made this event over 99.9% of audit rows. Recorded +successes carry `config=` and +`reason=first_since_start|token_changed|config_changed|daily` in their details. +`TestAgentConfigFetchAuditsNewDeliveriesAndEveryFailure` and +`TestAgentConfigFetchAuditTrackerRecordsOnlyNewDeliveries` pin the rule. Agent profile delete and unassign clients must now also route canonical `204` success handling through shared allowed-status helpers in `frontend-modern/src/api/responseUtils.ts` instead of open-coding local diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 4aa2124c9c..bd3a2eeae3 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -2674,6 +2674,17 @@ turning inactive rows into recovery state or preloading their inventories. The surface contract pins the route query map, the inactive recovery null query, and the direct Protection query parameters. +### Agent config fetch auditing leaves recovery evidence unchanged + +`/api/agents/agent/{id}/config` now records a successful fetch in the security +audit log only when it is new audit information (the agent's first delivery +after start, a token or desired-config change, or once a day), while every +failed fetch is still recorded. This changes no storage or recovery path: +recovery actions keep their own action audits, dispatch receipts, and +verification records, and only routine successful config polls stop producing +one security-audit row each. The rule is pinned by +`internal/api/host_agent_removal_lifecycle_integration_test.go`. + ### Fresh agent command policy preserves the existing recovery authority boundary New self-hosted install-command tokens project their explicit command-policy diff --git a/frontend-modern/public/docs/AUDIT_LOGGING.md b/frontend-modern/public/docs/AUDIT_LOGGING.md index 15ff2bdd32..b6d397f85f 100644 --- a/frontend-modern/public/docs/AUDIT_LOGGING.md +++ b/frontend-modern/public/docs/AUDIT_LOGGING.md @@ -31,7 +31,7 @@ Pulse automatically captures the following events: | `agent_profile_assigned` | Agent profile assignments | Profile `production` assigned to agent | | `agent_profile_unassigned` | Agent profile removals | Profile removed from agent | | `user_roles_updated` | RBAC role assignments changed | Updated roles for user jane: [operator] | -| `agent_config_fetch` | Agent configuration retrieval attempts | Agent config fetched successfully | +| `agent_config_fetch` | Every failed agent configuration fetch. A successful fetch is recorded on the agent's first delivery after Pulse starts, when its token or delivered configuration changes, and otherwise once a day; agents poll every minute, so repeat polls are not recorded | `agent_id=… token_id=… config=sha256:… reason=config_changed` | Each event includes: - **Timestamp** (UTC) diff --git a/internal/api/agent_config_fetch_audit.go b/internal/api/agent_config_fetch_audit.go new file mode 100644 index 0000000000..3d03d4de17 --- /dev/null +++ b/internal/api/agent_config_fetch_audit.go @@ -0,0 +1,80 @@ +package api + +import ( + "sync" + "time" +) + +// agentConfigFetchAuditInterval re-records an unchanged successful config +// delivery at most this often per agent, so the audit trail still shows +// continued access. +const agentConfigFetchAuditInterval = 24 * time.Hour + +// maxAgentConfigFetchAudits bounds remembered deliveries. Beyond it, entries +// older than agentConfigFetchAuditInterval are dropped; they would be +// re-recorded on their next fetch anyway. +const maxAgentConfigFetchAudits = 4096 + +// agentConfigFetchAuditTracker decides which successful agent config fetches +// are audit events. Agents poll their config every minute, and recording each +// poll made agent_config_fetch over 99.9% of audit rows (about 1,440 a day per +// agent), burying the logins and failures the log exists to show. Failed +// fetches are always audited and never pass through here. +type agentConfigFetchAuditTracker struct { + mu sync.Mutex + last map[agentConfigFetchAuditKey]agentConfigFetchAuditEntry +} + +type agentConfigFetchAuditKey struct { + orgID string + agentID string +} + +type agentConfigFetchAuditEntry struct { + tokenID string + configHash string + auditedAt time.Time +} + +func newAgentConfigFetchAuditTracker() *agentConfigFetchAuditTracker { + return &agentConfigFetchAuditTracker{last: make(map[agentConfigFetchAuditKey]agentConfigFetchAuditEntry)} +} + +// observe records a successful delivery and reports whether it is new audit +// information, with the reason: the agent's first delivery since startup, a +// different token or delivered config, or an unchanged delivery last audited +// at least agentConfigFetchAuditInterval ago. A nil tracker audits every +// delivery. +func (t *agentConfigFetchAuditTracker) observe(orgID, agentID, tokenID, configHash string, now time.Time) (string, bool) { + if t == nil { + return "", true + } + key := agentConfigFetchAuditKey{orgID: orgID, agentID: agentID} + entry := agentConfigFetchAuditEntry{tokenID: tokenID, configHash: configHash, auditedAt: now} + + t.mu.Lock() + defer t.mu.Unlock() + previous, seen := t.last[key] + reason := "" + switch { + case !seen: + reason = "first_since_start" + case previous.tokenID != tokenID: + reason = "token_changed" + case previous.configHash != configHash: + reason = "config_changed" + case now.Sub(previous.auditedAt) >= agentConfigFetchAuditInterval: + reason = "daily" + default: + return "", false + } + if !seen && len(t.last) >= maxAgentConfigFetchAudits { + for staleKey, stale := range t.last { + if now.Sub(stale.auditedAt) >= agentConfigFetchAuditInterval { + delete(t.last, staleKey) + } + } + } + t.last[key] = entry + return reason, true +} diff --git a/internal/api/agent_ingest.go b/internal/api/agent_ingest.go index 452c41ab77..5d43fa882c 100644 --- a/internal/api/agent_ingest.go +++ b/internal/api/agent_ingest.go @@ -43,6 +43,8 @@ type UnifiedAgentHandlers struct { // server upgrade on their next report instead of their next hourly update // check. Empty (the default) omits it from acks. serverVersion string + + configFetchAudits *agentConfigFetchAuditTracker } // SetServerVersion supplies the running server version to include on report @@ -63,7 +65,10 @@ func trimUnifiedAgentRoutePath(path string) string { // NewUnifiedAgentHandlers constructs a new handler set for Pulse Unified Agent ingest. func NewUnifiedAgentHandlers(mtm *monitoring.MultiTenantMonitor, m *monitoring.Monitor, hub *websocket.Hub) *UnifiedAgentHandlers { - return &UnifiedAgentHandlers{baseAgentHandlers: newBaseAgentHandlers(mtm, m, hub)} + return &UnifiedAgentHandlers{ + baseAgentHandlers: newBaseAgentHandlers(mtm, m, hub), + configFetchAudits: newAgentConfigFetchAuditTracker(), + } } // HandleReport ingests Pulse Unified Agent reports. @@ -607,8 +612,18 @@ func (h *UnifiedAgentHandlers) handleGetConfig(w http.ResponseWriter, r *http.Re return } - LogAuditEventForTenant(GetOrgID(r.Context()), "agent_config_fetch", auth.GetUser(r.Context()), GetClientIP(r), r.URL.Path, true, - fmt.Sprintf("agent_id=%s token_id=%s", agentID, tokenID(record))) + configHash := "" + if signedConfig.DesiredConfig != nil { + configHash = signedConfig.DesiredConfig.Hash + } + orgID := GetOrgID(r.Context()) + if reason, audit := h.configFetchAudits.observe(orgID, agentID, tokenID(record), configHash, time.Now()); audit { + details := fmt.Sprintf("agent_id=%s token_id=%s config=%s", agentID, tokenID(record), configHash) + if reason != "" { + details += " reason=" + reason + } + LogAuditEventForTenant(orgID, "agent_config_fetch", auth.GetUser(r.Context()), GetClientIP(r), r.URL.Path, true, details) + } } func tokenID(record *config.APITokenRecord) string { diff --git a/internal/api/host_agent_removal_lifecycle_integration_test.go b/internal/api/host_agent_removal_lifecycle_integration_test.go index 502a80c385..c32a44cd9b 100644 --- a/internal/api/host_agent_removal_lifecycle_integration_test.go +++ b/internal/api/host_agent_removal_lifecycle_integration_test.go @@ -16,6 +16,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" + "github.com/rcourtman/pulse-go-rewrite/pkg/audit" ) type hostRemovalLifecycleHTTPRuntime struct { @@ -647,3 +648,73 @@ func TestHostAgentRenameKeepsCommandGateAlignedWithChannelAdmission(t *testing.T t.Fatal("channel admission admitted a different agent ID on a matching hostname") } } + +// Agents poll their config every minute. Only deliveries that tell an auditor +// something new are recorded, while every failed fetch still is. +func TestAgentConfigFetchAuditsNewDeliveriesAndEveryFailure(t *testing.T) { + capture := &auditCaptureLogger{} + prevLogger := audit.GetLogger() + prevManager := GetTenantAuditManager() + audit.SetLogger(capture) + SetTenantAuditManager(nil) + t.Cleanup(func() { + audit.SetLogger(prevLogger) + SetTenantAuditManager(prevManager) + }) + + handler, monitor := newUnifiedAgentHandlers(t, nil) + hostID := seedUnifiedAgentHost(t, monitor) + monitorState(t, monitor).UpsertHost(models.Host{ID: hostID, Hostname: "node-1", TokenID: "runtime-token"}) + fetch := func(scopes ...string) int { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/agents/agent/"+hostID+"/config", nil) + attachAPITokenRecord(req, &config.APITokenRecord{ID: "runtime-token", Scopes: scopes}) + rec := httptest.NewRecorder() + handler.HandleConfig(rec, req) + return rec.Code + } + + for i := 0; i < 3; i++ { + if code := fetch(config.ScopeAgentConfigRead, config.ScopeAgentReport); code != http.StatusOK { + t.Fatalf("config fetch %d status = %d, want 200", i, code) + } + } + commandsEnabled := true + if err := monitor.UpdateHostAgentConfig(hostID, &commandsEnabled); err != nil { + t.Fatalf("UpdateHostAgentConfig: %v", err) + } + for i := 0; i < 2; i++ { + if code := fetch(config.ScopeAgentConfigRead, config.ScopeAgentReport); code != http.StatusOK { + t.Fatalf("config fetch after change %d status = %d, want 200", i, code) + } + } + for i := 0; i < 2; i++ { + if code := fetch(config.ScopeMonitoringRead); code == http.StatusOK { + t.Fatalf("config fetch with only %s succeeded", config.ScopeMonitoringRead) + } + } + + capture.mu.Lock() + events := append([]audit.Event(nil), capture.events...) + capture.mu.Unlock() + var successes []string + failures := 0 + for _, event := range events { + if event.EventType != "agent_config_fetch" { + continue + } + if event.Success { + successes = append(successes, event.Details) + } else { + failures++ + } + } + if len(successes) != 2 || + !strings.Contains(successes[0], "reason=first_since_start") || + !strings.Contains(successes[1], "reason=config_changed") { + t.Fatalf("successful fetch audit events = %q, want the first delivery and the config change", successes) + } + if failures != 2 { + t.Fatalf("failed fetch audit events = %d, want every failure", failures) + } +} diff --git a/internal/api/unified_agent_handlers_test.go b/internal/api/unified_agent_handlers_test.go index ffab87690b..4abb3b8a6f 100644 --- a/internal/api/unified_agent_handlers_test.go +++ b/internal/api/unified_agent_handlers_test.go @@ -552,3 +552,36 @@ func TestUnifiedAgentHandlers_HandleLinkUnlink(t *testing.T) { t.Fatalf("unlink status = %d, want 200: %s", unlinkRec.Code, unlinkRec.Body.String()) } } + +func TestAgentConfigFetchAuditTrackerRecordsOnlyNewDeliveries(t *testing.T) { + tracker := newAgentConfigFetchAuditTracker() + start := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) + for _, step := range []struct { + name string + token string + hash string + at time.Duration + reason string + audit bool + }{ + {"first delivery", "tok-a", "sha256:1", 0, "first_since_start", true}, + {"unchanged poll", "tok-a", "sha256:1", time.Minute, "", false}, + {"config change", "tok-a", "sha256:2", 2 * time.Minute, "config_changed", true}, + {"unchanged after change", "tok-a", "sha256:2", 3 * time.Minute, "", false}, + {"token rotation", "tok-b", "sha256:2", 4 * time.Minute, "token_changed", true}, + {"just under a day", "tok-b", "sha256:2", 4*time.Minute + agentConfigFetchAuditInterval - time.Second, "", false}, + {"a day since last audit", "tok-b", "sha256:2", 4*time.Minute + agentConfigFetchAuditInterval, "daily", true}, + } { + reason, audit := tracker.observe("default", "agent-1", step.token, step.hash, start.Add(step.at)) + if audit != step.audit || reason != step.reason { + t.Fatalf("%s: audit=%v reason=%q, want audit=%v reason=%q", step.name, audit, reason, step.audit, step.reason) + } + } + if reason, audit := tracker.observe("other-org", "agent-1", "tok-b", "sha256:2", start.Add(5*time.Minute)); !audit || reason != "first_since_start" { + t.Fatalf("same agent ID in another org: audit=%v reason=%q, want its own first delivery", audit, reason) + } + var unset *agentConfigFetchAuditTracker + if _, audit := unset.observe("default", "agent-1", "tok-a", "sha256:1", start); !audit { + t.Fatal("a handler without a tracker must audit every delivery") + } +}