diff --git a/docs/release-control/v6/internal/records/host-identity-fork-heal-on-reenrollment-2026-08-24.md b/docs/release-control/v6/internal/records/host-identity-fork-heal-on-reenrollment-2026-08-24.md index 2e435143a..fadffc1fd 100644 --- a/docs/release-control/v6/internal/records/host-identity-fork-heal-on-reenrollment-2026-08-24.md +++ b/docs/release-control/v6/internal/records/host-identity-fork-heal-on-reenrollment-2026-08-24.md @@ -48,3 +48,25 @@ patch. - `-d3b-83adbde6` — twice-forked: a base over 40 characters is truncated before the next suffix lands, leaving an amputated first suffix plus a full second one. + +## Prevention boundary landed (2026-08-27) + +New re-enrollments no longer fork merely because the retiring agent delivered +one final in-flight report after the replacement install token was created. +`staleHostIdentityForReenrollment` now admits that overlap for at most the old +agent's own health window, and only for a Pulse-issued install token with one +unambiguous same-machine, equivalent-hostname candidate. A changed non-empty +report IP, an active identity conflict, an arbitrary API token, multiple +matching records, or longer overlap still takes the clone-safe fork. The +handoff also retires bindings for the superseded token so a late old process +cannot reclaim the stable identity. + +Regression coverage is in `TestInstallHandoffReusesIdentityAcrossFinalInFlightReport` +and `TestInstallHandoffRequiresTrustedUnambiguousIdentityEvidence` alongside +the existing #1654 lifecycle cases. + +This is the source-side prevention slice. Hosts already persisted under one or +more suffixed identities still require a separately governed migration that +preserves identity-keyed profile, metadata, availability, history, and alert +state. The coverage gap therefore remains triaged rather than being declared +closed by prevention alone. diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 5ff53f046..ad598ab63 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -9716,6 +9716,11 @@ "path": "internal/monitoring/monitor_agents.go", "kind": "file" }, + { + "repo": "pulse", + "path": "internal/monitoring/monitor_host_agents_test.go", + "kind": "file" + }, { "repo": "pulse", "path": "internal/servicediscovery/service.go", diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index ad07738db..8bb85c5a8 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -432,6 +432,17 @@ or reconcile command permissions. Frontend install surfaces may choose whether commands are requested for a newly minted credential but must not compose install-token scope lists themselves. +The same server-issued metadata is the only token evidence that may authorize +a host-identity handoff during re-enrollment. A final report from the retiring +agent may arrive after the replacement token was minted, so the token-creation +boundary includes at most one health window of that existing agent. Reuse of +the existing identity additionally requires one unambiguous record with the +same machine ID and equivalent hostname; differing non-empty report IPs or an +active identity conflict veto the handoff. An arbitrary API token, multiple +matching identities, or an agent that remains live beyond that window must +retain the clone-safe fork. Once accepted, bindings for superseded tokens are +retired so a late old process cannot overwrite the replacement identity. + PVE node setup shared boundaries that render or copy `PulseMonitor` permissions must treat `VM.GuestAgent.Audit` plus `VM.GuestAgent.FileRead` as the PVE 9+ primary contract, with `VM.Monitor` retained only as the legacy PVE diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index b77ee0219..95a4dee30 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -2991,14 +2991,20 @@ allowance is the only path that clears that lineage. Focused proofs are `internal/api/host_agent_removal_lifecycle_integration_test.go`; the concurrency proof must also pass under the Go race detector. -Fresh-install reconciliation also applies before a tombstone exists. A token -created after a stale same-machine, same-normalized-hostname observation is -explicit re-enrollment evidence: monitoring preserves the established host ID, -removes older duplicate generations and token bindings, and keeps physical-disk -resource identities attached to that host. A generation observed at or after -the new token's creation remains live and is never removed by this rule. -`internal/monitoring/monitor_host_agents_test.go` proves stable-ID reuse, -duplicate cleanup, and the live-generation guard. +Fresh-install reconciliation also applies before a tombstone exists. A +Pulse-issued install token created after a same-machine, +same-normalized-hostname observation is explicit re-enrollment evidence: +monitoring preserves the established host ID, removes older duplicate +generations and token bindings, and keeps physical-disk resource identities +attached to that host. The retiring process may deliver one final in-flight +report after token creation, so eligibility extends through at most that +agent's own health window. One unambiguous candidate is required; differing +non-empty report IPs, an active identity conflict, multiple candidates, an +arbitrary API token, or reports beyond the overlap window preserve the +clone-safe fork. Accepted handoffs retire the old token binding so a late old +process cannot overwrite the replacement. `internal/monitoring/monitor_host_agents_test.go` +proves stable-ID reuse, overlap handoff, clone-safety vetoes, duplicate cleanup, +and the live-generation guard. ### Native pool-health collection and appliance isolation diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index ceb24db22..fb670b63e 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -9,6 +9,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/agentupdate" + "github.com/rcourtman/pulse-go-rewrite/internal/api/agentbinding" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/logging" "github.com/rcourtman/pulse-go-rewrite/internal/mock" @@ -2350,39 +2351,70 @@ func normalizedSecurityStrings(values []string) []string { } // staleHostIdentityForReenrollment recognizes explicit reinstall intent before -// a new token is bound. Reusing the stale record's ID keeps serial-less -// physical-disk identities stable across reinstall while the token creation -// boundary prevents a foreign token from taking over a live host. +// a new token is bound. A stopped agent may send a final in-flight report just +// after the replacement token is minted, so allow one of its own health windows +// of overlap. The supported install issuer, stable machine/hostname, report-IP, +// and identity-conflict checks prevent an arbitrary token or visible clone from +// taking over a live host. func staleHostIdentityForReenrollment(report agentshost.Report, tokenRecord *config.APITokenRecord, hosts []models.Host) string { if tokenRecord == nil || tokenRecord.CreatedAt.IsZero() { return "" } machineID := strings.TrimSpace(report.Host.MachineID) hostname := strings.TrimSpace(report.Host.Hostname) + reportIP := strings.TrimSpace(report.Host.ReportIP) tokenID := strings.TrimSpace(tokenRecord.ID) if machineID == "" || hostname == "" || tokenID == "" { return "" } + if !agentbinding.CanBindInstallToken(tokenRecord, machineID, hostname) { + return "" + } - var bestID string - var bestLastSeen time.Time + var candidateID string for _, candidate := range hosts { if strings.TrimSpace(candidate.MachineID) != machineID || !unifiedresources.HostnamesEquivalent(candidate.Hostname, hostname) || strings.TrimSpace(candidate.TokenID) == tokenID || - !candidate.LastSeen.Before(tokenRecord.CreatedAt) { + candidate.IdentityConflict != nil { continue } - candidateID := strings.TrimSpace(candidate.ID) - if candidateID == "" { + candidateReportIP := strings.TrimSpace(candidate.ReportIP) + if reportIP != "" && candidateReportIP != "" && !strings.EqualFold(reportIP, candidateReportIP) { continue } - if bestID == "" || candidate.LastSeen.After(bestLastSeen) { - bestID = candidateID - bestLastSeen = candidate.LastSeen + if candidate.LastSeen.After(tokenRecord.CreatedAt.Add(hostAgentHealthWindow(candidate.IntervalSeconds))) { + continue } + matchedID := strings.TrimSpace(candidate.ID) + if matchedID == "" { + continue + } + if candidateID != "" && candidateID != matchedID { + return "" + } + candidateID = matchedID + } + return candidateID +} + +func (m *Monitor) retireSupersededHostTokenBindings(hostID, currentTokenID string) { + hostID = strings.TrimSpace(hostID) + currentTokenID = strings.TrimSpace(currentTokenID) + if hostID == "" { + return + } + m.mu.Lock() + defer m.mu.Unlock() + for key, boundID := range m.hostTokenBindings { + if strings.TrimSpace(boundID) != hostID { + continue + } + if key == currentTokenID || strings.HasPrefix(key, currentTokenID+":") { + continue + } + delete(m.hostTokenBindings, key) } - return bestID } // hostRenameHealSource returns the host record that keeps a renamed machine's @@ -2676,6 +2708,9 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. identifier = bindingID } m.mu.Unlock() + if reusedStaleID != "" { + m.retireSupersededHostTokenBindings(reusedStaleID, tokenID) + } } } diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index c3d486abd..b895f4522 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/api/agentbinding" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/models" @@ -4673,6 +4674,17 @@ func issue1654Report(timestamp time.Time) agentshost.Report { } } +func issue1654InstallToken(id string, createdAt time.Time) *config.APITokenRecord { + return &config.APITokenRecord{ + ID: id, + CreatedAt: createdAt, + Metadata: map[string]string{ + "install_type": "host", + "issued_via": agentbinding.IssuedViaConfig, + }, + } +} + func TestIssue1654FreshInstallReusesStalePhysicalHostIdentity(t *testing.T) { monitor := issue1654Monitor() createdAt := time.Now().UTC() @@ -4688,7 +4700,7 @@ func TestIssue1654FreshInstallReusesStalePhysicalHostIdentity(t *testing.T) { host, err := monitor.ApplyHostReport( issue1654Report(createdAt.Add(time.Minute)), - &config.APITokenRecord{ID: "fresh-token", CreatedAt: createdAt}, + issue1654InstallToken("fresh-token", createdAt), ) if err != nil { t.Fatalf("ApplyHostReport() error = %v", err) @@ -4707,6 +4719,114 @@ func TestIssue1654FreshInstallReusesStalePhysicalHostIdentity(t *testing.T) { } } +func TestInstallHandoffReusesIdentityAcrossFinalInFlightReport(t *testing.T) { + monitor := issue1654Monitor() + createdAt := time.Now().UTC().Add(-2 * time.Minute) + monitor.state.UpsertHost(models.Host{ + ID: "original-host-id", + Hostname: "disk-host.local", + MachineID: "machine-stable", + ReportIP: "192.0.2.10", + TokenID: "old-token", + LastSeen: createdAt.Add(time.Minute), + IntervalSeconds: 30, + Status: "online", + }) + monitor.hostTokenBindings["old-token:disk-host.local"] = "original-host-id" + report := issue1654Report(time.Now().UTC()) + report.Host.ReportIP = "192.0.2.10" + + host, err := monitor.ApplyHostReport(report, issue1654InstallToken("fresh-token", createdAt)) + if err != nil { + t.Fatalf("ApplyHostReport() error = %v", err) + } + if host.ID != "original-host-id" { + t.Fatalf("host ID = %q, want stable original-host-id", host.ID) + } + if got := len(monitor.state.GetHosts()); got != 1 { + t.Fatalf("host count = %d, want 1", got) + } + if _, ok := monitor.hostTokenBindings["old-token:disk-host.local"]; ok { + t.Fatal("overlapped old-token binding survived install handoff") + } +} + +func TestInstallHandoffRequiresTrustedUnambiguousIdentityEvidence(t *testing.T) { + createdAt := time.Now().UTC() + report := issue1654Report(createdAt.Add(time.Minute)) + report.Host.ReportIP = "192.0.2.10" + candidate := models.Host{ + ID: "original-host-id", + Hostname: "disk-host.local", + MachineID: "machine-stable", + ReportIP: "192.0.2.10", + TokenID: "old-token", + LastSeen: createdAt.Add(time.Minute), + IntervalSeconds: 30, + } + + for _, test := range []struct { + name string + token *config.APITokenRecord + mutate func(*agentshost.Report, *models.Host) + secondary *models.Host + }{ + { + name: "arbitrary API token", + token: &config.APITokenRecord{ID: "fresh-token", CreatedAt: createdAt}, + }, + { + name: "report IP changed", + token: issue1654InstallToken("fresh-token", createdAt), + mutate: func(report *agentshost.Report, _ *models.Host) { + report.Host.ReportIP = "192.0.2.11" + }, + }, + { + name: "identity conflict active", + token: issue1654InstallToken("fresh-token", createdAt), + mutate: func(_ *agentshost.Report, candidate *models.Host) { + candidate.IdentityConflict = &models.HostIdentityConflict{Hostnames: []string{"disk-host.local", "clone.local"}} + }, + }, + { + name: "overlap exceeds health window", + token: issue1654InstallToken("fresh-token", createdAt), + mutate: func(_ *agentshost.Report, candidate *models.Host) { + candidate.LastSeen = createdAt.Add(4 * time.Minute) + }, + }, + { + name: "multiple matching identities", + token: issue1654InstallToken("fresh-token", createdAt), + secondary: &models.Host{ + ID: "another-host-id", + Hostname: "disk-host.local", + MachineID: "machine-stable", + ReportIP: "192.0.2.10", + TokenID: "another-old-token", + LastSeen: createdAt.Add(time.Minute), + IntervalSeconds: 30, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + testReport := report + testCandidate := candidate + if test.mutate != nil { + test.mutate(&testReport, &testCandidate) + } + hosts := []models.Host{testCandidate} + if test.secondary != nil { + hosts = append(hosts, *test.secondary) + } + if got := staleHostIdentityForReenrollment(testReport, test.token, hosts); got != "" { + t.Fatalf("staleHostIdentityForReenrollment() = %q, want no handoff", got) + } + }) + } +} + func TestIssue1654ExistingDuplicateGenerationIsSuperseded(t *testing.T) { monitor := issue1654Monitor() createdAt := time.Now().UTC() @@ -4750,19 +4870,27 @@ func TestIssue1654ExistingDuplicateGenerationIsSuperseded(t *testing.T) { func TestIssue1654LivePreexistingAgentIsNotSuperseded(t *testing.T) { monitor := issue1654Monitor() - createdAt := time.Now().UTC() + createdAt := time.Now().UTC().Add(-5 * time.Minute) monitor.state.UpsertHost(models.Host{ - ID: "still-live-host", - Hostname: "disk-host.local", - MachineID: "machine-stable", - TokenID: "old-token", - LastSeen: createdAt.Add(time.Minute), - Status: "online", + ID: "still-live-host", + Hostname: "disk-host.local", + MachineID: "machine-stable", + TokenID: "old-token", + LastSeen: createdAt.Add(4 * time.Minute), + IntervalSeconds: 30, + Status: "online", }) + if got := staleHostIdentityForReenrollment( + issue1654Report(createdAt.Add(5*time.Minute)), + issue1654InstallToken("fresh-token", createdAt), + monitor.state.GetHosts(), + ); got != "" { + t.Fatalf("live candidate unexpectedly eligible for install handoff: %q", got) + } _, err := monitor.ApplyHostReport( - issue1654Report(createdAt.Add(2*time.Minute)), - &config.APITokenRecord{ID: "fresh-token", CreatedAt: createdAt}, + issue1654Report(createdAt.Add(5*time.Minute)), + issue1654InstallToken("fresh-token", createdAt), ) if err != nil { t.Fatalf("ApplyHostReport() error = %v", err)