Files
pulse/internal/monitoring/docker_host_identity.go
T
rcourtman 3c31aa4805 Stop Docker records flip-flopping under shared same-hostname tokens
The Docker analog of the #1753 estate was still broken: two live
machines reusing one short hostname and one pasted unified install
token collapsed into a single flip-flopping DockerHost record, because
the hostname+token identity fallback adopts a record whose machine ID
disagrees with the report's. That fold is deliberate for recreated
containers (whose /etc/machine-id regenerates), so it cannot simply be
guarded by machine-ID inequality - the discriminator is a revisit: a
recreated container transitions to its new machine ID exactly once,
while two live machines alternate. Removing the collapsed record then
revoked the shared token unconditionally, rejecting every surviving
module - host reports included, since a unified install shares one
credential - with 401 "Unauthorized access attempt".

Consult the identity flap tracker before the hostname fallbacks adopt a
machine-ID-disagreeing record: a report whose machine ID returns to a
value already seen behind that identity is a second live machine and is
not folded. The machine whose identifiers minted the record reclaims it,
so the first site keeps its record and history, and the other site falls
through to the token binding check, converging on the documented "Each
Docker / Podman module must use a unique API token" rejection instead of
silently overwriting the record every cycle. RemoveDockerHost now skips
token revocation while any host or Docker record still authenticates
with the credential, mirroring the host-agent removal guard.

Regression coverage: an end-to-end router test walking the two-site
shared-token estate (host + Docker reports, alternating cycles, removal)
asserting the first site's identity stays stable, the second site gets
the unique-token guidance, and the shared token survives removal; a
router test proving removal of a machine's Docker record keeps the
unified token its host record still uses; and a state-layer test pinning
the reclaim/no-flip-flop convergence. Recreated-container adoption and
the existing token-uniqueness rejections keep their tests unchanged.
2026-09-01 15:26:48 +01:00

458 lines
14 KiB
Go

package monitoring
import (
"crypto/sha1"
"encoding/hex"
"fmt"
"strings"
"unicode"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
)
// tokenHintFromRecord returns a redacted token hint for display purposes.
func tokenHintFromRecord(record *config.APITokenRecord) string {
if record == nil {
return ""
}
switch {
case record.Prefix != "" && record.Suffix != "":
return fmt.Sprintf("%s…%s", record.Prefix, record.Suffix)
case record.Prefix != "":
return record.Prefix + "…"
case record.Suffix != "":
return "…" + record.Suffix
default:
return ""
}
}
// resolveDockerHostIdentifier determines a unique identifier for a Docker host
// based on its report and existing hosts. Returns the identifier, fallback identifiers,
// the existing host (if matched), and whether a match was found. machineRevisit
// (optional) reports whether folding the report's machine ID into a candidate
// identity would alternate back to a machine already seen behind it; see
// findMatchingDockerHost.
func resolveDockerHostIdentifier(report agentsdocker.Report, tokenRecord *config.APITokenRecord, hosts []*unifiedresources.DockerHostView, machineRevisit func(identifier, machineID string) bool) (string, []string, *unifiedresources.DockerHostView, bool) {
base := strings.TrimSpace(report.AgentKey())
fallbacks := uniqueNonEmptyStrings(
base,
strings.TrimSpace(report.Agent.ID),
strings.TrimSpace(report.Host.MachineID),
strings.TrimSpace(report.Host.Hostname),
)
if existing, ok := findMatchingDockerHost(hosts, report, tokenRecord, machineRevisit); ok {
return dockerHostStableID(existing), fallbacks, existing, true
}
identifier := base
if identifier == "" {
identifier = strings.TrimSpace(report.Host.MachineID)
}
if identifier == "" {
identifier = strings.TrimSpace(report.Host.Hostname)
}
if identifier == "" {
identifier = strings.TrimSpace(report.Agent.ID)
}
if identifier == "" {
identifier = fallbackDockerHostID(report, tokenRecord)
}
if identifier == "" {
identifier = "docker-host"
}
if dockerHostIDExists(identifier, hosts) {
identifier = generateDockerHostIdentifier(identifier, report, tokenRecord, hosts)
}
return identifier, fallbacks, nil, false
}
// findMatchingDockerHost searches for an existing host that matches the report.
//
// The hostname-based fallbacks deliberately fold a report whose machine ID
// disagrees with the candidate record: containerized agents regenerate
// /etc/machine-id on recreation and must keep their identity. But two live
// machines reusing one short hostname and one shared install token also land
// in those fallbacks, and folding them collapses two sites into one
// flip-flopping record (the Docker analog of #1753). machineRevisit is the
// discriminator: a recreated container transitions to its new machine ID
// exactly once, while two live machines alternate, so a report whose machine
// ID *returns* to a value already seen behind the candidate identity is proof
// of a second machine and must not be folded.
func findMatchingDockerHost(hosts []*unifiedresources.DockerHostView, report agentsdocker.Report, tokenRecord *config.APITokenRecord, machineRevisit func(identifier, machineID string) bool) (*unifiedresources.DockerHostView, bool) {
agentID := strings.TrimSpace(report.Agent.ID)
tokenID := ""
if tokenRecord != nil {
tokenID = strings.TrimSpace(tokenRecord.ID)
}
machineID := strings.TrimSpace(report.Host.MachineID)
hostname := strings.TrimSpace(report.Host.Hostname)
if agentID != "" {
for _, host := range hosts {
if host == nil || strings.TrimSpace(host.AgentID()) != agentID {
continue
}
existingToken := strings.TrimSpace(host.TokenID())
if tokenID == "" || existingToken == tokenID {
if dockerHostIdentityConflicts(host, report) {
continue
}
return host, true
}
}
}
if machineID != "" && hostname != "" {
for _, host := range hosts {
if host == nil {
continue
}
if strings.TrimSpace(host.MachineID()) == machineID &&
unifiedresources.HostnamesEquivalent(host.Hostname(), hostname) {
if tokenID == "" || strings.TrimSpace(host.TokenID()) == tokenID {
return host, true
}
}
}
}
// Fallback: match by Hostname and Token only (when MachineID is missing)
// This fixes issues where containerized agents without persistent machine-id
// reconnect with the same token but are treated as new agents.
if hostname != "" && tokenID != "" {
for _, host := range hosts {
if host == nil {
continue
}
if unifiedresources.HostnamesEquivalent(host.Hostname(), hostname) &&
strings.TrimSpace(host.TokenID()) == tokenID {
if dockerHostMachineIDRevisits(host, agentID, machineID, machineRevisit) {
continue
}
return host, true
}
}
}
if machineID != "" && tokenID == "" {
for _, host := range hosts {
if host == nil {
continue
}
if strings.TrimSpace(host.MachineID()) == machineID && strings.TrimSpace(host.TokenID()) == "" {
return host, true
}
}
}
if hostname != "" && tokenID == "" {
for _, host := range hosts {
if host == nil {
continue
}
if unifiedresources.HostnamesEquivalent(host.Hostname(), hostname) &&
strings.TrimSpace(host.TokenID()) == "" {
if dockerHostMachineIDRevisits(host, agentID, machineID, machineRevisit) {
continue
}
return host, true
}
}
}
return nil, false
}
// dockerHostMachineIDRevisits reports whether folding a report with the given
// machine ID into the candidate host would return the record to a machine
// identity it has already alternated away from - two live machines behind one
// hostname, not one recreated container. See findMatchingDockerHost.
//
// A record whose stable ID is derived from the report's own machine or agent
// ID belongs to the reporting machine: the returning owner reclaims it, and
// the *other* machine is the one that splits off to its own identity. This
// keeps each site on the record its identifiers minted, so removing one
// site's record never blocks the sibling through a shared alias.
func dockerHostMachineIDRevisits(host *unifiedresources.DockerHostView, agentID, machineID string, machineRevisit func(identifier, machineID string) bool) bool {
if machineRevisit == nil || host == nil || machineID == "" {
return false
}
existingMachineID := strings.TrimSpace(host.MachineID())
if existingMachineID == "" || existingMachineID == machineID {
return false
}
stableID := dockerHostStableID(host)
if stableID != "" && (stableID == machineID || (agentID != "" && stableID == agentID)) {
return false
}
return machineRevisit(stableID, machineID)
}
// dockerHostIdentityConflicts reports whether an incoming report is clearly from
// a different physical/swarm node than the existing Docker host record, so a
// shared token+agentID reused from another node is not collapsed into one host.
func dockerHostIdentityConflicts(existing *unifiedresources.DockerHostView, report agentsdocker.Report) bool {
if existing == nil {
return false
}
existingMachineID := strings.TrimSpace(existing.MachineID())
reportMachineID := strings.TrimSpace(report.Host.MachineID)
if existingMachineID != "" && reportMachineID != "" && existingMachineID != reportMachineID {
return true
}
existingSwarmNodeID := ""
if sw := existing.Swarm(); sw != nil {
existingSwarmNodeID = strings.TrimSpace(sw.NodeID)
}
reportSwarmNodeID := ""
if report.Host.Swarm != nil {
reportSwarmNodeID = strings.TrimSpace(report.Host.Swarm.NodeID)
}
if existingSwarmNodeID != "" && reportSwarmNodeID != "" && existingSwarmNodeID != reportSwarmNodeID {
return true
}
// Without stronger stable IDs, a hostname change is ambiguous and should not
// be treated as the same reporting host automatically.
existingHostname := strings.TrimSpace(existing.Hostname())
reportHostname := strings.TrimSpace(report.Host.Hostname)
if existingMachineID == "" && reportMachineID == "" &&
existingSwarmNodeID == "" && reportSwarmNodeID == "" &&
existingHostname != "" && reportHostname != "" &&
!unifiedresources.HostnamesEquivalent(existingHostname, reportHostname) {
return true
}
return false
}
func dockerHostStableID(host *unifiedresources.DockerHostView) string {
if host == nil {
return ""
}
if id := strings.TrimSpace(host.HostSourceID()); id != "" {
return id
}
return strings.TrimSpace(host.ID())
}
func resolveDockerTokenBindingIdentity(identifier string, report agentsdocker.Report, previous *unifiedresources.DockerHostView, hasPrevious bool) (string, []string) {
preferred := ""
if hasPrevious {
preferred = dockerHostStableID(previous)
}
if preferred == "" {
preferred = strings.TrimSpace(report.Agent.ID)
}
if preferred == "" {
preferred = strings.TrimSpace(identifier)
}
if preferred == "" {
preferred = strings.TrimSpace(report.Host.MachineID)
}
if preferred == "" {
preferred = strings.TrimSpace(report.Host.Hostname)
}
aliases := uniqueNonEmptyStrings(
preferred,
identifier,
report.Agent.ID,
report.Host.MachineID,
report.Host.Hostname,
)
if previous != nil {
aliases = uniqueNonEmptyStrings(append(aliases,
dockerHostStableID(previous),
previous.AgentID(),
previous.MachineID(),
previous.Hostname(),
)...)
}
return preferred, aliases
}
func dockerTokenBindingMatches(boundAgentID string, aliases []string) bool {
boundAgentID = strings.TrimSpace(boundAgentID)
if boundAgentID == "" {
return false
}
for _, alias := range aliases {
if boundAgentID == strings.TrimSpace(alias) {
return true
}
}
return false
}
// dockerHostIDExists checks if a host ID is already in use.
func dockerHostIDExists(id string, hosts []*unifiedresources.DockerHostView) bool {
if strings.TrimSpace(id) == "" {
return false
}
for _, host := range hosts {
if dockerHostStableID(host) == id {
return true
}
}
return false
}
// generateDockerHostIdentifier creates a unique identifier by appending suffixes.
func generateDockerHostIdentifier(base string, report agentsdocker.Report, tokenRecord *config.APITokenRecord, hosts []*unifiedresources.DockerHostView) string {
if strings.TrimSpace(base) == "" {
base = fallbackDockerHostID(report, tokenRecord)
}
if strings.TrimSpace(base) == "" {
base = "docker-host"
}
used := make(map[string]struct{}, len(hosts))
for _, host := range hosts {
if id := dockerHostStableID(host); id != "" {
used[id] = struct{}{}
}
}
suffixes := dockerHostSuffixCandidates(report, tokenRecord)
for _, suffix := range suffixes {
candidate := fmt.Sprintf("%s::%s", base, suffix)
if _, exists := used[candidate]; !exists {
return candidate
}
}
seed := strings.Join(suffixes, "|")
if strings.TrimSpace(seed) == "" {
seed = base
}
sum := sha1.Sum([]byte(seed))
hashSuffix := fmt.Sprintf("hash-%s", hex.EncodeToString(sum[:6]))
candidate := fmt.Sprintf("%s::%s", base, hashSuffix)
if _, exists := used[candidate]; !exists {
return candidate
}
for idx := 2; ; idx++ {
candidate = fmt.Sprintf("%s::%d", base, idx)
if _, exists := used[candidate]; !exists {
return candidate
}
}
}
// dockerHostSuffixCandidates returns candidate suffixes for generating unique IDs.
func dockerHostSuffixCandidates(report agentsdocker.Report, tokenRecord *config.APITokenRecord) []string {
candidates := make([]string, 0, 5)
if tokenRecord != nil {
if sanitized := sanitizeDockerHostSuffix(tokenRecord.ID); sanitized != "" {
candidates = append(candidates, "token-"+sanitized)
}
}
if agentID := sanitizeDockerHostSuffix(report.Agent.ID); agentID != "" {
candidates = append(candidates, "agent-"+agentID)
}
if machineID := sanitizeDockerHostSuffix(report.Host.MachineID); machineID != "" {
candidates = append(candidates, "machine-"+machineID)
}
hostNameSanitized := sanitizeDockerHostSuffix(report.Host.Hostname)
if hostNameSanitized != "" {
candidates = append(candidates, "host-"+hostNameSanitized)
}
hostDisplay := sanitizeDockerHostSuffix(report.Host.Name)
if hostDisplay != "" && hostDisplay != hostNameSanitized {
candidates = append(candidates, "name-"+hostDisplay)
}
return uniqueNonEmptyStrings(candidates...)
}
// sanitizeDockerHostSuffix cleans a string for use as a host ID suffix.
func sanitizeDockerHostSuffix(value string) string {
value = strings.ToLower(strings.TrimSpace(value))
if value == "" {
return ""
}
var builder strings.Builder
builder.Grow(len(value))
lastHyphen := false
runeCount := 0
for _, r := range value {
if runeCount >= 48 {
break
}
switch {
case unicode.IsLetter(r) || unicode.IsDigit(r):
builder.WriteRune(r)
lastHyphen = false
runeCount++
default:
if !lastHyphen {
builder.WriteRune('-')
lastHyphen = true
runeCount++
}
}
}
result := strings.Trim(builder.String(), "-")
if result == "" {
return ""
}
return result
}
// fallbackDockerHostID generates a hash-based ID when no better identifier exists.
func fallbackDockerHostID(report agentsdocker.Report, tokenRecord *config.APITokenRecord) string {
seedParts := dockerHostSuffixCandidates(report, tokenRecord)
if len(seedParts) == 0 {
seedParts = uniqueNonEmptyStrings(
report.Host.Hostname,
report.Host.MachineID,
report.Agent.ID,
)
}
if len(seedParts) == 0 {
return ""
}
seed := strings.Join(seedParts, "|")
sum := sha1.Sum([]byte(seed))
return fmt.Sprintf("docker-host-%s", hex.EncodeToString(sum[:6]))
}
// uniqueNonEmptyStrings returns unique non-empty strings in order of first appearance.
func uniqueNonEmptyStrings(values ...string) []string {
seen := make(map[string]struct{}, len(values))
result := make([]string, 0, len(values))
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
result = append(result, value)
}
return result
}