Integrate trust-gate reliability fixes

This commit is contained in:
rcourtman
2026-07-20 14:54:36 +01:00
parent fbebdddb7d
commit b6a74576bc
72 changed files with 4755 additions and 421 deletions
+47 -15
View File
@@ -167,6 +167,13 @@ func run(ctx context.Context, args []string, getenv func(string) string) error {
return nil
}
if err := secureAgentStateDir(cfg.StateDir); err != nil {
logger.Warn().
Err(err).
Str("path", cfg.StateDir).
Msg("Failed to enforce owner-only agent state directory permissions")
}
// 2b. Compute Agent ID if missing (needed for remote config)
// We replicate the logic from hostagent.New to ensure we get the same ID
lookupHostname := strings.TrimSpace(cfg.HostnameOverride)
@@ -587,6 +594,20 @@ func configureAgentLogger(cfg Config) (zerolog.Logger, func(), error) {
}, nil
}
func secureAgentStateDir(path string) error {
path = strings.TrimSpace(path)
if path == "" {
return nil
}
if err := os.MkdirAll(path, 0o700); err != nil {
return fmt.Errorf("create state directory: %w", err)
}
if err := os.Chmod(path, 0o700); err != nil {
return fmt.Errorf("chmod state directory: %w", err)
}
return nil
}
// readAgentIDFile reads a persisted agent identifier from the given path.
func readAgentIDFile(path string) (string, error) {
if path == "" {
@@ -1062,8 +1083,19 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
// URL; the startup warning is emitted once the logger exists in run().
securityutil.SetOperatorPlaintextHTTPConsent(*allowPlaintextHTTPFlag)
// Resolve token with priority: --token > --token-file > env > default file
token := resolveToken(*tokenFlag, *tokenFileFlag, envToken)
// Resolve the state directory before any implicit token or identity path.
// A custom instance must never borrow the default instance's credentials.
stateDir := strings.TrimSpace(*stateDirFlag)
if stateDir == "" {
stateDir = defaultAgentStateDir()
}
agentIDFile := strings.TrimSpace(*agentIDFileFlag)
if agentIDFile == "" {
agentIDFile = filepath.Join(stateDir, "agent-id")
}
// Resolve token with priority: --token > --token-file > env > state-dir file.
token := resolveToken(*tokenFlag, *tokenFileFlag, envToken, stateDir)
observers, err := agenttarget.LoadObservers(strings.TrimSpace(*observersFileFlag), pulseURL)
if err != nil {
return Config{}, fmt.Errorf("load observer destinations: %w", err)
@@ -1071,11 +1103,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
// When --enroll is set and a runtime token already exists from a previous
// enrollment, use it instead of the bootstrap token embedded in the service
// config. This ensures the agent survives restarts after enrollment.
stateDir := strings.TrimSpace(*stateDirFlag)
if stateDir == "" {
stateDir = defaultAgentStateDir()
}
// config. This ensures the agent survives process and server restarts.
if *enrollFlag {
runtimeTokenPath := filepath.Join(stateDir, "runtime.token")
if content, err := os.ReadFile(runtimeTokenPath); err == nil {
@@ -1141,7 +1169,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
Interval: interval,
HostnameOverride: strings.TrimSpace(*hostnameFlag),
AgentID: strings.TrimSpace(*agentIDFlag),
AgentIDFile: strings.TrimSpace(*agentIDFileFlag),
AgentIDFile: agentIDFile,
Tags: tags,
InsecureSkipVerify: *insecureFlag,
AllowPlaintextHTTP: *allowPlaintextHTTPFlag,
@@ -1294,11 +1322,11 @@ func resolveEnableCommands(enableFlag, disableFlag bool, envEnable, envDisable s
// 1. --token flag (direct value)
// 2. --token-file flag (read from file)
// 3. PULSE_TOKEN environment variable
// 4. Default token file under the platform state directory
// 4. Token file under the resolved state directory
//
// Reading from a file is more secure than CLI args as tokens won't appear in `ps` output.
func resolveToken(tokenFlag, tokenFileFlag, envToken string) string {
return resolveTokenInternal(tokenFlag, tokenFileFlag, envToken, os.ReadFile)
func resolveToken(tokenFlag, tokenFileFlag, envToken, stateDir string) string {
return resolveTokenInternal(tokenFlag, tokenFileFlag, envToken, stateDir, os.ReadFile)
}
// defaultAgentStateDir mirrors where each platform's installer keeps agent
@@ -1318,7 +1346,7 @@ func defaultTokenFilePath() string {
return filepath.Join(defaultAgentStateDir(), "token")
}
func resolveTokenInternal(tokenFlag, tokenFileFlag, envToken string, readFile func(string) ([]byte, error)) string {
func resolveTokenInternal(tokenFlag, tokenFileFlag, envToken, stateDir string, readFile func(string) ([]byte, error)) string {
// 1. Direct token from --token flag
if t := strings.TrimSpace(tokenFlag); t != "" {
return t
@@ -1338,9 +1366,13 @@ func resolveTokenInternal(tokenFlag, tokenFileFlag, envToken string, readFile fu
return t
}
// 4. Default token file (most secure method for service installs)
defaultTokenFile := defaultTokenFilePath()
if content, err := readFile(defaultTokenFile); err == nil {
// 4. Token file in the resolved state directory. When stateDir is custom,
// do not fall through to the default instance's token.
tokenFile := filepath.Join(strings.TrimSpace(stateDir), "token")
if strings.TrimSpace(stateDir) == "" {
tokenFile = defaultTokenFilePath()
}
if content, err := readFile(tokenFile); err == nil {
if t := strings.TrimSpace(string(content)); t != "" {
return t
}
+94 -6
View File
@@ -793,10 +793,14 @@ func TestResolveEnableCommands(t *testing.T) {
}
func TestResolveToken(t *testing.T) {
customStateDir := filepath.Join(string(filepath.Separator), "custom", "pulse-agent")
fakeReadFile := func(path string) ([]byte, error) {
if path == defaultTokenFilePath() {
return []byte("default-token"), nil
}
if path == filepath.Join(customStateDir, "token") {
return []byte("custom-token"), nil
}
if path == "valid-file" {
return []byte("file-token"), nil
}
@@ -808,12 +812,14 @@ func TestResolveToken(t *testing.T) {
tokenFlag string
tokenFileFlag string
envToken string
stateDir string
expected string
}{
{"flag priority", "flag-token", "valid-file", "env-token", "flag-token"},
{"file priority", "", "valid-file", "env-token", "file-token"},
{"env priority", "", "", "env-token", "env-token"},
{"default file priority", "", "", "", "default-token"},
{"flag priority", "flag-token", "valid-file", "env-token", customStateDir, "flag-token"},
{"file priority", "", "valid-file", "env-token", customStateDir, "file-token"},
{"env priority", "", "", "env-token", customStateDir, "env-token"},
{"default file priority", "", "", "", defaultAgentStateDir(), "default-token"},
{"custom state file priority", "", "", "", customStateDir, "custom-token"},
}
// Update the test cases to avoid the default file if we want to test empty
@@ -826,7 +832,7 @@ func TestResolveToken(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := resolveTokenInternal(tc.tokenFlag, tc.tokenFileFlag, tc.envToken, fakeReadFile)
got := resolveTokenInternal(tc.tokenFlag, tc.tokenFileFlag, tc.envToken, tc.stateDir, fakeReadFile)
if got != tc.expected {
t.Fatalf("%s: expected %q, got %q", tc.name, tc.expected, got)
}
@@ -834,11 +840,18 @@ func TestResolveToken(t *testing.T) {
}
t.Run("truly empty", func(t *testing.T) {
got := resolveTokenInternal("", "", "", fakeReadFileNoDefault)
got := resolveTokenInternal("", "", "", customStateDir, fakeReadFileNoDefault)
if got != "" {
t.Fatalf("expected empty, got %q", got)
}
})
t.Run("custom state never borrows default token", func(t *testing.T) {
got := resolveTokenInternal("", "", "", filepath.Join(string(filepath.Separator), "missing-custom"), fakeReadFile)
if got != "" {
t.Fatalf("custom state unexpectedly borrowed default token %q", got)
}
})
}
func TestCleanupDockerAgent(t *testing.T) {
@@ -923,6 +936,9 @@ func TestLoadConfig(t *testing.T) {
if cfg.StateDir != defaultAgentStateDir() {
t.Errorf("expected platform state directory %q, got %q", defaultAgentStateDir(), cfg.StateDir)
}
if cfg.AgentIDFile != filepath.Join(defaultAgentStateDir(), "agent-id") {
t.Errorf("expected default agent ID file under state directory, got %q", cfg.AgentIDFile)
}
})
t.Run("env overrides", func(t *testing.T) {
@@ -995,6 +1011,61 @@ func TestLoadConfig(t *testing.T) {
if cfg.StateDir != "/custom/pulse-state" {
t.Errorf("expected explicit state directory, got %q", cfg.StateDir)
}
if cfg.AgentIDFile != "/custom/pulse-state/agent-id" {
t.Errorf("expected agent ID file under explicit state directory, got %q", cfg.AgentIDFile)
}
})
t.Run("custom state directory supplies implicit token", func(t *testing.T) {
stateDir := t.TempDir()
if err := os.WriteFile(filepath.Join(stateDir, "token"), []byte("custom-state-token\n"), 0600); err != nil {
t.Fatal(err)
}
cfg, err := loadConfig([]string{
"-url", "http://pulse.example.com",
"-state-dir", stateDir,
}, func(s string) string { return "" })
if err != nil {
t.Fatal(err)
}
if cfg.APIToken != "custom-state-token" {
t.Fatalf("expected implicit custom-state token, got %q", cfg.APIToken)
}
})
t.Run("custom enrollment restart prefers persisted runtime token", func(t *testing.T) {
stateDir := t.TempDir()
if err := os.WriteFile(filepath.Join(stateDir, "token"), []byte("bootstrap-token"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(stateDir, "runtime.token"), []byte("runtime-token"), 0600); err != nil {
t.Fatal(err)
}
cfg, err := loadConfig([]string{
"-url", "http://pulse.example.com",
"-state-dir", stateDir,
"-enroll",
}, func(s string) string { return "" })
if err != nil {
t.Fatal(err)
}
if cfg.APIToken != "runtime-token" {
t.Fatalf("expected persisted runtime token after restart, got %q", cfg.APIToken)
}
})
t.Run("explicit agent ID file overrides state-derived path", func(t *testing.T) {
cfg, err := loadConfig([]string{
"-token", "test-token",
"-state-dir", "/custom/pulse-state",
"-agent-id-file", "/identity/agent-id",
}, func(s string) string { return "" })
if err != nil {
t.Fatal(err)
}
if cfg.AgentIDFile != "/identity/agent-id" {
t.Errorf("expected explicit agent ID file, got %q", cfg.AgentIDFile)
}
})
t.Run("token optional when enrollment disabled", func(t *testing.T) {
@@ -2187,6 +2258,23 @@ func TestAgentIDFilePersistence(t *testing.T) {
})
}
func TestSecureAgentStateDir(t *testing.T) {
stateDir := filepath.Join(t.TempDir(), "state")
if err := os.MkdirAll(stateDir, 0755); err != nil {
t.Fatal(err)
}
if err := secureAgentStateDir(stateDir); err != nil {
t.Fatal(err)
}
info, err := os.Stat(stateDir)
if err != nil {
t.Fatal(err)
}
if got := info.Mode().Perm(); runtime.GOOS != "windows" && got != 0700 {
t.Fatalf("state directory mode = %o, want 700", got)
}
}
type stubTypedContainerUpdater struct {
calls int
}
+31
View File
@@ -284,6 +284,37 @@ go run ./cmd/patrol-qualify \
-artifacts tmp/patrol-qualification/<model-and-revision>
```
### Local-provider cold-start matrix
Manual-run release validation must also cover a cold local model independently
of finding-quality qualification. This matrix needs no cloud API key: use a
local Ollama model that passes Patrol preflight (for example a locally installed
`qwen3:8b`) and an already authenticated local Pulse session.
| Case | Preparation | First provider progress target | Required observations |
|---|---|---:|---|
| Warm control | Keep the model loaded | 05 seconds | POST returns one accepted `run_id`; status immediately reports the same `current_run_id` |
| Short cold load | Unload, then restart immediately | about 15 seconds | no start-timeout toast; one provider execution; one matching history record |
| Medium cold load | Unload and apply representative local memory pressure | about 30 seconds | accepted run remains running before provider progress; SSE reconnect does not retrigger POST |
| Long cold load | Use a deliberately cold model/runtime on representative hardware | about 60 seconds | no false error; completion or provider failure is recorded against the accepted `run_id` |
For Ollama, unload without deleting the model:
```sh
curl -fsS http://127.0.0.1:11434/api/generate \
-H 'Content-Type: application/json' \
-d '{"model":"qwen3:8b","keep_alive":0}'
```
For every row, capture the `POST /api/ai/patrol/run` response, poll
`GET /api/ai/patrol/status`, observe `/api/ai/patrol/stream`, and finally query
`GET /api/ai/patrol/runs?limit=30`. Pass only when the accepted ID is immediately
visible in status, the provider is invoked once, exactly one terminal history
record has that ID, browser/network and HTTP rejection errors remain distinct
from a recorded provider/runtime failure, and cancel/retry does not reuse stale
client tracking. The provider request timeout remains the terminal bound; a
quiet local model is never treated as a failed backend start.
`live-suite` selects every checked-in scenario for the requested track. The
remediation track still requires `--authorize-remediation`; selecting the
track does not broaden mutation authority.
@@ -338,6 +338,19 @@ update, profile rollout, command reachability, or fleet-control authority.
restart or OS reboot persistence, and complete uninstall cleanup through
the reusable lifecycle harness under `scripts/installtests/`.
26. `scripts/install.sh` shared with `deployment-installability`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
`--state-dir` is a whole-lifecycle ownership boundary, not only a runtime
flag. The resolved directory owns the protected bootstrap token,
enrollment runtime token, server-acknowledged `agent-id`, buffered and
command-receipt state, `connection.env`, and the saved offline installer.
Generated service definitions must carry that same directory and its token
file through install, process restart, server restart, update,
re-enrollment, and uninstall. Explicit state wins over discovered service
state, which wins over platform defaults; a custom instance must never
borrow token or identity files from the default instance. A changed
bootstrap token may clear the old enrollment runtime token to express
re-enrollment, while an unchanged token and tokenless update must preserve
it. Default platform paths remain valid migration inputs for installations
created before this contract.
Legacy update recovery is cross-platform lifecycle continuity. Linux may
read procfs or a systemd unit, while FreeBSD and pfSense must recover the
same URL, token, feature, identity, and trust arguments from the live
@@ -294,6 +294,15 @@ TLS floor in the dynamic config.
path must run the reusable lifecycle harness rather than stopping at a
parser check or foreground self-test.
8. `scripts/install.sh` shared with `agent-lifecycle`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
A caller-supplied `--state-dir` must remain canonical across rendered
systemd, launchd, OpenRC, rc.d, SysV, NAS wrapper, bootstrap, and reference
environment artifacts. Update and uninstall without a repeated custom path
must discover it from the active process or managed service before looking
at default-path state; explicit custom-path operations must not fall back
to another default instance. `connection.env` records the canonical state
and token-file paths without storing the token value, update rewrites the
same secure service shape, and uninstall removes the discovered canonical
directory rather than only `/var/lib/pulse-agent`.
Existing-agent update commands copied from the settings UI must use the
installer-owned `--update` mode rather than serializing a fresh enrollment
token into platform notice links. In `--update` mode, `scripts/install.sh`
@@ -1266,6 +1266,28 @@ The Proxmox polling runtime in `internal/monitoring/monitor_pve.go` must
evaluate disk alerts only after that merged disk view exists, so
controller-backed disks do not lose health and endurance coverage between
collection and alerting.
Wide SAS inventories use the same trust boundary. The host collector must
fan out per-disk SMART reads with bounded concurrency so a single report
deadline cannot truncate a controller-sized suffix of the inventory, and it
must preserve the Linux controller plus HCTL (or controller-member target)
alongside each reading. When Proxmox exposes a SAS address as `serial` while
smartctl exposes the drive serial, the smartctl serial is the canonical
hardware identity; exact device-path correlation is permitted only inside an
already-linked host/node parent and must fail closed when topology is
ambiguous. Direct SATA, SAS, and NVMe device fallback IDs retain their legacy
shape, while multiple controller members behind one block path add their
controller target to the fallback identity. Per-member I/O must never inherit
an aggregate controller counter.
Disk identity, temperature, I/O, controller association, and pool membership
also carry typed collection state from `pkg/diskinventory`: `available`,
transiently `unavailable`, provider/controller `unsupported`, or unexpectedly
`missing`. Normalization may retain the last known value when the current
observation is not available, but it must preserve the current state and
reason so API and UI consumers do not present retained evidence as freshly
collected. Unified-resource physical-disk round trips must retain named
`StorageGroup` membership rather than degrading it to the generic `Used`
filesystem label.
That same host-agent temperature boundary must prefer a recent linked host-agent
payload over legacy SSH collection once the agent provides any usable CPU, NVMe,
GPU, or SMART temperature reading. `internal/monitoring/monitor_polling_node_helpers.go`
@@ -351,6 +351,14 @@ the `white_label` branding entitlement.
retain that sink when remote configuration changes log level, and never
place runtime tokens or enrollment secrets in the service command or log
output.
Custom agent state directories share that same credential boundary: the
directory is owner-only, token, runtime-token, identity, and connection
files are mode `0600`, and implicit token lookup is confined to the
resolved directory instead of falling through to another instance's
default token. Installer health, lookup, and uninstall HTTP calls must feed
token headers through private curl configuration rather than exposing the
token in curl process arguments; generated service definitions and
`connection.env` may contain only protected token-file paths.
Global resource timeline reads through `/api/resources/timeline` are
adjacent monitoring-read surfaces, not a privacy bypass. Provider activity
filters may expose backend-authored task/event metadata, but the endpoint
+9 -3
View File
@@ -276,6 +276,8 @@ export interface FindingsTrustSummary {
export interface PatrolStatus {
runtime_state: PatrolRuntimeState;
running: boolean;
current_run_id?: string;
current_run_started_at?: string;
enabled: boolean;
last_patrol_at?: string;
last_activity_at?: string;
@@ -837,9 +839,13 @@ export interface PatrolRunScope {
* Patrol behaviour). Pass a scope to run a manual Targeted check scoped to
* specific resources e.g. a single alert's resource.
*/
export async function triggerPatrolRun(
scope?: PatrolRunScope,
): Promise<{ success: boolean; message: string }> {
export async function triggerPatrolRun(scope?: PatrolRunScope): Promise<{
success: boolean;
accepted?: boolean;
message: string;
run_id?: string;
started_at?: string;
}> {
const hasScope =
!!scope && ((scope.resource_ids?.length ?? 0) > 0 || (scope.resource_types?.length ?? 0) > 0);
return apiFetchJSON('/api/ai/patrol/run', {
@@ -56,6 +56,8 @@ export const DiskDetail: Component<DiskDetailProps> = (props) => {
attributeCards,
historyCharts,
metricResourceId,
collectionMessages,
liveIOAvailable,
} = useDiskDetailModel({
disk: () => props.disk,
});
@@ -115,6 +117,12 @@ export const DiskDetail: Component<DiskDetailProps> = (props) => {
</div>
</div>
<Show when={collectionMessages().length > 0}>
<div class={STORAGE_DETAIL_EMPTY_CLASS} role="status">
<For each={collectionMessages()}>{(message) => <p>{message}</p>}</For>
</div>
</Show>
{/* SMART attribute cards */}
<Show when={diskData().smartAttributes}>
<div class={STORAGE_DISK_DETAIL_ATTRIBUTE_GRID_CLASS}>
@@ -131,7 +139,7 @@ export const DiskDetail: Component<DiskDetailProps> = (props) => {
</Show>
{/* Live Performance Sparklines */}
<Show when={metricResourceId()}>
<Show when={metricResourceId() && liveIOAvailable()}>
<div class={STORAGE_DISK_DETAIL_SECTION_CLASS}>
<h4
class={`${STORAGE_DETAIL_SECTION_TITLE_CLASS} ${STORAGE_DISK_DETAIL_SECTION_HEADING_CLASS}`}
@@ -64,4 +64,35 @@ describe('DiskDetail', () => {
expect(Array.from(rangeSelector.options).map((option) => option.value)).not.toContain('14d');
expect(Array.from(rangeSelector.options).map((option) => option.value)).not.toContain('90d');
});
it('shows explicit collection status and does not render misleading live I/O', () => {
const disk = buildDisk();
disk.physicalDisk!.collection = {
serial: { state: 'available', source: 'smartctl' },
temperature: {
state: 'unavailable',
source: 'smartctl',
reason: 'collection deadline exceeded',
},
io: {
state: 'unsupported',
source: 'controller',
reason: 'per-member counters unavailable',
},
controller: { state: 'available', source: 'linux-sysfs' },
pool: { state: 'available', source: 'zpool-status' },
};
render(() => <DiskDetail disk={disk} nodes={[]} />);
expect(
screen.getByText('Temperature is temporarily unavailable: collection deadline exceeded'),
).toBeInTheDocument();
expect(
screen.getByText('Disk I/O is unsupported: per-member counters unavailable'),
).toBeInTheDocument();
expect(screen.queryByText('Live I/O (30m)')).not.toBeInTheDocument();
expect(screen.queryByText(/:diskread:/)).not.toBeInTheDocument();
expect(screen.queryByText(/:diskwrite:/)).not.toBeInTheDocument();
});
});
@@ -77,4 +77,35 @@ describe('useDiskDetailModel', () => {
expect(result.historyResourceId()).toBe('disk:truenas-main:sda');
});
it('disables live I/O only when the collector explicitly reports it unavailable', () => {
const [disk] = createSignal(
buildDisk({
physicalDisk: {
devPath: '/dev/sda',
model: 'SAS archive disk',
serial: 'ZR5TESTA0001',
diskType: 'sas',
temperature: 30,
collection: {
serial: { state: 'available', source: 'smartctl' },
temperature: { state: 'available', source: 'smartctl' },
io: {
state: 'unsupported',
source: 'controller',
reason: 'per-member counters unavailable',
},
controller: { state: 'available', source: 'linux-sysfs' },
pool: { state: 'available', source: 'zpool-status' },
},
},
} as Partial<Resource>),
);
const { result } = renderHook(() => useDiskDetailModel({ disk }));
expect(result.liveIOAvailable()).toBe(false);
expect(result.collectionMessages()).toEqual([
'Disk I/O is unsupported: per-member counters unavailable',
]);
});
});
@@ -1,6 +1,7 @@
import { Accessor, createMemo, createSignal } from 'solid-js';
import type { HistoryTimeRange } from '@/api/charts';
import {
getPhysicalDiskCollectionMessages,
extractPhysicalDiskPresentationData,
type PhysicalDiskPresentationData,
} from '@/features/storageBackups/diskPresentation';
@@ -29,6 +30,11 @@ export const useDiskDetailModel = (options: UseDiskDetailModelOptions) => {
);
const historyCharts = createMemo(() => getDiskDetailHistoryCharts(diskData()));
const metricResourceId = createMemo(() => historyResourceId());
const collectionMessages = createMemo(() => getPhysicalDiskCollectionMessages(diskData()));
const liveIOAvailable = createMemo(() => {
const state = diskData().collection?.io?.state;
return !state || state === 'available';
});
return {
chartRange,
@@ -38,5 +44,7 @@ export const useDiskDetailModel = (options: UseDiskDetailModelOptions) => {
attributeCards,
historyCharts,
metricResourceId,
collectionMessages,
liveIOAvailable,
};
};
@@ -284,6 +284,7 @@ export const buildProjectedOverrides = ({
const alertResourceIdCandidates = (resource: Resource): string[] =>
uniqueIds(
resource.id,
...(resource.canonicalIdentity?.supersededIds ?? []),
resource.metricsTarget?.resourceId,
resource.discoveryTarget?.resourceId,
resource.platformId,
@@ -4,14 +4,22 @@ import { guestOverrideIdCandidates, guestOverrideStorageId } from '../guestOverr
import type { Resource as TableResource } from './tableTypes';
import type { Override } from './types';
const exactOverrideIdentity = (resource?: Pick<TableResource, 'id'>) => ({
candidateIds: resource?.id ? [resource.id] : [],
storageId: resource?.id ?? '',
type OverrideIdentityResource = Pick<
TableResource,
'id' | 'type' | 'vmid' | 'node' | 'instance' | 'overrideIdCandidates' | 'overrideStorageId'
>;
const exactOverrideIdentity = (resource?: OverrideIdentityResource) => ({
candidateIds:
resource?.overrideIdCandidates && resource.overrideIdCandidates.length > 0
? resource.overrideIdCandidates
: resource?.id
? [resource.id]
: [],
storageId: resource?.overrideStorageId || resource?.id || '',
});
export const getOverridePersistenceIdentity = (
resource?: Pick<TableResource, 'id' | 'type' | 'vmid' | 'node' | 'instance'>,
) => {
export const getOverridePersistenceIdentity = (resource?: OverrideIdentityResource) => {
if (!resource || resource.type !== 'guest') {
return exactOverrideIdentity(resource);
}
@@ -25,7 +33,7 @@ export const getOverridePersistenceIdentity = (
export const findOverrideForResource = (
overrides: Override[],
resource?: Pick<TableResource, 'id' | 'type' | 'vmid' | 'node' | 'instance'>,
resource?: OverrideIdentityResource,
): Override | undefined => {
const { candidateIds } = getOverridePersistenceIdentity(resource);
return overrides.find((override) => candidateIds.includes(override.id));
@@ -33,7 +41,7 @@ export const findOverrideForResource = (
export const findRawOverrideConfigForResource = (
rawOverridesConfig: Record<string, RawOverrideConfig>,
resource?: Pick<TableResource, 'id' | 'type' | 'vmid' | 'node' | 'instance'>,
resource?: OverrideIdentityResource,
): RawOverrideConfig | undefined => {
const { candidateIds } = getOverridePersistenceIdentity(resource);
return candidateIds
@@ -43,7 +51,7 @@ export const findRawOverrideConfigForResource = (
export const stripOverrideCandidates = (
overrides: Override[],
resource?: Pick<TableResource, 'id' | 'type' | 'vmid' | 'node' | 'instance'>,
resource?: OverrideIdentityResource,
): Override[] => {
const { candidateIds } = getOverridePersistenceIdentity(resource);
if (candidateIds.length === 0) {
@@ -54,7 +62,7 @@ export const stripOverrideCandidates = (
export const stripRawOverrideCandidates = (
rawOverridesConfig: Record<string, RawOverrideConfig>,
resource?: Pick<TableResource, 'id' | 'type' | 'vmid' | 'node' | 'instance'>,
resource?: OverrideIdentityResource,
): Record<string, RawOverrideConfig> => {
const nextRawConfig = { ...rawOverridesConfig };
const { candidateIds } = getOverridePersistenceIdentity(resource);
@@ -0,0 +1,193 @@
import { renderHook } from '@solidjs/testing-library';
import { createSignal } from 'solid-js';
import { describe, expect, it, vi } from 'vitest';
import { buildProjectedOverrides } from '@/features/alerts/alertOverridesModel';
import type { Override } from '@/features/alerts/types';
import type { ThresholdsTableProps } from '@/features/alerts/thresholds/types';
import type { RawOverrideConfig } from '@/types/alerts';
import type { Resource } from '@/types/resource';
import { useThresholdsOverrideMutations } from '../useThresholdsOverrideMutations';
import { useThresholdsPlatformData } from '../useThresholdsPlatformData';
const canonicalID = 'agent-b9ed6d0e20e94eaf';
const legacyCanonicalID = 'agent-535886018cb53055';
const connectionID = 'truenas-connection-1';
const connectionBackedTrueNASResource = (machineId: string): Resource =>
({
id: canonicalID,
type: 'agent',
name: 'strawberrynas',
displayName: 'Strawberry NAS',
status: 'online',
platformType: 'truenas',
sources: ['truenas'],
identity: {
hostname: 'strawberrynas',
machineId,
},
truenas: {
hostname: 'strawberrynas',
},
metricsTarget: {
resourceType: 'agent',
resourceId: connectionID,
},
canonicalIdentity: {
primaryId: `agent:${connectionID}`,
aliases: [canonicalID, connectionID, 'strawberrynas'],
supersededIds: [legacyCanonicalID],
},
memory: {
current: 87,
},
}) as Resource;
const projectOverrides = (
rawConfig: Record<string, RawOverrideConfig>,
resource: Resource,
): Override[] =>
buildProjectedOverrides({
rawConfig,
nodeResources: [],
vmResources: [],
containerResources: [],
storageResources: [],
agentResourceList: [resource],
containerRuntimeResources: [resource],
getChildren: () => [],
pbsInstanceById: new Map(),
allResources: [resource],
});
describe('TrueNAS threshold persistence identity', () => {
it('re-homes a connection-target override onto the canonical ID and survives reload/refetch', () => {
const [resource, setResource] = createSignal(
connectionBackedTrueNASResource('serial-visible-on-first-poll'),
);
const initialRawConfig: Record<string, RawOverrideConfig> = {
[connectionID]: {
memory: {
trigger: 95,
clear: 90,
},
},
};
const [rawOverridesConfig, setRawOverridesConfig] = createSignal(initialRawConfig);
const [overrides, setOverrides] = createSignal<Override[]>(
projectOverrides(initialRawConfig, resource()),
);
const [editingId] = createSignal<string | null>(null);
const [editingThresholds] = createSignal<Record<string, number | undefined>>({
memory: 95,
});
const [editingNote] = createSignal('');
const [bulkEditIds] = createSignal<string[]>([]);
const setHasUnsavedChanges = vi.fn();
const cancelEdit = vi.fn();
const props = {
get allResources() {
return [resource()];
},
overrides,
setOverrides,
rawOverridesConfig,
setRawOverridesConfig,
trueNASDefaults: {
memory: 85,
},
trueNASDiskDefaults: {},
backupDefaults: () => ({ enabled: false, warningDays: 7, criticalDays: 14 }),
snapshotDefaults: () => ({
enabled: false,
warningDays: 30,
criticalDays: 45,
warningSizeGiB: 0,
criticalSizeGiB: 0,
}),
guestDisableConnectivity: () => false,
guestPoweredOffSeverity: () => 'warning' as const,
dockerDisableConnectivity: () => false,
dockerPoweredOffSeverity: () => 'warning' as const,
setHasUnsavedChanges,
} as unknown as ThresholdsTableProps;
const platform = renderHook(() =>
useThresholdsPlatformData({
props,
editingId,
searchTerm: () => '',
}),
);
const mutations = renderHook(() =>
useThresholdsOverrideMutations({
props,
resources: {
nodesWithOverrides: () => [],
agentsWithOverrides: () => [],
agentDisksWithOverrides: () => [],
dockerHostsWithOverrides: () => [],
guestsFlat: () => [],
dockerContainersFlat: () => [],
pbsServersWithOverrides: () => [],
pmgServersWithOverrides: () => [],
storageWithOverrides: () => [],
trueNASSystemsWithOverrides: platform.result.trueNASSystemsWithOverrides,
},
editingThresholds,
editingNote,
bulkEditIds,
cancelEdit,
updateBackupDefaults: vi.fn(),
updateSnapshotDefaults: vi.fn(),
}),
);
expect(platform.result.trueNASSystemsWithOverrides()).toEqual([
expect.objectContaining({
id: canonicalID,
hasOverride: true,
thresholds: {
memory: 95,
},
}),
]);
mutations.result.saveEdit(canonicalID);
expect(rawOverridesConfig()).toEqual({
[canonicalID]: {
memory: {
trigger: 95,
clear: 90,
},
},
});
expect(rawOverridesConfig()).not.toHaveProperty(connectionID);
// Model the alerts API JSON round-trip, followed by a resource refetch where
// TrueNAS changes the reported machine serial. The configured connection
// remains the durable identity, so the canonical resource ID must not move.
const persisted = JSON.parse(JSON.stringify(rawOverridesConfig())) as Record<
string,
RawOverrideConfig
>;
setRawOverridesConfig(persisted);
setResource(connectionBackedTrueNASResource('serial-missing-on-next-poll'));
setOverrides(projectOverrides(persisted, resource()));
expect(platform.result.trueNASSystemsWithOverrides()).toEqual([
expect.objectContaining({
id: canonicalID,
hasOverride: true,
thresholds: {
memory: 95,
},
}),
]);
expect(setHasUnsavedChanges).toHaveBeenCalledWith(true);
});
});
@@ -35,6 +35,7 @@ type AlertPlatformResourceType =
const resourceAlertIdCandidates = (resource: Resource): string[] =>
uniqueIds(
resource.id,
...(resource.canonicalIdentity?.supersededIds ?? []),
resource.metricsTarget?.resourceId,
resource.discoveryTarget?.resourceId,
resource.platformId,
@@ -113,7 +114,9 @@ const toTableResource = ({
const note = typeof override?.note === 'string' ? override.note : undefined;
return {
id: override?.id || candidates[0] || resource.id,
id: resourceAlertActionId(resource),
overrideIdCandidates: candidates,
overrideStorageId: resource.id,
name: getAlertResourceDisplayLabel(resource),
displayName: getAlertResourceDisplayLabel(resource),
rawName: resource.name,
@@ -7,6 +7,8 @@ export type ThresholdsActiveTab =
export interface Resource {
id: string;
overrideIdCandidates?: string[];
overrideStorageId?: string;
name: string;
displayName?: string;
policy?: ResourcePolicy;
@@ -0,0 +1,155 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { PatrolRunRecord, PatrolStatus } from '@/api/patrol';
import {
schedulePatrolRunAcceptanceReconciliation,
type PatrolRunAcceptanceOutcome,
} from '../patrolRunAcceptance';
const runningStatus = (runId: string): PatrolStatus =>
({
runtime_state: 'running',
running: true,
current_run_id: runId,
enabled: true,
}) as PatrolStatus;
const completedRun = (runId: string, status: 'healthy' | 'error' = 'healthy'): PatrolRunRecord =>
({
id: runId,
status,
error_count: status === 'error' ? 1 : 0,
}) as PatrolRunRecord;
describe('Patrol accepted-run reconciliation', () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
for (const firstProviderEventMs of [0, 15_000, 30_000, 60_000]) {
it(`keeps one accepted run authoritative when the first provider event arrives at ${firstProviderEventMs / 1000}s`, async () => {
const runId = `run-${firstProviderEventMs}`;
let providerEventObserved = false;
const history: PatrolRunRecord[] = [];
const errorToast = vi.fn();
const onResult = vi.fn<(outcome: PatrolRunAcceptanceOutcome) => void>((outcome) => {
if (outcome.kind === 'missing') errorToast();
});
const getStatus = vi.fn(async () =>
history.length > 0
? ({ ...runningStatus(runId), runtime_state: 'active', running: false } as PatrolStatus)
: runningStatus(runId),
);
const getHistory = vi.fn(async () => [...history]);
let providerCalls = 0;
providerCalls += 1;
setTimeout(() => {
providerEventObserved = true;
}, firstProviderEventMs);
setTimeout(() => {
history.push(completedRun(runId));
}, firstProviderEventMs + 1_000);
schedulePatrolRunAcceptanceReconciliation({
runId,
delayMs: 15_000,
refreshTimeoutMs: 5_000,
getStatus,
getHistory,
isCurrent: () => true,
onResult,
});
await vi.advanceTimersByTimeAsync(15_000);
expect(onResult).toHaveBeenCalledTimes(1);
expect(onResult.mock.calls[0][0].kind).toMatch(/running|recorded/);
expect(onResult).not.toHaveBeenCalledWith(expect.objectContaining({ kind: 'missing' }));
expect(errorToast).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(Math.max(0, firstProviderEventMs + 1_000 - 15_000));
expect(providerCalls).toBe(1);
expect(providerEventObserved).toBe(true);
expect(history).toHaveLength(1);
expect(history[0].id).toBe(runId);
expect(getStatus).toHaveBeenCalledTimes(1);
expect(getHistory).toHaveBeenCalledTimes(1);
});
}
it('preserves a provider/runtime failure as a recorded run instead of a start failure', async () => {
const onResult = vi.fn<(outcome: PatrolRunAcceptanceOutcome) => void>();
schedulePatrolRunAcceptanceReconciliation({
runId: 'run-provider-error',
delayMs: 15_000,
refreshTimeoutMs: 5_000,
getStatus: async () => ({ ...runningStatus('run-provider-error'), running: false }),
getHistory: async () => [completedRun('run-provider-error', 'error')],
isCurrent: () => true,
onResult,
});
await vi.advanceTimersByTimeAsync(15_000);
expect(onResult).toHaveBeenCalledWith(
expect.objectContaining({
kind: 'recorded',
run: expect.objectContaining({ id: 'run-provider-error', status: 'error' }),
}),
);
});
it('bounds hung reconciliation reads and reports a refresh failure', async () => {
const onResult = vi.fn<(outcome: PatrolRunAcceptanceOutcome) => void>();
schedulePatrolRunAcceptanceReconciliation({
runId: 'run-hung-read',
delayMs: 15_000,
refreshTimeoutMs: 5_000,
getStatus: () => new Promise(() => undefined),
getHistory: () => new Promise(() => undefined),
isCurrent: () => true,
onResult,
});
await vi.advanceTimersByTimeAsync(20_000);
expect(onResult).toHaveBeenCalledTimes(1);
expect(onResult.mock.calls[0][0].kind).toBe('refresh_failed');
});
it('cancels without reading status or history and permits a clean retry', async () => {
const firstStatus = vi.fn(async () => runningStatus('run-cancelled'));
const firstHistory = vi.fn(async () => [] as PatrolRunRecord[]);
const firstResult = vi.fn<(outcome: PatrolRunAcceptanceOutcome) => void>();
const cancel = schedulePatrolRunAcceptanceReconciliation({
runId: 'run-cancelled',
delayMs: 15_000,
refreshTimeoutMs: 5_000,
getStatus: firstStatus,
getHistory: firstHistory,
isCurrent: () => true,
onResult: firstResult,
});
cancel();
await vi.advanceTimersByTimeAsync(15_000);
expect(firstStatus).not.toHaveBeenCalled();
expect(firstHistory).not.toHaveBeenCalled();
expect(firstResult).not.toHaveBeenCalled();
const retryResult = vi.fn<(outcome: PatrolRunAcceptanceOutcome) => void>();
schedulePatrolRunAcceptanceReconciliation({
runId: 'run-retry',
delayMs: 15_000,
refreshTimeoutMs: 5_000,
getStatus: async () => runningStatus('run-retry'),
getHistory: async () => [],
isCurrent: () => true,
onResult: retryResult,
});
await vi.advanceTimersByTimeAsync(15_000);
expect(retryResult).toHaveBeenCalledWith(expect.objectContaining({ kind: 'running' }));
});
});
@@ -5,6 +5,7 @@ import {
PATROL_REFRESH_TIMEOUT_MS,
buildPatrolSettingsReadinessFailure,
openPatrolAssistantWorkflowHandoff,
patrolStartFailureMessage,
recordPatrolControlStarterActivity,
recordPatrolWorkflowStarterActivity,
resolvePatrolAutonomyLevelForSave,
@@ -70,6 +71,20 @@ describe('usePatrolIntelligenceState', () => {
expect(patrolIntelligenceStateSource).toContain('PATROL_MANUAL_SYNC_TIMEOUT_MS');
});
it('keeps browser/network start failures distinct from backend rejections', () => {
expect(patrolStartFailureMessage(new TypeError('Failed to fetch'))).toBe(
'Could not reach Pulse to start Patrol: Failed to fetch',
);
expect(
patrolStartFailureMessage(
Object.assign(new Error('Patrol is already running.'), {
code: 'patrol_already_running',
status: 409,
}),
),
).toBe('Patrol is already running.');
});
it('fails soft when Patrol data refreshes reject', () => {
expect(patrolIntelligenceStateSource).toContain('const [patrolLoadError, setPatrolLoadError]');
expect(patrolIntelligenceStateSource).toContain('rememberPatrolLoadError');
@@ -0,0 +1,100 @@
import type { PatrolRunRecord, PatrolStatus } from '@/api/patrol';
export type PatrolRunAcceptanceOutcome =
| { kind: 'running'; status: PatrolStatus }
| { kind: 'recorded'; run: PatrolRunRecord }
| { kind: 'missing' }
| { kind: 'refresh_failed'; error: unknown };
interface PatrolRunAcceptanceReconciliationOptions {
runId: string;
delayMs: number;
refreshTimeoutMs: number;
getStatus: () => Promise<PatrolStatus | null>;
getHistory: () => Promise<PatrolRunRecord[]>;
isCurrent: () => boolean;
onResult: (outcome: PatrolRunAcceptanceOutcome) => void;
}
const timeoutError = () => new Error('Patrol acceptance reconciliation timed out');
async function settleReconciliationReads(
getStatus: () => Promise<PatrolStatus | null>,
getHistory: () => Promise<PatrolRunRecord[]>,
timeoutMs: number,
): Promise<
| {
status: PromiseSettledResult<PatrolStatus | null>;
history: PromiseSettledResult<PatrolRunRecord[]>;
}
| { error: unknown }
> {
let timeout: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
Promise.allSettled([getStatus(), getHistory()]).then(([status, history]) => ({
status,
history,
})),
new Promise<{ error: unknown }>((resolve) => {
timeout = setTimeout(() => resolve({ error: timeoutError() }), timeoutMs);
}),
]);
} finally {
if (timeout !== undefined) clearTimeout(timeout);
}
}
export function schedulePatrolRunAcceptanceReconciliation(
options: PatrolRunAcceptanceReconciliationOptions,
): () => void {
let cancelled = false;
const timer = setTimeout(() => {
void settleReconciliationReads(
options.getStatus,
options.getHistory,
options.refreshTimeoutMs,
).then((reads) => {
if (cancelled || !options.isCurrent()) return;
if ('error' in reads) {
options.onResult({ kind: 'refresh_failed', error: reads.error });
return;
}
const status = reads.status.status === 'fulfilled' ? reads.status.value : null;
const history = reads.history.status === 'fulfilled' ? reads.history.value : null;
const statusOwnsAcceptedRun =
status?.running === true &&
(!options.runId || !status.current_run_id || status.current_run_id === options.runId);
if (statusOwnsAcceptedRun) {
options.onResult({ kind: 'running', status });
return;
}
const recordedRun = options.runId
? history?.find((run) => run.id === options.runId)
: undefined;
if (recordedRun) {
options.onResult({ kind: 'recorded', run: recordedRun });
return;
}
const refreshError =
reads.status.status === 'rejected'
? reads.status.reason
: reads.history.status === 'rejected'
? reads.history.reason
: undefined;
if (refreshError !== undefined) {
options.onResult({ kind: 'refresh_failed', error: refreshError });
return;
}
options.onResult({ kind: 'missing' });
});
}, options.delayMs);
return () => {
cancelled = true;
clearTimeout(timer);
};
}
@@ -40,6 +40,7 @@ import {
} from '@/stores/license';
import { PATROL_AUTONOMY_FEATURE_KEY } from './patrolAutonomyAvailability';
import type { AISettings } from '@/types/ai';
import { apiErrorStatus } from '@/api/responseUtils';
import {
hasFindingInvestigationHandoffPointer,
isPatrolRuntimeFinding,
@@ -63,6 +64,7 @@ import {
type PatrolAssistantFindingHandoff,
type PatrolConfigurationFailureInput,
} from './patrolInvestigationContextModel';
import { schedulePatrolRunAcceptanceReconciliation } from './patrolRunAcceptance';
type PatrolTab = 'findings' | 'history';
@@ -93,6 +95,8 @@ const recordPatrolWorkflowStarterActivityForSurface = async (
export const PATROL_REFRESH_TIMEOUT_MS = 15000;
export const PATROL_MANUAL_SYNC_TIMEOUT_MS = 5000;
export const PATROL_ACCEPTANCE_RECONCILE_MS = 15000;
export const PATROL_ACCEPTANCE_REFRESH_TIMEOUT_MS = 5000;
export function recordPatrolWorkflowStarterActivity(): void {
void recordPatrolWorkflowStarterActivityForSurface(
@@ -132,6 +136,11 @@ export function openPatrolAssistantWorkflowHandoff(
const patrolErrorMessage = (error: unknown, fallback: string) =>
error instanceof Error && error.message.trim() ? error.message : fallback;
export const patrolStartFailureMessage = (error: unknown): string =>
apiErrorStatus(error) === null
? `Could not reach Pulse to start Patrol: ${patrolErrorMessage(error, 'network request failed')}`
: patrolErrorMessage(error, 'Pulse rejected the Patrol run');
const buildReadinessDetails = (
readiness: NonNullable<AISettings['patrol_readiness']>,
): Record<string, string> => {
@@ -239,6 +248,7 @@ export function usePatrolIntelligenceState() {
const [isUpdatingAutonomy, setIsUpdatingAutonomy] = createSignal(false);
const [activityRefreshTrigger, setActivityRefreshTrigger] = createSignal(0);
const [manualRunRequested, setManualRunRequested] = createSignal(false);
const [acceptedManualRunId, setAcceptedManualRunId] = createSignal('');
const [patrolEnabledLocal, setPatrolEnabledLocal] = createSignal<boolean>(true);
const [liveRunStartedAt, setLiveRunStartedAt] = createSignal('');
const [investigationBudget, setInvestigationBudget] = createSignal(15);
@@ -250,22 +260,29 @@ export function usePatrolIntelligenceState() {
const [assistantHandoffFindingId, setAssistantHandoffFindingId] = createSignal('');
const [patrolLoadError, setPatrolLoadError] = createSignal('');
let safetyTimerRef: ReturnType<typeof setTimeout> | undefined;
let cancelAcceptanceReconciliation: (() => void) | undefined;
let findingScrollTimerRef: ReturnType<typeof setTimeout> | undefined;
let refreshTimeoutRef: ReturnType<typeof setTimeout> | undefined;
let manualRefreshTimeoutRef: ReturnType<typeof setTimeout> | undefined;
let refreshRequestId = 0;
let manualRefreshRequestId = 0;
let manualRunRequestId = 0;
let refreshInterval: ReturnType<typeof setInterval>;
let approvalPollInterval: ReturnType<typeof setInterval>;
const clearSafetyTimer = () => {
if (safetyTimerRef !== undefined) {
clearTimeout(safetyTimerRef);
safetyTimerRef = undefined;
const clearAcceptanceReconcileTimer = () => {
if (cancelAcceptanceReconciliation !== undefined) {
cancelAcceptanceReconciliation();
cancelAcceptanceReconciliation = undefined;
}
};
const finishManualRunTracking = () => {
clearAcceptanceReconcileTimer();
setAcceptedManualRunId('');
setManualRunRequested(false);
};
const clearRefreshTimeout = () => {
if (refreshTimeoutRef !== undefined) {
clearTimeout(refreshTimeoutRef);
@@ -323,15 +340,12 @@ export function usePatrolIntelligenceState() {
const patrolStream = usePatrolStream({
running: () =>
patrolEnabledLocal() && ((patrolStatus()?.running ?? false) || manualRunRequested()),
onStart: () => {
clearSafetyTimer();
},
onComplete: () => {
setManualRunRequested(false);
finishManualRunTracking();
loadAllData();
},
onError: () => {
setManualRunRequested(false);
finishManualRunTracking();
loadAllData();
},
});
@@ -377,8 +391,9 @@ export function usePatrolIntelligenceState() {
const newValue = !previousValue;
setPatrolEnabledLocal(newValue);
if (!newValue) {
setManualRunRequested(false);
clearSafetyTimer();
manualRunRequestId += 1;
finishManualRunTracking();
setIsTriggeringPatrol(false);
}
try {
const data = await AIAPI.updateSettings({ patrol_enabled: newValue });
@@ -413,41 +428,60 @@ export function usePatrolIntelligenceState() {
) {
return;
}
const requestId = ++manualRunRequestId;
setIsTriggeringPatrol(true);
setManualRunRequested(true);
clearSafetyTimer();
safetyTimerRef = setTimeout(() => {
safetyTimerRef = undefined;
if (!manualRunRequested() || patrolStream.isStreaming()) {
return;
}
// The trigger request already succeeded, so a quiet stream can also mean
// a slow provider (a self-hosted model cold-loading can take well over
// 15s to emit its first event). Only report a failure when the refreshed
// status confirms no run is in progress; otherwise let the running state
// carry the button.
void loadAllData()
.catch(() => undefined)
.then(() => {
setManualRunRequested(false);
if (!patrolStream.isStreaming() && !(patrolStatus()?.running ?? false)) {
notificationStore.error(
'Patrol run did not start. The provider did not respond; check the AI provider settings or run the Patrol preflight.',
);
}
});
}, 15000);
clearAcceptanceReconcileTimer();
try {
await triggerPatrolRun();
const acceptance = await triggerPatrolRun();
if (requestId !== manualRunRequestId || !manualRunRequested()) {
return;
}
const runId = acceptance?.run_id?.trim() || '';
setAcceptedManualRunId(runId);
cancelAcceptanceReconciliation = schedulePatrolRunAcceptanceReconciliation({
runId,
delayMs: PATROL_ACCEPTANCE_RECONCILE_MS,
refreshTimeoutMs: PATROL_ACCEPTANCE_REFRESH_TIMEOUT_MS,
getStatus: refetchPatrolStatus,
getHistory: () => getPatrolRunHistory(30),
isCurrent: () =>
requestId === manualRunRequestId &&
manualRunRequested() &&
acceptedManualRunId() === runId,
onResult: (outcome) => {
cancelAcceptanceReconciliation = undefined;
finishManualRunTracking();
if (outcome.kind === 'running' || outcome.kind === 'recorded') {
setActivityRefreshTrigger((prev) => prev + 1);
return;
}
if (outcome.kind === 'refresh_failed') {
rememberPatrolLoadError(
outcome.error,
'Patrol accepted the run, but its live status could not be refreshed.',
);
return;
}
notificationStore.error(
runId
? `Pulse accepted Patrol run ${runId}, but the backend no longer reports it as running and did not record it in history.`
: 'Pulse accepted the Patrol run, but the backend no longer reports it as running and did not record it in history.',
);
},
});
} catch (err) {
console.error('Failed to trigger patrol run:', err);
setManualRunRequested(false);
notificationStore.error(patrolErrorMessage(err, 'Failed to start patrol run'));
clearSafetyTimer();
if (requestId === manualRunRequestId) {
finishManualRunTracking();
notificationStore.error(patrolStartFailureMessage(err));
}
return;
} finally {
setIsTriggeringPatrol(false);
if (requestId === manualRunRequestId) {
setIsTriggeringPatrol(false);
}
}
void loadAllData().catch((err) => {
@@ -993,7 +1027,8 @@ export function usePatrolIntelligenceState() {
onCleanup(() => {
stopPolling();
clearSafetyTimer();
manualRunRequestId += 1;
clearAcceptanceReconcileTimer();
clearManualRefreshTimeout();
if (findingScrollTimerRef !== undefined) {
clearTimeout(findingScrollTimerRef);
@@ -1005,6 +1040,7 @@ export function usePatrolIntelligenceState() {
activeTab,
activePatrolFindings,
activityRefreshTrigger,
acceptedManualRunId,
assistantHandoffFindingId,
autonomyLevel,
requestedAutonomyLevel,
@@ -18,6 +18,8 @@ import {
PHYSICAL_DISK_TABLE_CLASS,
PHYSICAL_DISK_TABLE_ROW_HOVER_CLASS,
getPhysicalDiskEmptyStatePresentation,
getPhysicalDiskCollectionMessages,
getPhysicalDiskFieldStatusMessage,
getPhysicalDiskHealthStatus,
getPhysicalDiskHealthSummary,
getPhysicalDiskHostLabel,
@@ -61,6 +63,47 @@ function makeDiskData(
}
describe('diskPresentation', () => {
it('distinguishes unsupported, unavailable, and unexpectedly missing disk evidence', () => {
expect(
getPhysicalDiskFieldStatusMessage('Disk I/O', {
state: 'unsupported',
source: 'controller',
reason: 'per-member counters unavailable',
}),
).toBe('Disk I/O is unsupported: per-member counters unavailable');
expect(
getPhysicalDiskFieldStatusMessage('Temperature', {
state: 'unavailable',
source: 'smartctl',
reason: 'collection deadline exceeded',
}),
).toBe('Temperature is temporarily unavailable: collection deadline exceeded');
expect(
getPhysicalDiskFieldStatusMessage('Serial number', {
state: 'missing',
source: 'smartctl',
reason: 'serial absent from successful response',
}),
).toBe('Serial number is unexpectedly missing: serial absent from successful response');
expect(
getPhysicalDiskCollectionMessages(
makeDiskData({
collection: {
serial: { state: 'available', source: 'smartctl' },
temperature: { state: 'unavailable', source: 'smartctl' },
io: { state: 'unsupported', source: 'controller' },
controller: { state: 'missing', source: 'linux-sysfs' },
pool: { state: 'available', source: 'zpool-status' },
},
}),
),
).toEqual([
'Temperature is temporarily unavailable.',
'Disk I/O is unsupported.',
'Controller association is unexpectedly missing.',
]);
});
it('returns critical presentation for failed disks', () => {
expect(PHYSICAL_DISK_EMPTY_CARD_CLASS).toBe('text-center');
expect(PHYSICAL_DISK_TABLE_CLASS).toBe('w-full table-fixed text-xs');
@@ -1,4 +1,8 @@
import type { Resource } from '@/types/resource';
import type {
PhysicalDiskCollectionStatus,
PhysicalDiskFieldStatus,
Resource,
} from '@/types/resource';
import {
getSourcePlatformLabel,
getSourcePlatformPresentation,
@@ -54,9 +58,12 @@ export interface PhysicalDiskPresentationData {
writeCount?: number;
errorCount?: number;
type: string;
controller?: string;
target?: string;
temperature: number;
rpm: number;
used: string;
collection?: PhysicalDiskCollectionStatus;
smartAttributes?: {
powerOnHours?: number;
powerCycles?: number;
@@ -296,6 +303,8 @@ export function extractPhysicalDiskPresentationData(
serial: pd.serial || '',
wwn: pd.wwn || '',
type: pd.diskType || '',
controller: pd.controller,
target: pd.target,
size: pd.sizeBytes || 0,
health: pd.health || 'UNKNOWN',
wearout: pd.wearout ?? -1,
@@ -309,6 +318,7 @@ export function extractPhysicalDiskPresentationData(
readCount: pd.readCount,
writeCount: pd.writeCount,
errorCount: pd.errorCount,
collection: pd.collection,
riskLevel: pd.risk?.level,
riskReasons,
smartAttributes: pd.smart
@@ -328,6 +338,36 @@ export function extractPhysicalDiskPresentationData(
};
}
export function getPhysicalDiskFieldStatusMessage(
label: string,
status: PhysicalDiskFieldStatus | null | undefined,
): string {
if (!status || status.state === 'available') return '';
const reason = status.reason?.trim();
switch (status.state) {
case 'unsupported':
return `${label} is unsupported${reason ? `: ${reason}` : '.'}`;
case 'unavailable':
return `${label} is temporarily unavailable${reason ? `: ${reason}` : '.'}`;
case 'missing':
return `${label} is unexpectedly missing${reason ? `: ${reason}` : '.'}`;
default:
return '';
}
}
export function getPhysicalDiskCollectionMessages(disk: PhysicalDiskPresentationData): string[] {
const collection = disk.collection;
if (!collection) return [];
return [
getPhysicalDiskFieldStatusMessage('Serial number', collection.serial),
getPhysicalDiskFieldStatusMessage('Temperature', collection.temperature),
getPhysicalDiskFieldStatusMessage('Disk I/O', collection.io),
getPhysicalDiskFieldStatusMessage('Controller association', collection.controller),
getPhysicalDiskFieldStatusMessage('Pool membership', collection.pool),
].filter((message): message is string => message.length > 0);
}
export function buildPhysicalDiskPresentationDataMap(
disks: Resource[],
): Map<string, PhysicalDiskPresentationData> {
@@ -1303,6 +1303,7 @@ describe('useUnifiedResources', () => {
platformId: 'pve1',
primaryId: 'node:instance-pve1',
aliases: ['node:instance-pve1', 'instance-pve1', 'tower.local'],
supersededIds: ['agent-retired'],
},
},
],
@@ -1324,6 +1325,7 @@ describe('useUnifiedResources', () => {
expect(result!.resources()[0].canonicalIdentity).toMatchObject({
primaryId: 'node:instance-pve1',
hostname: 'tower.local',
supersededIds: ['agent-retired'],
});
dispose();
@@ -469,6 +469,7 @@ type APIResource = {
platformId?: string;
primaryId?: string;
aliases?: string[];
supersededIds?: string[];
};
policy?: {
sensitivity?: string;
@@ -865,8 +865,11 @@ describe('AIIntelligence entitlement gating', () => {
it('surfaces backend readiness rejection when a stale manual run request reaches the server', async () => {
triggerPatrolRunMock.mockRejectedValue(
new Error(
'The selected Patrol model is a reasoning-only model family that commonly does not emit tool calls.',
Object.assign(
new Error(
'The selected Patrol model is a reasoning-only model family that commonly does not emit tool calls.',
),
{ code: 'patrol_readiness_not_ready', status: 409 },
),
);
+13
View File
@@ -1126,6 +1126,8 @@ export interface PhysicalDisk {
serial: string;
wwn: string;
type: 'nvme' | 'sata' | 'sas' | string;
controller?: string;
target?: string;
size: number;
health: 'PASSED' | 'FAILED' | 'UNKNOWN' | string;
wearout: number; // 0-100, 100 is best, -1 when the controller doesn't report it
@@ -1134,6 +1136,17 @@ export interface PhysicalDisk {
used: string;
lastChecked: string;
smartAttributes?: SMARTAttributes;
io?: {
device?: string;
readBytes?: number;
writeBytes?: number;
readOps?: number;
writeOps?: number;
readTimeMs?: number;
writeTimeMs?: number;
ioTimeMs?: number;
};
collection?: import('./resource').PhysicalDiskCollectionStatus;
}
export interface CPUInfo {
+32
View File
@@ -189,6 +189,7 @@ export interface ResourceCanonicalIdentity {
platformId?: string;
primaryId?: string;
aliases?: string[];
supersededIds?: string[];
}
export type ResourceSensitivity = 'public' | 'internal' | 'sensitive' | 'restricted';
@@ -451,6 +452,33 @@ export interface ResourcePhysicalDiskRisk {
reasons?: ResourceStorageRiskReason[];
}
export type PhysicalDiskFieldState = 'available' | 'unavailable' | 'unsupported' | 'missing';
export interface PhysicalDiskFieldStatus {
state: PhysicalDiskFieldState;
source?: string;
reason?: string;
}
export interface PhysicalDiskCollectionStatus {
serial?: PhysicalDiskFieldStatus;
temperature?: PhysicalDiskFieldStatus;
io?: PhysicalDiskFieldStatus;
controller?: PhysicalDiskFieldStatus;
pool?: PhysicalDiskFieldStatus;
}
export interface ResourcePhysicalDiskIO {
device?: string;
readBytes?: number;
writeBytes?: number;
readOps?: number;
writeOps?: number;
readTimeMs?: number;
writeTimeMs?: number;
ioTimeMs?: number;
}
export interface ResourceCephPoolMeta {
name: string;
storedBytes: number;
@@ -485,6 +513,8 @@ export interface ResourcePhysicalDiskMeta {
serial?: string;
wwn?: string;
diskType?: string;
controller?: string;
target?: string;
sizeBytes?: number;
health?: string;
wearout?: number;
@@ -498,6 +528,8 @@ export interface ResourcePhysicalDiskMeta {
readCount?: number;
writeCount?: number;
errorCount?: number;
io?: ResourcePhysicalDiskIO;
collection?: PhysicalDiskCollectionStatus;
smart?: {
powerOnHours?: number;
powerCycles?: number;
@@ -693,10 +693,12 @@ const mergeCanonicalIdentity = (
if (!incoming) return existing;
if (!existing) return incoming;
const aliases = mergeStringArrays(incoming.aliases, existing.aliases);
const supersededIds = mergeStringArrays(incoming.supersededIds, existing.supersededIds);
return {
...existing,
...incoming,
aliases,
supersededIds,
};
};
+26 -4
View File
@@ -51,14 +51,23 @@ func isDemoFindingID(id string) bool {
// yet (mock disabled again, another run in flight, or mock state not
// generated yet).
func (p *PatrolService) runDemoPatrolCycle(trigger TriggerReason) bool {
return p.runDemoPatrolCycleWithStart(trigger, nil)
}
func (p *PatrolService) runDemoPatrolCycleWithStart(trigger TriggerReason, acceptedStart *patrolRunStart) bool {
if p == nil || p.findings == nil || p.runHistoryStore == nil {
return false
}
if !IsDemoMode() {
return false
}
if !p.tryStartRun("full") {
return false
runStart := acceptedStart
if runStart == nil {
var accepted bool
runStart, accepted = p.beginRun("full")
if !accepted {
return false
}
}
defer p.endRun()
@@ -114,6 +123,12 @@ func (p *PatrolService) runDemoPatrolCycle(trigger TriggerReason) bool {
// retry or the next scheduled cycle will populate the surface once
// state exists.
log.Debug().Msg("demo patrol: no mock state available yet, skipping cycle")
if acceptedStart != nil {
p.recordAcceptedRunFailure(runStart, trigger,
"Patrol demo state unavailable",
"Pulse accepted the run, but demo infrastructure state was not available.")
return true
}
return false
}
@@ -181,10 +196,17 @@ func (p *PatrolService) runDemoPatrolCycle(trigger TriggerReason) bool {
// Presentational duration and token counts: deterministic, scaled to the
// size of the mock fleet so the history reads like a real analysis pass.
duration := time.Duration(35+resourceCount%40) * time.Second
runID := fmt.Sprintf("demo-run-%d", now.UnixNano())
startedAt := now.Add(-duration)
if acceptedStart != nil {
runID = runStart.id
startedAt = runStart.startedAt
duration = now.Sub(startedAt)
}
record := PatrolRunRecord{
ID: fmt.Sprintf("demo-run-%d", now.UnixNano()),
ID: runID,
Source: PatrolRunSourceDemo,
StartedAt: now.Add(-duration),
StartedAt: startedAt,
CompletedAt: now,
Duration: duration,
DurationMs: duration.Milliseconds(),
+19 -16
View File
@@ -88,22 +88,24 @@ const (
)
type PatrolStatus struct {
RuntimeState PatrolRuntimeState `json:"runtime_state"`
Running bool `json:"running"`
Enabled bool `json:"enabled"`
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"` // Last completed full patrol
LastActivityAt *time.Time `json:"last_activity_at,omitempty"` // Last completed Patrol activity of any kind
TriggerStatus *TriggerStatus `json:"trigger_status,omitempty"`
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
LastDuration time.Duration `json:"last_duration_ms"`
ResourcesChecked int `json:"resources_checked"`
FindingsCount int `json:"findings_count"`
ErrorCount int `json:"error_count"`
Healthy bool `json:"healthy"`
IntervalMs int64 `json:"interval_ms"` // Patrol interval in milliseconds
BlockedReason string `json:"blocked_reason,omitempty"`
BlockedCause PatrolFailureCause `json:"blocked_cause,omitempty"`
BlockedAt *time.Time `json:"blocked_at,omitempty"`
RuntimeState PatrolRuntimeState `json:"runtime_state"`
Running bool `json:"running"`
CurrentRunID string `json:"current_run_id,omitempty"`
CurrentRunStartedAt *time.Time `json:"current_run_started_at,omitempty"`
Enabled bool `json:"enabled"`
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"` // Last completed full patrol
LastActivityAt *time.Time `json:"last_activity_at,omitempty"` // Last completed Patrol activity of any kind
TriggerStatus *TriggerStatus `json:"trigger_status,omitempty"`
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
LastDuration time.Duration `json:"last_duration_ms"`
ResourcesChecked int `json:"resources_checked"`
FindingsCount int `json:"findings_count"`
ErrorCount int `json:"error_count"`
Healthy bool `json:"healthy"`
IntervalMs int64 `json:"interval_ms"` // Patrol interval in milliseconds
BlockedReason string `json:"blocked_reason,omitempty"`
BlockedCause PatrolFailureCause `json:"blocked_cause,omitempty"`
BlockedAt *time.Time `json:"blocked_at,omitempty"`
}
// PatrolFindingAssessment is the model's explicit evidence-grounded verdict
@@ -549,6 +551,7 @@ type PatrolService struct {
// Runtime state
running bool
runInProgress bool
currentRunID string
runStartedAt time.Time
stopCh chan struct{}
configChanged chan struct{} // Signal when config changes to reset ticker
+22 -4
View File
@@ -444,14 +444,32 @@ func (p *PatrolService) GetFindingsHistory(startTime *time.Time) []*Finding {
return findings
}
// ForcePatrol triggers an immediate patrol run.
// Uses context.Background() since this runs async after the HTTP response.
func (p *PatrolService) ForcePatrol(ctx context.Context) {
// PatrolRunAcceptance is the durable acknowledgement returned when a manual
// run has reserved the single Patrol execution slot.
type PatrolRunAcceptance struct {
RunID string
StartedAt time.Time
}
// ForcePatrol atomically accepts and starts an immediate patrol run.
// The reservation is visible through GetStatus before this method returns, so
// callers never have to infer backend acceptance from the first provider event.
// The detached context lets the accepted run continue after the HTTP request
// that initiated it has completed.
func (p *PatrolService) ForcePatrol(ctx context.Context) (PatrolRunAcceptance, bool) {
runCtx := context.Background()
if ctx != nil {
runCtx = context.WithoutCancel(ctx)
}
go p.runPatrolWithTrigger(runCtx, TriggerReasonManual, nil)
runStart, accepted := p.beginRun("full")
if !accepted {
return PatrolRunAcceptance{}, false
}
go p.runPatrolWithTriggerStart(runCtx, TriggerReasonManual, nil, runStart)
return PatrolRunAcceptance{
RunID: runStart.id,
StartedAt: runStart.startedAt,
}, true
}
// chatServiceExecutorAccessor is satisfied by *chat.Service, allowing patrol to
@@ -0,0 +1,133 @@
package ai
import (
"context"
"encoding/json"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/tools"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestForcePatrolAcceptanceOwnsStatusAndProducesOneHistoryRecord(t *testing.T) {
persistence := config.NewConfigPersistence(t.TempDir())
service := NewService(persistence, nil)
service.cfg = &config.AIConfig{Enabled: true, PatrolModel: "mock:model"}
service.provider = &mockProvider{}
providerRelease := make(chan struct{})
providerEntered := make(chan struct{})
var providerCalls atomic.Int32
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
service.SetChatService(&mockChatService{
executor: executor,
executePatrolStreamFunc: func(_ context.Context, _ PatrolExecuteRequest, callback ChatStreamCallback) (*PatrolStreamResponse, error) {
providerCalls.Add(1)
close(providerEntered)
<-providerRelease
content, err := json.Marshal(struct {
Text string `json:"text"`
}{Text: "No actionable issues found."})
if err != nil {
return nil, err
}
callback(ChatStreamEvent{Type: "content", Data: content})
return &PatrolStreamResponse{
Content: "No actionable issues found.",
InputTokens: 12,
OutputTokens: 5,
}, nil
},
})
patrol := NewPatrolService(service, &mockStateProvider{
state: models.StateSnapshot{
Nodes: []models.Node{{ID: "node-1", Name: "pve-1", Status: "online"}},
},
})
patrol.SetConfig(PatrolConfig{
Enabled: true,
AnalyzeNodes: true,
})
acceptance, accepted := patrol.ForcePatrol(context.Background())
if !accepted {
t.Fatal("manual run was not accepted")
}
if acceptance.RunID == "" || acceptance.StartedAt.IsZero() {
t.Fatalf("incomplete acceptance: %+v", acceptance)
}
status := patrol.GetStatus()
if !status.Running {
t.Fatal("accepted run was not synchronously visible as running")
}
if status.CurrentRunID != acceptance.RunID {
t.Fatalf("status run id = %q, want %q", status.CurrentRunID, acceptance.RunID)
}
if duplicate, duplicateAccepted := patrol.ForcePatrol(context.Background()); duplicateAccepted {
t.Fatalf("duplicate run was accepted: %+v", duplicate)
}
select {
case <-providerEntered:
case <-time.After(2 * time.Second):
t.Fatal("accepted run did not reach the controlled provider")
}
if calls := providerCalls.Load(); calls != 1 {
t.Fatalf("provider calls = %d, want 1", calls)
}
close(providerRelease)
deadline := time.Now().Add(3 * time.Second)
for patrol.GetStatus().Running && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if patrol.GetStatus().Running {
t.Fatal("manual run did not finish")
}
history := patrol.GetRunHistory(10)
if len(history) != 1 {
t.Fatalf("history records = %d, want 1: %+v", len(history), history)
}
if history[0].ID != acceptance.RunID {
t.Fatalf("history run id = %q, want %q", history[0].ID, acceptance.RunID)
}
if calls := providerCalls.Load(); calls != 1 {
t.Fatalf("provider calls after completion = %d, want 1", calls)
}
}
func TestAcceptedManualPatrolRecordsRuntimeStateFailure(t *testing.T) {
patrol := NewPatrolService(nil, nil)
patrol.SetConfig(PatrolConfig{Enabled: true})
acceptance, accepted := patrol.ForcePatrol(context.Background())
if !accepted {
t.Fatal("manual run was not accepted")
}
deadline := time.Now().Add(2 * time.Second)
for patrol.GetStatus().Running && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if patrol.GetStatus().Running {
t.Fatal("manual run did not terminate")
}
history := patrol.GetRunHistory(10)
if len(history) != 1 {
t.Fatalf("history records = %d, want 1: %+v", len(history), history)
}
if history[0].ID != acceptance.RunID {
t.Fatalf("history run id = %q, want %q", history[0].ID, acceptance.RunID)
}
if history[0].Status != "error" || history[0].ErrorCount != 1 {
t.Fatalf("history failure = status %q, errors %d", history[0].Status, history[0].ErrorCount)
}
}
+101 -16
View File
@@ -285,8 +285,51 @@ func (p *PatrolService) runPatrol(ctx context.Context) {
p.runPatrolWithTrigger(ctx, TriggerReasonScheduled, nil)
}
type patrolRunStart struct {
id string
startedAt time.Time
}
func (p *PatrolService) recordAcceptedRunFailure(start *patrolRunStart, trigger TriggerReason, summary, detail string) {
if start == nil || p.runHistoryStore == nil {
return
}
completedAt := time.Now()
duration := completedAt.Sub(start.startedAt)
p.runHistoryStore.Add(PatrolRunRecord{
ID: start.id,
StartedAt: start.startedAt,
CompletedAt: completedAt,
Duration: duration,
DurationMs: duration.Milliseconds(),
Type: "patrol",
TriggerReason: string(trigger),
FindingsSummary: "Analysis incomplete (1 error)",
FindingIDs: []string{},
ErrorCount: 1,
Status: "error",
ErrorSummary: summary,
ErrorDetail: detail,
})
p.mu.Lock()
p.lastActivity = completedAt
p.lastFullPatrol = completedAt
p.lastDuration = duration
p.resourcesChecked = 0
p.errorCount = 1
p.mu.Unlock()
}
// runPatrolWithTrigger executes a patrol run with trigger context
func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger TriggerReason, scope *PatrolScope) {
p.runPatrolWithTriggerStart(ctx, trigger, scope, nil)
}
// runPatrolWithTriggerStart executes a patrol run, optionally using a start
// reservation that was accepted synchronously by the manual-run API. Keeping
// acceptance and execution on the same reservation removes the interval where
// the API has returned success but status still reports no active run.
func (p *PatrolService) runPatrolWithTriggerStart(ctx context.Context, trigger TriggerReason, scope *PatrolScope, acceptedStart *patrolRunStart) {
p.mu.RLock()
cfg := p.config
breaker := p.circuitBreaker
@@ -296,21 +339,42 @@ func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger Trigge
// (not from the boot-time config snapshot) because release demo instances
// enable mock fixtures only after the license sync authorizes them.
if IsDemoMode() {
p.runDemoPatrolCycle(trigger)
if !p.runDemoPatrolCycleWithStart(trigger, acceptedStart) && acceptedStart != nil {
p.recordAcceptedRunFailure(acceptedStart, trigger,
"Patrol demo state unavailable",
"Pulse accepted the run, but demo infrastructure state was not available.")
}
return
}
if !cfg.Enabled {
if acceptedStart != nil {
p.recordAcceptedRunFailure(acceptedStart, trigger,
"Patrol disabled before execution",
"Pulse accepted the run, but Patrol was disabled before execution began.")
p.endRun()
}
return
}
if reason := strings.TrimSpace(cfg.RuntimeBlockedReason); reason != "" {
if acceptedStart != nil {
p.recordAcceptedRunFailure(acceptedStart, trigger,
"Patrol runtime became unavailable",
reason)
p.endRun()
}
p.setBlockedReasonWithCause(reason, cfg.RuntimeBlockedCause)
log.Info().Str("reason", reason).Str("cause", string(cfg.RuntimeBlockedCause)).Msg("AI Patrol: Skipping run - runtime readiness blocked")
return
}
if !p.tryStartRun("full") {
return
runStart := acceptedStart
if runStart == nil {
var accepted bool
runStart, accepted = p.beginRun("full")
if !accepted {
return
}
}
defer p.endRun()
@@ -320,8 +384,8 @@ func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger Trigge
log.Warn().Msg("AI Patrol: Circuit breaker is open (LLM calls blocked)")
}
start := time.Now()
runID := fmt.Sprintf("%d", start.UnixNano())
start := runStart.startedAt
runID := runStart.id
executionID := uuid.NewString()
patrolType := "patrol"
GetPatrolMetrics().RecordRun(string(trigger), "full")
@@ -358,6 +422,11 @@ func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger Trigge
// Get current state
if !p.hasPatrolRuntimeInputs() {
log.Warn().Msg("AI Patrol: No runtime state available")
if acceptedStart != nil {
p.recordAcceptedRunFailure(runStart, trigger,
"Patrol runtime state unavailable",
"Pulse accepted the run, but no infrastructure state was available for analysis.")
}
return
}
@@ -496,10 +565,10 @@ func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger Trigge
}
p.setBlockedReasonWithCause(reason, cause)
log.Info().Str("reason", reason).Str("cause", string(cause)).Msg("AI Patrol: Skipping run - AI unavailable")
return
}
{
runStats.errors++
runStats.errorSummary = "Patrol provider unavailable"
runStats.errorDetail = reason
} else {
p.clearBlockedReason()
// Ensure stream state is clean for this run before the first streamed event.
p.resetStreamForRun(runID)
@@ -792,7 +861,8 @@ func (p *PatrolService) runScopedPatrol(ctx context.Context, scope PatrolScope)
return
}
if !p.tryStartRun("scoped") {
runStart, accepted := p.beginRun("scoped")
if !accepted {
// Re-queue with backoff if retries remain
if scope.RetryCount < scopedPatrolMaxRetries {
scope.RetryCount++
@@ -826,8 +896,8 @@ func (p *PatrolService) runScopedPatrol(ctx context.Context, scope PatrolScope)
log.Warn().Msg("AI Patrol: Circuit breaker is open for scoped patrol (LLM calls blocked)")
}
start := time.Now()
runID := fmt.Sprintf("%d", start.UnixNano())
start := runStart.startedAt
runID := runStart.id
executionID := uuid.NewString()
GetPatrolMetrics().RecordRun(string(scope.Reason), "scoped")
var runStats struct {
@@ -1793,6 +1863,7 @@ func (p *PatrolService) GetStatus() PatrolStatus {
status := PatrolStatus{
RuntimeState: PatrolRuntimeStateActive,
Running: analysisInProgress,
CurrentRunID: p.currentRunID,
Enabled: p.config.Enabled,
LastDuration: p.lastDuration,
ResourcesChecked: p.resourcesChecked,
@@ -1802,6 +1873,10 @@ func (p *PatrolService) GetStatus() PatrolStatus {
BlockedReason: p.lastBlockedReason,
BlockedCause: p.lastBlockedCause,
}
if analysisInProgress && !p.runStartedAt.IsZero() {
startedAt := p.runStartedAt
status.CurrentRunStartedAt = &startedAt
}
if p.triggerManager != nil {
triggerStatus := p.triggerManager.GetStatus()
status.TriggerStatus = &triggerStatus
@@ -2730,7 +2805,7 @@ func (p *PatrolService) TriggerPatrolForAlert(alert *alerts.Alert) {
}
}
func (p *PatrolService) tryStartRun(kind string) bool {
func (p *PatrolService) beginRun(kind string) (*patrolRunStart, bool) {
p.mu.Lock()
if p.runInProgress {
// Detect stuck runs: if the current run has been going for >20 minutes,
@@ -2751,18 +2826,28 @@ func (p *PatrolService) tryStartRun(kind string) bool {
} else {
log.Debug().Str("kind", kind).Msg("AI Patrol: Run already in progress, skipping")
}
return false
return nil, false
}
}
startedAt := time.Now()
runID := fmt.Sprintf("%d", startedAt.UnixNano())
p.runInProgress = true
p.runStartedAt = time.Now()
p.currentRunID = runID
p.runStartedAt = startedAt
p.mu.Unlock()
return true
return &patrolRunStart{id: runID, startedAt: startedAt}, true
}
func (p *PatrolService) tryStartRun(kind string) bool {
_, accepted := p.beginRun(kind)
return accepted
}
func (p *PatrolService) endRun() {
p.mu.Lock()
p.runInProgress = false
p.currentRunID = ""
p.runStartedAt = time.Time{}
orch := p.investigationOrchestrator
p.mu.Unlock()
@@ -0,0 +1,69 @@
package alerts
import "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
// MigrateCanonicalOverrideKeys re-homes overrides written under a retired
// canonical resource ID onto the resource's current canonical ID. Only
// provider-declared superseded IDs participate: display aliases, hostnames,
// metric targets, and other lookup conveniences are never persistence keys.
//
// A current-key override wins over an old-key override, and an old ID is left
// untouched when it is still live or maps to more than one current resource.
// The function mutates config only when it can make an unambiguous migration.
func MigrateCanonicalOverrideKeys(config *AlertConfig, resources []unifiedresources.Resource) bool {
if config == nil || len(config.Overrides) == 0 || len(resources) == 0 {
return false
}
liveIDs := make(map[string]struct{}, len(resources))
successors := make(map[string]string)
ambiguous := make(map[string]struct{})
for _, resource := range resources {
currentID := unifiedresources.CanonicalResourceID(resource.ID)
if currentID == "" {
continue
}
liveIDs[currentID] = struct{}{}
for _, supersededID := range resource.SupersededCanonicalIDs {
oldID := unifiedresources.CanonicalResourceID(supersededID)
if oldID == "" || oldID == currentID {
continue
}
if existing, ok := successors[oldID]; ok && existing != currentID {
delete(successors, oldID)
ambiguous[oldID] = struct{}{}
continue
}
if _, conflict := ambiguous[oldID]; conflict {
continue
}
successors[oldID] = currentID
}
}
changed := false
overrides := make(map[string]ThresholdConfig, len(config.Overrides))
for resourceID, override := range config.Overrides {
overrides[resourceID] = override
}
for oldID, newID := range successors {
if _, stillLive := liveIDs[oldID]; stillLive {
continue
}
override, exists := overrides[oldID]
if !exists {
continue
}
if _, currentExists := overrides[newID]; !currentExists {
overrides[newID] = override
}
delete(overrides, oldID)
changed = true
}
if changed {
config.Overrides = overrides
}
return changed
}
@@ -0,0 +1,68 @@
package alerts
import (
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestMigrateCanonicalOverrideKeysRehomesUnambiguousSupersededIdentity(t *testing.T) {
const (
oldID = "agent-535886018cb53055"
newID = "agent-b9ed6d0e20e94eaf"
)
config := AlertConfig{
Overrides: map[string]ThresholdConfig{
oldID: {
Memory: &HysteresisThreshold{Trigger: 95, Clear: 90},
},
},
}
resources := []unifiedresources.Resource{{
ID: newID,
Type: unifiedresources.ResourceTypeAgent,
SupersededCanonicalIDs: []string{oldID},
}}
if !MigrateCanonicalOverrideKeys(&config, resources) {
t.Fatal("expected superseded TrueNAS override identity to migrate")
}
if _, exists := config.Overrides[oldID]; exists {
t.Fatalf("override remained under superseded identity %s", oldID)
}
override, exists := config.Overrides[newID]
if !exists || override.Memory == nil {
t.Fatalf("override missing under current canonical identity %s: %+v", newID, config.Overrides)
}
if override.Memory.Trigger != 95 || override.Memory.Clear != 90 {
t.Fatalf("migrated override changed threshold values: %+v", override.Memory)
}
}
func TestMigrateCanonicalOverrideKeysRefusesAmbiguousOrLiveIdentity(t *testing.T) {
const oldID = "agent-shared"
for name, resources := range map[string][]unifiedresources.Resource{
"ambiguous successor": {
{ID: "agent-a", SupersededCanonicalIDs: []string{oldID}},
{ID: "agent-b", SupersededCanonicalIDs: []string{oldID}},
},
"still live": {
{ID: oldID},
{ID: "agent-a", SupersededCanonicalIDs: []string{oldID}},
},
} {
t.Run(name, func(t *testing.T) {
config := AlertConfig{
Overrides: map[string]ThresholdConfig{
oldID: {Memory: &HysteresisThreshold{Trigger: 95, Clear: 90}},
},
}
if MigrateCanonicalOverrideKeys(&config, resources) {
t.Fatal("unsafe succession unexpectedly migrated")
}
if _, exists := config.Overrides[oldID]; !exists {
t.Fatal("unsafe succession removed the original override")
}
})
}
}
+66 -41
View File
@@ -4958,23 +4958,25 @@ func (h *AISettingsHandler) HandleOAuthDisconnect(w http.ResponseWriter, r *http
// PatrolStatusResponse is the response for GET /api/ai/patrol/status
type PatrolStatusResponse struct {
RuntimeState ai.PatrolRuntimeState `json:"runtime_state"`
Running bool `json:"running"`
Enabled bool `json:"enabled"`
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"`
LastActivityAt *time.Time `json:"last_activity_at,omitempty"`
TriggerStatus *ai.TriggerStatus `json:"trigger_status,omitempty"`
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
LastDurationMs int64 `json:"last_duration_ms"`
ResourcesChecked int `json:"resources_checked"`
FindingsCount int `json:"findings_count"`
ErrorCount int `json:"error_count"`
Healthy bool `json:"healthy"`
IntervalMs int64 `json:"interval_ms"` // Patrol interval in milliseconds
FixedCount int `json:"fixed_count"` // Number of issues remediated by Patrol
BlockedReason string `json:"blocked_reason,omitempty"`
BlockedCause string `json:"blocked_cause,omitempty"`
BlockedAt *time.Time `json:"blocked_at,omitempty"`
RuntimeState ai.PatrolRuntimeState `json:"runtime_state"`
Running bool `json:"running"`
CurrentRunID string `json:"current_run_id,omitempty"`
CurrentRunStartedAt *time.Time `json:"current_run_started_at,omitempty"`
Enabled bool `json:"enabled"`
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"`
LastActivityAt *time.Time `json:"last_activity_at,omitempty"`
TriggerStatus *ai.TriggerStatus `json:"trigger_status,omitempty"`
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
LastDurationMs int64 `json:"last_duration_ms"`
ResourcesChecked int `json:"resources_checked"`
FindingsCount int `json:"findings_count"`
ErrorCount int `json:"error_count"`
Healthy bool `json:"healthy"`
IntervalMs int64 `json:"interval_ms"` // Patrol interval in milliseconds
FixedCount int `json:"fixed_count"` // Number of issues remediated by Patrol
BlockedReason string `json:"blocked_reason,omitempty"`
BlockedCause string `json:"blocked_cause,omitempty"`
BlockedAt *time.Time `json:"blocked_at,omitempty"`
// License status for Pro feature gating
LicenseRequired bool `json:"license_required"` // True if Pro license needed for full features
LicenseStatus string `json:"license_status"` // "active", "expired", "grace_period", "none"
@@ -5356,26 +5358,28 @@ func (h *AISettingsHandler) HandleGetPatrolStatus(w http.ResponseWriter, r *http
}
response := PatrolStatusResponse{
RuntimeState: status.RuntimeState,
Running: status.Running,
Enabled: status.Enabled,
LastPatrolAt: status.LastPatrolAt,
LastActivityAt: status.LastActivityAt,
TriggerStatus: status.TriggerStatus,
NextPatrolAt: status.NextPatrolAt,
LastDurationMs: status.LastDuration.Milliseconds(),
ResourcesChecked: status.ResourcesChecked,
FindingsCount: status.FindingsCount,
ErrorCount: status.ErrorCount,
Healthy: status.Healthy,
IntervalMs: status.IntervalMs,
FixedCount: fixedCount,
BlockedReason: status.BlockedReason,
BlockedCause: patrolFailureCauseResponse(status.BlockedCause),
BlockedAt: status.BlockedAt,
LicenseRequired: !hasAutoFixFeature,
LicenseStatus: licenseStatus,
Readiness: ptrToPatrolReadiness(h.buildPatrolReadiness(r.Context(), aiService, true)),
RuntimeState: status.RuntimeState,
Running: status.Running,
CurrentRunID: status.CurrentRunID,
CurrentRunStartedAt: status.CurrentRunStartedAt,
Enabled: status.Enabled,
LastPatrolAt: status.LastPatrolAt,
LastActivityAt: status.LastActivityAt,
TriggerStatus: status.TriggerStatus,
NextPatrolAt: status.NextPatrolAt,
LastDurationMs: status.LastDuration.Milliseconds(),
ResourcesChecked: status.ResourcesChecked,
FindingsCount: status.FindingsCount,
ErrorCount: status.ErrorCount,
Healthy: status.Healthy,
IntervalMs: status.IntervalMs,
FixedCount: fixedCount,
BlockedReason: status.BlockedReason,
BlockedCause: patrolFailureCauseResponse(status.BlockedCause),
BlockedAt: status.BlockedAt,
LicenseRequired: !hasAutoFixFeature,
LicenseStatus: licenseStatus,
Readiness: ptrToPatrolReadiness(h.buildPatrolReadiness(r.Context(), aiService, true)),
}
if !hasAutoFixFeature {
response.UpgradeURL = upgradeURLForFeatureFromLicensing(featureAIAutoFixValue)
@@ -5819,6 +5823,12 @@ func (h *AISettingsHandler) HandleForcePatrol(w http.ResponseWriter, r *http.Req
return
}
if !patrol.GetStatus().Enabled {
writeErrorResponse(w, http.StatusConflict, "patrol_disabled",
"Patrol is disabled. Enable Patrol before starting a manual run.", nil)
return
}
// Cadence cap: Community tier is limited to 1 patrol run per hour.
// Patrol itself is free (ai_patrol), but higher cadence is gated behind Pro/Cloud.
if !aiService.HasLicenseFeature(featureAIAutoFixValue) {
@@ -5832,12 +5842,27 @@ func (h *AISettingsHandler) HandleForcePatrol(w http.ResponseWriter, r *http.Req
}
}
// Trigger patrol asynchronously
patrol.ForcePatrol(r.Context())
// Atomically reserve the execution slot before acknowledging the request.
// This makes backend acceptance independently observable from both provider
// progress events and the timing of the execution goroutine.
acceptance, accepted := patrol.ForcePatrol(r.Context())
if !accepted {
status := patrol.GetStatus()
details := map[string]string{}
if status.CurrentRunID != "" {
details["current_run_id"] = status.CurrentRunID
}
writeErrorResponse(w, http.StatusConflict, "patrol_already_running",
"Patrol is already running.", details)
return
}
response := map[string]interface{}{
"success": true,
"message": "Triggered patrol run",
"success": true,
"accepted": true,
"message": "Triggered patrol run",
"run_id": acceptance.RunID,
"started_at": acceptance.StartedAt,
}
if err := utils.WriteJSONResponse(w, response); err != nil {
@@ -954,11 +954,41 @@ func TestHandleForcePatrol_ConfigDisabled(t *testing.T) {
handler.HandleForcePatrol(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
if rec.Code != http.StatusConflict {
t.Fatalf("status = %d, want 409", rec.Code)
}
if !strings.Contains(rec.Body.String(), "Triggered patrol run") {
t.Fatalf("expected success message")
var payload APIError
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode error payload: %v", err)
}
if payload.Code != "patrol_disabled" {
t.Fatalf("code = %q, want patrol_disabled", payload.Code)
}
}
func TestHandleForcePatrolRejectsDuplicateAfterSynchronousAcceptance(t *testing.T) {
handler, patrol, _, _ := setupAIHandlerWithPatrol(t)
seedReadyAnthropicPatrolRuntime(t, handler)
setUnexportedField(t, patrol, "runInProgress", true)
setUnexportedField(t, patrol, "currentRunID", "run-active")
setUnexportedField(t, patrol, "runStartedAt", time.Now())
req := newLoopbackRequest(http.MethodPost, "/api/ai/patrol/run", nil)
rec := httptest.NewRecorder()
handler.HandleForcePatrol(rec, req)
if rec.Code != http.StatusConflict {
t.Fatalf("status = %d, want 409: %s", rec.Code, rec.Body.String())
}
var payload APIError
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode error payload: %v", err)
}
if payload.Code != "patrol_already_running" {
t.Fatalf("code = %q, want patrol_already_running", payload.Code)
}
if payload.Details["current_run_id"] != "run-active" {
t.Fatalf("current_run_id = %q, want run-active", payload.Details["current_run_id"])
}
}
@@ -1019,6 +1049,17 @@ func TestHandleForcePatrol_CommunityTierIgnoresRecentScopedActivityForFullPatrol
if strings.Contains(rec.Body.String(), "patrol_rate_limited") {
t.Fatalf("expected community force patrol to ignore scoped-only activity, got %s", rec.Body.String())
}
var response struct {
Accepted bool `json:"accepted"`
RunID string `json:"run_id"`
StartedAt string `json:"started_at"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("decode acceptance response: %v", err)
}
if !response.Accepted || response.RunID == "" || response.StartedAt == "" {
t.Fatalf("incomplete acceptance response: %+v", response)
}
}
func TestBuildManualScopedPatrolScope(t *testing.T) {
+89 -2
View File
@@ -29,6 +29,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/sensors"
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rs/zerolog"
gohost "github.com/shirou/gopsutil/v4/host"
)
@@ -913,6 +914,10 @@ func (a *Agent) persistReportQueue(queue *utils.Queue[agentshost.Report], fileNa
a.logger.Warn().Err(err).Str("dir", a.stateDir).Msg("Failed to create state dir for buffer persistence")
return
}
if err := os.Chmod(a.stateDir, 0700); err != nil {
a.logger.Warn().Err(err).Str("dir", a.stateDir).Msg("Failed to enforce state dir permissions for buffer persistence")
return
}
path := filepath.Join(a.stateDir, fileName)
tmpPath := path + ".tmp"
@@ -996,6 +1001,7 @@ func (a *Agent) buildReport(ctx context.Context) (agentshost.Report, error) {
// Collect S.M.A.R.T. disk data (best effort - don't fail if unavailable)
smartData := a.collectSMARTData(collectCtx, runtimeConfig.diskExclude)
if len(smartData) > 0 {
annotateSMARTWithDiskIO(smartData, snapshot.DiskIO)
sensorData.SMART = smartData
}
@@ -1210,6 +1216,10 @@ func (a *Agent) persistAgentID(agentID string) {
a.logger.Debug().Err(err).Msg("Failed to create state directory for agent-id")
return
}
if err := a.collector.Chmod(a.stateDir, 0700); err != nil {
a.logger.Debug().Err(err).Msg("Failed to enforce state directory permissions for agent-id")
return
}
agentIDPath := filepath.Join(a.stateDir, "agent-id")
if err := a.collector.WriteFile(agentIDPath, []byte(agentID), 0600); err != nil {
a.logger.Debug().Err(err).Msg("Failed to persist agent-id")
@@ -1924,10 +1934,13 @@ func (a *Agent) collectSMARTData(ctx context.Context, diskExclude []string) []ag
Serial: disk.Serial,
WWN: disk.WWN,
Type: disk.Type,
Controller: disk.Controller,
Target: disk.Target,
SizeBytes: disk.SizeBytes,
Temperature: disk.Temperature,
Health: disk.Health,
Standby: disk.Standby,
Collection: diskinventory.CloneStatus(disk.Collection),
}
if disk.Attributes != nil {
entry.Attributes = &agentshost.SMARTAttributes{
@@ -1947,9 +1960,20 @@ func (a *Agent) collectSMARTData(ctx context.Context, diskExclude []string) []ag
}
if pools, err := ZFSDiskPoolMap(ctx); err != nil {
for index := range result {
ensureAgentDiskCollection(&result[index]).Pool = diskinventory.Unavailable(
"zpool",
"ZFS pool membership could not be collected",
)
}
a.logger.Debug().Err(err).Msg("Failed to collect ZFS pool membership for SMART annotation")
} else if len(pools) > 0 {
annotateSMARTWithZFSPools(result, pools)
} else {
if len(pools) > 0 {
annotateSMARTWithZFSPools(result, pools)
}
for index := range result {
ensureAgentDiskCollection(&result[index]).Pool = diskinventory.Available("zpool")
}
}
a.logger.Debug().
@@ -1959,6 +1983,69 @@ func (a *Agent) collectSMARTData(ctx context.Context, diskExclude []string) []ag
return result
}
func ensureAgentDiskCollection(disk *agentshost.DiskSMART) *diskinventory.CollectionStatus {
if disk.Collection == nil {
disk.Collection = &diskinventory.CollectionStatus{}
}
return disk.Collection
}
// annotateSMARTWithDiskIO binds cumulative kernel counters to a SMART
// inventory entry only when the block-device association is unambiguous.
// Controller-member targets share one logical kernel device, so assigning that
// aggregate counter to any member would fabricate per-disk I/O.
func annotateSMARTWithDiskIO(smart []agentshost.DiskSMART, diskIO []agentshost.DiskIO) {
if len(smart) == 0 {
return
}
ioByDevice := make(map[string]agentshost.DiskIO, len(diskIO))
for _, entry := range diskIO {
device := diskinventory.DeviceToken(entry.Device)
if device != "" {
ioByDevice[strings.ToLower(device)] = entry
}
}
smartCountByDevice := make(map[string]int, len(smart))
for _, disk := range smart {
device := strings.ToLower(diskinventory.DeviceToken(disk.Device))
if device != "" {
smartCountByDevice[device]++
}
}
for index := range smart {
disk := &smart[index]
collection := ensureAgentDiskCollection(disk)
device := strings.ToLower(diskinventory.DeviceToken(disk.Device))
if isMultiplexedDeviceType(disk.Target) || (device != "" && smartCountByDevice[device] > 1) {
collection.IO = diskinventory.Unsupported(
"kernel_diskstats",
"controller target does not expose member-level I/O counters",
)
continue
}
ioEntry, ok := ioByDevice[device]
if !ok {
if len(diskIO) == 0 {
collection.IO = diskinventory.Unavailable(
"kernel_diskstats",
"disk I/O counters could not be collected",
)
} else {
collection.IO = diskinventory.Missing(
"kernel_diskstats",
"disk was present but its I/O counters were not reported",
)
}
continue
}
ioCopy := ioEntry
disk.IO = &ioCopy
collection.IO = diskinventory.Available("kernel_diskstats")
}
}
// runProxmoxSetup performs one-time Proxmox API token setup and node registration.
// Supports hosts with multiple Proxmox products (e.g., PVE + PBS on same host).
func (a *Agent) runProxmoxSetup(ctx context.Context) {
+6
View File
@@ -77,6 +77,8 @@ func (a *Agent) runEnrollmentLoop(ctx context.Context) error {
a.persistRuntimeToken(result.RuntimeToken)
canonicalID := strings.TrimSpace(result.AgentID)
if canonicalID != "" {
a.agentID = canonicalID
a.cfg.AgentID = canonicalID
a.persistAgentID(canonicalID)
}
a.logger.Info().
@@ -204,6 +206,10 @@ func (a *Agent) persistRuntimeToken(token string) {
a.logger.Warn().Err(err).Msg("Failed to create state directory for runtime token")
return
}
if err := a.collector.Chmod(a.stateDir, 0700); err != nil {
a.logger.Warn().Err(err).Msg("Failed to enforce state directory permissions for runtime token")
return
}
tokenPath := filepath.Join(a.stateDir, runtimeTokenFile)
if err := a.collector.WriteFile(tokenPath, []byte(token), 0600); err != nil {
a.logger.Warn().Err(err).Msg("Failed to persist runtime token")
+3
View File
@@ -85,6 +85,9 @@ func TestEnroll_Success(t *testing.T) {
if agent.cfg.APIToken != "runtime-tok-123" {
t.Errorf("expected token runtime-tok-123, got %s", agent.cfg.APIToken)
}
if agent.agentID != "host-test-host" || agent.cfg.AgentID != "host-test-host" {
t.Fatalf("running agent identity was not updated after enrollment: agent=%q config=%q", agent.agentID, agent.cfg.AgentID)
}
// Verify runtime token was persisted.
tokenPath := filepath.Join(stateDir, runtimeTokenFile)
@@ -0,0 +1,272 @@
//go:build !windows
package hostagent
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
type issue1595TopologyFixture struct {
Node string `json:"node"`
Instance string `json:"instance"`
AgentID string `json:"agentId"`
Model string `json:"model"`
SizeBytes int64 `json:"sizeBytes"`
Controllers []struct {
ID string `json:"id"`
Pool string `json:"pool"`
Disks []struct {
Device string `json:"device"`
Target string `json:"target"`
Serial string `json:"serial"`
ProviderSerial string `json:"providerSerial"`
Temperature int `json:"temperature"`
ReadBytes uint64 `json:"readBytes"`
WriteBytes uint64 `json:"writeBytes"`
IOTimeMs uint64 `json:"ioTimeMs"`
} `json:"disks"`
} `json:"controllers"`
}
func loadIssue1595TopologyFixture(t *testing.T) issue1595TopologyFixture {
t.Helper()
path := filepath.Join("..", "..", "testdata", "issue1595_sas_topology.json")
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture: %v", err)
}
var fixture issue1595TopologyFixture
if err := json.Unmarshal(data, &fixture); err != nil {
t.Fatalf("decode fixture: %v", err)
}
return fixture
}
func TestIssue1595CollectSMARTLocalPreservesTwentyFourSASDisksAcrossTwoHBAs(t *testing.T) {
fixture := loadIssue1595TopologyFixture(t)
type diskFixture struct {
controller string
target string
serial string
temperature int
}
byDevice := make(map[string]diskFixture)
var scan strings.Builder
var entries []os.DirEntry
for _, controller := range fixture.Controllers {
for _, disk := range controller.Disks {
byDevice[disk.Device] = diskFixture{
controller: controller.ID,
target: disk.Target,
serial: disk.Serial,
temperature: disk.Temperature,
}
fmt.Fprintf(&scan, "/dev/%s -d scsi # synthetic issue #1595 SAS disk\n", disk.Device)
entries = append(entries, fakeDirEntry{name: disk.Device})
}
}
if len(byDevice) != 24 {
t.Fatalf("fixture disk count = %d, want 24", len(byDevice))
}
origGOOS := runtimeGOOS
origReadDir := readDir
origReadFile := smartctlReadFile
origEvalSymlinks := smartctlEvalSymlinks
origRun := smartRunCommandOutput
origLookPath := execLookPath
origConcurrency := smartCollectionConcurrency
origThreshold := smartCollectionParallelThreshold
t.Cleanup(func() {
runtimeGOOS = origGOOS
readDir = origReadDir
smartctlReadFile = origReadFile
smartctlEvalSymlinks = origEvalSymlinks
smartRunCommandOutput = origRun
execLookPath = origLookPath
smartCollectionConcurrency = origConcurrency
smartCollectionParallelThreshold = origThreshold
})
runtimeGOOS = "linux"
smartCollectionConcurrency = 6
smartCollectionParallelThreshold = 12
readDir = func(path string) ([]os.DirEntry, error) {
if path == "/sys/block" {
return entries, nil
}
return nil, fs.ErrNotExist
}
smartctlReadFile = func(path string) ([]byte, error) {
device := issue1595DeviceFromSysfsPath(path)
if device == "" {
return nil, fs.ErrNotExist
}
switch filepath.Base(path) {
case "size":
return []byte(fmt.Sprintf("%d\n", fixture.SizeBytes/512)), nil
case "protocol":
return []byte("SAS\n"), nil
case "rotational":
return []byte("1\n"), nil
case "model":
return []byte(fixture.Model + "\n"), nil
default:
return nil, fs.ErrNotExist
}
}
smartctlEvalSymlinks = func(path string) (string, error) {
device := issue1595DeviceFromSysfsPath(path)
disk, ok := byDevice[device]
if !ok {
return "", fs.ErrNotExist
}
if strings.HasSuffix(path, "/device/subsystem") {
return "/sys/bus/scsi", nil
}
if strings.HasSuffix(path, "/device") || path == "/sys/block/"+device {
host := strings.Split(disk.target, ":")[0]
return fmt.Sprintf(
"/sys/devices/pci0000:00/%s/host%s/target%s/%s/block/%s",
disk.controller,
host,
strings.Join(strings.Split(disk.target, ":")[:3], ":"),
disk.target,
device,
), nil
}
return "", fs.ErrNotExist
}
execLookPath = func(string) (string, error) { return "smartctl", nil }
var active atomic.Int32
var maxActive atomic.Int32
smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) {
if len(args) == 1 && args[0] == "--scan-open" {
return []byte(scan.String()), nil
}
device := strings.TrimPrefix(args[len(args)-1], "/dev/")
disk, ok := byDevice[device]
if !ok {
return nil, fmt.Errorf("unexpected SMART target %q", device)
}
current := active.Add(1)
for {
previous := maxActive.Load()
if current <= previous || maxActive.CompareAndSwap(previous, current) {
break
}
}
defer active.Add(-1)
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(15 * time.Millisecond):
}
return json.Marshal(map[string]any{
"device": map[string]any{
"name": "/dev/" + device,
"type": "scsi",
"protocol": "SAS",
},
"model_name": fixture.Model,
"serial_number": disk.serial,
"user_capacity": map[string]any{"bytes": fixture.SizeBytes},
"smart_status": map[string]any{"passed": true},
"temperature": map[string]any{"current": disk.temperature},
})
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
results, err := CollectSMARTLocal(ctx, nil)
if err != nil {
t.Fatalf("CollectSMARTLocal() error = %v", err)
}
if len(results) != 24 {
t.Fatalf("collected disk count = %d, want 24", len(results))
}
if maxActive.Load() < 2 {
t.Fatalf("SMART collection did not use bounded concurrency; max active = %d", maxActive.Load())
}
for _, result := range results {
want := byDevice[result.Device]
if result.Serial != want.serial || result.Type != "sas" || result.Temperature != want.temperature {
t.Fatalf("disk %s lost SAS SMART identity: %+v", result.Device, result)
}
if result.Controller != want.controller || result.Target != want.target {
t.Fatalf("disk %s topology = %q/%q, want %q/%q", result.Device, result.Controller, result.Target, want.controller, want.target)
}
if result.Collection == nil ||
result.Collection.Serial.State != diskinventory.FieldAvailable ||
result.Collection.Temperature.State != diskinventory.FieldAvailable ||
result.Collection.Controller.State != diskinventory.FieldAvailable {
t.Fatalf("disk %s collection status = %+v", result.Device, result.Collection)
}
}
}
func TestIssue1595DiskIOAssociationMarksSharedControllerTargetsUnsupported(t *testing.T) {
smart := []agentshost.DiskSMART{
{Device: "sda [megaraid,0]", Controller: "sda", Target: "megaraid,0"},
{Device: "sda [megaraid,1]", Controller: "sda", Target: "megaraid,1"},
}
annotateSMARTWithDiskIO(smart, []agentshost.DiskIO{{Device: "sda", ReadBytes: 100}})
for _, disk := range smart {
if disk.IO != nil {
t.Fatalf("controller member %s received aggregate I/O counters", disk.Target)
}
if disk.Collection == nil || disk.Collection.IO.State != diskinventory.FieldUnsupported {
t.Fatalf("controller member %s I/O status = %+v, want unsupported", disk.Target, disk.Collection)
}
}
}
func TestLinuxBlockDeviceTopologyPreservesDirectNVMeAndSATAPCIController(t *testing.T) {
origEvalSymlinks := smartctlEvalSymlinks
t.Cleanup(func() { smartctlEvalSymlinks = origEvalSymlinks })
paths := map[string]string{
"/sys/block/sda/device": "/sys/devices/pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0",
"/sys/block/nvme0n1/device": "/sys/devices/pci0000:00/0000:00:1d.0/nvme/nvme0/nvme0n1",
}
smartctlEvalSymlinks = func(path string) (string, error) {
if resolved, ok := paths[path]; ok {
return resolved, nil
}
return "", fs.ErrNotExist
}
if controller, target := linuxBlockDeviceTopology("sda"); controller != "0000:00:17.0" || target != "0:0:0:0" {
t.Fatalf("SATA topology = %q/%q, want PCI controller and HCTL", controller, target)
}
if controller, target := linuxBlockDeviceTopology("nvme0n1"); controller != "0000:00:1d.0" || target != "" {
t.Fatalf("NVMe topology = %q/%q, want PCI controller without HCTL", controller, target)
}
}
func issue1595DeviceFromSysfsPath(path string) string {
const prefix = "/sys/block/"
if !strings.HasPrefix(path, prefix) {
return ""
}
remainder := strings.TrimPrefix(path, prefix)
if index := strings.IndexByte(remainder, '/'); index >= 0 {
remainder = remainder[:index]
}
return remainder
}
+205 -23
View File
@@ -15,10 +15,12 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/rs/zerolog/log"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters"
)
@@ -39,21 +41,27 @@ var (
timeNow = time.Now
runtimeGOOS = runtime.GOOS
smartCollectionConcurrency = 6
smartCollectionParallelThreshold = 12
)
// DiskSMART represents S.M.A.R.T. data for a single disk.
type DiskSMART struct {
Device string `json:"device"` // Block device name (e.g., sda, nvme0n1)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
SizeBytes int64 `json:"sizeBytes,omitempty"` // Capacity in bytes (0 when unknown)
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
Attributes *SMARTAttributes `json:"attributes,omitempty"`
LastUpdated time.Time `json:"lastUpdated"` // When this reading was taken
Device string `json:"device"` // Block device name (e.g., sda, nvme0n1)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
Controller string `json:"controller,omitempty"` // PCI/controller association when reported
Target string `json:"target,omitempty"` // HCTL or smartctl controller-member target
SizeBytes int64 `json:"sizeBytes,omitempty"` // Capacity in bytes (0 when unknown)
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
Collection *diskinventory.CollectionStatus `json:"collection,omitempty"`
Attributes *SMARTAttributes `json:"attributes,omitempty"`
LastUpdated time.Time `json:"lastUpdated"` // When this reading was taken
}
// SMARTAttributes holds normalized SMART attributes for both SATA and NVMe disks.
@@ -205,6 +213,7 @@ var (
smartTextCurrentTempRE = regexp.MustCompile(`(?i)^current(?: drive)? temperature:\s*(\d{1,3})\b`)
smartTextTemperatureRE = regexp.MustCompile(`(?i)^temperature:\s*(\d{1,3})\b`)
linuxDirectSATDeviceRE = regexp.MustCompile(`^(sd|hd)[a-z]+$`)
pciControllerAddressRE = regexp.MustCompile(`(?i)^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`)
)
type smartctlTarget struct {
@@ -236,6 +245,38 @@ func CollectSMARTLocal(ctx context.Context, diskExclude []string) ([]DiskSMART,
return nil, nil
}
type smartOutcome struct {
smart *DiskSMART
err error
}
outcomes := make([]smartOutcome, len(targets))
workerCount := smartCollectionConcurrency
if len(targets) < smartCollectionParallelThreshold {
workerCount = 1
}
if workerCount < 1 {
workerCount = 1
}
if workerCount > len(targets) {
workerCount = len(targets)
}
jobs := make(chan int)
var workers sync.WaitGroup
workers.Add(workerCount)
for worker := 0; worker < workerCount; worker++ {
go func() {
defer workers.Done()
for index := range jobs {
outcomes[index].smart, outcomes[index].err = collectSMARTTarget(ctx, targets[index])
}
}()
}
for index := range targets {
jobs <- index
}
close(jobs)
workers.Wait()
var results []DiskSMART
var missed []smartctlTarget
collected := make(map[string]struct{}, len(targets))
@@ -245,7 +286,10 @@ func CollectSMARTLocal(ctx context.Context, diskExclude []string) ([]DiskSMART,
if isMultiplexedDeviceType(target.DeviceType) && block != "" {
multiplexed[block] = struct{}{}
}
smart, err := collectSMARTTarget(ctx, target)
}
for index, target := range targets {
block := canonicalBlockDeviceForScanPath(target.Path)
smart, err := outcomes[index].smart, outcomes[index].err
if err != nil {
if errors.Is(err, errSMARTDataUnavailable) {
log.Debug().
@@ -801,23 +845,115 @@ func refineLinuxBlockDeviceIdentity(smart *DiskSMART, target smartctlTarget) {
if smart == nil || runtimeGOOS != "linux" {
return
}
block := canonicalBlockDeviceForScanPath(target.Path)
if block == "" {
return
}
// Disks addressed behind a multiplexing controller (megaraid,7; cciss,1;
// areca,1/1; ...) all share a single /dev path, so the smartctl scan label is
// the only thing that disambiguates them and /sys/block describes the array,
// not the member. Leave those as-is and trust the smartctl-reported capacity.
if isMultiplexedDeviceType(target.DeviceType) {
return
}
block := canonicalBlockDeviceForScanPath(target.Path)
if block == "" {
smart.Controller = block
smart.Target = strings.TrimSpace(target.DeviceType)
ensureControllerCollectionStatus(smart, "smartctl_scan")
return
}
smart.Device = block
smart.Controller, smart.Target = linuxBlockDeviceTopology(block)
ensureControllerCollectionStatus(smart, "sysfs")
if size := blockDeviceSizeBytes(block); size > 0 {
smart.SizeBytes = size
}
}
func ensureControllerCollectionStatus(smart *DiskSMART, source string) {
if smart.Collection == nil {
smart.Collection = &diskinventory.CollectionStatus{}
}
if smart.Controller != "" || smart.Target != "" {
smart.Collection.Controller = diskinventory.Available(source)
return
}
smart.Collection.Controller = diskinventory.Missing(source, "controller association was not reported")
}
// linuxBlockDeviceTopology derives a stable controller association and SCSI
// target from the resolved /sys/block device path. The controller prefers the
// PCI address immediately preceding hostN; the target is the terminal H:C:T:L
// segment. Neither value is fabricated when sysfs does not expose it.
func linuxBlockDeviceTopology(block string) (string, string) {
resolved, err := smartctlEvalSymlinks(filepath.Join("/sys/block", block, "device"))
if err != nil {
return "", ""
}
parts := strings.Split(filepath.Clean(resolved), string(filepath.Separator))
controller := ""
controllerFallback := ""
target := ""
for index, part := range parts {
if pciControllerAddressRE.MatchString(part) {
controller = part
}
if strings.HasPrefix(part, "host") && hasNumericSuffix(part, "host") && index > 0 {
controllerFallback = parts[index-1]
}
if isSCSITargetAddress(part) {
target = part
}
}
if controller == "" {
controller = controllerFallback
}
return controller, target
}
func isSCSITargetAddress(value string) bool {
parts := strings.Split(value, ":")
if len(parts) != 4 {
return false
}
for _, part := range parts {
if part == "" || !isAllDigits(part) {
return false
}
}
return true
}
func linuxBlockDeviceTransport(block string, target smartctlTarget) string {
for _, candidate := range []string{
readTrimmedFile(filepath.Join("/sys/block", block, "device", "protocol")),
readTrimmedFile(filepath.Join("/sys/block", block, "device", "transport")),
} {
switch normalized := strings.ToLower(strings.TrimSpace(candidate)); {
case strings.Contains(normalized, "nvme"):
return "nvme"
case strings.Contains(normalized, "sas"):
return "sas"
case strings.Contains(normalized, "sata"), strings.Contains(normalized, "ata"):
return "sata"
case strings.Contains(normalized, "usb"):
return "usb"
}
}
if strings.HasPrefix(block, "nvme") {
return "nvme"
}
deviceType := strings.ToLower(strings.TrimSpace(target.DeviceType))
switch {
case strings.HasPrefix(deviceType, "nvme"):
return "nvme"
case strings.HasPrefix(deviceType, "sat"):
return "sata"
default:
// Preserve the legacy direct sdX fallback when the kernel supplies no
// transport evidence. Successful smartctl probes still override this
// with their protocol field before this fallback is needed.
return "sata"
}
}
// linuxIdentityOnlyDisks builds identity-only entries for physical disks whose
// SMART probes produced nothing usable. A real disk that refuses SMART must
// still be listed — Proxmox's own disks/list shows it, and a monitoring view
@@ -863,17 +999,31 @@ func linuxIdentityOnlyDisks(missed []smartctlTarget, collected, multiplexed map[
continue
}
diskType := "sata"
if strings.HasPrefix(block, "nvme") {
diskType = "nvme"
serial := readTrimmedFile(filepath.Join("/sys/block", block, "device", "serial"))
controller, controllerTarget := linuxBlockDeviceTopology(block)
collection := &diskinventory.CollectionStatus{
Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data"),
}
if serial != "" {
collection.Serial = diskinventory.Available("sysfs")
} else {
collection.Serial = diskinventory.Missing("sysfs", "disk serial was not reported")
}
if controller != "" || controllerTarget != "" {
collection.Controller = diskinventory.Available("sysfs")
} else {
collection.Controller = diskinventory.Missing("sysfs", "controller association was not reported")
}
results = append(results, DiskSMART{
Device: block,
Model: readTrimmedFile(filepath.Join("/sys/block", block, "device", "model")),
Serial: readTrimmedFile(filepath.Join("/sys/block", block, "device", "serial")),
Type: diskType,
Serial: serial,
Type: linuxBlockDeviceTransport(block, target),
Controller: controller,
Target: controllerTarget,
SizeBytes: size,
Health: "UNKNOWN",
Collection: collection,
LastUpdated: timeNow(),
})
log.Debug().
@@ -1056,8 +1206,12 @@ func collectSMARTTarget(ctx context.Context, target smartctlTarget) (*DiskSMART,
exitCode := exitErr.ExitCode()
if (exitCode == smartctlStandbyExitStatus || exitCode&2 != 0) && len(output) == 0 {
standbyResult := &DiskSMART{
Device: filepath.Base(target.Path),
Standby: true,
Device: filepath.Base(target.Path),
Standby: true,
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Unavailable("smartctl", "disk is in standby"),
Temperature: diskinventory.Unavailable("smartctl", "disk is in standby"),
},
LastUpdated: timeNow(),
}
if runtimeGOOS == "freebsd" && i < len(attempts)-1 && target.DeviceType == "" {
@@ -1338,6 +1492,7 @@ func parseSMARTOutput(output []byte, target smartctlTarget) (*DiskSMART, error)
Type: detectDiskType(smartData),
Standby: isStandbyPowerMode(smartData.PowerMode),
LastUpdated: timeNow(),
Collection: &diskinventory.CollectionStatus{},
}
if smartData.WWN.NAA != 0 {
@@ -1380,6 +1535,19 @@ func parseSMARTOutput(output []byte, target smartctlTarget) (*DiskSMART, error)
}
applySMARTTextFallback(result, parseSMARTTextFallback(strings.Join(smartData.Smartctl.Output, "\n")))
if result.Serial != "" {
result.Collection.Serial = diskinventory.Available("smartctl")
} else {
result.Collection.Serial = diskinventory.Missing("smartctl", "disk serial was not reported")
}
switch {
case result.Temperature > 0:
result.Collection.Temperature = diskinventory.Available("smartctl")
case result.Standby:
result.Collection.Temperature = diskinventory.Unavailable("smartctl", "disk is in standby")
default:
result.Collection.Temperature = diskinventory.Unsupported("smartctl", "device did not expose a temperature reading")
}
if result.Health == "" {
result.Health = "UNKNOWN"
}
@@ -1403,6 +1571,7 @@ func parseSMARTTextOutput(text string, target smartctlTarget) (*DiskSMART, error
Health: fallback.Health,
Standby: fallback.Standby,
LastUpdated: timeNow(),
Collection: &diskinventory.CollectionStatus{},
}
if result.Type == "" {
switch {
@@ -1418,6 +1587,19 @@ func parseSMARTTextOutput(text string, target smartctlTarget) (*DiskSMART, error
if result.Health == "" {
result.Health = "UNKNOWN"
}
if result.Serial != "" {
result.Collection.Serial = diskinventory.Available("smartctl_text")
} else {
result.Collection.Serial = diskinventory.Missing("smartctl_text", "disk serial was not reported")
}
switch {
case result.Temperature > 0:
result.Collection.Temperature = diskinventory.Available("smartctl_text")
case result.Standby:
result.Collection.Temperature = diskinventory.Unavailable("smartctl_text", "disk is in standby")
default:
result.Collection.Temperature = diskinventory.Unsupported("smartctl_text", "device did not expose a temperature reading")
}
if result.Health == "UNKNOWN" && result.Temperature == 0 && !result.Standby {
return nil, errSMARTDataUnavailable
}
+15 -1
View File
@@ -4,6 +4,8 @@ import (
"encoding/json"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// ToFrontend converts a State to StateFrontend
@@ -925,11 +927,15 @@ func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFronte
Serial: disk.Serial,
WWN: disk.WWN,
Type: disk.Type,
Controller: disk.Controller,
Target: disk.Target,
SizeBytes: disk.SizeBytes,
Temperature: disk.Temperature,
Health: disk.Health,
Standby: disk.Standby,
Attributes: disk.Attributes,
IO: cloneDiskIOForFrontend(disk.IO),
Collection: diskinventory.CloneStatus(disk.Collection),
Attributes: cloneSMARTAttributes(disk.Attributes),
}
}
}
@@ -937,6 +943,14 @@ func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFronte
return &normalized
}
func cloneDiskIOForFrontend(src *DiskIO) *DiskIO {
if src == nil {
return nil
}
dest := *src
return &dest
}
func copyHostThermalState(src *HostThermalState) *HostThermalState {
if src == nil {
return nil
+12
View File
@@ -3,6 +3,8 @@ package models
import (
"reflect"
"time"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func cloneBoolPtr(src *bool) *bool {
@@ -237,6 +239,11 @@ func cloneHostDiskSMART(src []HostDiskSMART) []HostDiskSMART {
for i, disk := range src {
diskCopy := disk
diskCopy.Attributes = cloneSMARTAttributes(disk.Attributes)
diskCopy.Collection = diskinventory.CloneStatus(disk.Collection)
if disk.IO != nil {
ioCopy := *disk.IO
diskCopy.IO = &ioCopy
}
dest[i] = diskCopy
}
return dest
@@ -1102,6 +1109,11 @@ func cloneCephClusters(src []CephCluster) []CephCluster {
func clonePhysicalDisk(src PhysicalDisk) PhysicalDisk {
dest := src
dest.SmartAttributes = cloneSMARTAttributes(src.SmartAttributes)
dest.Collection = diskinventory.CloneStatus(src.Collection)
if src.IO != nil {
ioCopy := *src.IO
dest.IO = &ioCopy
}
return dest
}
+37 -28
View File
@@ -12,6 +12,7 @@ import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/proxmoxidentity"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// State represents the current state of all monitored resources
@@ -464,17 +465,21 @@ func (s HostSensorSummary) NormalizeCollections() HostSensorSummary {
// HostDiskSMART represents S.M.A.R.T. data for a disk from a host agent.
type HostDiskSMART struct {
Device string `json:"device"` // Block device name (e.g., sda, nvme0n1)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
SizeBytes int64 `json:"sizeBytes,omitempty"` // Capacity in bytes (0 when unknown)
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
Pool string `json:"pool,omitempty"` // ZFS pool this disk belongs to (empty if not a ZFS member)
Attributes *SMARTAttributes `json:"attributes,omitempty"`
Device string `json:"device"` // Block device name (e.g., sda, nvme0n1)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
Controller string `json:"controller,omitempty"` // Stable controller association when reported
Target string `json:"target,omitempty"` // Controller target/HCTL or smartctl member target
SizeBytes int64 `json:"sizeBytes,omitempty"` // Capacity in bytes (0 when unknown)
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
Pool string `json:"pool,omitempty"` // ZFS pool this disk belongs to (empty if not a ZFS member)
IO *DiskIO `json:"io,omitempty"`
Collection *diskinventory.CollectionStatus `json:"collection,omitempty"`
Attributes *SMARTAttributes `json:"attributes,omitempty"`
}
// SMARTAttributes holds normalized SMART attributes for both SATA and NVMe disks.
@@ -2395,23 +2400,27 @@ type CephServiceStatus struct {
// PhysicalDisk represents a physical disk on a node
type PhysicalDisk struct {
ID string `json:"id"` // "{instance}-{node}-{devpath}"
Node string `json:"node"`
Instance string `json:"instance"`
DevPath string `json:"devPath"` // /dev/nvme0n1, /dev/sda
Model string `json:"model"`
Serial string `json:"serial"`
WWN string `json:"wwn"` // World Wide Name
Type string `json:"type"` // nvme, sata, sas
Size int64 `json:"size"` // bytes
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
Wearout int `json:"wearout"` // SSD wear metric from Proxmox (0-100, -1 when unavailable)
Temperature int `json:"temperature"` // Celsius (if available)
RPM int `json:"rpm"` // 0 for SSDs
Used string `json:"used"` // Filesystem or partition usage
StorageGroup string `json:"storageGroup"` // Pool/VG/array this disk belongs to (e.g. ZFS pool name); empty if not matched
SmartAttributes *SMARTAttributes `json:"smartAttributes,omitempty"`
LastChecked time.Time `json:"lastChecked"`
ID string `json:"id"` // "{instance}-{node}-{devpath}"
Node string `json:"node"`
Instance string `json:"instance"`
DevPath string `json:"devPath"` // /dev/nvme0n1, /dev/sda
Model string `json:"model"`
Serial string `json:"serial"`
WWN string `json:"wwn"` // World Wide Name
Type string `json:"type"` // nvme, sata, sas
Controller string `json:"controller,omitempty"` // Controller association when reported
Target string `json:"target,omitempty"` // Controller target/HCTL when reported
Size int64 `json:"size"` // bytes
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
Wearout int `json:"wearout"` // SSD wear metric from Proxmox (0-100, -1 when unavailable)
Temperature int `json:"temperature"` // Celsius (if available)
RPM int `json:"rpm"` // 0 for SSDs
Used string `json:"used"` // Filesystem or partition usage
StorageGroup string `json:"storageGroup"` // Pool/VG/array this disk belongs to (e.g. ZFS pool name); empty if not matched
SmartAttributes *SMARTAttributes `json:"smartAttributes,omitempty"`
IO *DiskIO `json:"io,omitempty"`
Collection *diskinventory.CollectionStatus `json:"collection,omitempty"`
LastChecked time.Time `json:"lastChecked"`
}
// PBSInstance represents a Proxmox Backup Server instance
+19 -11
View File
@@ -1,6 +1,10 @@
package models
import "encoding/json"
import (
"encoding/json"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// Frontend-friendly type aliases with proper JSON tags
// These extend the base types with additional computed fields
@@ -798,16 +802,20 @@ func (s HostSensorSummaryFrontend) NormalizeCollections() HostSensorSummaryFront
// HostDiskSMARTFrontend represents S.M.A.R.T. data for a disk from a host agent.
type HostDiskSMARTFrontend struct {
Device string `json:"device"` // Device name (e.g., sda)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
SizeBytes int64 `json:"sizeBytes,omitempty"`
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
Attributes *SMARTAttributes `json:"attributes,omitempty"`
Device string `json:"device"` // Device name (e.g., sda)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
Controller string `json:"controller,omitempty"` // Controller association
Target string `json:"target,omitempty"` // Controller target/HCTL
SizeBytes int64 `json:"sizeBytes,omitempty"`
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
IO *DiskIO `json:"io,omitempty"`
Collection *diskinventory.CollectionStatus `json:"collection,omitempty"`
Attributes *SMARTAttributes `json:"attributes,omitempty"`
}
// StorageFrontend represents Storage with frontend-friendly field names
@@ -0,0 +1,314 @@
package monitoring
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
type issue1595TopologyFixture struct {
Node string `json:"node"`
Instance string `json:"instance"`
AgentID string `json:"agentId"`
Model string `json:"model"`
SizeBytes int64 `json:"sizeBytes"`
Controllers []struct {
ID string `json:"id"`
Pool string `json:"pool"`
Disks []struct {
Device string `json:"device"`
Target string `json:"target"`
Serial string `json:"serial"`
ProviderSerial string `json:"providerSerial"`
Temperature int `json:"temperature"`
ReadBytes uint64 `json:"readBytes"`
WriteBytes uint64 `json:"writeBytes"`
IOTimeMs uint64 `json:"ioTimeMs"`
} `json:"disks"`
} `json:"controllers"`
}
func loadIssue1595TopologyFixture(t *testing.T) issue1595TopologyFixture {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "issue1595_sas_topology.json"))
if err != nil {
t.Fatalf("read issue #1595 fixture: %v", err)
}
var fixture issue1595TopologyFixture
if err := json.Unmarshal(data, &fixture); err != nil {
t.Fatalf("decode issue #1595 fixture: %v", err)
}
return fixture
}
func TestIssue1595SASTopologySurvivesMergeRegistryAndReadState(t *testing.T) {
fixture := loadIssue1595TopologyFixture(t)
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
nodeID := fixture.Instance + "-" + fixture.Node
host := models.Host{
ID: fixture.AgentID,
Hostname: fixture.Node,
Status: "online",
LastSeen: now,
LinkedNodeID: nodeID,
}
node := models.Node{
ID: nodeID,
Name: fixture.Node,
Instance: fixture.Instance,
Status: "online",
LastSeen: now,
LinkedAgentID: fixture.AgentID,
}
var providerDisks []models.PhysicalDisk
expectedBySerial := make(map[string]models.PhysicalDisk)
for _, controller := range fixture.Controllers {
for index, disk := range controller.Disks {
device := "/dev/" + disk.Device
io := &models.DiskIO{
Device: disk.Device,
ReadBytes: disk.ReadBytes,
WriteBytes: disk.WriteBytes,
ReadOps: uint64(100 + index),
WriteOps: uint64(200 + index),
ReadTime: uint64(300 + index),
WriteTime: uint64(400 + index),
IOTime: disk.IOTimeMs,
}
host.DiskIO = append(host.DiskIO, *io)
host.Sensors.SMART = append(host.Sensors.SMART, models.HostDiskSMART{
Device: device,
Model: fixture.Model,
Serial: disk.Serial,
Type: "sas",
Controller: controller.ID,
Target: disk.Target,
SizeBytes: fixture.SizeBytes,
Temperature: disk.Temperature,
Health: "PASSED",
Pool: controller.Pool,
IO: io,
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Available("smartctl"),
Temperature: diskinventory.Available("smartctl"),
IO: diskinventory.Available("linux-diskstats"),
Controller: diskinventory.Available("linux-sysfs"),
Pool: diskinventory.Available("zpool-status"),
},
})
providerDisk := models.PhysicalDisk{
ID: fixture.Instance + "-" + fixture.Node + "-" + disk.Device,
Node: fixture.Node,
Instance: fixture.Instance,
DevPath: device,
Model: fixture.Model,
Serial: disk.ProviderSerial,
Type: "unknown",
Size: fixture.SizeBytes,
Health: "UNKNOWN",
Wearout: -1,
Used: "ZFS",
StorageGroup: controller.Pool,
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Available("pve-api"),
Temperature: diskinventory.Unsupported("pve-api", "provider does not report disk temperature"),
IO: diskinventory.Unsupported("pve-api", "provider does not report per-disk I/O"),
Controller: diskinventory.Missing("pve-api", "controller association absent"),
Pool: diskinventory.Available("pve-storage"),
},
LastChecked: now,
}
providerDisks = append(providerDisks, providerDisk)
want := providerDisk
want.Serial = disk.Serial
want.Type = "sas"
want.Controller = controller.ID
want.Target = disk.Target
want.Temperature = disk.Temperature
want.Health = "PASSED"
want.IO = io
expectedBySerial[disk.Serial] = want
}
}
merged := mergeHostAgentSMARTIntoDisks(providerDisks, []models.Node{node}, []models.Host{host})
if len(merged) != 24 {
t.Fatalf("merged disk count = %d, want 24", len(merged))
}
for _, disk := range merged {
want, ok := expectedBySerial[disk.Serial]
if !ok {
t.Fatalf("provider SAS address survived as stable serial for %s: %q", disk.DevPath, disk.Serial)
}
assertIssue1595PhysicalDisk(t, disk, want)
}
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{node},
Hosts: []models.Host{host},
PhysicalDisks: providerDisks,
})
resources := registry.ListByType(unifiedresources.ResourceTypePhysicalDisk)
if len(resources) != 24 {
t.Fatalf("registry physical disk count = %d, want 24 distinct disks", len(resources))
}
for _, resource := range resources {
if resource.PhysicalDisk == nil {
t.Fatalf("physical disk resource %q has no physicalDisk metadata", resource.ID)
}
_, ok := expectedBySerial[resource.PhysicalDisk.Serial]
if !ok {
t.Fatalf("registry serial = %q, want a smartctl serial", resource.PhysicalDisk.Serial)
}
if !hasIssue1595Source(resource.Sources, unifiedresources.SourceAgent) ||
!hasIssue1595Source(resource.Sources, unifiedresources.SourceProxmox) {
t.Fatalf("disk %q sources = %v, want agent and proxmox", resource.PhysicalDisk.Serial, resource.Sources)
}
}
views := registry.PhysicalDisks()
if len(views) != 24 {
t.Fatalf("read-state physical disk count = %d, want 24", len(views))
}
for _, view := range views {
want, ok := expectedBySerial[view.Serial()]
if !ok {
t.Fatalf("read-state serial = %q, want smartctl serial", view.Serial())
}
if view.MetricResourceID() != want.Serial {
t.Fatalf("disk %q metrics target = %q, want serial-stable target", want.Serial, view.MetricResourceID())
}
readBack := physicalDiskFromReadStateView(view)
assertIssue1595PhysicalDisk(t, readBack, want)
if readBack.Collection == nil ||
readBack.Collection.Serial.State != diskinventory.FieldAvailable ||
readBack.Collection.Temperature.State != diskinventory.FieldAvailable ||
readBack.Collection.IO.State != diskinventory.FieldAvailable ||
readBack.Collection.Controller.State != diskinventory.FieldAvailable ||
readBack.Collection.Pool.State != diskinventory.FieldAvailable {
t.Fatalf("disk %q collection status = %+v", readBack.Serial, readBack.Collection)
}
}
}
func TestPhysicalDiskUnavailableEvidenceIsRetainedWithoutPretendingItWasCollected(t *testing.T) {
previous := models.PhysicalDisk{
Serial: "ZR5TESTA0001",
Controller: "0000:03:00.0",
Target: "6:0:0:0",
Temperature: 30,
StorageGroup: "tank-a",
IO: &models.DiskIO{Device: "sda", ReadBytes: 1000},
}
current := models.PhysicalDisk{
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Missing("smartctl", "serial absent from successful response"),
Temperature: diskinventory.Unavailable("smartctl", "collection deadline exceeded"),
IO: diskinventory.Unsupported("controller", "per-member counters unavailable"),
Controller: diskinventory.Unavailable("linux-sysfs", "topology lookup failed"),
Pool: diskinventory.Unavailable("zpool-status", "command failed"),
},
}
got := preserveUnavailablePhysicalDiskEvidence(current, previous)
if got.Serial != previous.Serial ||
got.Controller != previous.Controller ||
got.Target != previous.Target ||
got.Temperature != previous.Temperature ||
got.StorageGroup != previous.StorageGroup ||
got.IO == nil ||
got.IO.ReadBytes != previous.IO.ReadBytes {
t.Fatalf("unavailable evidence was discarded: %+v", got)
}
if got.Collection.Temperature.State != diskinventory.FieldUnavailable ||
got.Collection.IO.State != diskinventory.FieldUnsupported ||
got.Collection.Serial.State != diskinventory.FieldMissing {
t.Fatalf("retained values concealed current collection state: %+v", got.Collection)
}
got.IO.ReadBytes = 2000
if previous.IO.ReadBytes != 1000 {
t.Fatal("retained I/O evidence aliases the previous snapshot")
}
}
func TestTrustedSMARTSerialPromotionDoesNotRewriteSATAOrNVMeIdentity(t *testing.T) {
for _, diskType := range []string{"sata", "nvme"} {
t.Run(diskType, func(t *testing.T) {
device := "/dev/sda"
if diskType == "nvme" {
device = "/dev/nvme0n1"
}
disks := []models.PhysicalDisk{{
ID: "provider-disk",
Node: "node",
DevPath: device,
Serial: "PROVIDER-SERIAL",
Type: diskType,
}}
hosts := []models.Host{{
ID: "agent",
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{{
Device: device,
Serial: "SMARTCTL-SERIAL",
Type: diskType,
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Available("smartctl"),
},
}},
},
}}
got := mergeHostAgentSMARTIntoDisks(
disks,
[]models.Node{{Name: "node", LinkedAgentID: "agent"}},
hosts,
)[0]
if got.Serial != "PROVIDER-SERIAL" || got.Type != diskType {
t.Fatalf("%s identity changed during SAS remediation: %+v", diskType, got)
}
})
}
}
func assertIssue1595PhysicalDisk(t *testing.T, got, want models.PhysicalDisk) {
t.Helper()
if got.Serial != want.Serial ||
got.Type != "sas" ||
got.Controller != want.Controller ||
got.Target != want.Target ||
got.Temperature != want.Temperature ||
got.StorageGroup != want.StorageGroup ||
got.IO == nil ||
got.IO.ReadBytes != want.IO.ReadBytes ||
got.IO.WriteBytes != want.IO.WriteBytes ||
got.IO.ReadOps != want.IO.ReadOps ||
got.IO.WriteOps != want.IO.WriteOps ||
got.IO.ReadTime != want.IO.ReadTime ||
got.IO.WriteTime != want.IO.WriteTime ||
got.IO.IOTime != want.IO.IOTime {
t.Fatalf("disk %s lost trusted inventory data:\n got %+v\n want %+v", got.DevPath, got, want)
}
}
func hasIssue1595Source(sources []unifiedresources.DataSource, want unifiedresources.DataSource) bool {
for _, source := range sources {
if source == want {
return true
}
}
return false
}
+116 -10
View File
@@ -29,6 +29,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/system"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
@@ -390,7 +391,7 @@ func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) [
for _, temp := range smartTemps {
if temp.WWN != "" && strings.EqualFold(temp.WWN, updated[i].WWN) {
if temp.Temperature > 0 && !temp.StandbySkipped {
updated[i].Temperature = temp.Temperature
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
log.Debug().
Str("disk", updated[i].DevPath).
Str("wwn", updated[i].WWN).
@@ -407,7 +408,7 @@ func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) [
for _, temp := range smartTemps {
if temp.Serial != "" && strings.EqualFold(temp.Serial, updated[i].Serial) {
if temp.Temperature > 0 && !temp.StandbySkipped {
updated[i].Temperature = temp.Temperature
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
log.Debug().
Str("disk", updated[i].DevPath).
Str("serial", updated[i].Serial).
@@ -426,7 +427,7 @@ func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) [
normalizedTempDev := normalizeSMARTDeviceIdentifier(temp.Device)
if normalizedTempDev != "" && normalizedTempDev == normalizedDevPath {
if temp.Temperature > 0 && !temp.StandbySkipped {
updated[i].Temperature = temp.Temperature
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
log.Debug().
Str("disk", updated[i].DevPath).
Int("temp", temp.Temperature).
@@ -466,7 +467,7 @@ func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) [
continue
}
updated[diskIdx].Temperature = int(math.Round(tempVal))
setPhysicalDiskTemperature(&updated[diskIdx], int(math.Round(tempVal)), "proxmox_node_nvme")
log.Debug().
Str("disk", updated[diskIdx].DevPath).
Int("temp", updated[diskIdx].Temperature).
@@ -477,6 +478,17 @@ func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) [
return updated
}
func setPhysicalDiskTemperature(disk *models.PhysicalDisk, temperature int, source string) {
if disk == nil || temperature <= 0 {
return
}
disk.Temperature = temperature
if disk.Collection == nil {
disk.Collection = &diskinventory.CollectionStatus{}
}
disk.Collection.Temperature = diskinventory.Available(source)
}
// mergeHostAgentSMARTIntoDisks merges SMART temperature data from linked host agents
// into physical disks for Proxmox nodes. This allows disk temps collected by the
// pulse-agent running on a PVE node to populate the Physical Disks view.
@@ -522,7 +534,7 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No
continue
}
// Find matching SMART entry by WWN, serial, or device path
// Find matching SMART entry by WWN, serial, or topology-scoped device path.
var matched *models.HostDiskSMART
// Try to match by WWN (most reliable)
@@ -550,29 +562,38 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No
normalizedDevPath := normalizeSMARTDeviceIdentifier(updated[i].DevPath)
for j := range smartData {
normalizedDiskDev := normalizeSMARTDeviceIdentifier(smartData[j].Device)
if normalizedDiskDev != "" && normalizedDiskDev == normalizedDevPath {
if normalizedDiskDev != "" &&
normalizedDiskDev == normalizedDevPath &&
diskTopologyCompatible(updated[i].Controller, updated[i].Target, smartData[j].Controller, smartData[j].Target) {
matched = &smartData[j]
break
}
}
}
if matched == nil || matched.Standby {
if matched == nil {
continue
}
if strings.TrimSpace(updated[i].Model) == "" && strings.TrimSpace(matched.Model) != "" {
updated[i].Model = strings.TrimSpace(matched.Model)
}
if strings.TrimSpace(updated[i].Serial) == "" && strings.TrimSpace(matched.Serial) != "" {
if strings.TrimSpace(matched.Serial) != "" &&
(strings.TrimSpace(updated[i].Serial) == "" || shouldPromoteHostAgentSerial(updated[i], *matched)) {
updated[i].Serial = strings.TrimSpace(matched.Serial)
}
if strings.TrimSpace(updated[i].WWN) == "" && strings.TrimSpace(matched.WWN) != "" {
updated[i].WWN = strings.TrimSpace(matched.WWN)
}
if strings.TrimSpace(updated[i].Type) == "" && strings.TrimSpace(matched.Type) != "" {
if shouldPromoteHostAgentDiskType(updated[i].Type, matched.Type) {
updated[i].Type = strings.TrimSpace(matched.Type)
}
if strings.TrimSpace(updated[i].Controller) == "" && strings.TrimSpace(matched.Controller) != "" {
updated[i].Controller = strings.TrimSpace(matched.Controller)
}
if strings.TrimSpace(updated[i].Target) == "" && strings.TrimSpace(matched.Target) != "" {
updated[i].Target = strings.TrimSpace(matched.Target)
}
if updated[i].Size <= 0 && matched.SizeBytes > 0 {
updated[i].Size = matched.SizeBytes
}
@@ -581,7 +602,7 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No
}
// Merge temperature if not already set
if updated[i].Temperature == 0 && matched.Temperature > 0 {
if !matched.Standby && updated[i].Temperature == 0 && matched.Temperature > 0 {
updated[i].Temperature = matched.Temperature
log.Debug().
Str("device", updated[i].DevPath).
@@ -598,6 +619,11 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No
}
}
}
if matched.IO != nil {
ioCopy := *matched.IO
updated[i].IO = &ioCopy
}
updated[i].Collection = diskinventory.MergeStatus(updated[i].Collection, matched.Collection)
if (strings.TrimSpace(updated[i].Health) == "" || strings.EqualFold(updated[i].Health, "unknown")) && strings.TrimSpace(matched.Health) != "" {
updated[i].Health = matched.Health
@@ -607,6 +633,61 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No
return updated
}
func diskTopologyCompatible(leftController, leftTarget, rightController, rightTarget string) bool {
leftController = strings.TrimSpace(leftController)
leftTarget = strings.TrimSpace(leftTarget)
rightController = strings.TrimSpace(rightController)
rightTarget = strings.TrimSpace(rightTarget)
if leftController != "" && rightController != "" && !strings.EqualFold(leftController, rightController) {
return false
}
if leftTarget != "" && rightTarget != "" && !strings.EqualFold(leftTarget, rightTarget) {
return false
}
return true
}
func shouldPromoteHostAgentSerial(disk models.PhysicalDisk, smart models.HostDiskSMART) bool {
if smart.Collection != nil &&
strings.EqualFold(strings.TrimSpace(smart.Type), "sas") &&
smart.Collection.Serial.State == diskinventory.FieldAvailable &&
strings.HasPrefix(strings.ToLower(strings.TrimSpace(smart.Collection.Serial.Source)), "smartctl") {
return true
}
// Older agents did not carry collection provenance. Proxmox commonly puts
// a 64-bit SAS address in its serial field; a non-address smartctl serial is
// the actual drive identity and must replace that transport address.
return strings.EqualFold(strings.TrimSpace(disk.Type), "sas") &&
strings.EqualFold(strings.TrimSpace(smart.Type), "sas") &&
looksLikeSASAddress(disk.Serial) &&
!looksLikeSASAddress(smart.Serial)
}
func looksLikeSASAddress(value string) bool {
value = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(value)), "0x")
if len(value) != 16 {
return false
}
for _, char := range value {
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
return false
}
}
return true
}
func shouldPromoteHostAgentDiskType(existing, incoming string) bool {
incoming = strings.ToLower(strings.TrimSpace(incoming))
if incoming == "" {
return false
}
existing = strings.ToLower(strings.TrimSpace(existing))
if existing == "" || existing == "unknown" || existing == "scsi" {
return true
}
return false
}
func deriveWearoutFromSMARTAttributes(attrs *models.SMARTAttributes) int {
if attrs == nil || attrs.PercentageUsed == nil {
return -1
@@ -636,17 +717,38 @@ func physicalDiskFromReadStateView(view *unifiedresources.PhysicalDiskView) mode
Serial: view.Serial(),
WWN: view.WWN(),
Type: view.DiskType(),
Controller: view.Controller(),
Target: view.Target(),
Size: view.SizeBytes(),
Health: view.Health(),
Wearout: view.Wearout(),
Temperature: view.Temperature(),
RPM: view.RPM(),
Used: view.Used(),
StorageGroup: view.StorageGroup(),
SmartAttributes: smartAttributesFromUnifiedMeta(view.SMART()),
IO: physicalDiskIOFromUnifiedMeta(view.IO()),
Collection: diskinventory.CloneStatus(view.Collection()),
LastChecked: view.LastSeen(),
}
}
func physicalDiskIOFromUnifiedMeta(in *unifiedresources.PhysicalDiskIOMeta) *models.DiskIO {
if in == nil {
return nil
}
return &models.DiskIO{
Device: in.Device,
ReadBytes: in.ReadBytes,
WriteBytes: in.WriteBytes,
ReadOps: in.ReadOps,
WriteOps: in.WriteOps,
ReadTime: in.ReadTimeMs,
WriteTime: in.WriteTimeMs,
IOTime: in.IOTimeMs,
}
}
func smartAttributesFromUnifiedMeta(in *unifiedresources.SMARTMeta) *models.SMARTAttributes {
if in == nil {
return nil
@@ -3443,11 +3545,15 @@ func hostSensorsFromReadStateView(sensors *unifiedresources.HostSensorMeta) mode
Serial: smart.Serial,
WWN: smart.WWN,
Type: smart.Type,
Controller: smart.Controller,
Target: smart.Target,
SizeBytes: smart.SizeBytes,
Temperature: smart.Temperature,
Health: smart.Health,
Standby: smart.Standby,
Pool: smart.Pool,
IO: physicalDiskIOFromUnifiedMeta(smart.IO),
Collection: diskinventory.CloneStatus(smart.Collection),
Attributes: smartAttributesCopy(smart.Attributes),
})
}
+11 -1
View File
@@ -2678,14 +2678,24 @@ func hostDiskIOMetricResourceID(host models.Host, io models.DiskIO, proxmoxDisks
return ""
}
smartMetricID := ""
for _, disk := range host.Sensors.SMART {
if disk.Standby {
continue
}
if strings.EqualFold(normalizeHostDiskDevice(disk.Device), device) {
return unifiedresources.HostSMARTDiskSourceID(host, disk)
candidate := unifiedresources.HostSMARTDiskSourceID(host, disk)
if smartMetricID != "" && smartMetricID != candidate {
// Multiple controller members share this kernel block path.
// The counter belongs to the aggregate device, not any member.
return ""
}
smartMetricID = candidate
}
}
if smartMetricID != "" {
return smartMetricID
}
for _, pd := range proxmoxDisks {
if pd.device == "" || pd.metricID == "" {
@@ -0,0 +1,64 @@
package monitoring
import (
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestSyncUnifiedResourceAlertsPersistsCanonicalOverrideSuccession(t *testing.T) {
const (
oldID = "agent-535886018cb53055"
newID = "agent-b9ed6d0e20e94eaf"
)
dataDir := t.TempDir()
persistence := config.NewConfigPersistence(dataDir)
manager := alerts.NewManagerWithDataDir(dataDir)
t.Cleanup(manager.Stop)
alertConfig := manager.GetConfig()
alertConfig.Enabled = false
alertConfig.Overrides = map[string]alerts.ThresholdConfig{
oldID: {
Memory: &alerts.HysteresisThreshold{Trigger: 95, Clear: 90},
},
}
manager.UpdateConfig(alertConfig)
if err := persistence.SaveAlertConfig(manager.GetConfig()); err != nil {
t.Fatalf("seed legacy alert config: %v", err)
}
monitor := &Monitor{
alertManager: manager,
configPersist: persistence,
state: models.NewState(),
}
monitor.syncUnifiedResourceAlertsToState([]unifiedresources.Resource{{
ID: newID,
Type: unifiedresources.ResourceTypeAgent,
SupersededCanonicalIDs: []string{oldID},
}})
inMemory := manager.GetConfig()
if _, exists := inMemory.Overrides[oldID]; exists {
t.Fatalf("in-memory override remained under superseded identity %s", oldID)
}
if override := inMemory.Overrides[newID]; override.Memory == nil || override.Memory.Trigger != 95 {
t.Fatalf("in-memory override missing under canonical identity %s: %+v", newID, override)
}
reloaded, err := config.NewConfigPersistence(dataDir).LoadAlertConfig()
if err != nil {
t.Fatalf("reload migrated alert config: %v", err)
}
if _, exists := reloaded.Overrides[oldID]; exists {
t.Fatalf("persisted override remained under superseded identity %s", oldID)
}
if override := reloaded.Overrides[newID]; override.Memory == nil || override.Memory.Trigger != 95 {
t.Fatalf("reloaded override missing under canonical identity %s: %+v", newID, override)
}
}
+12
View File
@@ -3,6 +3,7 @@ package monitoring
import (
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
@@ -95,6 +96,17 @@ func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.
return
}
config := m.alertManager.GetConfig()
if alerts.MigrateCanonicalOverrideKeys(&config, resources) {
if m.configPersist == nil {
log.Warn().Msg("cannot persist canonical alert override migration without config persistence")
} else if err := m.configPersist.SaveAlertConfig(config); err != nil {
log.Error().Err(err).Msg("failed to persist canonical alert override migration")
} else {
m.alertManager.UpdateConfig(config)
}
}
m.alertManager.CheckUnifiedResourceMetrics(resources)
m.alertManager.SyncUnifiedResourceIncidents(resources)
m.syncAlertsToState()
+17
View File
@@ -11,6 +11,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/models"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rs/zerolog/log"
)
@@ -616,11 +617,27 @@ func convertAgentSMARTToModels(smart []agentshost.DiskSMART) []models.HostDiskSM
Serial: disk.Serial,
WWN: disk.WWN,
Type: disk.Type,
Controller: disk.Controller,
Target: disk.Target,
SizeBytes: disk.SizeBytes,
Temperature: disk.Temperature,
Health: disk.Health,
Standby: disk.Standby,
Pool: disk.Pool,
Collection: diskinventory.CloneStatus(disk.Collection),
}
if disk.IO != nil {
ioCopy := models.DiskIO{
Device: disk.IO.Device,
ReadBytes: disk.IO.ReadBytes,
WriteBytes: disk.IO.WriteBytes,
ReadOps: disk.IO.ReadOps,
WriteOps: disk.IO.WriteOps,
ReadTime: disk.IO.ReadTime,
WriteTime: disk.IO.WriteTime,
IOTime: disk.IO.IOTime,
}
entry.IO = &ioCopy
}
if disk.Attributes != nil {
entry.Attributes = convertAgentSMARTAttributes(disk.Attributes)
+75 -13
View File
@@ -14,6 +14,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog"
@@ -1003,10 +1004,13 @@ func (m *Monitor) maybePollPhysicalDisksAsync(
// knows which ZFS pool (if any) it belongs to. Errors are
// non-fatal; we simply leave StorageGroup empty.
var poolAssignment *diskPoolAssignment
poolStatus := diskinventory.Unavailable("proxmox_zfs", "ZFS pool monitoring is disabled")
if zfsPoolingEnabled {
if pools, pErr := pveClient.GetZFSPoolsWithDetails(nodeCtx, node.Node); pErr == nil {
poolAssignment = buildDiskPoolAssignment(pools)
poolStatus = diskinventory.Available("proxmox_zfs")
} else {
poolStatus = diskinventory.Unavailable("proxmox_zfs", "ZFS pool membership query failed")
log.Debug().
Err(pErr).
Str("node", node.Node).
@@ -1020,21 +1024,32 @@ func (m *Monitor) maybePollPhysicalDisksAsync(
for _, disk := range disks {
diskID := fmt.Sprintf("%s-%s-%s", inst, node.Node, strings.ReplaceAll(disk.DevPath, "/", "-"))
physicalDisk := models.PhysicalDisk{
ID: diskID,
Node: node.Node,
Instance: inst,
DevPath: disk.DevPath,
Model: disk.Model,
Serial: disk.Serial,
WWN: disk.WWN,
Type: disk.Type,
Size: disk.Size,
Health: disk.Health,
Wearout: disk.Wearout,
RPM: disk.RPM,
Used: disk.Used,
ID: diskID,
Node: node.Node,
Instance: inst,
DevPath: disk.DevPath,
Model: disk.Model,
Serial: disk.Serial,
WWN: disk.WWN,
Type: disk.Type,
Size: disk.Size,
Health: disk.Health,
Wearout: disk.Wearout,
RPM: disk.RPM,
Used: disk.Used,
Collection: &diskinventory.CollectionStatus{
Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature"),
IO: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose physical-disk I/O counters"),
Controller: diskinventory.Missing("proxmox_disks", "controller association was not reported"),
Pool: poolStatus,
},
LastChecked: time.Now(),
}
if strings.TrimSpace(disk.Serial) != "" {
physicalDisk.Collection.Serial = diskinventory.Available("proxmox_disks")
} else {
physicalDisk.Collection.Serial = diskinventory.Missing("proxmox_disks", "disk serial was not reported")
}
if poolAssignment != nil {
physicalDisk.StorageGroup = poolAssignment.lookup(physicalDisk)
}
@@ -1058,6 +1073,11 @@ func (m *Monitor) maybePollPhysicalDisksAsync(
allDisks = mergeNVMeTempsIntoDisks(allDisks, nodesFromState)
allDisks = mergeHostAgentSMARTIntoDisks(allDisks, nodesFromState, hosts)
for index := range allDisks {
if previous, ok := existingDisksMap[allDisks[index].ID]; ok {
allDisks[index] = preserveUnavailablePhysicalDiskEvidence(allDisks[index], previous)
}
}
for _, disk := range allDisks {
if !polledNodes[disk.Node] {
continue
@@ -1134,11 +1154,15 @@ func physicalDisksFromHostAgentSMART(inst, nodeName string, smartEntries []model
Serial: strings.TrimSpace(smart.Serial),
WWN: strings.TrimSpace(smart.WWN),
Type: strings.TrimSpace(smart.Type),
Controller: strings.TrimSpace(smart.Controller),
Target: strings.TrimSpace(smart.Target),
Size: smart.SizeBytes,
Health: health,
Wearout: deriveWearoutFromSMARTAttributes(smart.Attributes),
Temperature: smart.Temperature,
StorageGroup: strings.TrimSpace(smart.Pool),
IO: cloneDiskIO(smart.IO),
Collection: diskinventory.CloneStatus(smart.Collection),
SmartAttributes: smartAttributesCopy(smart.Attributes),
LastChecked: now,
})
@@ -1146,6 +1170,44 @@ func physicalDisksFromHostAgentSMART(inst, nodeName string, smartEntries []model
return disks
}
func cloneDiskIO(in *models.DiskIO) *models.DiskIO {
if in == nil {
return nil
}
out := *in
return &out
}
func preserveUnavailablePhysicalDiskEvidence(current, previous models.PhysicalDisk) models.PhysicalDisk {
if current.Collection == nil {
return current
}
if current.Collection.Serial.State != diskinventory.FieldAvailable &&
strings.TrimSpace(current.Serial) == "" {
current.Serial = previous.Serial
}
if current.Collection.Temperature.State != diskinventory.FieldAvailable &&
current.Temperature <= 0 {
current.Temperature = previous.Temperature
}
if current.Collection.Controller.State != diskinventory.FieldAvailable {
if current.Controller == "" {
current.Controller = previous.Controller
}
if current.Target == "" {
current.Target = previous.Target
}
}
if current.Collection.Pool.State != diskinventory.FieldAvailable &&
strings.TrimSpace(current.StorageGroup) == "" {
current.StorageGroup = previous.StorageGroup
}
if current.Collection.IO.State != diskinventory.FieldAvailable && current.IO == nil {
current.IO = cloneDiskIO(previous.IO)
}
return current
}
func proxmoxDiskFromPhysicalDisk(disk models.PhysicalDisk) proxmox.Disk {
return proxmox.Disk{
DevPath: disk.DevPath,
+57
View File
@@ -3,6 +3,7 @@ package truenas
import (
"context"
"math"
"slices"
"strings"
"testing"
"time"
@@ -681,6 +682,46 @@ func newConnectionProvider(t *testing.T, fixtures FixtureSnapshot, connectionID
return provider
}
func TestConnectionBackedSystemCanonicalIDSurvivesReconnectWithChangedReportedIdentity(t *testing.T) {
previous := IsFeatureEnabled()
SetFeatureEnabled(true)
t.Cleanup(func() {
SetFeatureEnabled(previous)
})
const connectionID = "conn-stable"
first := DefaultFixtures()
first.System.Hostname = "truenas-before-reconnect"
first.System.MachineID = "serial-before-reconnect"
second := DefaultFixtures()
second.System.Hostname = "truenas-after-reconnect"
second.System.MachineID = "serial-after-reconnect"
canonicalIDs := make([]string, 0, 2)
for _, fixtures := range []FixtureSnapshot{first, second} {
registry := unifiedresources.NewRegistry(unifiedresources.NewMemoryStore())
registry.IngestRecords(
unifiedresources.SourceTrueNAS,
newConnectionProvider(t, fixtures, connectionID).Records(),
)
systemID := unifiedresources.SourceSpecificID(
unifiedresources.ResourceTypeAgent,
unifiedresources.SourceTrueNAS,
"system:"+connectionID,
)
system := mustResourceByID(t, registry.List(), systemID)
canonicalIDs = append(canonicalIDs, system.ID)
}
if canonicalIDs[0] != canonicalIDs[1] {
t.Fatalf(
"connection-backed canonical ID drifted across reconnect/refetch: before=%s after=%s",
canonicalIDs[0],
canonicalIDs[1],
)
}
}
func TestRegistryIngestRecordsKeepsSameHostnameSystemsDistinct(t *testing.T) {
previous := IsFeatureEnabled()
SetFeatureEnabled(true)
@@ -800,6 +841,22 @@ func TestIngestRecordsSucceedLegacyHostnameScopedCanonicalIDs(t *testing.T) {
if system.Agent == nil || system.Agent.AgentID != "conn-a" {
t.Fatalf("expected connection-scoped system record, got %+v", system.Agent)
}
if system.Canonical == nil || !slices.Contains(system.Canonical.Aliases, legacySystemID) {
t.Fatalf(
"expected connection-backed system %s to expose superseded override identity %s, got %+v",
newSystemID,
legacySystemID,
system.Canonical,
)
}
if !slices.Contains(system.Canonical.SupersededIDs, legacySystemID) {
t.Fatalf(
"expected connection-backed system %s to publish explicit superseded identity %s, got %+v",
newSystemID,
legacySystemID,
system.Canonical.SupersededIDs,
)
}
for oldID, newID := range map[string]string{legacySystemID: newSystemID, legacyPoolID: newPoolID} {
if _, found, err := store.GetResourceOperatorState(oldID); err != nil || found {
+29
View File
@@ -12,6 +12,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt"
"github.com/rcourtman/pulse-go-rewrite/internal/platformsupport"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func resourceFromProxmoxNode(node models.Node, linkedHost *models.Host) (Resource, ResourceIdentity) {
@@ -267,11 +268,15 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) {
Serial: s.Serial,
WWN: s.WWN,
Type: s.Type,
Controller: s.Controller,
Target: s.Target,
SizeBytes: s.SizeBytes,
Temperature: s.Temperature,
Health: s.Health,
Standby: s.Standby,
Pool: s.Pool,
IO: physicalDiskIOToMeta(s.IO),
Collection: diskinventory.CloneStatus(s.Collection),
Attributes: cloneSMARTAttributes(s.Attributes),
}
}
@@ -839,6 +844,8 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res
Serial: serial,
WWN: strings.TrimSpace(disk.WWN),
DiskType: diskType,
Controller: strings.TrimSpace(disk.Controller),
Target: strings.TrimSpace(disk.Target),
SizeBytes: sizeBytes,
Health: health,
Wearout: -1,
@@ -851,6 +858,8 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res
ReadCount: unraidDiskCounter(unraidDisk, "read"),
WriteCount: unraidDiskCounter(unraidDisk, "write"),
ErrorCount: unraidDiskCounter(unraidDisk, "error"),
IO: physicalDiskIOToMeta(disk.IO),
Collection: diskinventory.CloneStatus(disk.Collection),
SMART: convertSMARTAttributes(disk.Attributes),
Risk: physicalDiskRiskFromAssessment(assessment),
},
@@ -2005,6 +2014,8 @@ func resourceFromPhysicalDisk(disk models.PhysicalDisk) (Resource, ResourceIdent
Serial: disk.Serial,
WWN: disk.WWN,
DiskType: disk.Type,
Controller: disk.Controller,
Target: disk.Target,
SizeBytes: disk.Size,
Health: disk.Health,
Wearout: disk.Wearout,
@@ -2012,6 +2023,8 @@ func resourceFromPhysicalDisk(disk models.PhysicalDisk) (Resource, ResourceIdent
RPM: disk.RPM,
Used: disk.Used,
StorageGroup: disk.StorageGroup,
IO: physicalDiskIOToMeta(disk.IO),
Collection: diskinventory.CloneStatus(disk.Collection),
Risk: physicalDiskRiskFromAssessment(assessment),
}
@@ -2047,6 +2060,22 @@ func resourceFromPhysicalDisk(disk models.PhysicalDisk) (Resource, ResourceIdent
return resource, identity
}
func physicalDiskIOToMeta(in *models.DiskIO) *PhysicalDiskIOMeta {
if in == nil {
return nil
}
return &PhysicalDiskIOMeta{
Device: strings.TrimSpace(in.Device),
ReadBytes: in.ReadBytes,
WriteBytes: in.WriteBytes,
ReadOps: in.ReadOps,
WriteOps: in.WriteOps,
ReadTimeMs: in.ReadTime,
WriteTimeMs: in.WriteTime,
IOTimeMs: in.IOTime,
}
}
func convertSMARTAttributes(attrs *models.SMARTAttributes) *SMARTMeta {
if attrs == nil {
return nil
@@ -19,11 +19,12 @@ func RefreshCanonicalIdentity(resource *Resource) {
}
resource.Canonical = &CanonicalIdentity{
DisplayName: displayName,
Hostname: hostname,
PlatformID: platformID,
PrimaryID: primaryID,
Aliases: aliases,
DisplayName: displayName,
Hostname: hostname,
PlatformID: platformID,
PrimaryID: primaryID,
Aliases: aliases,
SupersededIDs: uniqueTrimmed(resource.SupersededCanonicalIDs...),
}
}
@@ -79,7 +80,8 @@ func canonicalProxmoxNodePrimaryID(resource Resource) string {
}
func canonicalAliases(resource Resource, primaryID, platformID, hostname string) []string {
values := []string{
values := append([]string{}, resource.SupersededCanonicalIDs...)
values = append(values,
primaryID,
targetResourceID(resource.MetricsTarget),
targetAgentID(resource.DiscoveryTarget),
@@ -99,7 +101,7 @@ func canonicalAliases(resource Resource, primaryID, platformID, hostname string)
hostname,
strings.TrimSpace(resource.Identity.MachineID),
strings.TrimSpace(resource.ID),
}
)
return uniqueTrimmed(values...)
}
+12
View File
@@ -4,6 +4,7 @@ import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func cloneResourcePtr(in *Resource) *Resource {
@@ -23,6 +24,7 @@ func cloneResource(in *Resource) Resource {
out.DiscoveryTarget = cloneDiscoveryTarget(in.DiscoveryTarget)
out.DiscoveryReadiness = cloneResourceDiscoveryReadiness(in.DiscoveryReadiness)
out.MetricsTarget = cloneMetricsTarget(in.MetricsTarget)
out.SupersededCanonicalIDs = cloneStringSlice(in.SupersededCanonicalIDs)
out.PlatformScopes = cloneStringSlice(in.PlatformScopes)
out.Sources = cloneDataSourceSlice(in.Sources)
out.SourceStatus = cloneSourceStatusMap(in.SourceStatus)
@@ -554,11 +556,21 @@ func clonePhysicalDiskMeta(in *PhysicalDiskMeta) *PhysicalDiskMeta {
}
out := *in
out.TemperatureAggregate = cloneTemperatureAggregateMeta(in.TemperatureAggregate)
out.IO = clonePhysicalDiskIOMeta(in.IO)
out.Collection = diskinventory.CloneStatus(in.Collection)
out.SMART = cloneSMARTMeta(in.SMART)
out.Risk = clonePhysicalDiskRisk(in.Risk)
return &out
}
func clonePhysicalDiskIOMeta(in *PhysicalDiskIOMeta) *PhysicalDiskIOMeta {
if in == nil {
return nil
}
out := *in
return &out
}
func cloneTemperatureAggregateMeta(in *TemperatureAggregateMeta) *TemperatureAggregateMeta {
if in == nil {
return nil
@@ -1696,8 +1696,9 @@ func TestCanonicalIdentityIsCanonicalResourceField(t *testing.T) {
requiredSnippets := []string{
"json:\"canonicalIdentity,omitempty\"",
"type CanonicalIdentity struct {",
"DisplayName string `json:\"displayName,omitempty\"`",
"Aliases []string `json:\"aliases,omitempty\"`",
"DisplayName string `json:\"displayName,omitempty\"`",
"Aliases []string `json:\"aliases,omitempty\"`",
"SupersededIDs []string `json:\"supersededIds,omitempty\"`",
}
for _, snippet := range requiredSnippets {
if !strings.Contains(typesSource, snippet) {
+40 -11
View File
@@ -5,6 +5,7 @@ import (
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// PreferredPhysicalDiskMetricID returns the canonical history key used for
@@ -22,11 +23,14 @@ func PreferredPhysicalDiskMetricID(serial, wwn, fallback string) string {
func HostSMARTDiskSourceID(host models.Host, disk models.HostDiskSMART) string {
device := normalizePhysicalDiskDeviceToken(disk.Device)
fallback := ""
if device != "" {
fallback = fmt.Sprintf("%s:%s", strings.TrimSpace(host.ID), device)
}
return PreferredPhysicalDiskMetricID(disk.Serial, disk.WWN, fallback)
return diskinventory.PreferredID(
disk.Serial,
disk.WWN,
strings.TrimSpace(host.ID),
device,
disk.Controller,
disk.Target,
)
}
func HostUnraidDiskSourceID(host models.Host, disk models.HostUnraidDisk) string {
@@ -41,6 +45,9 @@ func HostUnraidDiskSourceID(host models.Host, disk models.HostUnraidDisk) string
}
func PhysicalDiskMetricID(disk models.PhysicalDisk) string {
if strings.TrimSpace(disk.Serial) != "" || strings.TrimSpace(disk.WWN) != "" {
return PreferredPhysicalDiskMetricID(disk.Serial, disk.WWN, "")
}
fallback := strings.TrimSpace(disk.ID)
if fallback == "" && strings.TrimSpace(disk.DevPath) != "" {
fallback = fmt.Sprintf(
@@ -50,20 +57,42 @@ func PhysicalDiskMetricID(disk models.PhysicalDisk) string {
strings.ReplaceAll(strings.TrimSpace(disk.DevPath), "/", "-"),
)
}
return PreferredPhysicalDiskMetricID(disk.Serial, disk.WWN, fallback)
if diskinventory.IsControllerMemberTarget(disk.Target) {
return diskinventory.PreferredID(
"",
"",
fallback,
disk.DevPath,
disk.Controller,
disk.Target,
)
}
return strings.TrimSpace(fallback)
}
func PhysicalDiskMetaMetricID(disk *PhysicalDiskMeta, fallback string) string {
if disk == nil {
return strings.TrimSpace(fallback)
}
if serial := strings.TrimSpace(disk.Serial); serial != "" {
return serial
}
if wwn := strings.TrimSpace(disk.WWN); wwn != "" {
return wwn
}
if diskinventory.IsControllerMemberTarget(disk.Target) && strings.TrimSpace(fallback) != "" {
return diskinventory.PreferredID(
"",
"",
strings.TrimSpace(fallback),
disk.DevPath,
disk.Controller,
disk.Target,
)
}
return PreferredPhysicalDiskMetricID(disk.Serial, disk.WWN, fallback)
}
func normalizePhysicalDiskDeviceToken(device string) string {
device = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(device), "/dev/"))
if fields := strings.Fields(device); len(fields) > 0 {
device = fields[0]
}
return strings.TrimSpace(device)
return diskinventory.DeviceToken(device)
}
+143
View File
@@ -13,6 +13,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters"
)
@@ -525,6 +526,7 @@ func (rr *ResourceRegistry) IngestRecords(source DataSource, records []IngestRec
}
}
newID := rr.ingest(source, sourceID, resource, record.Identity)
rr.retainSupersededCanonicalIDs(newID, record.SupersededCanonicalIDs)
for _, superseded := range record.SupersededCanonicalIDs {
superseded = CanonicalResourceID(superseded)
if superseded == "" || newID == "" || superseded == newID {
@@ -552,6 +554,35 @@ func (rr *ResourceRegistry) IngestRecords(source DataSource, records []IngestRec
rr.mu.Unlock()
}
// retainSupersededCanonicalIDs keeps record-declared identity eras on the
// live resource. Canonical metadata exposes them as aliases, and alert
// configuration migration consumes the explicit field so display aliases
// such as hostnames are never mistaken for persistence keys.
func (rr *ResourceRegistry) retainSupersededCanonicalIDs(resourceID string, supersededIDs []string) {
resourceID = CanonicalResourceID(resourceID)
if resourceID == "" || len(supersededIDs) == 0 {
return
}
rr.mu.Lock()
defer rr.mu.Unlock()
resource := rr.resources[resourceID]
if resource == nil {
return
}
ids := append([]string(nil), resource.SupersededCanonicalIDs...)
for _, supersededID := range supersededIDs {
supersededID = CanonicalResourceID(supersededID)
if supersededID == "" || supersededID == resourceID {
continue
}
ids = append(ids, supersededID)
}
resource.SupersededCanonicalIDs = uniqueTrimmed(ids...)
}
// applyRecordSuccessions re-keys operator-owned store rows from canonical IDs
// that ingested records declared superseded (IngestRecord.SupersededCanonicalIDs)
// onto the records' current canonical IDs. Mirrors the guards of pin-driven
@@ -2524,6 +2555,13 @@ func (rr *ResourceRegistry) findMatch(identity ResourceIdentity, resourceType Re
}
func (rr *ResourceRegistry) resolveLinkedResource(source DataSource, sourceID string, resource Resource) string {
if resource.Type == ResourceTypePhysicalDisk &&
(source == SourceAgent || source == SourceProxmox) {
if linked := rr.resolveLinkedPhysicalDisk(source, resource); linked != "" {
return linked
}
}
switch source {
case SourceProxmox:
if resource.Proxmox != nil && resource.Proxmox.LinkedAgentID != "" {
@@ -2562,6 +2600,76 @@ func (rr *ResourceRegistry) resolveLinkedResource(source DataSource, sourceID st
return ""
}
// resolveLinkedPhysicalDisk correlates agent and Proxmox observations inside
// one already-linked host boundary. This is the durable join for cases where
// Proxmox reports a SAS address in its serial field while smartctl reports the
// drive's real serial. Device paths are safe only inside the common parent and
// only when the topology match is unique.
func (rr *ResourceRegistry) resolveLinkedPhysicalDisk(source DataSource, incoming Resource) string {
if incoming.PhysicalDisk == nil || incoming.ParentID == nil {
return ""
}
parentID := strings.TrimSpace(*incoming.ParentID)
if parentID == "" {
return ""
}
incomingDevice := strings.ToLower(normalizePhysicalDiskDeviceToken(incoming.PhysicalDisk.DevPath))
incomingSerial := strings.TrimSpace(incoming.PhysicalDisk.Serial)
incomingWWN := strings.TrimSpace(incoming.PhysicalDisk.WWN)
matchID := ""
for resourceID, existing := range rr.resources {
if existing == nil ||
existing.Type != ResourceTypePhysicalDisk ||
existing.PhysicalDisk == nil ||
existing.ParentID == nil ||
strings.TrimSpace(*existing.ParentID) != parentID ||
hasDataSource(existing.Sources, source) {
continue
}
if source == SourceProxmox && !hasDataSource(existing.Sources, SourceAgent) {
continue
}
if source == SourceAgent && !hasDataSource(existing.Sources, SourceProxmox) {
continue
}
identityMatch :=
(incomingSerial != "" && strings.EqualFold(incomingSerial, existing.PhysicalDisk.Serial)) ||
(incomingWWN != "" && strings.EqualFold(incomingWWN, existing.PhysicalDisk.WWN))
existingDevice := strings.ToLower(normalizePhysicalDiskDeviceToken(existing.PhysicalDisk.DevPath))
agentReportsSAS := (source == SourceProxmox && strings.EqualFold(existing.PhysicalDisk.DiskType, "sas")) ||
(source == SourceAgent && strings.EqualFold(incoming.PhysicalDisk.DiskType, "sas"))
deviceMatch := agentReportsSAS &&
incomingDevice != "" &&
incomingDevice == existingDevice &&
physicalDiskTopologyCompatible(incoming.PhysicalDisk, existing.PhysicalDisk)
if !identityMatch && !deviceMatch {
continue
}
if matchID != "" && matchID != resourceID {
return ""
}
matchID = resourceID
}
return matchID
}
func physicalDiskTopologyCompatible(left, right *PhysicalDiskMeta) bool {
if left == nil || right == nil {
return false
}
if left.Controller != "" && right.Controller != "" &&
!strings.EqualFold(strings.TrimSpace(left.Controller), strings.TrimSpace(right.Controller)) {
return false
}
if left.Target != "" && right.Target != "" &&
!strings.EqualFold(strings.TrimSpace(left.Target), strings.TrimSpace(right.Target)) {
return false
}
return true
}
type availabilityLinkResolution struct {
ResourceID string
State AvailabilityCorrelationState
@@ -2939,6 +3047,24 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou
previous := existing.PhysicalDisk
existing.PhysicalDisk = mergePhysicalDiskData(existing.PhysicalDisk, incoming.PhysicalDisk)
if source == SourceProxmox && previous != nil && hasDataSource(existing.Sources, SourceAgent) {
if previous.Serial != "" &&
(previous.Collection == nil ||
(previous.Collection.Serial.State == diskinventory.FieldAvailable &&
strings.HasPrefix(strings.ToLower(previous.Collection.Serial.Source), "smartctl"))) {
existing.PhysicalDisk.Serial = previous.Serial
}
if previous.WWN != "" {
existing.PhysicalDisk.WWN = previous.WWN
}
if previous.DiskType != "" {
existing.PhysicalDisk.DiskType = previous.DiskType
}
if previous.Controller != "" {
existing.PhysicalDisk.Controller = previous.Controller
}
if previous.Target != "" {
existing.PhysicalDisk.Target = previous.Target
}
if previous.Temperature > 0 {
existing.PhysicalDisk.Temperature = previous.Temperature
}
@@ -2949,6 +3075,13 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou
smart := *previous.SMART
existing.PhysicalDisk.SMART = &smart
}
if previous.IO != nil {
existing.PhysicalDisk.IO = clonePhysicalDiskIOMeta(previous.IO)
}
existing.PhysicalDisk.Collection = diskinventory.MergeStatus(
incoming.PhysicalDisk.Collection,
previous.Collection,
)
}
}
if existing.PhysicalDisk != nil && (mergedPhysicalDisk || len(incoming.Incidents) > 0) {
@@ -3181,6 +3314,12 @@ func mergePhysicalDiskData(existing *PhysicalDiskMeta, incoming *PhysicalDiskMet
if incoming.DiskType != "" {
merged.DiskType = incoming.DiskType
}
if incoming.Controller != "" {
merged.Controller = incoming.Controller
}
if incoming.Target != "" {
merged.Target = incoming.Target
}
if incoming.SizeBytes > 0 {
merged.SizeBytes = incoming.SizeBytes
}
@@ -3223,6 +3362,10 @@ func mergePhysicalDiskData(existing *PhysicalDiskMeta, incoming *PhysicalDiskMet
if incoming.ErrorCount > 0 {
merged.ErrorCount = incoming.ErrorCount
}
if incoming.IO != nil {
merged.IO = clonePhysicalDiskIOMeta(incoming.IO)
}
merged.Collection = diskinventory.MergeStatus(merged.Collection, incoming.Collection)
if incoming.SMART != nil {
smart := *incoming.SMART
merged.SMART = &smart
+69 -40
View File
@@ -7,6 +7,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// Resource represents a unified resource aggregated across multiple data sources.
@@ -23,9 +24,15 @@ type Resource struct {
DiscoveryReadiness *ResourceDiscoveryReadiness `json:"discoveryReadiness,omitempty"`
MetricsTarget *MetricsTarget `json:"metricsTarget,omitempty"`
Canonical *CanonicalIdentity `json:"canonicalIdentity,omitempty"`
Policy *ResourcePolicy `json:"policy,omitempty"`
AISafeSummary string `json:"aiSafeSummary,omitempty"`
PlatformScopes []string `json:"platformScopes,omitempty"`
// SupersededCanonicalIDs carries retired canonical IDs that are known to
// identify this same resource. Providers declare them on IngestRecord when
// an identity derivation changes; the registry retains them so alert
// overrides and other operator-authored configuration can migrate onto the
// current canonical ID instead of becoming orphaned.
SupersededCanonicalIDs []string `json:"-"`
Policy *ResourcePolicy `json:"policy,omitempty"`
AISafeSummary string `json:"aiSafeSummary,omitempty"`
PlatformScopes []string `json:"platformScopes,omitempty"`
Sources []DataSource `json:"sources"`
SourceStatus map[DataSource]SourceStatus `json:"sourceStatus,omitempty"`
@@ -151,11 +158,12 @@ type MetricsTarget struct {
// unified resource so frontend surfaces do not need to reconstruct labels and
// host hints from source-specific facets.
type CanonicalIdentity struct {
DisplayName string `json:"displayName,omitempty"`
Hostname string `json:"hostname,omitempty"`
PlatformID string `json:"platformId,omitempty"`
PrimaryID string `json:"primaryId,omitempty"`
Aliases []string `json:"aliases,omitempty"`
DisplayName string `json:"displayName,omitempty"`
Hostname string `json:"hostname,omitempty"`
PlatformID string `json:"platformId,omitempty"`
PrimaryID string `json:"primaryId,omitempty"`
Aliases []string `json:"aliases,omitempty"`
SupersededIDs []string `json:"supersededIds,omitempty"`
}
// ResourceType represents the kind of resource.
@@ -438,27 +446,44 @@ type ResourceIncident struct {
// PhysicalDiskMeta contains physical disk-specific metadata.
type PhysicalDiskMeta struct {
DevPath string `json:"devPath"`
Model string `json:"model,omitempty"`
Serial string `json:"serial,omitempty"`
WWN string `json:"wwn,omitempty"`
DiskType string `json:"diskType"` // nvme, sata, sas
SizeBytes int64 `json:"sizeBytes"`
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
Wearout int `json:"wearout"` // 0-100, -1 unavailable
Temperature int `json:"temperature"` // Celsius
TemperatureAggregate *TemperatureAggregateMeta `json:"temperatureAggregate,omitempty"`
RPM int `json:"rpm"`
Used string `json:"used,omitempty"`
StorageRole string `json:"storageRole,omitempty"`
StorageGroup string `json:"storageGroup,omitempty"`
StorageState string `json:"storageState,omitempty"`
SpunDown bool `json:"spunDown,omitempty"`
ReadCount int64 `json:"readCount,omitempty"`
WriteCount int64 `json:"writeCount,omitempty"`
ErrorCount int64 `json:"errorCount,omitempty"`
SMART *SMARTMeta `json:"smart,omitempty"`
Risk *PhysicalDiskRisk `json:"risk,omitempty"`
DevPath string `json:"devPath"`
Model string `json:"model,omitempty"`
Serial string `json:"serial,omitempty"`
WWN string `json:"wwn,omitempty"`
DiskType string `json:"diskType"` // nvme, sata, sas
Controller string `json:"controller,omitempty"`
Target string `json:"target,omitempty"`
SizeBytes int64 `json:"sizeBytes"`
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
Wearout int `json:"wearout"` // 0-100, -1 unavailable
Temperature int `json:"temperature"` // Celsius
TemperatureAggregate *TemperatureAggregateMeta `json:"temperatureAggregate,omitempty"`
RPM int `json:"rpm"`
Used string `json:"used,omitempty"`
StorageRole string `json:"storageRole,omitempty"`
StorageGroup string `json:"storageGroup,omitempty"`
StorageState string `json:"storageState,omitempty"`
SpunDown bool `json:"spunDown,omitempty"`
ReadCount int64 `json:"readCount,omitempty"`
WriteCount int64 `json:"writeCount,omitempty"`
ErrorCount int64 `json:"errorCount,omitempty"`
IO *PhysicalDiskIOMeta `json:"io,omitempty"`
Collection *diskinventory.CollectionStatus `json:"collection,omitempty"`
SMART *SMARTMeta `json:"smart,omitempty"`
Risk *PhysicalDiskRisk `json:"risk,omitempty"`
}
// PhysicalDiskIOMeta preserves the cumulative kernel counters attributed to a
// physical disk. Rates are derived separately by the monitoring layer.
type PhysicalDiskIOMeta struct {
Device string `json:"device,omitempty"`
ReadBytes uint64 `json:"readBytes,omitempty"`
WriteBytes uint64 `json:"writeBytes,omitempty"`
ReadOps uint64 `json:"readOps,omitempty"`
WriteOps uint64 `json:"writeOps,omitempty"`
ReadTimeMs uint64 `json:"readTimeMs,omitempty"`
WriteTimeMs uint64 `json:"writeTimeMs,omitempty"`
IOTimeMs uint64 `json:"ioTimeMs,omitempty"`
}
// TemperatureAggregateMeta stores recent aggregate temperature history for a
@@ -568,17 +593,21 @@ type HostThermalState struct {
// HostSMARTMeta describes a disk's SMART data.
type HostSMARTMeta struct {
Device string `json:"device"`
Model string `json:"model,omitempty"`
Serial string `json:"serial,omitempty"`
WWN string `json:"wwn,omitempty"`
Type string `json:"type,omitempty"`
SizeBytes int64 `json:"sizeBytes,omitempty"`
Temperature int `json:"temperature"`
Health string `json:"health"`
Standby bool `json:"standby,omitempty"`
Pool string `json:"pool,omitempty"`
Attributes *models.SMARTAttributes `json:"attributes,omitempty"`
Device string `json:"device"`
Model string `json:"model,omitempty"`
Serial string `json:"serial,omitempty"`
WWN string `json:"wwn,omitempty"`
Type string `json:"type,omitempty"`
Controller string `json:"controller,omitempty"`
Target string `json:"target,omitempty"`
SizeBytes int64 `json:"sizeBytes,omitempty"`
Temperature int `json:"temperature"`
Health string `json:"health"`
Standby bool `json:"standby,omitempty"`
Pool string `json:"pool,omitempty"`
IO *PhysicalDiskIOMeta `json:"io,omitempty"`
Collection *diskinventory.CollectionStatus `json:"collection,omitempty"`
Attributes *models.SMARTAttributes `json:"attributes,omitempty"`
}
// HostRAIDDeviceMeta describes a device in a RAID array.
+36
View File
@@ -7,6 +7,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// Metric helpers (nil-safe).
@@ -2049,6 +2050,20 @@ func (v PhysicalDiskView) DiskType() string {
return v.r.PhysicalDisk.DiskType
}
func (v PhysicalDiskView) Controller() string {
if v.r == nil || v.r.PhysicalDisk == nil {
return ""
}
return strings.TrimSpace(v.r.PhysicalDisk.Controller)
}
func (v PhysicalDiskView) Target() string {
if v.r == nil || v.r.PhysicalDisk == nil {
return ""
}
return strings.TrimSpace(v.r.PhysicalDisk.Target)
}
func (v PhysicalDiskView) SizeBytes() int64 {
if v.r == nil || v.r.PhysicalDisk == nil {
return 0
@@ -2113,6 +2128,13 @@ func (v PhysicalDiskView) Used() string {
return v.r.PhysicalDisk.Used
}
func (v PhysicalDiskView) StorageGroup() string {
if v.r == nil || v.r.PhysicalDisk == nil {
return ""
}
return strings.TrimSpace(v.r.PhysicalDisk.StorageGroup)
}
func (v PhysicalDiskView) SMART() *SMARTMeta {
if v.r == nil || v.r.PhysicalDisk == nil || v.r.PhysicalDisk.SMART == nil {
return nil
@@ -2120,6 +2142,20 @@ func (v PhysicalDiskView) SMART() *SMARTMeta {
return cloneSMARTMeta(v.r.PhysicalDisk.SMART)
}
func (v PhysicalDiskView) IO() *PhysicalDiskIOMeta {
if v.r == nil || v.r.PhysicalDisk == nil {
return nil
}
return clonePhysicalDiskIOMeta(v.r.PhysicalDisk.IO)
}
func (v PhysicalDiskView) Collection() *diskinventory.CollectionStatus {
if v.r == nil || v.r.PhysicalDisk == nil {
return nil
}
return diskinventory.CloneStatus(v.r.PhysicalDisk.Collection)
}
func (v PhysicalDiskView) MetricResourceID() string {
if v.r == nil {
return ""
+20 -12
View File
@@ -1,6 +1,10 @@
package host
import "time"
import (
"time"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// Report represents the payload sent by the host module of pulse-agent.
type Report struct {
@@ -227,17 +231,21 @@ const (
// DiskSMART represents S.M.A.R.T. data for a single disk.
type DiskSMART struct {
Device string `json:"device"` // Block device name (e.g., sda, nvme0n1)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
SizeBytes int64 `json:"sizeBytes,omitempty"` // Capacity in bytes (0 when unknown)
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
Pool string `json:"pool,omitempty"` // ZFS pool this disk belongs to (empty if not a ZFS member)
Attributes *SMARTAttributes `json:"attributes,omitempty"`
Device string `json:"device"` // Block device name (e.g., sda, nvme0n1)
Model string `json:"model,omitempty"` // Disk model
Serial string `json:"serial,omitempty"` // Serial number
WWN string `json:"wwn,omitempty"` // World Wide Name
Type string `json:"type,omitempty"` // Transport type: sata, sas, nvme
Controller string `json:"controller,omitempty"` // Stable controller association when the OS reports one
Target string `json:"target,omitempty"` // Controller target/HCTL or smartctl member target
SizeBytes int64 `json:"sizeBytes,omitempty"` // Capacity in bytes (0 when unknown)
Temperature int `json:"temperature"` // Temperature in Celsius
Health string `json:"health,omitempty"` // PASSED, FAILED, UNKNOWN
Standby bool `json:"standby,omitempty"` // True if disk was in standby
Pool string `json:"pool,omitempty"` // ZFS pool this disk belongs to (empty if not a ZFS member)
IO *DiskIO `json:"io,omitempty"` // Cumulative kernel counters when attributable to this disk
Collection *diskinventory.CollectionStatus `json:"collection,omitempty"`
Attributes *SMARTAttributes `json:"attributes,omitempty"`
}
// SMARTAttributes holds normalized SMART attributes for both SATA and NVMe disks.
+56
View File
@@ -0,0 +1,56 @@
package diskinventory
import (
"fmt"
"strings"
)
// DeviceToken returns the kernel block-device token from either a canonical
// /dev path or a legacy smartctl display label such as "sda [scsi]".
func DeviceToken(device string) string {
device = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(device), "/dev/"))
if fields := strings.Fields(device); len(fields) > 0 {
device = fields[0]
}
return strings.TrimSpace(device)
}
// PreferredID selects stable hardware identity first and scopes topology
// fallbacks to the reporting host. Controller and target are included only
// when present, preserving legacy direct-device IDs for SATA/NVMe disks.
func PreferredID(serial, wwn, scope, device, controller, target string) string {
if serial = strings.TrimSpace(serial); serial != "" {
return serial
}
if wwn = strings.TrimSpace(wwn); wwn != "" {
return wwn
}
scope = strings.TrimSpace(scope)
device = DeviceToken(device)
controller = strings.TrimSpace(controller)
target = strings.TrimSpace(target)
if scope == "" || device == "" {
return ""
}
if !IsControllerMemberTarget(target) {
return fmt.Sprintf("%s:%s", scope, device)
}
return fmt.Sprintf("%s:%s@%s/%s", scope, device, controller, target)
}
// IsControllerMemberTarget reports whether target addresses one member behind
// a shared controller block path (for example megaraid,7 or cciss,1).
func IsControllerMemberTarget(target string) bool {
target = strings.TrimSpace(target)
index := strings.IndexByte(target, ',')
if index < 0 || index+1 >= len(target) {
return false
}
for _, char := range target[index+1:] {
if char < '0' || char > '9' {
return false
}
}
return true
}
+34
View File
@@ -0,0 +1,34 @@
package diskinventory
import "testing"
func TestPreferredIDPreservesDirectDeviceFallbacks(t *testing.T) {
for _, test := range []struct {
name string
device string
controller string
target string
want string
}{
{name: "sata", device: "/dev/sda", want: "host:sda"},
{name: "nvme", device: "nvme0n1", want: "host:nvme0n1"},
{name: "direct sas hctl", device: "sdb", controller: "0000:03:00.0", target: "6:0:0:0", want: "host:sdb"},
{name: "controller member", device: "sdc [megaraid,7]", controller: "sdc", target: "megaraid,7", want: "host:sdc@sdc/megaraid,7"},
} {
t.Run(test.name, func(t *testing.T) {
got := PreferredID("", "", "host", test.device, test.controller, test.target)
if got != test.want {
t.Fatalf("PreferredID() = %q, want %q", got, test.want)
}
})
}
}
func TestPreferredIDKeepsExistingHardwareIdentityPriority(t *testing.T) {
if got := PreferredID(" SERIAL ", "WWN", "host", "sda", "controller", "megaraid,7"); got != "SERIAL" {
t.Fatalf("serial identity = %q, want SERIAL", got)
}
if got := PreferredID("", " WWN ", "host", "sda", "controller", "megaraid,7"); got != "WWN" {
t.Fatalf("WWN identity = %q, want WWN", got)
}
}
+100
View File
@@ -0,0 +1,100 @@
package diskinventory
import "strings"
// FieldState describes whether a physical-disk field was observed during the
// current collection pass. It deliberately separates provider limitations
// from transient collection failures and from fields that should have been
// present but were not.
type FieldState string
const (
FieldAvailable FieldState = "available"
FieldUnavailable FieldState = "unavailable"
FieldUnsupported FieldState = "unsupported"
FieldMissing FieldState = "missing"
)
// FieldStatus carries collection state and provenance for one disk signal.
type FieldStatus struct {
State FieldState `json:"state"`
Source string `json:"source,omitempty"`
Reason string `json:"reason,omitempty"`
}
// CollectionStatus is the field-level payload carried by collection evidence;
// it is not a separate trust posture or mutation lifecycle. Operational trust
// may wrap this payload in its shared EvidenceEnvelope, while Collection Trust
// remains responsible for evaluating the observation. Empty means the report
// predates this contract.
type CollectionStatus struct {
Serial FieldStatus `json:"serial,omitempty"`
Temperature FieldStatus `json:"temperature,omitempty"`
IO FieldStatus `json:"io,omitempty"`
Controller FieldStatus `json:"controller,omitempty"`
Pool FieldStatus `json:"pool,omitempty"`
}
func Available(source string) FieldStatus {
return FieldStatus{State: FieldAvailable, Source: strings.TrimSpace(source)}
}
func Unavailable(source, reason string) FieldStatus {
return FieldStatus{
State: FieldUnavailable,
Source: strings.TrimSpace(source),
Reason: strings.TrimSpace(reason),
}
}
func Unsupported(source, reason string) FieldStatus {
return FieldStatus{
State: FieldUnsupported,
Source: strings.TrimSpace(source),
Reason: strings.TrimSpace(reason),
}
}
func Missing(source, reason string) FieldStatus {
return FieldStatus{
State: FieldMissing,
Source: strings.TrimSpace(source),
Reason: strings.TrimSpace(reason),
}
}
func CloneStatus(status *CollectionStatus) *CollectionStatus {
if status == nil {
return nil
}
clone := *status
return &clone
}
// MergeStatus keeps an available observation over a weaker state while still
// allowing a current available observation to replace older provenance.
func MergeStatus(existing, incoming *CollectionStatus) *CollectionStatus {
if existing == nil {
return CloneStatus(incoming)
}
if incoming == nil {
return CloneStatus(existing)
}
merged := *existing
merged.Serial = mergeFieldStatus(merged.Serial, incoming.Serial)
merged.Temperature = mergeFieldStatus(merged.Temperature, incoming.Temperature)
merged.IO = mergeFieldStatus(merged.IO, incoming.IO)
merged.Controller = mergeFieldStatus(merged.Controller, incoming.Controller)
merged.Pool = mergeFieldStatus(merged.Pool, incoming.Pool)
return &merged
}
func mergeFieldStatus(existing, incoming FieldStatus) FieldStatus {
if incoming.State == "" {
return existing
}
if existing.State == FieldAvailable && incoming.State != FieldAvailable {
return existing
}
return incoming
}
+26
View File
@@ -0,0 +1,26 @@
package diskinventory
import "testing"
func TestMergeStatusPrefersAvailableEvidencePerField(t *testing.T) {
existing := &CollectionStatus{
Serial: Available("smartctl"),
Temperature: Unavailable("smartctl", "deadline exceeded"),
}
incoming := &CollectionStatus{
Serial: Unsupported("provider", "not exposed"),
Temperature: Available("provider"),
IO: Missing("kernel", "counter absent"),
}
got := MergeStatus(existing, incoming)
if got.Serial.State != FieldAvailable || got.Serial.Source != "smartctl" {
t.Fatalf("available serial evidence was downgraded: %+v", got.Serial)
}
if got.Temperature.State != FieldAvailable || got.Temperature.Source != "provider" {
t.Fatalf("available temperature evidence did not win: %+v", got.Temperature)
}
if got.IO.State != FieldMissing {
t.Fatalf("missing I/O state was not retained: %+v", got.IO)
}
}
+270 -81
View File
@@ -94,11 +94,14 @@ KUBECONFIG_PATH="" # Path to kubeconfig file for Kubernetes monitoring
KUBE_INCLUDE_ALL_PODS="false"
KUBE_INCLUDE_ALL_DEPLOYMENTS="false"
DISK_EXCLUDES=() # Array for multiple --disk-exclude values
STATE_DIR="/var/lib/pulse-agent" # Persistent state directory (overridden per platform)
DEFAULT_STATE_DIR="/var/lib/pulse-agent"
STATE_DIR="$DEFAULT_STATE_DIR" # Persistent state directory (overridden per platform)
STATE_DIR_SOURCE="default" # default, explicit, recovered, or platform
CURL_CA_BUNDLE="" # Path to CA bundle for curl and agent TLS (sets SSL_CERT_FILE)
NON_INTERACTIVE="false"
TOKEN_FILE_PATH="" # Path to file containing the token
RUNTIME_TOKEN_FILE="" # Secure token file passed to the installed service
RUNTIME_TOKEN_CHANGED="false"
OUTPUT_FORMAT="text" # "text" (default) or "json"
PREFLIGHT_ONLY="false"
INSTALL_SIGNATURE_NAMESPACE="pulse-install"
@@ -147,6 +150,33 @@ log_error() {
printf "[ERROR] %s\n" "$1"
fi
}
# Feed the API token to curl through a private config file. Passing a header
# value with -H would expose the token in the transient curl process argv.
curl_with_pulse_token() {
local config_file=""
local curl_rc=0
if [[ -z "$PULSE_TOKEN" ]]; then
curl "$@"
return $?
fi
case "$PULSE_TOKEN" in
*$'\r'*|*$'\n'*) return 2 ;;
esac
config_file=$(mktemp)
TMP_FILES+=("$config_file")
chmod 600 "$config_file"
printf 'header = "X-API-Token: %s"\n' "$PULSE_TOKEN" > "$config_file"
if curl --config "$config_file" "$@"; then
curl_rc=0
else
curl_rc=$?
fi
rm -f "$config_file"
return "$curl_rc"
}
url_encode() {
local input="$1"
local output=""
@@ -385,11 +415,10 @@ verify_agent_server_registration() {
return 1
fi
if [[ -n "$PULSE_TOKEN" ]]; then lookup_args+=(-H "X-API-Token: ${PULSE_TOKEN}"); fi
if [[ "$INSECURE" == "true" ]]; then lookup_args+=(-k); fi
if [[ -n "$CURL_CA_BUNDLE" ]]; then lookup_args+=(--cacert "$CURL_CA_BUNDLE"); fi
lookup_out=$(curl "${lookup_args[@]}" "${PULSE_URL}/api/agents/agent/lookup?hostname=$(url_encode "$lookup_hostname")" 2>/dev/null || true)
lookup_out=$(curl_with_pulse_token "${lookup_args[@]}" "${PULSE_URL}/api/agents/agent/lookup?hostname=$(url_encode "$lookup_hostname")" 2>/dev/null || true)
lookup_status="${lookup_out##*$'\n'}"
lookup_body="${lookup_out%$'\n'*}"
@@ -1055,6 +1084,42 @@ portable_sed_in_place() {
sed -i '' "$expr" "$target" 2>/dev/null || sed -i "$expr" "$target" 2>/dev/null || true
}
select_platform_state_dir() {
local platform_default="$1"
if [[ "${STATE_DIR_SOURCE:-default}" == "default" ]]; then
STATE_DIR="$platform_default"
STATE_DIR_SOURCE="platform"
fi
}
discover_state_dir_from_saved_installer() {
local script_path="${1:-$0}"
local script_dir=""
if [[ "${STATE_DIR_SOURCE:-default}" != "default" || ! -f "$script_path" ]]; then
return 1
fi
script_dir=$(cd "$(dirname "$script_path")" 2>/dev/null && pwd -P) || return 1
if [[ -f "$script_dir/connection.env" ]]; then
STATE_DIR="$script_dir"
STATE_DIR_SOURCE="recovered"
return 0
fi
return 1
}
remove_agent_state_dir() {
local state_dir="${1:-$STATE_DIR}"
if [[ -z "$state_dir" || "$state_dir" != /* || "$state_dir" == "/" ||
"$state_dir" == *$'\r'* || "$state_dir" == *$'\n'* ]]; then
log_warn "Refusing to remove invalid agent state directory: ${state_dir:-<empty>}"
return 1
fi
rm -rf -- "$state_dir"
}
detect_qnap_data_volume() {
local qnap_vol=""
local candidate=""
@@ -1081,11 +1146,14 @@ detect_qnap_data_volume() {
find_qnap_state_dir() {
local candidate=""
if [[ -n "$STATE_DIR" ]] && [[ "$STATE_DIR" != "/var/lib/pulse-agent" ]]; then
if [[ -d "$STATE_DIR" ]] || [[ -f "$STATE_DIR/connection.env" ]] || [[ -f "$STATE_DIR/agent-id" ]]; then
printf '%s\n' "$STATE_DIR"
return 0
fi
if [[ -n "$STATE_DIR" && "${STATE_DIR_SOURCE:-default}" != "default" ]]; then
printf '%s\n' "$STATE_DIR"
return 0
fi
if [[ -n "$STATE_DIR" ]] && [[ "$STATE_DIR" != "/var/lib/pulse-agent" ]] && \
{ [[ -d "$STATE_DIR" ]] || [[ -f "$STATE_DIR/connection.env" ]] || [[ -f "$STATE_DIR/agent-id" ]]; }; then
printf '%s\n' "$STATE_DIR"
return 0
fi
candidate=$(detect_qnap_data_volume || true)
@@ -1533,28 +1601,50 @@ build_plist_program_arguments() {
ensure_runtime_token_file() {
local state_dir="${1:-$STATE_DIR}"
local token_file="${state_dir}/token"
local previous_token=""
local old_umask=""
local token_tmp=""
RUNTIME_TOKEN_FILE=""
RUNTIME_TOKEN_CHANGED="false"
if [[ -z "$PULSE_TOKEN" ]]; then
rm -f "$token_file" 2>/dev/null || true
log_info "No API token provided; installer will configure token-optional agent runtime."
return 0
fi
mkdir -p "$state_dir"
local old_umask=""
old_umask=$(umask)
umask 077
if ! printf '%s' "$PULSE_TOKEN" > "$token_file"; then
mkdir -p "$state_dir"
chmod 700 "$state_dir"
if [[ -f "$token_file" && ! -L "$token_file" ]]; then
previous_token=$(cat "$token_file" 2>/dev/null || true)
fi
if [[ "$previous_token" != "$PULSE_TOKEN" ]]; then
RUNTIME_TOKEN_CHANGED="true"
fi
token_tmp=$(mktemp "${state_dir}/.token.XXXXXX")
TMP_FILES+=("$token_tmp")
if ! printf '%s' "$PULSE_TOKEN" > "$token_tmp"; then
umask "$old_umask"
fail "Failed to write runtime token file: $token_file" "$EXIT_GENERAL"
fi
umask "$old_umask"
chmod 600 "$token_file"
chmod 600 "$token_tmp"
if [[ "$(id -u 2>/dev/null || echo 1)" == "0" ]]; then
chown root:root "$token_file" 2>/dev/null || true
chown root:root "$token_tmp" 2>/dev/null || true
fi
if ! mv -f "$token_tmp" "$token_file"; then
umask "$old_umask"
fail "Failed to install runtime token file: $token_file" "$EXIT_GENERAL"
fi
umask "$old_umask"
RUNTIME_TOKEN_FILE="$token_file"
# A changed bootstrap token is explicit re-enrollment intent. Preserve the
# runtime token across ordinary restarts and tokenless updates, but do not
# let a stale runtime token shadow fresh enrollment credentials.
if [[ "${ENROLL:-false}" == "true" && "$RUNTIME_TOKEN_CHANGED" == "true" ]]; then
rm -f "${state_dir}/runtime.token" 2>/dev/null || true
fi
log_info "Token stored securely at $token_file (mode 600)"
}
@@ -1609,7 +1699,11 @@ recover_token_from_default_agent_token_file() {
# v5.1.x Linux services could omit --token and --token-file because the
# Go agent read this default file itself.
for token_path in "${STATE_DIR%/}/token" "/var/lib/pulse-agent/token" "$TRUENAS_STATE_DIR/token"; do
local token_paths=("${STATE_DIR%/}/token")
if [[ "${STATE_DIR_SOURCE:-default}" == "default" ]]; then
token_paths+=("${DEFAULT_STATE_DIR:-/var/lib/pulse-agent}/token" "$TRUENAS_STATE_DIR/token")
fi
for token_path in "${token_paths[@]}"; do
[[ -n "$token_path" && -f "$token_path" ]] || continue
recovered_token=$(cat "$token_path" 2>/dev/null || true)
if [[ -n "$recovered_token" ]]; then
@@ -1623,6 +1717,15 @@ recover_token_from_default_agent_token_file() {
recover_connection_state() {
local file="$1"
local saved_state_dir=""
saved_state_dir=$(read_connection_state_value "$file" "PULSE_STATE_DIR")
if [[ -n "$saved_state_dir" && "$saved_state_dir" == /* && "$saved_state_dir" != "/" &&
"$saved_state_dir" != *$'\r'* && "$saved_state_dir" != *$'\n'* &&
"${STATE_DIR_SOURCE:-default}" == "default" ]]; then
STATE_DIR="$saved_state_dir"
STATE_DIR_SOURCE="recovered"
fi
if [[ -z "$PULSE_URL" ]]; then
PULSE_URL=$(read_connection_state_value "$file" "PULSE_URL")
@@ -1734,7 +1837,12 @@ apply_recovered_agent_arg_value() {
RECOVERED_AGENT_ARG_STATE="true"
;;
state-dir)
if [[ -n "$value" && "$STATE_DIR" == "/var/lib/pulse-agent" ]]; then STATE_DIR="$value"; fi
if [[ -n "$value" && "$value" == /* && "$value" != "/" &&
"$value" != *$'\r'* && "$value" != *$'\n'* &&
"${STATE_DIR_SOURCE:-default}" == "default" ]]; then
STATE_DIR="$value"
STATE_DIR_SOURCE="recovered"
fi
RECOVERED_AGENT_ARG_STATE="true"
;;
kubeconfig)
@@ -1893,6 +2001,16 @@ recover_connection_state_from_env_stream() {
fi
RECOVERED_AGENT_ENV_STATE="true"
;;
PULSE_STATE_DIR=*)
value="${env_line#*=}"
if [[ -n "$value" && "$value" == /* && "$value" != "/" &&
"$value" != *$'\r'* && "$value" != *$'\n'* &&
"${STATE_DIR_SOURCE:-default}" == "default" ]]; then
STATE_DIR="$value"
STATE_DIR_SOURCE="recovered"
fi
RECOVERED_AGENT_ENV_STATE="true"
;;
PULSE_AGENT_ID=*)
value="${env_line#*=}"
if [[ -z "$AGENT_ID" ]]; then AGENT_ID="$value"; fi
@@ -2109,6 +2227,36 @@ recover_connection_state_from_systemd_unit() {
return 1
}
launchd_agent_arg_stream() {
local plist_path="${1:-/Library/LaunchDaemons/com.pulse.agent.plist}"
[[ -f "$plist_path" ]] || return 1
awk '
/<key>ProgramArguments<\/key>/ { in_program_args = 1; next }
in_program_args && /<\/array>/ { exit }
in_program_args && /<string>/ {
value = $0
sub(/^.*<string>/, "", value)
sub(/<\/string>.*$/, "", value)
gsub(/&amp;/, "\\&", value)
gsub(/&lt;/, "<", value)
gsub(/&gt;/, ">", value)
gsub(/&quot;/, "\"", value)
gsub(/&apos;/, "\047", value)
print value
}
' "$plist_path"
}
recover_connection_state_from_launchd_plist() {
local plist_path="${1:-/Library/LaunchDaemons/com.pulse.agent.plist}"
if recover_connection_state_from_arg_stream < <(launchd_agent_arg_stream "$plist_path"); then
return 0
fi
return 1
}
recover_connection_state_from_service_scripts() {
local candidate=""
local line=""
@@ -2176,6 +2324,10 @@ recover_connection_state_from_existing_agent() {
log_info "Recovered connection details from the existing Pulse Agent service."
return 0
fi
if recover_connection_state_from_launchd_plist; then
log_info "Recovered connection details from the existing Pulse Agent launchd service."
return 0
fi
if recover_connection_state_from_service_scripts; then
log_info "Recovered connection details from the existing Pulse Agent service script."
return 0
@@ -2187,18 +2339,24 @@ recover_connection_state_from_existing_agent() {
find_connection_state_file() {
local conn_env=""
local qnap_state_dir=""
local conn_paths=("${STATE_DIR%/}/connection.env")
for conn_env in /var/lib/pulse-agent/connection.env /boot/config/plugins/pulse-agent/connection.env "$TRUENAS_STATE_DIR/connection.env"; do
if [[ "${STATE_DIR_SOURCE:-default}" == "default" ]]; then
conn_paths+=("${DEFAULT_STATE_DIR:-/var/lib/pulse-agent}/connection.env" /boot/config/plugins/pulse-agent/connection.env "$TRUENAS_STATE_DIR/connection.env")
fi
for conn_env in "${conn_paths[@]}"; do
if [[ -f "$conn_env" ]]; then
printf '%s\n' "$conn_env"
return 0
fi
done
qnap_state_dir=$(find_qnap_state_dir || true)
if [[ -n "$qnap_state_dir" ]] && [[ -f "$qnap_state_dir/connection.env" ]]; then
printf '%s\n' "$qnap_state_dir/connection.env"
return 0
if [[ "${STATE_DIR_SOURCE:-default}" == "default" ]]; then
qnap_state_dir=$(find_qnap_state_dir || true)
if [[ -n "$qnap_state_dir" ]] && [[ -f "$qnap_state_dir/connection.env" ]]; then
printf '%s\n' "$qnap_state_dir/connection.env"
return 0
fi
fi
return 1
@@ -2207,11 +2365,17 @@ find_connection_state_file() {
recover_agent_id_from_state_file() {
local aid_path=""
local qnap_state_dir=""
local aid_paths=(/var/lib/pulse-agent/agent-id /boot/config/plugins/pulse-agent/agent-id "$TRUENAS_STATE_DIR/agent-id")
local aid_paths=("${STATE_DIR%/}/agent-id")
qnap_state_dir=$(find_qnap_state_dir || true)
if [[ -n "$qnap_state_dir" ]]; then
aid_paths+=("$qnap_state_dir/agent-id")
if [[ "${STATE_DIR_SOURCE:-default}" == "default" ]]; then
aid_paths+=("${DEFAULT_STATE_DIR:-/var/lib/pulse-agent}/agent-id" /boot/config/plugins/pulse-agent/agent-id "$TRUENAS_STATE_DIR/agent-id")
fi
if [[ "${STATE_DIR_SOURCE:-default}" == "default" ]]; then
qnap_state_dir=$(find_qnap_state_dir || true)
if [[ -n "$qnap_state_dir" ]]; then
aid_paths+=("$qnap_state_dir/agent-id")
fi
fi
for aid_path in "${aid_paths[@]}"; do
@@ -2228,22 +2392,31 @@ recover_agent_id_from_state_file() {
save_connection_info() {
local state_dir="$1"
local conn_env="${state_dir}/connection.env"
local conn_tmp=""
local old_umask=""
old_umask=$(umask)
umask 077
mkdir -p "$state_dir"
chmod 700 "$state_dir"
# Save connection details so uninstall can deregister without --url/--token.
# Single-quote values to prevent shell interpretation on read-back.
# Legacy connection files may contain PULSE_TOKEN, but new installs persist
# only the protected token file path.
: > "$conn_env"
write_connection_state_value "$conn_env" "PULSE_URL" "$PULSE_URL"
write_connection_state_value "$conn_env" "PULSE_TOKEN_FILE" "$RUNTIME_TOKEN_FILE"
write_connection_state_value "$conn_env" "PULSE_AGENT_ID" "$AGENT_ID"
write_connection_state_value "$conn_env" "PULSE_HOSTNAME" "$HOSTNAME_OVERRIDE"
conn_tmp=$(mktemp "${state_dir}/.connection.env.XXXXXX")
TMP_FILES+=("$conn_tmp")
write_connection_state_value "$conn_tmp" "PULSE_STATE_DIR" "$state_dir"
write_connection_state_value "$conn_tmp" "PULSE_URL" "$PULSE_URL"
write_connection_state_value "$conn_tmp" "PULSE_TOKEN_FILE" "$RUNTIME_TOKEN_FILE"
write_connection_state_value "$conn_tmp" "PULSE_AGENT_ID" "$AGENT_ID"
write_connection_state_value "$conn_tmp" "PULSE_HOSTNAME" "$HOSTNAME_OVERRIDE"
if [[ "$INSECURE" == "true" ]]; then
write_connection_state_value "$conn_env" "PULSE_INSECURE_SKIP_VERIFY" "true"
write_connection_state_value "$conn_tmp" "PULSE_INSECURE_SKIP_VERIFY" "true"
fi
write_connection_state_value "$conn_env" "PULSE_SERVER_FINGERPRINT" "$SERVER_FINGERPRINT"
write_connection_state_value "$conn_env" "PULSE_CACERT" "$CURL_CA_BUNDLE"
chmod 600 "$conn_env"
write_connection_state_value "$conn_tmp" "PULSE_SERVER_FINGERPRINT" "$SERVER_FINGERPRINT"
write_connection_state_value "$conn_tmp" "PULSE_CACERT" "$CURL_CA_BUNDLE"
chmod 600 "$conn_tmp"
mv -f "$conn_tmp" "$conn_env"
umask "$old_umask"
# Save a copy of this install script for offline uninstall.
# When run via "curl | bash", $0 is /dev/stdin — not a usable file.
# Try local copy first, then download a fresh copy from the server.
@@ -2304,7 +2477,7 @@ while [[ $# -gt 0 ]]; do
--uninstall) UNINSTALL="true"; shift ;;
--agent-id) AGENT_ID="$2"; shift 2 ;;
--hostname) HOSTNAME_OVERRIDE="$2"; shift 2 ;;
--state-dir) STATE_DIR="$2"; shift 2 ;;
--state-dir) STATE_DIR="$2"; STATE_DIR_SOURCE="explicit"; shift 2 ;;
--kube-include-all-pods) KUBE_INCLUDE_ALL_PODS="true"; shift ;;
--kube-include-all-deployments) KUBE_INCLUDE_ALL_DEPLOYMENTS="true"; shift ;;
--disk-exclude) DISK_EXCLUDES+=("$2"); shift 2 ;;
@@ -2317,6 +2490,13 @@ while [[ $# -gt 0 ]]; do
esac
done
discover_state_dir_from_saved_installer "$0" || true
if [[ -z "$STATE_DIR" || "$STATE_DIR" != /* || "$STATE_DIR" == "/" ||
"$STATE_DIR" == *$'\r'* || "$STATE_DIR" == *$'\n'* ]]; then
fail "--state-dir must be an absolute, non-root path." "$EXIT_MISSING_ARGS"
fi
if [[ -n "$OBSERVERS_FILE" ]]; then
if [[ "$OBSERVERS_FILE" != /* ]]; then
fail "Observer config path must be absolute: ${OBSERVERS_FILE}" "$EXIT_MISSING_ARGS"
@@ -2358,33 +2538,48 @@ if [[ -n "$PULSE_URL" ]]; then
PULSE_URL="${PULSE_URL%/}"
fi
# --- Update State Recovery ---
# Update commands are intentionally tokenless for already-installed agents. The
# installer reuses the canonical saved connection state instead of asking the
# operator to mint a new install token for a host Pulse already knows about.
if [[ "$UPDATE_ONLY" == "true" ]]; then
# --- Installed Lifecycle State Recovery ---
# An explicit state directory is authoritative. Without one, inspect the active
# process/service first so a custom installation wins over stale default-path
# artifacts, then merge its canonical connection.env and agent-id state.
if [[ "$UPDATE_ONLY" == "true" || "$UNINSTALL" == "true" ]]; then
if [[ "$STATE_DIR_SOURCE" != "explicit" ]]; then
recover_connection_state_from_existing_agent || true
fi
local lifecycle_conn_env=""
lifecycle_conn_env=$(find_connection_state_file || true)
if [[ -n "$lifecycle_conn_env" ]]; then
log_info "Recovering connection details from ${lifecycle_conn_env}..."
recover_connection_state "$lifecycle_conn_env"
fi
if update_connection_state_incomplete; then
local update_conn_env=""
update_conn_env=$(find_connection_state_file || true)
if [[ -n "$update_conn_env" ]]; then
log_info "Recovering connection details from ${update_conn_env}..."
recover_connection_state "$update_conn_env"
fi
if update_connection_state_incomplete; then
recover_connection_state_from_existing_agent || true
fi
if [[ -n "$PULSE_URL" && -n "$PULSE_TOKEN" ]]; then
:
elif [[ -z "$PULSE_URL" || -z "$PULSE_TOKEN" ]]; then
fail "No existing Pulse Agent connection state found. Use the install command instead." "$EXIT_MISSING_ARGS"
fi
if [[ -z "$AGENT_ID" ]]; then
AGENT_ID=$(recover_agent_id_from_state_file || true)
if [[ -n "$AGENT_ID" ]]; then
log_info "Recovered agent ID from persisted agent-id state."
fi
recover_connection_state_from_existing_agent || true
fi
if [[ -z "$AGENT_ID" ]]; then
AGENT_ID=$(recover_agent_id_from_state_file || true)
if [[ -n "$AGENT_ID" ]]; then
log_info "Recovered agent ID from persisted agent-id state."
fi
fi
if [[ "$UPDATE_ONLY" == "true" && ( -z "$PULSE_URL" || -z "$PULSE_TOKEN" ) ]]; then
fail "No existing Pulse Agent connection state found. Use the install command instead." "$EXIT_MISSING_ARGS"
fi
fi
if [[ -z "$STATE_DIR" || "$STATE_DIR" != /* || "$STATE_DIR" == "/" ||
"$STATE_DIR" == *$'\r'* || "$STATE_DIR" == *$'\n'* ]]; then
fail "Recovered Pulse Agent state directory is invalid." "$EXIT_MISSING_ARGS"
fi
if [[ "$STATE_DIR_SOURCE" == "explicit" || "$STATE_DIR_SOURCE" == "recovered" ]]; then
TRUENAS_STATE_DIR="$STATE_DIR"
TRUENAS_LOG_DIR="$TRUENAS_STATE_DIR/logs"
TRUENAS_BOOTSTRAP_SCRIPT="$TRUENAS_STATE_DIR/bootstrap-pulse-agent.sh"
TRUENAS_ENV_FILE="$TRUENAS_STATE_DIR/pulse-agent.env"
fi
if [[ -n "$PULSE_URL" ]]; then
@@ -2553,18 +2748,6 @@ if [[ "$UNINSTALL" == "true" ]]; then
log_info "Uninstalling ${AGENT_NAME} and cleaning up legacy agents..."
local qnap_state_dir=""
# Recover connection details from the canonical installer-owned state artifact
# if command line input is only partial. Read keys explicitly instead of
# sourcing the file so uninstall cannot execute persisted shell content.
if [[ -z "$PULSE_URL" || -z "$PULSE_TOKEN" || -z "$AGENT_ID" || -z "$HOSTNAME_OVERRIDE" || -z "$CURL_CA_BUNDLE" || "$INSECURE" != "true" ]]; then
local conn_env=""
conn_env=$(find_connection_state_file || true)
if [[ -n "$conn_env" ]]; then
log_info "Recovering connection details from ${conn_env}..."
recover_connection_state "$conn_env"
fi
fi
# Try to notify the Pulse server about uninstallation if we have connection details
# This ensures the agent record is removed and any linked PVE nodes are updated immediately.
if [[ -n "$PULSE_URL" ]]; then
@@ -2572,7 +2755,10 @@ if [[ "$UNINSTALL" == "true" ]]; then
# Priority: agent-id file (canonical) > hostname API lookup (fallback)
if [[ -z "$AGENT_ID" ]]; then
local aid_path=""
local aid_paths=(/var/lib/pulse-agent/agent-id /boot/config/plugins/pulse-agent/agent-id "$TRUENAS_STATE_DIR/agent-id")
local aid_paths=("${STATE_DIR%/}/agent-id")
if [[ "$STATE_DIR_SOURCE" == "default" ]]; then
aid_paths+=("$DEFAULT_STATE_DIR/agent-id" /boot/config/plugins/pulse-agent/agent-id "$TRUENAS_STATE_DIR/agent-id")
fi
qnap_state_dir=$(find_qnap_state_dir || true)
if [[ -n "$qnap_state_dir" ]]; then
aid_paths+=("$qnap_state_dir/agent-id")
@@ -2596,11 +2782,10 @@ if [[ "$UNINSTALL" == "true" ]]; then
fi
if [[ -n "$LOOKUP_HOSTNAME" ]]; then
LOOKUP_ARGS=(-fsSL --connect-timeout 5)
if [[ -n "$PULSE_TOKEN" ]]; then LOOKUP_ARGS+=(-H "X-API-Token: ${PULSE_TOKEN}"); fi
if [[ "$INSECURE" == "true" ]]; then LOOKUP_ARGS+=(-k); fi
if [[ -n "$CURL_CA_BUNDLE" ]]; then LOOKUP_ARGS+=(--cacert "$CURL_CA_BUNDLE"); fi
LOOKUP_HOSTNAME_ESCAPED=$(url_encode "$LOOKUP_HOSTNAME")
LOOKUP_RESP=$(curl "${LOOKUP_ARGS[@]}" "${PULSE_URL}/api/agents/agent/lookup?hostname=${LOOKUP_HOSTNAME_ESCAPED}" 2>/dev/null || true)
LOOKUP_RESP=$(curl_with_pulse_token "${LOOKUP_ARGS[@]}" "${PULSE_URL}/api/agents/agent/lookup?hostname=${LOOKUP_HOSTNAME_ESCAPED}" 2>/dev/null || true)
if [[ -n "$LOOKUP_RESP" ]]; then
# Extract .agent.id from JSON (portable, no jq dependency)
AGENT_ID=$(echo "$LOOKUP_RESP" | grep -o '"id"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"id"[[:space:]]*:[[:space:]]*"//; s/"$//' || true)
@@ -2614,12 +2799,11 @@ if [[ "$UNINSTALL" == "true" ]]; then
if [[ -n "$AGENT_ID" ]]; then
log_info "Notifying Pulse server to unregister agent ID: ${AGENT_ID}..."
CURL_ARGS=(-fsSL --connect-timeout 5 -X POST -H "Content-Type: application/json")
if [[ -n "$PULSE_TOKEN" ]]; then CURL_ARGS+=(-H "X-API-Token: ${PULSE_TOKEN}"); fi
if [[ "$INSECURE" == "true" ]]; then CURL_ARGS+=(-k); fi
if [[ -n "$CURL_CA_BUNDLE" ]]; then CURL_ARGS+=(--cacert "$CURL_CA_BUNDLE"); fi
# Send unregistration request (ignore errors as we are uninstalling anyway)
curl "${CURL_ARGS[@]}" -d "{\"agentId\": \"${AGENT_ID}\"}" "${PULSE_URL}/api/agents/agent/uninstall" >/dev/null 2>&1 || true
curl_with_pulse_token "${CURL_ARGS[@]}" -d "{\"agentId\": \"${AGENT_ID}\"}" "${PULSE_URL}/api/agents/agent/uninstall" >/dev/null 2>&1 || true
fi
fi
@@ -2640,7 +2824,7 @@ if [[ "$UNINSTALL" == "true" ]]; then
# Remove legacy binaries
# Remove agent state directory (contains agent ID, proxmox registration state, etc.)
rm -rf /var/lib/pulse-agent
remove_agent_state_dir "$STATE_DIR"
# Remove log files
rm -f /var/log/pulse-agent.log
@@ -3171,12 +3355,12 @@ if [[ -f /etc/unraid-version ]]; then
# Unraid's /boot is FAT32 (no execute permission), so we store the binary there
# for persistence but copy it to RAM disk (/usr/local/bin) for execution
UNRAID_STORAGE_DIR="/boot/config/plugins/pulse-agent"
select_platform_state_dir "/boot/config/plugins/pulse-agent"
UNRAID_STORAGE_DIR="$STATE_DIR"
UNRAID_STORED_BINARY="${UNRAID_STORAGE_DIR}/${BINARY_NAME}"
RUNTIME_BINARY="${INSTALL_DIR}/${BINARY_NAME}"
GO_SCRIPT="/boot/config/go"
STATE_DIR="$UNRAID_STORAGE_DIR"
mkdir -p "$UNRAID_STORAGE_DIR"
# Copy binary to persistent storage (for survival across reboots)
@@ -3285,7 +3469,7 @@ if [[ -f /sbin/getcfg ]] || [[ -f /etc/config/qpkg.conf ]]; then
fail "Could not find a writable QNAP data volume. Is a storage volume configured?"
fi
STATE_DIR="${QNAP_VOL}/.pulse-agent"
select_platform_state_dir "${QNAP_VOL}/.pulse-agent"
QNAP_STORED_BINARY="${STATE_DIR}/${BINARY_NAME}"
RUNTIME_BINARY="${INSTALL_DIR}/${BINARY_NAME}"
WRAPPER_SCRIPT="${STATE_DIR}/start-pulse-agent.sh"
@@ -3351,7 +3535,11 @@ fi
# Note: /data may have exec=off on some TrueNAS systems. We try multiple runtime locations.
if [[ "$TRUENAS" == true ]]; then
log_info "Configuring TrueNAS SCALE/CORE installation..."
STATE_DIR="$TRUENAS_STATE_DIR"
select_platform_state_dir "$TRUENAS_STATE_DIR"
TRUENAS_STATE_DIR="$STATE_DIR"
TRUENAS_LOG_DIR="$TRUENAS_STATE_DIR/logs"
TRUENAS_BOOTSTRAP_SCRIPT="$TRUENAS_STATE_DIR/bootstrap-pulse-agent.sh"
TRUENAS_ENV_FILE="$TRUENAS_STATE_DIR/pulse-agent.env"
# Stop any existing agent before we modify binaries
# The runtime binary may be in /root/bin or /var/tmp, not just INSTALL_DIR
@@ -3454,6 +3642,7 @@ if [[ "$TRUENAS" == true ]]; then
# Store environment/config for reference
cat > "$TRUENAS_ENV_FILE" <<EOF
# Pulse Agent configuration (for reference)
PULSE_STATE_DIR=${STATE_DIR}
PULSE_URL=${PULSE_URL}
PULSE_TOKEN_FILE=${RUNTIME_TOKEN_FILE}
PULSE_INTERVAL=${INTERVAL}
@@ -0,0 +1,456 @@
package installtests
import (
"bytes"
"compress/gzip"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"syscall"
"testing"
"time"
)
type agentLifecycleControlPlane struct {
mu sync.Mutex
online bool
bootstrapToken string
runtimeToken string
canonicalID string
enrollmentCount int
reportCount int
lastReportToken string
lastReportID string
lastCommands bool
}
type agentLifecycleSnapshot struct {
online bool
enrollmentCount int
reportCount int
lastReportToken string
lastReportID string
lastCommands bool
}
func (s *agentLifecycleControlPlane) setCredentials(bootstrapToken, runtimeToken, canonicalID string) {
s.mu.Lock()
defer s.mu.Unlock()
s.bootstrapToken = bootstrapToken
s.runtimeToken = runtimeToken
s.canonicalID = canonicalID
}
func (s *agentLifecycleControlPlane) setOnline(online bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.online = online
}
func (s *agentLifecycleControlPlane) snapshot() agentLifecycleSnapshot {
s.mu.Lock()
defer s.mu.Unlock()
return agentLifecycleSnapshot{
online: s.online,
enrollmentCount: s.enrollmentCount,
reportCount: s.reportCount,
lastReportToken: s.lastReportToken,
lastReportID: s.lastReportID,
lastCommands: s.lastCommands,
}
}
func (s *agentLifecycleControlPlane) serveHTTP(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
if !s.online {
s.mu.Unlock()
http.Error(w, "server restarting", http.StatusServiceUnavailable)
return
}
bootstrapToken := s.bootstrapToken
runtimeToken := s.runtimeToken
canonicalID := s.canonicalID
s.mu.Unlock()
switch {
case r.URL.Path == "/api/agents/agent/lookup":
http.Error(w, "not found", http.StatusNotFound)
case strings.HasPrefix(r.URL.Path, "/api/agents/agent/") && strings.HasSuffix(r.URL.Path, "/config"):
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"success":true,"config":{}}`)
case r.URL.Path == "/api/agents/agent/enroll":
if got := r.Header.Get("X-API-Token"); got != bootstrapToken {
http.Error(w, "bad bootstrap token", http.StatusUnauthorized)
return
}
var payload struct {
CommandsEnabled bool `json:"commandsEnabled"`
}
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
s.mu.Lock()
s.enrollmentCount++
s.lastCommands = payload.CommandsEnabled
s.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"agentId": canonicalID,
"runtimeToken": runtimeToken,
})
case r.URL.Path == "/api/agents/agent/report":
token := r.Header.Get("X-API-Token")
if token != runtimeToken {
http.Error(w, "bad runtime token", http.StatusUnauthorized)
return
}
reportID := ""
if gz, err := gzip.NewReader(r.Body); err == nil {
var report struct {
Agent struct {
ID string `json:"id"`
} `json:"agent"`
}
if json.NewDecoder(gz).Decode(&report) == nil {
reportID = report.Agent.ID
}
_ = gz.Close()
}
s.mu.Lock()
s.reportCount++
s.lastReportToken = token
s.lastReportID = reportID
s.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"success": true,
"agentId": canonicalID,
})
default:
http.NotFound(w, r)
}
}
func waitForLifecycleState(t *testing.T, timeout time.Duration, describe string, predicate func(agentLifecycleSnapshot) bool, state *agentLifecycleControlPlane) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
snapshot := state.snapshot()
if predicate(snapshot) {
return
}
time.Sleep(100 * time.Millisecond)
}
final := state.snapshot()
t.Fatalf("timed out waiting for %s; enrollments=%d reports=%d report_id=%q commands=%t",
describe, final.enrollmentCount, final.reportCount, final.lastReportID, final.lastCommands)
}
func buildLifecycleAgent(t *testing.T) string {
t.Helper()
binaryPath := filepath.Join(t.TempDir(), "pulse-agent")
cmd := exec.Command("go", "build", "-o", binaryPath, "./cmd/pulse-agent")
cmd.Dir = repoFile()
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("build pulse-agent: %v\n%s", err, output)
}
return binaryPath
}
func renderLifecycleService(t *testing.T, stateDir, stateSource, unitPath, pulseURL, token string, commandsEnabled, recoverExisting bool) {
t.Helper()
commandFlag := "false"
if commandsEnabled {
commandFlag = "true"
}
recovery := ""
if recoverExisting {
recovery = `
PULSE_URL=""
PULSE_TOKEN=""
AGENT_ID=""
HOSTNAME_OVERRIDE=""
INSECURE="false"
SERVER_FINGERPRINT=""
CURL_CA_BUNDLE=""
recover_connection_state "$STATE_DIR/connection.env"
`
}
script := `
set -euo pipefail
STATE_DIR="` + stateDir + `"
DEFAULT_STATE_DIR="` + stateDir + `"
STATE_DIR_SOURCE="` + stateSource + `"
TRUENAS_STATE_DIR="` + filepath.Join(filepath.Dir(stateDir), "truenas-state") + `"
PULSE_URL="` + pulseURL + `"
IFS= read -r PULSE_TOKEN
INTERVAL="1s"
ENABLE_HOST="true"
ENABLE_DOCKER="false"
DOCKER_EXPLICIT="true"
ENABLE_KUBERNETES="false"
KUBECONFIG_PATH=""
ENABLE_PROXMOX="false"
PROXMOX_TYPE=""
INSECURE="false"
SERVER_FINGERPRINT=""
OBSERVERS_FILE=""
ENABLE_COMMANDS="` + commandFlag + `"
HEALTH_ADDR_SET="true"
HEALTH_ADDR=""
ENROLL="true"
KUBE_INCLUDE_ALL_PODS="false"
KUBE_INCLUDE_ALL_DEPLOYMENTS="false"
AGENT_ID=""
HOSTNAME_OVERRIDE="state-lifecycle-host"
DISK_EXCLUDES=()
CURL_CA_BUNDLE=""
RUNTIME_TOKEN_FILE=""
RUNTIME_TOKEN_CHANGED="false"
SYSTEMD_ENV_LINES=""
SHELL_EXPORT_LINES=""
SAVED_INSTALL_SCRIPT=""
NON_INTERACTIVE="true"
log_info() { :; }
log_warn() { :; }
fail() { printf 'FAIL:%s\n' "$1" >&2; return 99; }
curl() { return 1; }
` + extractInstallShellFunction(t, "write_connection_state_value") + `
` + extractInstallShellFunction(t, "read_connection_state_value") + `
` + extractInstallShellFunction(t, "recover_token_from_default_agent_token_file") + `
` + extractInstallShellFunction(t, "recover_connection_state") + `
` + extractInstallShellFunction(t, "ensure_runtime_token_file") + `
` + extractInstallShellFunction(t, "build_exec_arg_items") + `
` + extractInstallShellFunction(t, "join_exec_arg_items") + `
` + extractInstallShellFunction(t, "build_exec_args") + `
` + extractInstallShellFunction(t, "systemd_agent_requires_lxc_attach") + `
` + extractInstallShellFunction(t, "render_systemd_agent_unit") + `
` + extractInstallShellFunction(t, "save_connection_info") + recovery + `
ensure_runtime_token_file "$STATE_DIR"
build_exec_args
render_systemd_agent_unit "` + unitPath + `" "/test/pulse-agent" "$EXEC_ARGS" "network-online.target" "network-online.target" "root" ""
save_connection_info "$STATE_DIR"
`
cmd := exec.Command("bash", "-c", script)
cmd.Stdin = strings.NewReader(token + "\n")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("render lifecycle service: %v\n%s", err, out)
}
}
type runningLifecycleAgent struct {
cmd *exec.Cmd
logs *bytes.Buffer
}
func startLifecycleAgent(t *testing.T, binaryPath, pulseURL, stateDir string, commandsEnabled bool) *runningLifecycleAgent {
t.Helper()
args := []string{
"--url", pulseURL,
"--token-file", filepath.Join(stateDir, "token"),
"--state-dir", stateDir,
"--interval", "1s",
"--hostname", "state-lifecycle-host",
"--enable-host",
"--enable-docker=false",
"--disable-auto-update",
"--health-addr", "",
"--enroll",
}
if commandsEnabled {
args = append(args, "--enable-commands")
}
logs := &bytes.Buffer{}
cmd := exec.Command(binaryPath, args...)
cmd.Env = append(os.Environ(), "PULSE_AGENT_CONFIG_SIGNATURE_REQUIRED=false")
cmd.Stdout = logs
cmd.Stderr = logs
if err := cmd.Start(); err != nil {
t.Fatalf("start pulse-agent: %v", err)
}
t.Cleanup(func() {
if cmd.Process != nil {
_ = cmd.Process.Kill()
_, _ = cmd.Process.Wait()
}
})
for _, arg := range cmd.Args {
if strings.Contains(arg, "bootstrap-") || strings.Contains(arg, "runtime-") {
t.Fatalf("agent argv leaked token: %q", cmd.Args)
}
}
return &runningLifecycleAgent{cmd: cmd, logs: logs}
}
func (p *runningLifecycleAgent) stop(t *testing.T) string {
t.Helper()
if p.cmd.Process == nil {
return p.logs.String()
}
if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatalf("signal pulse-agent: %v", err)
}
done := make(chan error, 1)
go func() { done <- p.cmd.Wait() }()
select {
case err := <-done:
if err != nil {
t.Fatalf("pulse-agent shutdown: %v\n%s", err, p.logs.String())
}
case <-time.After(10 * time.Second):
_ = p.cmd.Process.Kill()
t.Fatalf("pulse-agent did not stop\n%s", p.logs.String())
}
p.cmd.Process = nil
return p.logs.String()
}
func assertPrivateLifecycleFile(t *testing.T, path string, want os.FileMode) {
t.Helper()
info, err := os.Stat(path)
if err != nil {
t.Fatalf("stat %s: %v", path, err)
}
if runtime.GOOS != "windows" && info.Mode().Perm() != want {
t.Fatalf("%s mode = %o, want %o", path, info.Mode().Perm(), want)
}
}
func TestPulseAgentStateDirLifecycleIntegration(t *testing.T) {
if testing.Short() {
t.Skip("real agent lifecycle integration")
}
binaryPath := buildLifecycleAgent(t)
for _, tc := range []struct {
name string
commandsEnabled bool
customState bool
}{
{name: "default_state_commands_disabled", commandsEnabled: false, customState: false},
{name: "custom_state_commands_enabled", commandsEnabled: true, customState: true},
} {
t.Run(tc.name, func(t *testing.T) {
root := t.TempDir()
stateName := "default-state"
if tc.customState {
stateName = "custom-state"
}
stateDir := filepath.Join(root, stateName)
unitPath := filepath.Join(root, "pulse-agent.service")
controlPlane := &agentLifecycleControlPlane{online: true}
controlPlane.setCredentials("bootstrap-one", "runtime-one", "agent-one")
server := httptest.NewServer(http.HandlerFunc(controlPlane.serveHTTP))
defer server.Close()
stateSource := "default"
if tc.customState {
stateSource = "explicit"
}
renderLifecycleService(t, stateDir, stateSource, unitPath, server.URL, "bootstrap-one", tc.commandsEnabled, false)
unit, err := os.ReadFile(unitPath)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(unit), "--state-dir "+stateDir) ||
!strings.Contains(string(unit), "--token-file "+filepath.Join(stateDir, "token")) {
t.Fatalf("generated service does not own canonical state paths:\n%s", unit)
}
if strings.Contains(string(unit), "bootstrap-one") || strings.Contains(string(unit), "runtime-one") {
t.Fatalf("generated service leaked a token:\n%s", unit)
}
if tc.commandsEnabled != strings.Contains(string(unit), "--enable-commands") {
t.Fatalf("generated service command mode mismatch:\n%s", unit)
}
proc := startLifecycleAgent(t, binaryPath, server.URL, stateDir, tc.commandsEnabled)
waitForLifecycleState(t, 20*time.Second, "initial enrollment and report", func(state agentLifecycleSnapshot) bool {
return state.enrollmentCount == 1 && state.reportCount >= 1 &&
state.lastReportToken == "runtime-one" && state.lastReportID == "agent-one" &&
state.lastCommands == tc.commandsEnabled
}, controlPlane)
logOutput := proc.stop(t)
for path, mode := range map[string]os.FileMode{
stateDir: 0700,
filepath.Join(stateDir, "token"): 0600,
filepath.Join(stateDir, "runtime.token"): 0600,
filepath.Join(stateDir, "agent-id"): 0600,
filepath.Join(stateDir, "connection.env"): 0600,
} {
assertPrivateLifecycleFile(t, path, mode)
}
beforeRestart := controlPlane.snapshot()
proc = startLifecycleAgent(t, binaryPath, server.URL, stateDir, tc.commandsEnabled)
waitForLifecycleState(t, 20*time.Second, "restart report with persisted identity", func(state agentLifecycleSnapshot) bool {
return state.reportCount > beforeRestart.reportCount &&
state.lastReportToken == "runtime-one" && state.lastReportID == "agent-one"
}, controlPlane)
if got := controlPlane.snapshot().enrollmentCount; got != 1 {
t.Fatalf("ordinary restart re-enrolled unexpectedly: enrollments=%d", got)
}
controlPlane.setOnline(false)
reportsBeforeOutage := controlPlane.snapshot().reportCount
time.Sleep(1500 * time.Millisecond)
controlPlane.setOnline(true)
waitForLifecycleState(t, 20*time.Second, "report recovery after server restart", func(state agentLifecycleSnapshot) bool {
return state.reportCount > reportsBeforeOutage && state.lastReportToken == "runtime-one"
}, controlPlane)
logOutput += proc.stop(t)
renderLifecycleService(t, stateDir, stateSource, unitPath, server.URL, "bootstrap-one", tc.commandsEnabled, true)
if _, err := os.Stat(filepath.Join(stateDir, "runtime.token")); err != nil {
t.Fatalf("update did not preserve runtime enrollment token: %v", err)
}
controlPlane.setCredentials("bootstrap-two", "runtime-two", "agent-two")
renderLifecycleService(t, stateDir, stateSource, unitPath, server.URL, "bootstrap-two", tc.commandsEnabled, false)
if _, err := os.Stat(filepath.Join(stateDir, "runtime.token")); !os.IsNotExist(err) {
t.Fatalf("fresh bootstrap did not clear stale runtime token: %v", err)
}
proc = startLifecycleAgent(t, binaryPath, server.URL, stateDir, tc.commandsEnabled)
waitForLifecycleState(t, 20*time.Second, "re-enrollment and canonical report", func(state agentLifecycleSnapshot) bool {
return state.enrollmentCount == 2 && state.lastReportToken == "runtime-two" &&
state.lastReportID == "agent-two" && state.lastCommands == tc.commandsEnabled
}, controlPlane)
logOutput += proc.stop(t)
for _, secret := range []string{"bootstrap-one", "runtime-one", "bootstrap-two", "runtime-two"} {
if strings.Contains(logOutput, secret) {
t.Fatalf("agent logs leaked %q:\n%s", secret, logOutput)
}
}
removeScript := `
set -euo pipefail
STATE_DIR="` + stateDir + `"
log_warn() { :; }
` + extractInstallShellFunction(t, "remove_agent_state_dir") + `
remove_agent_state_dir "$STATE_DIR"
`
if out, err := exec.Command("bash", "-c", removeScript).CombinedOutput(); err != nil {
t.Fatalf("uninstall state cleanup: %v\n%s", err, out)
}
if _, err := os.Stat(stateDir); !os.IsNotExist(err) {
t.Fatalf("uninstall did not remove canonical state directory: %v", err)
}
connectionState, err := os.ReadFile(filepath.Join(root, stateName, "connection.env"))
if err == nil {
t.Fatalf("connection state survived uninstall: %s", connectionState)
}
})
}
}
+517 -15
View File
@@ -54,7 +54,7 @@ func TestInstallSHPersistsAndVerifiesServerFingerprint(t *testing.T) {
`verify_pinned_server_certificate() {`,
`openssl s_client -connect "$target" -servername "$host"`,
`EXEC_ARG_ITEMS+=(--server-fingerprint "$SERVER_FINGERPRINT")`,
`write_connection_state_value "$conn_env" "PULSE_SERVER_FINGERPRINT" "$SERVER_FINGERPRINT"`,
`write_connection_state_value "$conn_tmp" "PULSE_SERVER_FINGERPRINT" "$SERVER_FINGERPRINT"`,
`SERVER_FINGERPRINT=$(read_connection_state_value "$file" "PULSE_SERVER_FINGERPRINT")`,
`if [[ "$OS" == "linux" || "$OS" == "freebsd" ]]; then`,
}
@@ -373,6 +373,8 @@ func TestBuildPlistProgramArgumentsUsesSharedExecArgs(t *testing.T) {
`<string>--health-addr</string>`,
`<string>--hostname</string>`,
`<string>Richard's Mac &amp; Mini</string>`,
`<string>--state-dir</string>`,
`<string>/var/lib/pulse-agent</string>`,
`<string>--disk-exclude</string>`,
`<string>Time Machine</string>`,
}
@@ -655,12 +657,13 @@ func TestInstallSHPersistsIdentityInConnectionEnv(t *testing.T) {
`read_connection_state_value() {`,
`recover_connection_state() {`,
`find_connection_state_file() {`,
`write_connection_state_value "$conn_env" "PULSE_TOKEN_FILE" "$RUNTIME_TOKEN_FILE"`,
`write_connection_state_value "$conn_env" "PULSE_AGENT_ID" "$AGENT_ID"`,
`write_connection_state_value "$conn_env" "PULSE_HOSTNAME" "$HOSTNAME_OVERRIDE"`,
`write_connection_state_value "$conn_env" "PULSE_INSECURE_SKIP_VERIFY" "true"`,
`write_connection_state_value "$conn_env" "PULSE_CACERT" "$CURL_CA_BUNDLE"`,
`recover_connection_state "$conn_env"`,
`write_connection_state_value "$conn_tmp" "PULSE_STATE_DIR" "$state_dir"`,
`write_connection_state_value "$conn_tmp" "PULSE_TOKEN_FILE" "$RUNTIME_TOKEN_FILE"`,
`write_connection_state_value "$conn_tmp" "PULSE_AGENT_ID" "$AGENT_ID"`,
`write_connection_state_value "$conn_tmp" "PULSE_HOSTNAME" "$HOSTNAME_OVERRIDE"`,
`write_connection_state_value "$conn_tmp" "PULSE_INSECURE_SKIP_VERIFY" "true"`,
`write_connection_state_value "$conn_tmp" "PULSE_CACERT" "$CURL_CA_BUNDLE"`,
`recover_connection_state "$lifecycle_conn_env"`,
}
for _, needle := range required {
if !strings.Contains(script, needle) {
@@ -677,9 +680,10 @@ func TestInstallSHRecoversSavedStateForPartialUninstallContext(t *testing.T) {
script := string(content)
needles := []string{
`if [[ -z "$PULSE_URL" || -z "$PULSE_TOKEN" || -z "$AGENT_ID" || -z "$HOSTNAME_OVERRIDE" || -z "$CURL_CA_BUNDLE" || "$INSECURE" != "true" ]]; then`,
`# Recover connection details from the canonical installer-owned state artifact`,
`conn_env=$(find_connection_state_file || true)`,
`if [[ "$UPDATE_ONLY" == "true" || "$UNINSTALL" == "true" ]]; then`,
`# An explicit state directory is authoritative.`,
`lifecycle_conn_env=$(find_connection_state_file || true)`,
`recover_connection_state "$lifecycle_conn_env"`,
}
for _, needle := range needles {
if !strings.Contains(script, needle) {
@@ -699,12 +703,13 @@ func TestInstallSHSupportsSavedStateUpdateMode(t *testing.T) {
`--update Update an existing agent using saved connection state`,
`UPDATE_ONLY="false"`,
`--update) UPDATE_ONLY="true"; shift ;;`,
`if [[ "$UPDATE_ONLY" == "true" ]]; then`,
`update_conn_env=$(find_connection_state_file || true)`,
`recover_connection_state "$update_conn_env"`,
`if [[ "$UPDATE_ONLY" == "true" || "$UNINSTALL" == "true" ]]; then`,
`lifecycle_conn_env=$(find_connection_state_file || true)`,
`recover_connection_state "$lifecycle_conn_env"`,
`recover_connection_state_from_existing_agent() {`,
`recover_connection_state_from_running_agent`,
`recover_connection_state_from_systemd_unit`,
`recover_connection_state_from_launchd_plist`,
`recover_connection_state_from_service_scripts`,
`running_agent_arg_stream() {`,
`running_agent_env_stream() {`,
@@ -720,7 +725,7 @@ func TestInstallSHSupportsSavedStateUpdateMode(t *testing.T) {
`[[ "$RECOVERED_AGENT_ENV_STATE" == "true" ]] && recovered_connection_state_ready`,
`if update_connection_state_incomplete; then`,
`recover_connection_state_from_existing_agent || true`,
`if [[ -n "$PULSE_URL" && -n "$PULSE_TOKEN" ]]; then`,
`if [[ "$UPDATE_ONLY" == "true" && ( -z "$PULSE_URL" || -z "$PULSE_TOKEN" ) ]]; then`,
`recover_agent_id_from_state_file() {`,
`AGENT_ID=$(recover_agent_id_from_state_file || true)`,
`No existing Pulse Agent connection state found. Use the install command instead.`,
@@ -1430,7 +1435,7 @@ func TestInstallSHUsesCanonicalQNAPBootstrapRenderer(t *testing.T) {
`remove_qnap_autorun_block() {`,
`write_qnap_wrapper_script() {`,
`append_qnap_autorun_block() {`,
`STATE_DIR="${QNAP_VOL}/.pulse-agent"`,
`select_platform_state_dir "${QNAP_VOL}/.pulse-agent"`,
`write_qnap_wrapper_script "$WRAPPER_SCRIPT" "$RUNTIME_BINARY" "$QNAP_STORED_BINARY"`,
`append_qnap_autorun_block "$AUTORUN_PATH" "$WRAPPER_SCRIPT" "$STATE_DIR"`,
`complete_installation_flow "$STATE_DIR" "Installation complete! Agent is running." "Upgrade complete! Agent is running." "tail -f /var/log/${AGENT_NAME}.log"`,
@@ -2228,6 +2233,501 @@ func TestStateDirFlagIsAcceptedByInstallerParser(t *testing.T) {
}
}
func TestInstallSHCustomStateDirOwnsTokenAndEnrollmentContinuity(t *testing.T) {
stateDir := filepath.Join(t.TempDir(), "custom-state")
if err := os.MkdirAll(stateDir, 0755); err != nil {
t.Fatal(err)
}
tokenPath := filepath.Join(stateDir, "token")
runtimePath := filepath.Join(stateDir, "runtime.token")
if err := os.WriteFile(tokenPath, []byte("aaa111"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(runtimePath, []byte("runtime-one"), 0600); err != nil {
t.Fatal(err)
}
ensureToken := extractInstallShellFunction(t, "ensure_runtime_token_file")
run := func(token string) string {
t.Helper()
script := `
set -euo pipefail
PULSE_TOKEN="` + token + `"
STATE_DIR="` + stateDir + `"
ENROLL="true"
RUNTIME_TOKEN_FILE=""
RUNTIME_TOKEN_CHANGED="false"
log_info() { :; }
` + ensureToken + `
ensure_runtime_token_file "$STATE_DIR"
printf 'changed=%s runtime=%s token_file=%s\n' \
"$RUNTIME_TOKEN_CHANGED" "$([[ -f "$STATE_DIR/runtime.token" ]] && echo present || echo missing)" "$RUNTIME_TOKEN_FILE"
`
out, err := exec.Command("bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
return string(out)
}
if got := run("aaa111"); !strings.Contains(got, "changed=false runtime=present") {
t.Fatalf("unchanged bootstrap token did not preserve enrollment runtime state:\n%s", got)
}
if got := run("bbb222"); !strings.Contains(got, "changed=true runtime=missing") {
t.Fatalf("changed bootstrap token did not force re-enrollment:\n%s", got)
}
for path, want := range map[string]os.FileMode{
stateDir: 0700,
tokenPath: 0600,
} {
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if got := info.Mode().Perm(); got != want {
t.Fatalf("%s mode = %o, want %o", path, got, want)
}
}
}
func TestInstallSHExplicitCustomStateNeverFallsBackToDefaultInstance(t *testing.T) {
root := t.TempDir()
customState := filepath.Join(root, "custom")
defaultState := filepath.Join(root, "default")
for _, dir := range []string{customState, defaultState} {
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
}
customConn := filepath.Join(customState, "connection.env")
if err := os.WriteFile(customConn, []byte("PULSE_URL='https://custom.example'\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(defaultState, "connection.env"), []byte("PULSE_URL='https://default.example'\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(defaultState, "agent-id"), []byte("default-agent"), 0600); err != nil {
t.Fatal(err)
}
script := `
set -euo pipefail
STATE_DIR="` + customState + `"
STATE_DIR_SOURCE="explicit"
DEFAULT_STATE_DIR="` + defaultState + `"
TRUENAS_STATE_DIR="` + filepath.Join(root, "truenas") + `"
` + extractInstallShellFunction(t, "find_connection_state_file") + `
` + extractInstallShellFunction(t, "recover_agent_id_from_state_file") + `
printf 'connection=%s\n' "$(find_connection_state_file)"
rm -f "$STATE_DIR/connection.env"
printf 'fallback_connection=%s\n' "$(find_connection_state_file || true)"
printf 'fallback_agent=%s\n' "$(recover_agent_id_from_state_file || true)"
`
out, err := exec.Command("bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
got := string(out)
if !strings.Contains(got, "connection="+customConn) {
t.Fatalf("custom connection state was not preferred:\n%s", got)
}
if !strings.Contains(got, "fallback_connection=\n") || !strings.Contains(got, "fallback_agent=\n") {
t.Fatalf("explicit custom state borrowed default instance state:\n%s", got)
}
}
func TestInstallSHSavedInstallerDiscoversItsCustomStateDir(t *testing.T) {
stateDir := filepath.Join(t.TempDir(), "saved custom state")
if err := os.MkdirAll(stateDir, 0700); err != nil {
t.Fatal(err)
}
installerPath := filepath.Join(stateDir, "install.sh")
if err := os.WriteFile(installerPath, []byte("#!/usr/bin/env bash\n"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(stateDir, "connection.env"), []byte("PULSE_URL='https://pulse.example'\n"), 0600); err != nil {
t.Fatal(err)
}
script := `
set -euo pipefail
STATE_DIR="/var/lib/pulse-agent"
STATE_DIR_SOURCE="default"
` + extractInstallShellFunction(t, "discover_state_dir_from_saved_installer") + `
discover_state_dir_from_saved_installer "` + installerPath + `"
printf 'state=%s source=%s\n' "$STATE_DIR" "$STATE_DIR_SOURCE"
`
out, err := exec.Command("bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
got := string(out)
resolvedStateDir, err := filepath.EvalSymlinks(stateDir)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(got, "state="+resolvedStateDir) || !strings.Contains(got, "source=recovered") {
t.Fatalf("saved installer did not discover adjacent custom state:\n%s", got)
}
}
func TestInstallSHDiscoversCustomStateDirFromGeneratedSystemdUnit(t *testing.T) {
for _, commandsEnabled := range []bool{false, true} {
t.Run(map[bool]string{false: "commands_disabled", true: "commands_enabled"}[commandsEnabled], func(t *testing.T) {
root := t.TempDir()
customState := filepath.Join(root, "custom state")
defaultState := filepath.Join(root, "default")
if err := os.MkdirAll(customState, 0700); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(defaultState, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(customState, "token"), []byte("custom123"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(defaultState, "token"), []byte("default999"), 0600); err != nil {
t.Fatal(err)
}
commandArg := ""
if commandsEnabled {
commandArg = " --enable-commands"
}
unitPath := filepath.Join(root, "pulse-agent.service")
unit := `[Service]
ExecStart=/usr/local/bin/pulse-agent --url https://custom.example --token-file ` +
strings.ReplaceAll(filepath.Join(customState, "token"), " ", `\ `) +
` --state-dir ` + strings.ReplaceAll(customState, " ", `\ `) + commandArg + "\n"
if err := os.WriteFile(unitPath, []byte(unit), 0644); err != nil {
t.Fatal(err)
}
script := `
set -euo pipefail
AGENT_NAME="pulse-agent"
PULSE_URL=""
PULSE_TOKEN=""
INTERVAL="30s"
INTERVAL_EXPLICIT="false"
ENABLE_HOST="true"
HOST_EXPLICIT="false"
ENABLE_DOCKER=""
DOCKER_EXPLICIT="false"
ENABLE_KUBERNETES=""
KUBERNETES_EXPLICIT="false"
KUBECONFIG_PATH=""
ENABLE_PROXMOX=""
PROXMOX_EXPLICIT="false"
PROXMOX_TYPE=""
INSECURE="false"
ENABLE_COMMANDS="false"
ENROLL="false"
HEALTH_ADDR=""
HEALTH_ADDR_SET="false"
AGENT_ID=""
HOSTNAME_OVERRIDE=""
STATE_DIR="` + defaultState + `"
STATE_DIR_SOURCE="default"
DEFAULT_STATE_DIR="` + defaultState + `"
TRUENAS_STATE_DIR="` + filepath.Join(root, "truenas") + `"
CURL_CA_BUNDLE=""
SERVER_FINGERPRINT=""
OBSERVERS_FILE=""
KUBE_INCLUDE_ALL_PODS="false"
KUBE_INCLUDE_ALL_DEPLOYMENTS="false"
DISK_EXCLUDES=()
log_warn() { :; }
systemctl() { printf '%s\n' "` + unitPath + `"; }
` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + `
` + extractInstallShellFunction(t, "normalize_recovered_agent_arg_key") + `
` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + `
` + extractInstallShellFunction(t, "recovered_connection_state_ready") + `
` + extractInstallShellFunction(t, "recover_token_from_default_agent_token_file") + `
` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + `
` + extractInstallShellFunction(t, "recover_connection_state_from_env_stream") + `
` + extractInstallShellFunction(t, "split_recovered_shell_words") + `
` + extractInstallShellFunction(t, "recover_connection_state_from_systemd_unit") + `
` + extractInstallShellFunction(t, "remove_agent_state_dir") + `
recover_connection_state_from_systemd_unit
printf 'state=%s source=%s url=%s token=%s commands=%s\n' \
"$STATE_DIR" "$STATE_DIR_SOURCE" "$PULSE_URL" "$PULSE_TOKEN" "$ENABLE_COMMANDS"
remove_agent_state_dir "$STATE_DIR"
`
out, err := exec.Command("bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
got := string(out)
for _, want := range []string{
"state=" + customState,
"source=recovered",
"url=https://custom.example",
"token=custom123",
"commands=" + map[bool]string{false: "false", true: "true"}[commandsEnabled],
} {
if !strings.Contains(got, want) {
t.Fatalf("systemd discovery missing %q:\n%s", want, got)
}
}
if strings.Contains(got, "default999") {
t.Fatalf("systemd discovery borrowed default token:\n%s", got)
}
if _, err := os.Stat(customState); !os.IsNotExist(err) {
t.Fatalf("uninstall did not remove discovered custom state: %v", err)
}
if _, err := os.Stat(defaultState); err != nil {
t.Fatalf("uninstall removed the default instance instead of discovered custom state: %v", err)
}
})
}
}
func TestInstallSHDiscoversCustomStateDirFromGeneratedLaunchdPlist(t *testing.T) {
root := t.TempDir()
customState := filepath.Join(root, "custom & state")
if err := os.MkdirAll(customState, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(customState, "token"), []byte("launch123"), 0600); err != nil {
t.Fatal(err)
}
plistPath := filepath.Join(root, "com.pulse.agent.plist")
escapedState := strings.ReplaceAll(customState, "&", "&amp;")
plist := `<?xml version="1.0" encoding="UTF-8"?>
<plist version="1.0"><dict>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/pulse-agent</string>
<string>--url</string>
<string>https://launch.example</string>
<string>--token-file</string>
<string>` + escapedState + `/token</string>
<string>--state-dir</string>
<string>` + escapedState + `</string>
<string>--enable-commands</string>
</array>
</dict></plist>`
if err := os.WriteFile(plistPath, []byte(plist), 0644); err != nil {
t.Fatal(err)
}
script := `
set -euo pipefail
PULSE_URL=""
PULSE_TOKEN=""
INTERVAL="30s"
INTERVAL_EXPLICIT="false"
ENABLE_HOST="true"
HOST_EXPLICIT="false"
ENABLE_DOCKER=""
DOCKER_EXPLICIT="false"
ENABLE_KUBERNETES=""
KUBERNETES_EXPLICIT="false"
KUBECONFIG_PATH=""
ENABLE_PROXMOX=""
PROXMOX_EXPLICIT="false"
PROXMOX_TYPE=""
INSECURE="false"
ENABLE_COMMANDS="false"
ENROLL="false"
HEALTH_ADDR=""
HEALTH_ADDR_SET="false"
AGENT_ID=""
HOSTNAME_OVERRIDE=""
STATE_DIR="/var/lib/pulse-agent"
STATE_DIR_SOURCE="default"
DEFAULT_STATE_DIR="/var/lib/pulse-agent"
TRUENAS_STATE_DIR="/data/pulse-agent"
CURL_CA_BUNDLE=""
SERVER_FINGERPRINT=""
OBSERVERS_FILE=""
KUBE_INCLUDE_ALL_PODS="false"
KUBE_INCLUDE_ALL_DEPLOYMENTS="false"
DISK_EXCLUDES=()
` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + `
` + extractInstallShellFunction(t, "normalize_recovered_agent_arg_key") + `
` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + `
` + extractInstallShellFunction(t, "recovered_connection_state_ready") + `
` + extractInstallShellFunction(t, "recover_token_from_default_agent_token_file") + `
` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + `
` + extractInstallShellFunction(t, "launchd_agent_arg_stream") + `
` + extractInstallShellFunction(t, "recover_connection_state_from_launchd_plist") + `
recover_connection_state_from_launchd_plist "` + plistPath + `"
printf 'state=%s url=%s token=%s commands=%s\n' "$STATE_DIR" "$PULSE_URL" "$PULSE_TOKEN" "$ENABLE_COMMANDS"
`
out, err := exec.Command("bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
got := string(out)
for _, want := range []string{
"state=" + customState,
"url=https://launch.example",
"token=launch123",
"commands=true",
} {
if !strings.Contains(got, want) {
t.Fatalf("launchd discovery missing %q:\n%s", want, got)
}
}
}
func TestInstallSHConnectionEnvPersistsCanonicalStateDirWithoutTokenValue(t *testing.T) {
stateDir := filepath.Join(t.TempDir(), "state")
script := `
set -euo pipefail
STATE_DIR="` + stateDir + `"
PULSE_URL="https://pulse.example.com"
PULSE_TOKEN="deadbeef"
RUNTIME_TOKEN_FILE="$STATE_DIR/token"
AGENT_ID="agent-custom"
HOSTNAME_OVERRIDE="host-custom"
INSECURE="false"
SERVER_FINGERPRINT=""
CURL_CA_BUNDLE=""
SAVED_INSTALL_SCRIPT=""
` + extractInstallShellFunction(t, "write_connection_state_value") + `
` + extractInstallShellFunction(t, "save_connection_info") + `
curl() { return 1; }
save_connection_info "$STATE_DIR"
cat "$STATE_DIR/connection.env"
`
out, err := exec.Command("bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
got := string(out)
if !strings.Contains(got, "PULSE_STATE_DIR='"+stateDir+"'") ||
!strings.Contains(got, "PULSE_TOKEN_FILE='"+filepath.Join(stateDir, "token")+"'") {
t.Fatalf("connection.env did not persist canonical state paths:\n%s", got)
}
if strings.Contains(got, "PULSE_TOKEN='") || strings.Contains(got, "deadbeef") {
t.Fatalf("connection.env leaked the token value:\n%s", got)
}
info, err := os.Stat(filepath.Join(stateDir, "connection.env"))
if err != nil {
t.Fatal(err)
}
if gotMode := info.Mode().Perm(); gotMode != 0600 {
t.Fatalf("connection.env mode = %o, want 600", gotMode)
}
}
func TestInstallSHStateWritesReplaceSymlinksAtomically(t *testing.T) {
stateDir := filepath.Join(t.TempDir(), "state")
if err := os.MkdirAll(stateDir, 0700); err != nil {
t.Fatal(err)
}
victimToken := filepath.Join(t.TempDir(), "victim-token")
victimConnection := filepath.Join(t.TempDir(), "victim-connection")
if err := os.WriteFile(victimToken, []byte("victim-token-content"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(victimConnection, []byte("victim-connection-content"), 0600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(victimToken, filepath.Join(stateDir, "token")); err != nil {
t.Fatal(err)
}
if err := os.Symlink(victimConnection, filepath.Join(stateDir, "connection.env")); err != nil {
t.Fatal(err)
}
script := `
set -euo pipefail
STATE_DIR="` + stateDir + `"
PULSE_URL="https://pulse.example.com"
IFS= read -r PULSE_TOKEN
RUNTIME_TOKEN_FILE=""
RUNTIME_TOKEN_CHANGED="false"
ENROLL="false"
AGENT_ID="agent-one"
HOSTNAME_OVERRIDE="host-one"
INSECURE="false"
SERVER_FINGERPRINT=""
CURL_CA_BUNDLE=""
SAVED_INSTALL_SCRIPT=""
NON_INTERACTIVE="true"
TMP_FILES=()
log_info() { :; }
curl() { return 1; }
` + extractInstallShellFunction(t, "write_connection_state_value") + `
` + extractInstallShellFunction(t, "ensure_runtime_token_file") + `
` + extractInstallShellFunction(t, "save_connection_info") + `
ensure_runtime_token_file "$STATE_DIR"
save_connection_info "$STATE_DIR"
`
cmd := exec.Command("bash", "-c", script)
cmd.Stdin = strings.NewReader("new-token-content\n")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
for path, want := range map[string]string{
victimToken: "victim-token-content",
victimConnection: "victim-connection-content",
} {
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(got) != want {
t.Fatalf("state write followed symlink and modified %s: %q", path, got)
}
}
for _, path := range []string{filepath.Join(stateDir, "token"), filepath.Join(stateDir, "connection.env")} {
info, err := os.Lstat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode()&os.ModeSymlink != 0 {
t.Fatalf("%s remained a symlink after atomic replacement", path)
}
}
}
func TestInstallSHCurlTokenTransportKeepsSecretOutOfArgv(t *testing.T) {
recordDir := t.TempDir()
argsPath := filepath.Join(recordDir, "args")
configPath := filepath.Join(recordDir, "config")
script := `
set -euo pipefail
PULSE_TOKEN="feedface"
curl() {
printf '%s\n' "$@" > "` + argsPath + `"
while [[ $# -gt 0 ]]; do
if [[ "$1" == "--config" ]]; then
cp "$2" "` + configPath + `"
shift 2
continue
fi
shift
done
}
` + extractInstallShellFunction(t, "curl_with_pulse_token") + `
curl_with_pulse_token -sS https://pulse.example.com/api/agents/agent/lookup
`
out, err := exec.Command("bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bash: %v\n%s", err, out)
}
args, err := os.ReadFile(argsPath)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(args), "feedface") || strings.Contains(string(args), "X-API-Token") {
t.Fatalf("curl argv leaked token material:\n%s", args)
}
config, err := os.ReadFile(configPath)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(config), "X-API-Token: feedface") {
t.Fatalf("private curl config did not carry auth header:\n%s", config)
}
}
func TestSetupUpdateCommandHonorsRCChannelAndCustomPaths(t *testing.T) {
tmpDir := t.TempDir()
updatePath := filepath.Join(tmpDir, "update")
@@ -3783,6 +4283,7 @@ func TestSetupAutoUpdatesPreservesRCChannelWhenUpdatingExistingConfig(t *testing
// actionable error instead of leaving a silent 401 loop behind (issue #1515).
func TestInstallSHVerifyAgentServerRegistrationDetectsRejectedToken(t *testing.T) {
urlEncode := extractInstallShellFunction(t, "url_encode")
curlWithPulseToken := extractInstallShellFunction(t, "curl_with_pulse_token")
verifyFn := extractInstallShellFunction(t, "verify_agent_server_registration")
cases := []struct {
@@ -3816,6 +4317,7 @@ func TestInstallSHVerifyAgentServerRegistrationDetectsRejectedToken(t *testing.T
HOSTNAME_OVERRIDE="omv"
INSECURE="false"
CURL_CA_BUNDLE=""
` + curlWithPulseToken + `
` + urlEncode + `
` + verifyFn + `
verify_agent_server_registration
+47
View File
@@ -0,0 +1,47 @@
{
"issue": 1595,
"description": "Synthetic public-evidence topology: 24 same-model SAS disks, two HBAs, two ZFS pools. Serial values and counters are synthetic.",
"node": "nuclear",
"instance": "pve-lab",
"agentId": "agent-nuclear",
"model": "ST18000NM019J",
"sizeBytes": 18000207937536,
"controllers": [
{
"id": "0000:03:00.0",
"pool": "tank-a",
"disks": [
{"device":"sda","target":"6:0:0:0","serial":"ZR5TESTA0001","providerSerial":"5000c500a0000001","temperature":30,"readBytes":1000000,"writeBytes":2000000,"ioTimeMs":100},
{"device":"sdb","target":"6:0:1:0","serial":"ZR5TESTA0002","providerSerial":"5000c500a0000002","temperature":31,"readBytes":1100000,"writeBytes":2100000,"ioTimeMs":110},
{"device":"sdc","target":"6:0:2:0","serial":"ZR5TESTA0003","providerSerial":"5000c500a0000003","temperature":32,"readBytes":1200000,"writeBytes":2200000,"ioTimeMs":120},
{"device":"sdd","target":"6:0:3:0","serial":"ZR5TESTA0004","providerSerial":"5000c500a0000004","temperature":33,"readBytes":1300000,"writeBytes":2300000,"ioTimeMs":130},
{"device":"sde","target":"6:0:4:0","serial":"ZR5TESTA0005","providerSerial":"5000c500a0000005","temperature":34,"readBytes":1400000,"writeBytes":2400000,"ioTimeMs":140},
{"device":"sdf","target":"6:0:5:0","serial":"ZR5TESTA0006","providerSerial":"5000c500a0000006","temperature":35,"readBytes":1500000,"writeBytes":2500000,"ioTimeMs":150},
{"device":"sdg","target":"6:0:6:0","serial":"ZR5TESTA0007","providerSerial":"5000c500a0000007","temperature":36,"readBytes":1600000,"writeBytes":2600000,"ioTimeMs":160},
{"device":"sdh","target":"6:0:7:0","serial":"ZR5TESTA0008","providerSerial":"5000c500a0000008","temperature":37,"readBytes":1700000,"writeBytes":2700000,"ioTimeMs":170},
{"device":"sdi","target":"6:0:8:0","serial":"ZR5TESTA0009","providerSerial":"5000c500a0000009","temperature":38,"readBytes":1800000,"writeBytes":2800000,"ioTimeMs":180},
{"device":"sdj","target":"6:0:9:0","serial":"ZR5TESTA0010","providerSerial":"5000c500a0000010","temperature":39,"readBytes":1900000,"writeBytes":2900000,"ioTimeMs":190},
{"device":"sdk","target":"6:0:10:0","serial":"ZR5TESTA0011","providerSerial":"5000c500a0000011","temperature":40,"readBytes":2000000,"writeBytes":3000000,"ioTimeMs":200},
{"device":"sdl","target":"6:0:11:0","serial":"ZR5TESTA0012","providerSerial":"5000c500a0000012","temperature":41,"readBytes":2100000,"writeBytes":3100000,"ioTimeMs":210}
]
},
{
"id": "0000:04:00.0",
"pool": "tank-b",
"disks": [
{"device":"sdm","target":"7:0:0:0","serial":"ZR5TESTB0001","providerSerial":"5000c500b0000001","temperature":30,"readBytes":3000000,"writeBytes":4000000,"ioTimeMs":300},
{"device":"sdn","target":"7:0:1:0","serial":"ZR5TESTB0002","providerSerial":"5000c500b0000002","temperature":31,"readBytes":3100000,"writeBytes":4100000,"ioTimeMs":310},
{"device":"sdo","target":"7:0:2:0","serial":"ZR5TESTB0003","providerSerial":"5000c500b0000003","temperature":32,"readBytes":3200000,"writeBytes":4200000,"ioTimeMs":320},
{"device":"sdp","target":"7:0:3:0","serial":"ZR5TESTB0004","providerSerial":"5000c500b0000004","temperature":33,"readBytes":3300000,"writeBytes":4300000,"ioTimeMs":330},
{"device":"sdq","target":"7:0:4:0","serial":"ZR5TESTB0005","providerSerial":"5000c500b0000005","temperature":34,"readBytes":3400000,"writeBytes":4400000,"ioTimeMs":340},
{"device":"sdr","target":"7:0:5:0","serial":"ZR5TESTB0006","providerSerial":"5000c500b0000006","temperature":35,"readBytes":3500000,"writeBytes":4500000,"ioTimeMs":350},
{"device":"sds","target":"7:0:6:0","serial":"ZR5TESTB0007","providerSerial":"5000c500b0000007","temperature":36,"readBytes":3600000,"writeBytes":4600000,"ioTimeMs":360},
{"device":"sdt","target":"7:0:7:0","serial":"ZR5TESTB0008","providerSerial":"5000c500b0000008","temperature":37,"readBytes":3700000,"writeBytes":4700000,"ioTimeMs":370},
{"device":"sdu","target":"7:0:8:0","serial":"ZR5TESTB0009","providerSerial":"5000c500b0000009","temperature":38,"readBytes":3800000,"writeBytes":4800000,"ioTimeMs":380},
{"device":"sdv","target":"7:0:9:0","serial":"ZR5TESTB0010","providerSerial":"5000c500b0000010","temperature":39,"readBytes":3900000,"writeBytes":4900000,"ioTimeMs":390},
{"device":"sdw","target":"7:0:10:0","serial":"ZR5TESTB0011","providerSerial":"5000c500b0000011","temperature":40,"readBytes":4000000,"writeBytes":5000000,"ioTimeMs":400},
{"device":"sdx","target":"7:0:11:0","serial":"ZR5TESTB0012","providerSerial":"5000c500b0000012","temperature":41,"readBytes":4100000,"writeBytes":5100000,"ioTimeMs":410}
]
}
]
}