diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index f09880c2b9..4ef2ff262c 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,30 @@ ## Purpose +### Platform connection discovery follows tenant storage identity + +TrueNAS and VMware pollers enumerate validated, sorted persistence directory +keys, including `default`, independently of `org.json` metadata. A missing, +malformed or foreign metadata ID cannot prevent a saved connection from +acquiring a provider, redirect its config read, or block authoritative +disable/removal in an unrelated tenant. Configs, credentials, runtime statuses +and discovered records remain keyed by that storage identity and connection +ID. Existing path validation, disabled-connection pruning, encrypted config +loading and poll intervals are unchanged. A directory-list failure still +returns no partial inventory; a connection-config read failure does not gain +an old-credential fallback. Metadata-dependent account and authentication +consumers retain strict organization loading: storage discovery is not +authorization, membership or a substitute for request tenant resolution. + +The storage-key, TrueNAS and VMware discovery tests pin malformed/absent IDs, +metadata redirection and unrelated corruption, plus disable/removal without +additional polling. API controls use the actual saved Test, periodic poller +and redacted list response: a probe cannot manufacture runtime evidence, and +ordinary polling supplies transport, attempt/success and observed inventory. +These synthetic source controls reproduce a family matching #2382's second +operator response, not that operator's cause, the original downgrade-resolved +regression, native appliance acceptance or release availability. + ### Lightweight detached History reads Guest, node, disk and storage History readers allocate one exact-sized snapshot diff --git a/internal/api/truenas_handlers_test.go b/internal/api/truenas_handlers_test.go index f1ba5961ed..bb8a1af321 100644 --- a/internal/api/truenas_handlers_test.go +++ b/internal/api/truenas_handlers_test.go @@ -8,6 +8,8 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "strconv" "strings" "testing" @@ -1552,3 +1554,78 @@ func trueNASInstanceFromRawURL(t *testing.T, id string, rawURL string, enabled b PollIntervalSecs: 60, } } + +// This is an API + real client + periodic poller fixture, not native appliance +// acceptance or attribution of either operator's #2382 installation. +func TestTrueNASHandlers_SavedProbeAndPollingSurviveUnrelatedMetadata(t *testing.T) { + setTrueNASFeatureForTest(t, true) + setMockModeForTest(t, false) + for _, tc := range []struct{ name, metadataDir, body string }{ + {name: "default_empty_id", body: `{}`}, + {name: "default_foreign_id", body: `{"id":"metadata-redirect"}`}, + {name: "unrelated_corrupt_metadata", metadataDir: "orgs/broken-peer", body: `{broken`}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := &config.Config{DataPath: t.TempDir()} + handler, persistence, _ := newTrueNASHandlersForTest(t, cfg) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v2.0/system/info": + _, _ = w.Write([]byte(`{"hostname":"fixture-nas","version":"TrueNAS-SCALE-24.10.2","system_serial":"FIXTURE-NAS"}`)) + case "/api/v2.0/pool", "/api/v2.0/pool/dataset", "/api/v2.0/disk", "/api/v2.0/alert/list": + _, _ = w.Write([]byte(`[]`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + connection := trueNASInstanceFromRawURL(t, "fixture-saved-id", server.URL, true) + if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil { + t.Fatal(err) + } + dir := filepath.Join(cfg.DataPath, tc.metadataDir) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "org.json"), []byte(tc.body), 0o600); err != nil { + t.Fatal(err) + } + poller := monitoring.NewTrueNASPoller(config.NewMultiTenantPersistence(cfg.DataPath), 50*time.Millisecond, nil) + handler.getPoller = func(context.Context) *monitoring.TrueNASPoller { return poller } + probe := httptest.NewRecorder() + handler.HandleTestSavedConnection(probe, httptest.NewRequest(http.MethodPost, "/api/truenas/connections/fixture-saved-id/test", nil)) + if probe.Code != http.StatusOK { + t.Fatalf("saved probe status = %d: %s", probe.Code, probe.Body.String()) + } + before := poller.ConnectionSummaries("default", []config.TrueNASInstance{connection})[connection.ID] + if before.Poll.LastSuccessAt != nil || before.Observed != nil || before.Transport != nil { + t.Fatal("standalone saved probe manufactured runtime evidence") + } + poller.Start(context.Background()) + t.Cleanup(poller.Stop) + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + summary := poller.ConnectionSummaries("default", []config.TrueNASInstance{connection})[connection.ID] + if summary.Poll.LastSuccessAt != nil { + break + } + time.Sleep(10 * time.Millisecond) + } + listed := httptest.NewRecorder() + handler.HandleList(listed, httptest.NewRequest(http.MethodGet, "/api/truenas/connections", nil)) + var records []trueNASConnectionResponse + if listed.Code != http.StatusOK || json.Unmarshal(listed.Body.Bytes(), &records) != nil || len(records) != 1 { + t.Fatalf("invalid list response: %d %s", listed.Code, listed.Body.String()) + } + result := records[0] + if result.Poll == nil || result.Poll.IntervalSeconds != 60 || result.Poll.LastSuccessAt == nil || + result.Observed == nil || result.Observed.Host != "fixture-nas" || result.Transport == nil || !result.Transport.Connected { + t.Fatalf("API did not expose actual saved polling: %s", listed.Body.String()) + } + if result.APIKey != "********" || strings.Contains(listed.Body.String(), connection.APIKey) { + t.Fatal("connected runtime summary disclosed stored credentials") + } + }) + } +} diff --git a/internal/config/multi_tenant.go b/internal/config/multi_tenant.go index 1ddc3b3f7d..4bf173b705 100644 --- a/internal/config/multi_tenant.go +++ b/internal/config/multi_tenant.go @@ -210,8 +210,12 @@ func (mtp *MultiTenantPersistence) SaveOrganization(org *models.Organization) er return nil } -// ListOrganizations returns all known organizations (including the default org). -func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization, error) { +// ListOrganizationIDs returns sorted persistence keys, including the default +// organization, without reading organization metadata. Background inventory +// readers use these keys so one malformed org.json cannot interrupt unrelated +// monitoring or redirect a connection into another tenant's storage. +// This is storage discovery, not a membership or authorization decision. +func (mtp *MultiTenantPersistence) ListOrganizationIDs() ([]string, error) { orgIDs := map[string]struct{}{ "default": {}, } @@ -239,9 +243,18 @@ func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization, sortedIDs = append(sortedIDs, orgID) } sort.Strings(sortedIDs) + return sortedIDs, nil +} - orgs := make([]*models.Organization, 0, len(sortedIDs)) - for _, orgID := range sortedIDs { +// ListOrganizations returns all known organizations (including the default org). +// Metadata-dependent consumers retain strict loading of every organization. +func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization, error) { + orgIDs, err := mtp.ListOrganizationIDs() + if err != nil { + return nil, err + } + orgs := make([]*models.Organization, 0, len(orgIDs)) + for _, orgID := range orgIDs { org, loadErr := mtp.LoadOrganization(orgID) if loadErr != nil { return nil, fmt.Errorf("failed to load organization %s: %w", orgID, loadErr) diff --git a/internal/config/multi_tenant_test.go b/internal/config/multi_tenant_test.go index 118d479be0..0445cab471 100644 --- a/internal/config/multi_tenant_test.go +++ b/internal/config/multi_tenant_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -118,3 +119,73 @@ func TestResolveTenantOrgDirRejectsBlankBaseDir(t *testing.T) { t.Fatal("expected blank base dir to be rejected") } } + +func TestMultiTenantPersistence_ListOrganizationIDsUsesStorageKeysOnly(t *testing.T) { + root := t.TempDir() + mtp := NewMultiTenantPersistence(root) + for orgID, body := range map[string]string{ + "default": `{"id":"metadata-redirect"}`, + "alpha": `{broken`, + "zeta": `{}`, + } { + persistence, err := mtp.GetPersistence(orgID) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "org.json"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + orgsDir := filepath.Join(root, "orgs") + for _, invalid := range []string{"bad name", "bad:org"} { + if err := os.Mkdir(filepath.Join(orgsDir, invalid), 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(orgsDir, "stray-file"), []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(t.TempDir(), filepath.Join(orgsDir, "linked-tenant")); err != nil { + t.Fatal(err) + } + got, err := mtp.ListOrganizationIDs() + if err != nil || !reflect.DeepEqual(got, []string{"alpha", "default", "zeta"}) { + t.Fatalf("storage IDs = %v, %v", got, err) + } + if _, err := os.Stat(filepath.Join(orgsDir, "metadata-redirect")); !os.IsNotExist(err) { + t.Fatal("enumeration created a tenant from a metadata ID") + } + // Metadata-dependent consumers, including account/authentication flows, + // retain strict decoding: the new storage-key API is not authorization. + if orgs, err := mtp.ListOrganizations(); err == nil || orgs != nil { + t.Fatal("metadata list must still fail closed on malformed org.json") + } + body, err := os.ReadFile(filepath.Join(orgsDir, "alpha", "org.json")) + if err != nil || string(body) != "{broken" { + t.Fatal("enumeration repaired or overwrote metadata") + } +} + +func TestMultiTenantPersistence_ListOrganizationIDsMissingDirectoryAndFailure(t *testing.T) { + t.Run("missing_directory_has_default_without_writes", func(t *testing.T) { + root := filepath.Join(t.TempDir(), "absent") + mtp := NewMultiTenantPersistence(root) + got, err := mtp.ListOrganizationIDs() + if err != nil || !reflect.DeepEqual(got, []string{"default"}) { + t.Fatalf("missing directory IDs = %v, %v", got, err) + } + if _, err := os.Stat(root); !os.IsNotExist(err) { + t.Fatal("ID enumeration created persistence or metadata") + } + }) + t.Run("directory_failure_returns_no_partial_inventory", func(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "orgs"), []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + got, err := NewMultiTenantPersistence(root).ListOrganizationIDs() + if err == nil || got != nil { + t.Fatalf("broken organization directory returned partial inventory: %v, %v", got, err) + } + }) +} diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.go index debcdc8b7a..e8e627fe71 100644 --- a/internal/monitoring/truenas_poller.go +++ b/internal/monitoring/truenas_poller.go @@ -185,7 +185,7 @@ func (p *TrueNASPoller) syncConnections() { return } - orgs, err := p.multiTenant.ListOrganizations() + orgIDs, err := p.multiTenant.ListOrganizationIDs() if err != nil { log.Warn(). Str("component", "truenas_poller"). @@ -196,17 +196,10 @@ func (p *TrueNASPoller) syncConnections() { } // orgID -> connID -> instance - configured := make(map[string]map[string]config.TrueNASInstance, len(orgs)) - active := make(map[string]map[string]config.TrueNASInstance, len(orgs)) + configured := make(map[string]map[string]config.TrueNASInstance, len(orgIDs)) + active := make(map[string]map[string]config.TrueNASInstance, len(orgIDs)) - for _, org := range orgs { - if org == nil { - continue - } - orgID := strings.TrimSpace(org.ID) - if orgID == "" { - continue - } + for _, orgID := range orgIDs { persistence, err := p.multiTenant.GetPersistence(orgID) if err != nil || persistence == nil { log.Warn(). diff --git a/internal/monitoring/truenas_poller_test.go b/internal/monitoring/truenas_poller_test.go index 290814828e..505ddcc393 100644 --- a/internal/monitoring/truenas_poller_test.go +++ b/internal/monitoring/truenas_poller_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "net/url" "os" + "path/filepath" "strconv" "strings" "sync/atomic" @@ -2720,3 +2721,229 @@ func TestTrueNASPartialReportingPipeline(t *testing.T) { t.Fatalf("sufficiently covered local History lost Thermals: %+v", points) } } + +func setDiscoveryOrgMetadata(t *testing.T, persistence *config.ConfigPersistence, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "org.json"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +func discoveryTenant(t *testing.T, mtp *config.MultiTenantPersistence, orgID string) *config.ConfigPersistence { + t.Helper() + persistence, err := mtp.GetPersistence(orgID) + if err != nil { + t.Fatal(err) + } + return persistence +} + +func TestTrueNASPollerStorageIdentityDiscovery(t *testing.T) { + for _, tc := range []struct { + name, orgID, metadata, peerMetadata string + }{ + {name: "normal_missing_metadata", orgID: "default"}, + {name: "default_malformed_metadata", orgID: "default", metadata: "{broken"}, + {name: "default_missing_metadata_id", orgID: "default", metadata: `{}`}, + {name: "default_metadata_redirect", orgID: "default", metadata: `{"id":"metadata-redirect"}`}, + {name: "unrelated_malformed_metadata", orgID: "default", peerMetadata: "{broken"}, + {name: "tenant_malformed_metadata", orgID: "tenant-a", metadata: "{broken"}, + {name: "tenant_metadata_collision", orgID: "tenant-a", metadata: `{"id":"tenant-b"}`, peerMetadata: `{"id":"tenant-b"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + server := newTrueNASMockServer(t, "storage-owner-a") + t.Cleanup(server.Close) + mtp, _ := newTestTenantPersistence(t) + persistence := discoveryTenant(t, mtp, tc.orgID) + connection := trueNASInstanceForServer(t, "same-local-id", server.URL(), true) + connection.Username = "fixture-owner" + if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil { + t.Fatal(err) + } + if tc.metadata != "" { + setDiscoveryOrgMetadata(t, persistence, tc.metadata) + } + if tc.peerMetadata != "" { + peer := discoveryTenant(t, mtp, "tenant-b") + setDiscoveryOrgMetadata(t, peer, tc.peerMetadata) + } + + // An actual standalone probe reads system.info even when the + // poller's organization discovery cannot reach this saved config. + probe, err := truenas.NewClient(truenas.ClientConfig{ + Host: connection.Host, Port: connection.Port, + UseHTTPS: connection.UseHTTPS, APIKey: connection.APIKey, Username: connection.Username, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + defer probe.Close() + if err := probe.TestConnection(ctx); err != nil { + t.Fatalf("standalone probe failed: %v", err) + } + + poller := NewTrueNASPoller(mtp, 0, nil) + t.Cleanup(func() { + for _, byConnection := range poller.providersByOrg { + for _, provider := range byConnection { + provider.Close() + } + } + }) + poller.syncConnections() + if poller.providersByOrg[tc.orgID][connection.ID] == nil { + projection, _ := json.Marshal(poller.ConnectionSummaries(tc.orgID, []config.TrueNASInstance{connection})[connection.ID]) + t.Fatalf("standalone probe succeeded but saved provider is absent: %s", projection) + } + poller.pollAll(ctx) + summary := poller.ConnectionSummaries(tc.orgID, []config.TrueNASInstance{connection})[connection.ID] + if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Poll.LastAttemptAt == nil || + summary.Observed == nil || summary.Observed.Host != "storage-owner-a" || summary.Transport == nil || + !summary.Transport.Connected || !hasTrueNASHostForOrg(poller, tc.orgID, "storage-owner-a") { + t.Fatalf("ordinary polling did not reach saved connection: %+v", summary) + } + for _, other := range []string{"default", "tenant-b", "metadata-redirect"} { + if other != tc.orgID && len(poller.GetCurrentRecordsForOrg(other)) != 0 { + t.Fatalf("resources escaped storage owner into %s", other) + } + } + if _, err := os.Stat(filepath.Join(mtp.BaseDataDir(), "orgs", "metadata-redirect")); !os.IsNotExist(err) { + t.Fatal("metadata created an unrelated tenant directory") + } + }) + } +} + +func TestTrueNASPollerStorageDiscoveryAppliesRevocationWithBrokenPeerMetadata(t *testing.T) { + for _, action := range []string{"disable", "remove"} { + t.Run(action, func(t *testing.T) { + server := newTrueNASMockServer(t, "revocation-owner") + t.Cleanup(server.Close) + mtp, persistence := newTestTenantPersistence(t) + connection := trueNASInstanceForServer(t, "revocation-id", server.URL(), true) + if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil { + t.Fatal(err) + } + poller := NewTrueNASPoller(mtp, 0, nil) + t.Cleanup(func() { + for _, byConnection := range poller.providersByOrg { + for _, provider := range byConnection { + provider.Close() + } + } + }) + poller.syncConnections() + poller.pollAll(context.Background()) + if !hasTrueNASHostForOrg(poller, "default", "revocation-owner") { + t.Fatal("initial healthy fixture did not poll") + } + before := server.RequestCount() + setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), "{broken") + next := []config.TrueNASInstance{} + if action == "disable" { + connection.Enabled = false + next = append(next, connection) + } + if err := persistence.SaveTrueNASConfig(next); err != nil { + t.Fatal(err) + } + poller.syncConnections() + if poller.providersByOrg["default"][connection.ID] != nil { + t.Fatal("unrelated org metadata blocked authoritative connection revocation") + } + poller.pollAll(context.Background()) + if server.RequestCount() != before || len(poller.GetCurrentRecordsForOrg("default")) != 0 { + t.Fatal("revoked connection still polled or contributed current records") + } + if action == "remove" && poller.statusByOrg["default"][connection.ID] != nil { + t.Fatal("removed connection retained runtime state") + } + }) + } +} + +func TestTrueNASPollerStorageIdentitySeparatesTenantsAndTracksRemoval(t *testing.T) { + mtp, _ := newTestTenantPersistence(t) + servers := map[string]*trueNASMockServer{} + for _, orgID := range []string{"tenant-a", "tenant-b"} { + server := newTrueNASMockServer(t, orgID+"-nas") + t.Cleanup(server.Close) + servers[orgID] = server + connection := trueNASInstanceForServer(t, "same-local-id", server.URL(), true) + persistence := discoveryTenant(t, mtp, orgID) + if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil { + t.Fatal(err) + } + setDiscoveryOrgMetadata(t, persistence, `{"id":"tenant-b"}`) + } + poller := NewTrueNASPoller(mtp, 0, nil) + t.Cleanup(func() { + for _, byConnection := range poller.providersByOrg { + for _, provider := range byConnection { + provider.Close() + } + } + }) + poller.syncConnections() + poller.pollAll(context.Background()) + for _, orgID := range []string{"tenant-a", "tenant-b"} { + if !hasTrueNASHostForOrg(poller, orgID, orgID+"-nas") { + t.Fatalf("missing own tenant's resources: %s", orgID) + } + for _, other := range []string{"tenant-a", "tenant-b"} { + if other != orgID && hasTrueNASHostForOrg(poller, orgID, other+"-nas") { + t.Fatal("same local connection ID merged different tenant inventories") + } + } + } + retained := poller.providersByOrg["tenant-b"]["same-local-id"] + removedRequests := servers["tenant-a"].RequestCount() + if err := mtp.DeleteOrganization("tenant-a"); err != nil { + t.Fatal(err) + } + poller.syncConnections() + poller.pollAll(context.Background()) + if len(poller.GetCurrentRecordsForOrg("tenant-a")) != 0 || len(poller.providersByOrg["tenant-a"]) != 0 || + len(poller.statusByOrg["tenant-a"]) != 0 || servers["tenant-a"].RequestCount() != removedRequests { + t.Fatal("removed tenant retained runtime inventory or kept polling") + } + if poller.providersByOrg["tenant-b"]["same-local-id"] != retained || + !hasTrueNASHostForOrg(poller, "tenant-b", "tenant-b-nas") { + t.Fatal("removed tenant disrupted a different storage owner's provider") + } +} + +func TestTrueNASPollerStorageDiscoveryDoesNotReuseUnreadableConnectionConfig(t *testing.T) { + server := newTrueNASMockServer(t, "config-read-owner") + t.Cleanup(server.Close) + mtp, persistence := newTestTenantPersistence(t) + connection := trueNASInstanceForServer(t, "config-read-id", server.URL(), true) + if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil { + t.Fatal(err) + } + poller := NewTrueNASPoller(mtp, 0, nil) + t.Cleanup(func() { + for _, byConnection := range poller.providersByOrg { + for _, provider := range byConnection { + provider.Close() + } + } + }) + poller.syncConnections() + poller.pollAll(context.Background()) + if !hasTrueNASHostForOrg(poller, "default", "config-read-owner") { + t.Fatal("initial fixture did not poll") + } + before := server.RequestCount() + if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "truenas.enc"), []byte("{broken"), 0o600); err != nil { + t.Fatal(err) + } + poller.syncConnections() + poller.pollAll(context.Background()) + if poller.providersByOrg["default"][connection.ID] != nil || server.RequestCount() != before || + len(poller.GetCurrentRecordsForOrg("default")) != 0 { + t.Fatal("unreadable authoritative config reused old credentials or records") + } +} diff --git a/internal/monitoring/vmware_poller.go b/internal/monitoring/vmware_poller.go index 572a40aa13..fc4a7667bd 100644 --- a/internal/monitoring/vmware_poller.go +++ b/internal/monitoring/vmware_poller.go @@ -193,7 +193,7 @@ func (p *VMwarePoller) syncConnections() { return } - orgs, err := p.multiTenant.ListOrganizations() + orgIDs, err := p.multiTenant.ListOrganizationIDs() if err != nil { log.Warn(). Str("component", "vmware_poller"). @@ -203,15 +203,8 @@ func (p *VMwarePoller) syncConnections() { return } - active := make(map[string]map[string]config.VMwareVCenterInstance, len(orgs)) - for _, org := range orgs { - if org == nil { - continue - } - orgID := strings.TrimSpace(org.ID) - if orgID == "" { - continue - } + active := make(map[string]map[string]config.VMwareVCenterInstance, len(orgIDs)) + for _, orgID := range orgIDs { persistence, err := p.multiTenant.GetPersistence(orgID) if err != nil || persistence == nil { log.Warn(). diff --git a/internal/monitoring/vmware_poller_test.go b/internal/monitoring/vmware_poller_test.go index 448562421d..92a3a649a4 100644 --- a/internal/monitoring/vmware_poller_test.go +++ b/internal/monitoring/vmware_poller_test.go @@ -3,6 +3,8 @@ package monitoring import ( "context" "errors" + "os" + "path/filepath" "strings" "testing" "time" @@ -481,3 +483,104 @@ func captureVMwarePollerLogs(t *testing.T) *monitoringLogCapture { t.Helper() return newMonitoringLogCapture(t) } + +func TestVMwarePollerStorageIdentityDiscovery(t *testing.T) { + for _, tc := range []struct{ name, metadata, peerMetadata string }{ + {name: "normal_missing_metadata"}, + {name: "malformed_metadata", metadata: "{broken"}, + {name: "missing_metadata_id", metadata: `{}`}, + {name: "metadata_redirect", metadata: `{"id":"metadata-redirect"}`}, + {name: "unrelated_malformed_metadata", peerMetadata: "{broken"}, + } { + t.Run(tc.name, func(t *testing.T) { + mtp, persistence := newTestTenantPersistence(t) + connection := config.NewVMwareVCenterInstance() + connection.ID = "vc-storage-id" + connection.Host = "fixture-vcenter" + connection.Username = "fixture-owner" + connection.Password = "fixture-password" + if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { + t.Fatal(err) + } + if tc.metadata != "" { + setDiscoveryOrgMetadata(t, persistence, tc.metadata) + } + if tc.peerMetadata != "" { + setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), tc.peerMetadata) + } + poller := NewVMwarePoller(mtp, 0) + poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) { + return vmware.NewProvider(vmware.InventorySnapshot{ + ConnectionID: instance.ID, CollectedAt: time.Now().UTC(), + Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}}, + }), nil + } + t.Cleanup(func() { + for _, byConnection := range poller.providersByOrg { + for _, provider := range byConnection { + provider.Close() + } + } + }) + poller.syncConnections() + poller.pollAll(context.Background()) + summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] + if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Observed == nil || + summary.Observed.Hosts != 1 || len(poller.GetCurrentRecordsForOrg("default")) != 1 { + t.Fatalf("saved VMware config was not discovered independently of metadata: %+v", summary) + } + if _, err := os.Stat(filepath.Join(mtp.BaseDataDir(), "orgs", "metadata-redirect")); !os.IsNotExist(err) { + t.Fatal("metadata created an unrelated tenant directory") + } + }) + } +} + +func TestVMwarePollerStorageDiscoveryAppliesRevocationWithBrokenPeerMetadata(t *testing.T) { + for _, action := range []string{"disable", "remove", "unreadable_config"} { + t.Run(action, func(t *testing.T) { + mtp, persistence := newTestTenantPersistence(t) + connection := config.NewVMwareVCenterInstance() + connection.ID = "vc-storage-id" + connection.Host = "fixture-vcenter" + connection.Username = "fixture-owner" + connection.Password = "fixture-password" + if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { + t.Fatal(err) + } + poller := NewVMwarePoller(mtp, 0) + poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) { + return vmware.NewProvider(vmware.InventorySnapshot{ + ConnectionID: instance.ID, CollectedAt: time.Now().UTC(), + Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}}, + }), nil + } + poller.syncConnections() + poller.pollAll(context.Background()) + if len(poller.GetCurrentRecordsForOrg("default")) != 1 { + t.Fatal("initial fixture did not poll") + } + setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), "{broken") + switch action { + case "disable": + connection.Enabled = false + if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { + t.Fatal(err) + } + case "remove": + if err := persistence.SaveVMwareConfig(nil); err != nil { + t.Fatal(err) + } + case "unreadable_config": + if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "vmware.enc"), []byte("{broken"), 0o600); err != nil { + t.Fatal(err) + } + } + poller.syncConnections() + poller.pollAll(context.Background()) + if poller.providersByOrg["default"][connection.ID] != nil || len(poller.GetCurrentRecordsForOrg("default")) != 0 { + t.Fatal("unrelated metadata blocked fail-closed config reconciliation") + } + }) + } +}