diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index c88a249966..56791f981f 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -643,6 +643,23 @@ synthetic poll, async ingest, requested and state-filtered posture reads, and clock correction. No clock mutation, backup execution or native recovery is established by these fixtures. +The existing recovery rollup API uses the same upper clock boundary. Its shared +`ComputeVerifyIntentAt` classifier returns `unknown` when the success or newest +verification date is later than evaluation, instead of a false `verified` or a +stale-age claim. The existing seven-day inclusive verification window and +`verified`/`stale`/`unknown` wire values are unchanged. Both persisted and in-memory +rollups retain the original success, verification and attempt timestamps and +subject/provider scope; a normal corrected source date recovers on the next read. +This is a machine-API assessment correction, not a new frontend surface or an +installed/native backup recovery claim. + +`recovery_test.go` covers future, absent, invalid, equal-time and window-boundary +inputs, extreme timestamps, five provider types and in-place correction without +rewriting evidence. `recovery_handlers_test.go` exercises real persisted writes, +org-isolated rollup reads and the canonical/compatibility HTTP payload through +future and corrected dates. The current frontend does not consume this rollup +intent; Coverage's separately assessed posture and its pixels are unchanged. + ### Canonical posture recognises recovery's Proxmox subject aliases Protection posture selects stored points and publishes its subject ID using diff --git a/internal/api/recovery_handlers_test.go b/internal/api/recovery_handlers_test.go index fe8c6d66f1..baee5216e1 100644 --- a/internal/api/recovery_handlers_test.go +++ b/internal/api/recovery_handlers_test.go @@ -1,9 +1,11 @@ package api import ( + "bytes" "context" "database/sql" "encoding/json" + "fmt" "net/http" "net/http/httptest" "net/url" @@ -703,3 +705,112 @@ func joinCSV(parts []string) string { func timePtr(t time.Time) *time.Time { return &t } + +// Exercise the shared classifier through real persisted writes, tenant-scoped +// rollup reads and the public wire projection. Future source dates are evidence, +// not verified protection; ordinary date correction must recover in place. +func TestHandleListRollupsFutureClockEvidenceAndCorrection(t *testing.T) { + setMockModeForTest(t, false) + now := time.Now().UTC().Truncate(time.Millisecond) + recent, future := now.Add(-time.Hour), now.Add(24*time.Hour) + for _, provider := range []recovery.Provider{recovery.ProviderProxmoxPVE, recovery.ProviderProxmoxPBS} { + for _, verified := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/verified=%t", provider, verified), func(t *testing.T) { + manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir())) + store, err := manager.StoreForOrg("default") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + other, err := manager.StoreForOrg("clock-other") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = other.Close() }) + point := recovery.RecoveryPoint{ID: "clock-point", Provider: provider, ProviderScope: "clock-source", Kind: recovery.KindBackup, Mode: recovery.ModeRemote, + Outcome: recovery.OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest"} + otherPoint := point + otherPoint.CompletedAt = &recent + if err := other.UpsertPoints(context.Background(), []recovery.RecoveryPoint{otherPoint}); err != nil { + t.Fatal(err) + } + handler := NewRecoveryHandlers(manager) + read := func(org string) recoveryRollupPayload { + t.Helper() + request := httptest.NewRequest(http.MethodGet, "/api/recovery/rollups?limit=500", nil) + request = request.WithContext(context.WithValue(request.Context(), OrgIDContextKey, org)) + response := httptest.NewRecorder() + handler.HandleListRollups(response, request) + if response.Code != http.StatusOK { + t.Fatalf("status=%d, body=%s", response.Code, response.Body.String()) + } + var payload struct { + Data []recoveryRollupPayload `json:"data"` + Meta struct { + Total int `json:"total"` + } `json:"meta"` + } + if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil { + t.Fatal(err) + } + if len(payload.Data) != 1 || payload.Meta.Total != 1 { + t.Fatalf("response=%s", response.Body.String()) + } + return payload.Data[0] + } + check := func(got recoveryRollupPayload, intent recovery.VerifyIntent, at time.Time) { + t.Helper() + if got.VerifyIntent != intent { + t.Errorf("intent=%q, want %q", got.VerifyIntent, intent) + } + if got.RollupID != "res:clock-guest" || got.ItemResourceID != "clock-guest" || got.SubjectResourceID != got.ItemResourceID || len(got.Providers) != 1 || got.Providers[0] != provider || len(got.Platforms) != 1 || got.Platforms[0] != provider { + t.Errorf("canonical/compatibility scope changed: %+v", got) + } + if got.LastSuccessAt == nil || !got.LastSuccessAt.Equal(at) || got.LastAttemptAt == nil || !got.LastAttemptAt.Equal(at) || got.LastOutcome != recovery.OutcomeSuccess { + t.Errorf("source evidence changed: %+v", got) + } + if verified { + if got.LastVerifiedAt == nil || !got.LastVerifiedAt.Equal(at) { + t.Errorf("verification date changed: %+v", got) + } + } else if got.LastVerifiedAt != nil { + t.Error("unobserved verification gained a timestamp") + } + } + if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil { + t.Fatal(err) + } + before, err := json.Marshal(point) + if err != nil { + t.Fatal(err) + } + check(read("default"), recovery.VerifyIntentUnknown, future) + check(read("default"), recovery.VerifyIntentUnknown, future) + wantCorrected := recovery.VerifyIntentStale + if verified { + wantCorrected = recovery.VerifyIntentVerified + } + check(read("clock-other"), wantCorrected, recent) + stored, total, err := store.ListPoints(context.Background(), recovery.ListPointsOptions{Page: 1, Limit: 50}) + if err != nil || total != 1 || len(stored) != 1 { + t.Fatalf("point read: %d/%d/%v", total, len(stored), err) + } + // Persistence adds a derived display projection, but the measured completion, + // verification, source identity and scope must survive these read-only calls. + if stored[0].CompletedAt == nil || !stored[0].CompletedAt.Equal(future) || stored[0].Verified == nil || *stored[0].Verified != verified || stored[0].ID != point.ID || stored[0].ProviderScope != point.ProviderScope { + t.Fatalf("stored evidence changed: %+v", stored[0]) + } + after, err := json.Marshal(point) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("read changed the source input") + } + point.CompletedAt = &recent + if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil { + t.Fatal(err) + } + check(read("default"), wantCorrected, recent) + check(read("clock-other"), wantCorrected, recent) + }) + } + } +} diff --git a/internal/recovery/model/types.go b/internal/recovery/model/types.go index 27970b9611..8a1e56ce11 100644 --- a/internal/recovery/model/types.go +++ b/internal/recovery/model/types.go @@ -121,13 +121,13 @@ type VerifyIntent string const ( // VerifyIntentVerified means a verification-bearing successful recovery - // point exists within the staleness window. + // point exists at or before evaluation time within the staleness window. VerifyIntentVerified VerifyIntent = "verified" // VerifyIntentStale means a successful backup exists for the subject but // no verification-bearing point has landed within the staleness window. VerifyIntentStale VerifyIntent = "stale" - // VerifyIntentUnknown means there is no information one way or the other - // (no successful backup, or no verification-bearing points at all). + // VerifyIntentUnknown means there is no successful backup, or a reported + // success/verification date is in the future and its age cannot be assessed. VerifyIntentUnknown VerifyIntent = "unknown" ) diff --git a/internal/recovery/recovery_test.go b/internal/recovery/recovery_test.go index 4c82821cc1..f8512f23ca 100644 --- a/internal/recovery/recovery_test.go +++ b/internal/recovery/recovery_test.go @@ -1,6 +1,8 @@ package recovery import ( + "math" + "reflect" "strings" "testing" "time" @@ -1247,3 +1249,87 @@ func TestComputeVerifyIntentAt_BoundaryEqualsWindow(t *testing.T) { t.Fatalf("beyond boundary lastVerifiedAt = nil, want non-nil so UI can render history") } } + +func TestRollupVerificationClockBoundaries(t *testing.T) { + now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC) + recent := now.Add(-time.Hour).UnixMilli() + boundary := now.Add(-BackupVerifyStaleWindow).UnixMilli() + for _, tc := range []struct { + name string + success, verified int64 + want VerifyIntent + }{ + {"recent", recent, recent, VerifyIntentVerified}, + {"at-evaluation", now.UnixMilli(), now.UnixMilli(), VerifyIntentVerified}, + {"inclusive-window", boundary, boundary, VerifyIntentVerified}, + {"past-window", recent, boundary - 1, VerifyIntentStale}, + {"no-verification", recent, 0, VerifyIntentStale}, + {"invalid-verification", recent, -1, VerifyIntentStale}, + {"no-success", 0, recent, VerifyIntentUnknown}, + {"invalid-success", -1, recent, VerifyIntentUnknown}, + {"future-verification", recent, now.UnixMilli() + 1, VerifyIntentUnknown}, + {"future-success", now.UnixMilli() + 1, recent, VerifyIntentUnknown}, + {"future-success-unverified", now.UnixMilli() + 1, 0, VerifyIntentUnknown}, + {"both-future", now.UnixMilli() + 1, now.UnixMilli() + 1, VerifyIntentUnknown}, + {"extreme-future-verification", recent, math.MaxInt64, VerifyIntentUnknown}, + {"extreme-future-success", math.MaxInt64, recent, VerifyIntentUnknown}, + } { + t.Run(tc.name, func(t *testing.T) { + intent, verified := ComputeVerifyIntentAt(tc.success, tc.verified, now) + if intent != tc.want { + t.Errorf("intent=%q, want %q", intent, tc.want) + } + if tc.verified <= 0 { + if verified != nil { + t.Fatalf("unobserved verification became %v", verified) + } + } else if verified == nil || verified.UnixMilli() != tc.verified { + t.Fatalf("source verification date rewritten: %v, want %d", verified, tc.verified) + } + }) + } +} + +func TestRollupFutureDateCorrectionPreservesEvidence(t *testing.T) { + now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC) + recent, future := now.Add(-time.Hour), now.Add(time.Hour) + verified := true + for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS, ProviderProxmoxPMG, ProviderTrueNAS, ProviderKubernetes} { + t.Run(string(provider), func(t *testing.T) { + point := RecoveryPoint{ID: "clock-backup", Provider: provider, Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess, + CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest", ProviderScope: "clock-estate"} + points := []RecoveryPoint{point} + before := []RecoveryPoint{point} + rollups := BuildRollupsFromPointsAt(points, now) + if len(rollups) != 1 { + t.Fatalf("rollups=%+v", rollups) + } + r := rollups[0] + if r.VerifyIntent != VerifyIntentUnknown { + t.Errorf("future intent=%q, want unknown", r.VerifyIntent) + } + if r.LastSuccessAt == nil || !r.LastSuccessAt.Equal(future) || r.LastVerifiedAt == nil || !r.LastVerifiedAt.Equal(future) || r.LastAttemptAt == nil || !r.LastAttemptAt.Equal(future) { + t.Fatalf("future evidence lost: %+v", r) + } + if r.RollupID != "res:clock-guest" || r.SubjectResourceID != point.SubjectResourceID || len(r.Providers) != 1 || r.Providers[0] != provider || !reflect.DeepEqual(points, before) { + t.Fatalf("scope, identity or input changed: %+v", r) + } + // Merely collecting the same point again must not invent a new date. + again := BuildRollupsFromPointsAt(points, now) + if !reflect.DeepEqual(rollups, again) { + t.Fatal("ordinary reread changed evidence") + } + // Only a corrected source date (or actual time reaching the reported instant) + // makes this same verified point eligible for a verified assessment. + points[0].CompletedAt = &recent + corrected := BuildRollupsFromPointsAt(points, now)[0] + if corrected.VerifyIntent != VerifyIntentVerified || corrected.LastVerifiedAt == nil || !corrected.LastVerifiedAt.Equal(recent) || corrected.RollupID != r.RollupID { + t.Fatalf("corrected date did not recover in place: %+v", corrected) + } + atReportedTime := BuildRollupsFromPointsAt(before, future)[0] + if atReportedTime.VerifyIntent != VerifyIntentVerified || !atReportedTime.LastVerifiedAt.Equal(future) { + t.Fatal("evaluation did not recover at reported time") + } + }) + } +} diff --git a/internal/recovery/rollups.go b/internal/recovery/rollups.go index e1ee167898..cbbec8091d 100644 --- a/internal/recovery/rollups.go +++ b/internal/recovery/rollups.go @@ -21,10 +21,10 @@ const BackupVerifyStaleWindow = 7 * 24 * time.Hour // Semantics: // - verified : a verification-bearing point exists within window of now // - stale : a successful backup exists, but no verification within window -// - unknown : no successful backup at all (or no info) +// - unknown : no successful backup, or a success/verification date is in the future // // LastVerifiedAt is returned regardless of intent (when known) so consumers -// can surface "last verified X days ago" alongside the badge. +// can retain the source date even when clock uncertainty prevents an assessment. func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (VerifyIntent, *time.Time) { windowMs := BackupVerifyStaleWindow.Milliseconds() nowMs := now.UTC().UnixMilli() @@ -35,7 +35,10 @@ func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) ( lastVerifiedAt = &t } - if lastSuccessMs <= 0 { + // A future date proves no recency. Keep the reported date as evidence, + // but align the rollup with posture: clock uncertainty is neither verified + // nor stale. Check before subtraction so extreme dates cannot overflow age. + if lastSuccessMs <= 0 || lastSuccessMs > nowMs || lastVerifiedMs > nowMs { return VerifyIntentUnknown, lastVerifiedAt }