Keep future backup clocks out of verified rollup claims

Use the existing unknown assessment when success or verification dates lie beyond evaluation. Keep original dates and scope intact, retain inclusive window semantics, and recover through ordinary corrected reads.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-10-28 23:26:44 +00:00
parent 529d51ff23
commit 6fdfd1b2eb
5 changed files with 223 additions and 6 deletions
@@ -643,6 +643,23 @@ synthetic poll, async ingest, requested and state-filtered posture reads, and
clock correction. No clock mutation, backup execution or native recovery is
established by these fixtures.
The existing recovery rollup API uses the same upper clock boundary. Its shared
`ComputeVerifyIntentAt` classifier returns `unknown` when the success or newest
verification date is later than evaluation, instead of a false `verified` or a
stale-age claim. The existing seven-day inclusive verification window and
`verified`/`stale`/`unknown` wire values are unchanged. Both persisted and in-memory
rollups retain the original success, verification and attempt timestamps and
subject/provider scope; a normal corrected source date recovers on the next read.
This is a machine-API assessment correction, not a new frontend surface or an
installed/native backup recovery claim.
`recovery_test.go` covers future, absent, invalid, equal-time and window-boundary
inputs, extreme timestamps, five provider types and in-place correction without
rewriting evidence. `recovery_handlers_test.go` exercises real persisted writes,
org-isolated rollup reads and the canonical/compatibility HTTP payload through
future and corrected dates. The current frontend does not consume this rollup
intent; Coverage's separately assessed posture and its pixels are unchanged.
### Canonical posture recognises recovery's Proxmox subject aliases
Protection posture selects stored points and publishes its subject ID using
+111
View File
@@ -1,9 +1,11 @@
package api
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
@@ -703,3 +705,112 @@ func joinCSV(parts []string) string {
func timePtr(t time.Time) *time.Time {
return &t
}
// Exercise the shared classifier through real persisted writes, tenant-scoped
// rollup reads and the public wire projection. Future source dates are evidence,
// not verified protection; ordinary date correction must recover in place.
func TestHandleListRollupsFutureClockEvidenceAndCorrection(t *testing.T) {
setMockModeForTest(t, false)
now := time.Now().UTC().Truncate(time.Millisecond)
recent, future := now.Add(-time.Hour), now.Add(24*time.Hour)
for _, provider := range []recovery.Provider{recovery.ProviderProxmoxPVE, recovery.ProviderProxmoxPBS} {
for _, verified := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/verified=%t", provider, verified), func(t *testing.T) {
manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
store, err := manager.StoreForOrg("default")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
other, err := manager.StoreForOrg("clock-other")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = other.Close() })
point := recovery.RecoveryPoint{ID: "clock-point", Provider: provider, ProviderScope: "clock-source", Kind: recovery.KindBackup, Mode: recovery.ModeRemote,
Outcome: recovery.OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest"}
otherPoint := point
otherPoint.CompletedAt = &recent
if err := other.UpsertPoints(context.Background(), []recovery.RecoveryPoint{otherPoint}); err != nil {
t.Fatal(err)
}
handler := NewRecoveryHandlers(manager)
read := func(org string) recoveryRollupPayload {
t.Helper()
request := httptest.NewRequest(http.MethodGet, "/api/recovery/rollups?limit=500", nil)
request = request.WithContext(context.WithValue(request.Context(), OrgIDContextKey, org))
response := httptest.NewRecorder()
handler.HandleListRollups(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status=%d, body=%s", response.Code, response.Body.String())
}
var payload struct {
Data []recoveryRollupPayload `json:"data"`
Meta struct {
Total int `json:"total"`
} `json:"meta"`
}
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Data) != 1 || payload.Meta.Total != 1 {
t.Fatalf("response=%s", response.Body.String())
}
return payload.Data[0]
}
check := func(got recoveryRollupPayload, intent recovery.VerifyIntent, at time.Time) {
t.Helper()
if got.VerifyIntent != intent {
t.Errorf("intent=%q, want %q", got.VerifyIntent, intent)
}
if got.RollupID != "res:clock-guest" || got.ItemResourceID != "clock-guest" || got.SubjectResourceID != got.ItemResourceID || len(got.Providers) != 1 || got.Providers[0] != provider || len(got.Platforms) != 1 || got.Platforms[0] != provider {
t.Errorf("canonical/compatibility scope changed: %+v", got)
}
if got.LastSuccessAt == nil || !got.LastSuccessAt.Equal(at) || got.LastAttemptAt == nil || !got.LastAttemptAt.Equal(at) || got.LastOutcome != recovery.OutcomeSuccess {
t.Errorf("source evidence changed: %+v", got)
}
if verified {
if got.LastVerifiedAt == nil || !got.LastVerifiedAt.Equal(at) {
t.Errorf("verification date changed: %+v", got)
}
} else if got.LastVerifiedAt != nil {
t.Error("unobserved verification gained a timestamp")
}
}
if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {
t.Fatal(err)
}
before, err := json.Marshal(point)
if err != nil {
t.Fatal(err)
}
check(read("default"), recovery.VerifyIntentUnknown, future)
check(read("default"), recovery.VerifyIntentUnknown, future)
wantCorrected := recovery.VerifyIntentStale
if verified {
wantCorrected = recovery.VerifyIntentVerified
}
check(read("clock-other"), wantCorrected, recent)
stored, total, err := store.ListPoints(context.Background(), recovery.ListPointsOptions{Page: 1, Limit: 50})
if err != nil || total != 1 || len(stored) != 1 {
t.Fatalf("point read: %d/%d/%v", total, len(stored), err)
}
// Persistence adds a derived display projection, but the measured completion,
// verification, source identity and scope must survive these read-only calls.
if stored[0].CompletedAt == nil || !stored[0].CompletedAt.Equal(future) || stored[0].Verified == nil || *stored[0].Verified != verified || stored[0].ID != point.ID || stored[0].ProviderScope != point.ProviderScope {
t.Fatalf("stored evidence changed: %+v", stored[0])
}
after, err := json.Marshal(point)
if err != nil || !bytes.Equal(before, after) {
t.Fatal("read changed the source input")
}
point.CompletedAt = &recent
if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {
t.Fatal(err)
}
check(read("default"), wantCorrected, recent)
check(read("clock-other"), wantCorrected, recent)
})
}
}
}
+3 -3
View File
@@ -121,13 +121,13 @@ type VerifyIntent string
const (
// VerifyIntentVerified means a verification-bearing successful recovery
// point exists within the staleness window.
// point exists at or before evaluation time within the staleness window.
VerifyIntentVerified VerifyIntent = "verified"
// VerifyIntentStale means a successful backup exists for the subject but
// no verification-bearing point has landed within the staleness window.
VerifyIntentStale VerifyIntent = "stale"
// VerifyIntentUnknown means there is no information one way or the other
// (no successful backup, or no verification-bearing points at all).
// VerifyIntentUnknown means there is no successful backup, or a reported
// success/verification date is in the future and its age cannot be assessed.
VerifyIntentUnknown VerifyIntent = "unknown"
)
+86
View File
@@ -1,6 +1,8 @@
package recovery
import (
"math"
"reflect"
"strings"
"testing"
"time"
@@ -1247,3 +1249,87 @@ func TestComputeVerifyIntentAt_BoundaryEqualsWindow(t *testing.T) {
t.Fatalf("beyond boundary lastVerifiedAt = nil, want non-nil so UI can render history")
}
}
func TestRollupVerificationClockBoundaries(t *testing.T) {
now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
recent := now.Add(-time.Hour).UnixMilli()
boundary := now.Add(-BackupVerifyStaleWindow).UnixMilli()
for _, tc := range []struct {
name string
success, verified int64
want VerifyIntent
}{
{"recent", recent, recent, VerifyIntentVerified},
{"at-evaluation", now.UnixMilli(), now.UnixMilli(), VerifyIntentVerified},
{"inclusive-window", boundary, boundary, VerifyIntentVerified},
{"past-window", recent, boundary - 1, VerifyIntentStale},
{"no-verification", recent, 0, VerifyIntentStale},
{"invalid-verification", recent, -1, VerifyIntentStale},
{"no-success", 0, recent, VerifyIntentUnknown},
{"invalid-success", -1, recent, VerifyIntentUnknown},
{"future-verification", recent, now.UnixMilli() + 1, VerifyIntentUnknown},
{"future-success", now.UnixMilli() + 1, recent, VerifyIntentUnknown},
{"future-success-unverified", now.UnixMilli() + 1, 0, VerifyIntentUnknown},
{"both-future", now.UnixMilli() + 1, now.UnixMilli() + 1, VerifyIntentUnknown},
{"extreme-future-verification", recent, math.MaxInt64, VerifyIntentUnknown},
{"extreme-future-success", math.MaxInt64, recent, VerifyIntentUnknown},
} {
t.Run(tc.name, func(t *testing.T) {
intent, verified := ComputeVerifyIntentAt(tc.success, tc.verified, now)
if intent != tc.want {
t.Errorf("intent=%q, want %q", intent, tc.want)
}
if tc.verified <= 0 {
if verified != nil {
t.Fatalf("unobserved verification became %v", verified)
}
} else if verified == nil || verified.UnixMilli() != tc.verified {
t.Fatalf("source verification date rewritten: %v, want %d", verified, tc.verified)
}
})
}
}
func TestRollupFutureDateCorrectionPreservesEvidence(t *testing.T) {
now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
recent, future := now.Add(-time.Hour), now.Add(time.Hour)
verified := true
for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS, ProviderProxmoxPMG, ProviderTrueNAS, ProviderKubernetes} {
t.Run(string(provider), func(t *testing.T) {
point := RecoveryPoint{ID: "clock-backup", Provider: provider, Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess,
CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest", ProviderScope: "clock-estate"}
points := []RecoveryPoint{point}
before := []RecoveryPoint{point}
rollups := BuildRollupsFromPointsAt(points, now)
if len(rollups) != 1 {
t.Fatalf("rollups=%+v", rollups)
}
r := rollups[0]
if r.VerifyIntent != VerifyIntentUnknown {
t.Errorf("future intent=%q, want unknown", r.VerifyIntent)
}
if r.LastSuccessAt == nil || !r.LastSuccessAt.Equal(future) || r.LastVerifiedAt == nil || !r.LastVerifiedAt.Equal(future) || r.LastAttemptAt == nil || !r.LastAttemptAt.Equal(future) {
t.Fatalf("future evidence lost: %+v", r)
}
if r.RollupID != "res:clock-guest" || r.SubjectResourceID != point.SubjectResourceID || len(r.Providers) != 1 || r.Providers[0] != provider || !reflect.DeepEqual(points, before) {
t.Fatalf("scope, identity or input changed: %+v", r)
}
// Merely collecting the same point again must not invent a new date.
again := BuildRollupsFromPointsAt(points, now)
if !reflect.DeepEqual(rollups, again) {
t.Fatal("ordinary reread changed evidence")
}
// Only a corrected source date (or actual time reaching the reported instant)
// makes this same verified point eligible for a verified assessment.
points[0].CompletedAt = &recent
corrected := BuildRollupsFromPointsAt(points, now)[0]
if corrected.VerifyIntent != VerifyIntentVerified || corrected.LastVerifiedAt == nil || !corrected.LastVerifiedAt.Equal(recent) || corrected.RollupID != r.RollupID {
t.Fatalf("corrected date did not recover in place: %+v", corrected)
}
atReportedTime := BuildRollupsFromPointsAt(before, future)[0]
if atReportedTime.VerifyIntent != VerifyIntentVerified || !atReportedTime.LastVerifiedAt.Equal(future) {
t.Fatal("evaluation did not recover at reported time")
}
})
}
}
+6 -3
View File
@@ -21,10 +21,10 @@ const BackupVerifyStaleWindow = 7 * 24 * time.Hour
// Semantics:
// - verified : a verification-bearing point exists within window of now
// - stale : a successful backup exists, but no verification within window
// - unknown : no successful backup at all (or no info)
// - unknown : no successful backup, or a success/verification date is in the future
//
// LastVerifiedAt is returned regardless of intent (when known) so consumers
// can surface "last verified X days ago" alongside the badge.
// can retain the source date even when clock uncertainty prevents an assessment.
func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (VerifyIntent, *time.Time) {
windowMs := BackupVerifyStaleWindow.Milliseconds()
nowMs := now.UTC().UnixMilli()
@@ -35,7 +35,10 @@ func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (
lastVerifiedAt = &t
}
if lastSuccessMs <= 0 {
// A future date proves no recency. Keep the reported date as evidence,
// but align the rollup with posture: clock uncertainty is neither verified
// nor stale. Check before subtraction so extreme dates cannot overflow age.
if lastSuccessMs <= 0 || lastSuccessMs > nowMs || lastVerifiedMs > nowMs {
return VerifyIntentUnknown, lastVerifiedAt
}