mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-11-08 11:03:31 +00:00
Keep future backup clocks out of verified rollup claims
Use the existing unknown assessment when success or verification dates lie beyond evaluation. Keep original dates and scope intact, retain inclusive window semantics, and recover through ordinary corrected reads. Change-source: pulse-maintainer
This commit is contained in:
@@ -643,6 +643,23 @@ synthetic poll, async ingest, requested and state-filtered posture reads, and
|
||||
clock correction. No clock mutation, backup execution or native recovery is
|
||||
established by these fixtures.
|
||||
|
||||
The existing recovery rollup API uses the same upper clock boundary. Its shared
|
||||
`ComputeVerifyIntentAt` classifier returns `unknown` when the success or newest
|
||||
verification date is later than evaluation, instead of a false `verified` or a
|
||||
stale-age claim. The existing seven-day inclusive verification window and
|
||||
`verified`/`stale`/`unknown` wire values are unchanged. Both persisted and in-memory
|
||||
rollups retain the original success, verification and attempt timestamps and
|
||||
subject/provider scope; a normal corrected source date recovers on the next read.
|
||||
This is a machine-API assessment correction, not a new frontend surface or an
|
||||
installed/native backup recovery claim.
|
||||
|
||||
`recovery_test.go` covers future, absent, invalid, equal-time and window-boundary
|
||||
inputs, extreme timestamps, five provider types and in-place correction without
|
||||
rewriting evidence. `recovery_handlers_test.go` exercises real persisted writes,
|
||||
org-isolated rollup reads and the canonical/compatibility HTTP payload through
|
||||
future and corrected dates. The current frontend does not consume this rollup
|
||||
intent; Coverage's separately assessed posture and its pixels are unchanged.
|
||||
|
||||
### Canonical posture recognises recovery's Proxmox subject aliases
|
||||
|
||||
Protection posture selects stored points and publishes its subject ID using
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -703,3 +705,112 @@ func joinCSV(parts []string) string {
|
||||
func timePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
|
||||
// Exercise the shared classifier through real persisted writes, tenant-scoped
|
||||
// rollup reads and the public wire projection. Future source dates are evidence,
|
||||
// not verified protection; ordinary date correction must recover in place.
|
||||
func TestHandleListRollupsFutureClockEvidenceAndCorrection(t *testing.T) {
|
||||
setMockModeForTest(t, false)
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
recent, future := now.Add(-time.Hour), now.Add(24*time.Hour)
|
||||
for _, provider := range []recovery.Provider{recovery.ProviderProxmoxPVE, recovery.ProviderProxmoxPBS} {
|
||||
for _, verified := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/verified=%t", provider, verified), func(t *testing.T) {
|
||||
manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
|
||||
store, err := manager.StoreForOrg("default")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = store.Close() })
|
||||
other, err := manager.StoreForOrg("clock-other")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = other.Close() })
|
||||
point := recovery.RecoveryPoint{ID: "clock-point", Provider: provider, ProviderScope: "clock-source", Kind: recovery.KindBackup, Mode: recovery.ModeRemote,
|
||||
Outcome: recovery.OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest"}
|
||||
otherPoint := point
|
||||
otherPoint.CompletedAt = &recent
|
||||
if err := other.UpsertPoints(context.Background(), []recovery.RecoveryPoint{otherPoint}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := NewRecoveryHandlers(manager)
|
||||
read := func(org string) recoveryRollupPayload {
|
||||
t.Helper()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/recovery/rollups?limit=500", nil)
|
||||
request = request.WithContext(context.WithValue(request.Context(), OrgIDContextKey, org))
|
||||
response := httptest.NewRecorder()
|
||||
handler.HandleListRollups(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d, body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
Data []recoveryRollupPayload `json:"data"`
|
||||
Meta struct {
|
||||
Total int `json:"total"`
|
||||
} `json:"meta"`
|
||||
}
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Data) != 1 || payload.Meta.Total != 1 {
|
||||
t.Fatalf("response=%s", response.Body.String())
|
||||
}
|
||||
return payload.Data[0]
|
||||
}
|
||||
check := func(got recoveryRollupPayload, intent recovery.VerifyIntent, at time.Time) {
|
||||
t.Helper()
|
||||
if got.VerifyIntent != intent {
|
||||
t.Errorf("intent=%q, want %q", got.VerifyIntent, intent)
|
||||
}
|
||||
if got.RollupID != "res:clock-guest" || got.ItemResourceID != "clock-guest" || got.SubjectResourceID != got.ItemResourceID || len(got.Providers) != 1 || got.Providers[0] != provider || len(got.Platforms) != 1 || got.Platforms[0] != provider {
|
||||
t.Errorf("canonical/compatibility scope changed: %+v", got)
|
||||
}
|
||||
if got.LastSuccessAt == nil || !got.LastSuccessAt.Equal(at) || got.LastAttemptAt == nil || !got.LastAttemptAt.Equal(at) || got.LastOutcome != recovery.OutcomeSuccess {
|
||||
t.Errorf("source evidence changed: %+v", got)
|
||||
}
|
||||
if verified {
|
||||
if got.LastVerifiedAt == nil || !got.LastVerifiedAt.Equal(at) {
|
||||
t.Errorf("verification date changed: %+v", got)
|
||||
}
|
||||
} else if got.LastVerifiedAt != nil {
|
||||
t.Error("unobserved verification gained a timestamp")
|
||||
}
|
||||
}
|
||||
if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := json.Marshal(point)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check(read("default"), recovery.VerifyIntentUnknown, future)
|
||||
check(read("default"), recovery.VerifyIntentUnknown, future)
|
||||
wantCorrected := recovery.VerifyIntentStale
|
||||
if verified {
|
||||
wantCorrected = recovery.VerifyIntentVerified
|
||||
}
|
||||
check(read("clock-other"), wantCorrected, recent)
|
||||
stored, total, err := store.ListPoints(context.Background(), recovery.ListPointsOptions{Page: 1, Limit: 50})
|
||||
if err != nil || total != 1 || len(stored) != 1 {
|
||||
t.Fatalf("point read: %d/%d/%v", total, len(stored), err)
|
||||
}
|
||||
// Persistence adds a derived display projection, but the measured completion,
|
||||
// verification, source identity and scope must survive these read-only calls.
|
||||
if stored[0].CompletedAt == nil || !stored[0].CompletedAt.Equal(future) || stored[0].Verified == nil || *stored[0].Verified != verified || stored[0].ID != point.ID || stored[0].ProviderScope != point.ProviderScope {
|
||||
t.Fatalf("stored evidence changed: %+v", stored[0])
|
||||
}
|
||||
after, err := json.Marshal(point)
|
||||
if err != nil || !bytes.Equal(before, after) {
|
||||
t.Fatal("read changed the source input")
|
||||
}
|
||||
point.CompletedAt = &recent
|
||||
if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check(read("default"), wantCorrected, recent)
|
||||
check(read("clock-other"), wantCorrected, recent)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,13 +121,13 @@ type VerifyIntent string
|
||||
|
||||
const (
|
||||
// VerifyIntentVerified means a verification-bearing successful recovery
|
||||
// point exists within the staleness window.
|
||||
// point exists at or before evaluation time within the staleness window.
|
||||
VerifyIntentVerified VerifyIntent = "verified"
|
||||
// VerifyIntentStale means a successful backup exists for the subject but
|
||||
// no verification-bearing point has landed within the staleness window.
|
||||
VerifyIntentStale VerifyIntent = "stale"
|
||||
// VerifyIntentUnknown means there is no information one way or the other
|
||||
// (no successful backup, or no verification-bearing points at all).
|
||||
// VerifyIntentUnknown means there is no successful backup, or a reported
|
||||
// success/verification date is in the future and its age cannot be assessed.
|
||||
VerifyIntentUnknown VerifyIntent = "unknown"
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -1247,3 +1249,87 @@ func TestComputeVerifyIntentAt_BoundaryEqualsWindow(t *testing.T) {
|
||||
t.Fatalf("beyond boundary lastVerifiedAt = nil, want non-nil so UI can render history")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollupVerificationClockBoundaries(t *testing.T) {
|
||||
now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
|
||||
recent := now.Add(-time.Hour).UnixMilli()
|
||||
boundary := now.Add(-BackupVerifyStaleWindow).UnixMilli()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
success, verified int64
|
||||
want VerifyIntent
|
||||
}{
|
||||
{"recent", recent, recent, VerifyIntentVerified},
|
||||
{"at-evaluation", now.UnixMilli(), now.UnixMilli(), VerifyIntentVerified},
|
||||
{"inclusive-window", boundary, boundary, VerifyIntentVerified},
|
||||
{"past-window", recent, boundary - 1, VerifyIntentStale},
|
||||
{"no-verification", recent, 0, VerifyIntentStale},
|
||||
{"invalid-verification", recent, -1, VerifyIntentStale},
|
||||
{"no-success", 0, recent, VerifyIntentUnknown},
|
||||
{"invalid-success", -1, recent, VerifyIntentUnknown},
|
||||
{"future-verification", recent, now.UnixMilli() + 1, VerifyIntentUnknown},
|
||||
{"future-success", now.UnixMilli() + 1, recent, VerifyIntentUnknown},
|
||||
{"future-success-unverified", now.UnixMilli() + 1, 0, VerifyIntentUnknown},
|
||||
{"both-future", now.UnixMilli() + 1, now.UnixMilli() + 1, VerifyIntentUnknown},
|
||||
{"extreme-future-verification", recent, math.MaxInt64, VerifyIntentUnknown},
|
||||
{"extreme-future-success", math.MaxInt64, recent, VerifyIntentUnknown},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
intent, verified := ComputeVerifyIntentAt(tc.success, tc.verified, now)
|
||||
if intent != tc.want {
|
||||
t.Errorf("intent=%q, want %q", intent, tc.want)
|
||||
}
|
||||
if tc.verified <= 0 {
|
||||
if verified != nil {
|
||||
t.Fatalf("unobserved verification became %v", verified)
|
||||
}
|
||||
} else if verified == nil || verified.UnixMilli() != tc.verified {
|
||||
t.Fatalf("source verification date rewritten: %v, want %d", verified, tc.verified)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollupFutureDateCorrectionPreservesEvidence(t *testing.T) {
|
||||
now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
|
||||
recent, future := now.Add(-time.Hour), now.Add(time.Hour)
|
||||
verified := true
|
||||
for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS, ProviderProxmoxPMG, ProviderTrueNAS, ProviderKubernetes} {
|
||||
t.Run(string(provider), func(t *testing.T) {
|
||||
point := RecoveryPoint{ID: "clock-backup", Provider: provider, Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess,
|
||||
CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest", ProviderScope: "clock-estate"}
|
||||
points := []RecoveryPoint{point}
|
||||
before := []RecoveryPoint{point}
|
||||
rollups := BuildRollupsFromPointsAt(points, now)
|
||||
if len(rollups) != 1 {
|
||||
t.Fatalf("rollups=%+v", rollups)
|
||||
}
|
||||
r := rollups[0]
|
||||
if r.VerifyIntent != VerifyIntentUnknown {
|
||||
t.Errorf("future intent=%q, want unknown", r.VerifyIntent)
|
||||
}
|
||||
if r.LastSuccessAt == nil || !r.LastSuccessAt.Equal(future) || r.LastVerifiedAt == nil || !r.LastVerifiedAt.Equal(future) || r.LastAttemptAt == nil || !r.LastAttemptAt.Equal(future) {
|
||||
t.Fatalf("future evidence lost: %+v", r)
|
||||
}
|
||||
if r.RollupID != "res:clock-guest" || r.SubjectResourceID != point.SubjectResourceID || len(r.Providers) != 1 || r.Providers[0] != provider || !reflect.DeepEqual(points, before) {
|
||||
t.Fatalf("scope, identity or input changed: %+v", r)
|
||||
}
|
||||
// Merely collecting the same point again must not invent a new date.
|
||||
again := BuildRollupsFromPointsAt(points, now)
|
||||
if !reflect.DeepEqual(rollups, again) {
|
||||
t.Fatal("ordinary reread changed evidence")
|
||||
}
|
||||
// Only a corrected source date (or actual time reaching the reported instant)
|
||||
// makes this same verified point eligible for a verified assessment.
|
||||
points[0].CompletedAt = &recent
|
||||
corrected := BuildRollupsFromPointsAt(points, now)[0]
|
||||
if corrected.VerifyIntent != VerifyIntentVerified || corrected.LastVerifiedAt == nil || !corrected.LastVerifiedAt.Equal(recent) || corrected.RollupID != r.RollupID {
|
||||
t.Fatalf("corrected date did not recover in place: %+v", corrected)
|
||||
}
|
||||
atReportedTime := BuildRollupsFromPointsAt(before, future)[0]
|
||||
if atReportedTime.VerifyIntent != VerifyIntentVerified || !atReportedTime.LastVerifiedAt.Equal(future) {
|
||||
t.Fatal("evaluation did not recover at reported time")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,10 +21,10 @@ const BackupVerifyStaleWindow = 7 * 24 * time.Hour
|
||||
// Semantics:
|
||||
// - verified : a verification-bearing point exists within window of now
|
||||
// - stale : a successful backup exists, but no verification within window
|
||||
// - unknown : no successful backup at all (or no info)
|
||||
// - unknown : no successful backup, or a success/verification date is in the future
|
||||
//
|
||||
// LastVerifiedAt is returned regardless of intent (when known) so consumers
|
||||
// can surface "last verified X days ago" alongside the badge.
|
||||
// can retain the source date even when clock uncertainty prevents an assessment.
|
||||
func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (VerifyIntent, *time.Time) {
|
||||
windowMs := BackupVerifyStaleWindow.Milliseconds()
|
||||
nowMs := now.UTC().UnixMilli()
|
||||
@@ -35,7 +35,10 @@ func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (
|
||||
lastVerifiedAt = &t
|
||||
}
|
||||
|
||||
if lastSuccessMs <= 0 {
|
||||
// A future date proves no recency. Keep the reported date as evidence,
|
||||
// but align the rollup with posture: clock uncertainty is neither verified
|
||||
// nor stale. Check before subtraction so extreme dates cannot overflow age.
|
||||
if lastSuccessMs <= 0 || lastSuccessMs > nowMs || lastVerifiedMs > nowMs {
|
||||
return VerifyIntentUnknown, lastVerifiedAt
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user