From e44007012b34232dd8da219efb936bf6a2f0c204 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 1 Sep 2026 00:01:38 +0800 Subject: [PATCH] fix(scanner): recover legacy usage floor from backup (#6964) * fix(scanner): recover legacy usage floor from backup Allow scanner usage-floor startup and leadership fencing to use a valid legacy backup when the legacy primary read fails with a corruption-shaped error. Keep v2 primary read failures, stale metadata, transient I/O, and missing or invalid backups fail-closed. Co-Authored-By: heihutu * fix(scanner): cover legacy backup fencing gaps (#6966) * fix(scanner): recover legacy usage from valid backup * fix(scanner): recover legacy usage floor from backup Allow scanner usage-floor startup and leadership fencing to use a valid legacy backup when the legacy primary read fails with a corruption-shaped error. Keep v2 primary read failures, stale metadata, transient I/O, and missing or invalid backups fail-closed. Co-Authored-By: heihutu * fix(scanner): cover legacy backup fencing gaps --------- Co-authored-by: houseme Co-authored-by: heihutu --------- Co-authored-by: heihutu Co-authored-by: Henry Guo --- crates/scanner/src/data_usage_define.rs | 33 +++ crates/scanner/src/scanner/cycle_state.rs | 19 +- crates/scanner/src/scanner/leadership.rs | 37 ++- crates/scanner/src/scanner/tests.rs | 314 ++++++++++++++++++++++ crates/scanner/src/scanner/usage_store.rs | 29 +- 5 files changed, 421 insertions(+), 11 deletions(-) diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 9d72f72f2..35229ba6b 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -131,6 +131,39 @@ pub(crate) async fn read_config_with_revision( } } +pub(crate) fn usage_floor_primary_read_error_allows_backup(err: &Error) -> bool { + match err { + Error::FileCorrupt + | Error::CorruptedFormat + | Error::CorruptedBackend + | Error::PartMissingOrCorrupt + | Error::LessData + | Error::MoreData => true, + Error::Io(io_error) => { + matches!(io_error.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof) + || error_chain_has_usage_floor_corruption_signature(io_error) + } + _ => false, + } +} + +fn error_chain_has_usage_floor_corruption_signature(error: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(error); + while let Some(err) = current { + let message = err.to_string(); + if message.contains("InlineData value out of range") + || message.contains("InlineData key out of range") + || message.contains("insufficient data for metadata") + || message.contains("insufficient data for meta length") + || message.contains("insufficient data for CRC") + { + return true; + } + current = err.source(); + } + false +} + /// Read only the object revision without materializing its body. pub(crate) async fn read_config_revision(store: Arc, path: &str) -> StorageResult { match store diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index f8a11c6f3..99d2958d5 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -14,7 +14,9 @@ /// Scanner cycle-state codec, persisted usage floors, and cycle-state persistence. use super::*; use crate::ScannerGetObjectReader; -use crate::data_usage_define::{DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH}; +use crate::data_usage_define::{ + DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH, usage_floor_primary_read_error_allows_backup, +}; use crate::storage_api::owner::ObjectIO as _; use tokio::io::AsyncReadExt as _; @@ -1717,6 +1719,7 @@ pub(super) async fn persisted_usage_floor_for_startup( let backup_path = format!("{primary_path}.bkp"); let is_v2_path = primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str(); let mut recovered_primary_companion_epoch = None; + let mut primary_read_error = None; let primary_epoch = match read_config_with_revision(storeapi.clone(), primary_path).await { Ok((Some(data), revision)) => { let usage = serde_json::from_slice::(&data).map_err(|err| { @@ -1776,6 +1779,12 @@ pub(super) async fn persisted_usage_floor_for_startup( } } Ok((None, _)) => None, + Err(err) if !is_v2_path && usage_floor_primary_read_error_allows_backup(&err) => { + primary_read_error = Some(format!("failed to read scanner usage epoch floor from {primary_path}: {err}")); + invalid_baseline_path.get_or_insert_with(|| primary_path.to_string()); + unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string()); + None + } Err(err) => { return Err(ScannerError::Other(format!( "failed to read scanner usage epoch floor from {primary_path}: {err}" @@ -1855,6 +1864,14 @@ pub(super) async fn persisted_usage_floor_for_startup( ))); } } + if let Some(primary_read_error) = primary_read_error + && !any_found + { + return Err(ScannerError::Other(format!( + "{}; no valid scanner usage floor backup was available at {backup_path}", + primary_read_error + ))); + } if any_found { if bootstrap_pending { return Err(ScannerError::Other( diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index e728e7420..d273d133a 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -13,6 +13,7 @@ // limitations under the License. /// Leader-lock claiming, usage-epoch fencing, and lock-loss handling. use super::*; +use crate::data_usage_define::usage_floor_primary_read_error_allows_backup; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum ScannerLeadershipClaimReconcile { @@ -142,20 +143,34 @@ pub(super) async fn usage_snapshot_for_epoch_fence( } } - for path in [ - LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), - format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), - ] { - let (legacy, _) = read_config_with_revision(storeapi.clone(), &path) - .await - .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; + let legacy_primary_path = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(); + let legacy_backup_path = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()); + let mut legacy_primary_read_error = None; + for path in [&legacy_primary_path, &legacy_backup_path] { + let legacy = match read_config_with_revision(storeapi.clone(), path).await { + Ok((legacy, _)) => legacy, + Err(err) if path == &legacy_primary_path && usage_floor_primary_read_error_allows_backup(&err) => { + legacy_primary_read_error = Some(format!("failed to read legacy scanner usage epoch fence from {path}: {err}")); + continue; + } + Err(err) => { + return Err(ScannerError::Other(format!( + "failed to read legacy scanner usage epoch fence from {path}: {err}" + ))); + } + }; if let Some(legacy) = legacy.as_deref() { - let usage = decode_usage_snapshot_for_epoch_fence(legacy, &path, false)?; + let usage = decode_usage_snapshot_for_epoch_fence(legacy, path, false)?; if invalid_primary_epoch.is_none_or(|epoch| usage.scanner_epoch.unwrap_or_default() >= epoch) { return Ok(Some(usage)); } } } + if let Some(legacy_primary_read_error) = legacy_primary_read_error { + return Err(ScannerError::Other(format!( + "{legacy_primary_read_error}; no valid legacy scanner usage epoch fence backup was available at {legacy_backup_path}" + ))); + } // A missing usage snapshot is an uninitialized state, not an empty // snapshot. Leadership fencing may proceed without creating a plausible // default; the first authoritative scanner publication will create it. @@ -258,6 +273,12 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( Some(epoch) if epoch == claimed_epoch => return Ok(()), Some(_) | None => {} } + // A validated pre-marker legacy baseline needs an explicit complete + // identity before acquiring an epoch. Otherwise the v2 reader would + // reject the fenced value on its next startup. + if !usage.usage_snapshot_bootstrap_pending { + usage.usage_snapshot_complete = true; + } usage.scanner_epoch = Some(claimed_epoch); let data = serde_json::to_vec(&usage) .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index fcf815395..29a65a0e0 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -574,6 +574,7 @@ struct MemoryConfigStore { objects: Mutex>>, revisions: Mutex>, insert_after_gets: Mutex>>, + read_errors: Mutex>, delayed_gets: Mutex>, non_regular_objects: Mutex>, fail_put_number: Mutex>, @@ -623,6 +624,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore { _opts: &ObjectOptions, ) -> EcstoreResult { let key = memory_config_key(bucket, object); + if let Some(error) = self.read_errors.lock().await.get(&key).cloned() { + return Err(error); + } if let Some(delay) = self.delayed_gets.lock().await.remove(&key) { tokio::time::sleep(delay).await; } @@ -2902,6 +2906,147 @@ async fn scanner_usage_floor_recovers_from_incomplete_v2_primary_using_fenced_ba ); } +async fn seed_legacy_primary_read_error_with_backup(store: &Arc, error: EcstoreError, epoch: u64, cycle: u64) { + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + let legacy_backup = format!("{legacy_primary}.bkp"); + let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + backup.scanner_epoch = Some(epoch); + backup.scanner_cycle = Some(cycle); + + store + .read_errors + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, legacy_primary), error); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, &legacy_backup), + serde_json::to_vec(&backup).expect("legacy backup usage snapshot should encode"), + ); +} + +#[tokio::test] +async fn scanner_usage_floor_recovers_legacy_backup_after_primary_decode_error() { + let store = Arc::new(MemoryConfigStore::default()); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await; + + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("valid legacy backup should recover the startup floor"); + assert_eq!(state, PersistedUsageFloorStartup::Authoritative); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 19, + } + ); + assert_eq!( + persisted_usage_floor(store) + .await + .expect("valid legacy backup should recover the authoritative floor"), + floor + ); +} + +#[tokio::test] +async fn scanner_usage_floor_does_not_bootstrap_over_corrupt_legacy_primary_without_backup() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + store + .read_errors + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, legacy_primary), EcstoreError::FileCorrupt); + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("corrupt legacy primary without a valid backup must remain fail-closed"); + assert!(err.to_string().contains("no valid scanner usage floor backup"), "unexpected error: {err}"); +} + +#[tokio::test] +async fn scanner_usage_floor_does_not_fallback_to_legacy_after_corrupt_v2_primary() { + let store = Arc::new(MemoryConfigStore::default()); + let mut v2_backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + v2_backup.scanner_epoch = Some(8); + v2_backup.scanner_cycle = Some(11); + let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + legacy.scanner_epoch = Some(3); + legacy.scanner_cycle = Some(7); + store.read_errors.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + EcstoreError::FileCorrupt, + ); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str())), + serde_json::to_vec(&v2_backup).expect("v2 backup usage snapshot should encode"), + ); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"), + ); + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("corrupt v2 primary must not recover without a primary revision"); + assert!( + err.to_string().contains(&format!( + "failed to read scanner usage epoch floor from {}", + DATA_USAGE_OBJ_NAME_PATH.as_str() + )), + "unexpected error: {err}" + ); +} + +#[tokio::test] +async fn scanner_usage_floor_keeps_transient_primary_read_error_fail_closed() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + seed_legacy_primary_read_error_with_backup( + &store, + EcstoreError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "connection reset while reading usage primary", + )), + 19, + 41, + ) + .await; + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("transient primary errors must not be converted into backup recovery"); + assert!( + err.to_string() + .contains(&format!("failed to read scanner usage epoch floor from {legacy_primary}")), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("no valid scanner usage floor backup"), + "transient error should not enter corrupt-primary fallback: {err}" + ); +} + +#[tokio::test] +async fn scanner_usage_floor_keeps_outdated_primary_metadata_fail_closed() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::OutdatedXLMeta, 19, 41).await; + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("outdated primary metadata must not be converted into backup recovery"); + assert!( + err.to_string() + .contains(&format!("failed to read scanner usage epoch floor from {legacy_primary}")), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("no valid scanner usage floor backup"), + "outdated metadata should not enter corrupt-primary fallback: {err}" + ); +} + #[tokio::test] async fn scanner_usage_floor_does_not_bootstrap_over_incomplete_v2_primary() { let store = Arc::new(MemoryConfigStore::default()); @@ -3004,6 +3149,19 @@ async fn scanner_leadership_fencing_recovers_incomplete_v2_primary_from_backup() assert_eq!(recovered.scanner_cycle, Some(103)); } +#[tokio::test] +async fn scanner_usage_floor_leadership_fencing_recovers_legacy_backup_after_primary_decode_error() { + let store = Arc::new(MemoryConfigStore::default()); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await; + + let recovered = usage_snapshot_for_epoch_fence(store, None, false) + .await + .expect("a valid legacy backup should provide the fencing baseline") + .expect("the fencing baseline should be present"); + assert_eq!(recovered.scanner_epoch, Some(19)); + assert_eq!(recovered.scanner_cycle, Some(41)); +} + #[tokio::test] async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() { let store = Arc::new(MemoryConfigStore::default()); @@ -3955,6 +4113,162 @@ async fn scanner_defers_leadership_when_usage_snapshots_are_stably_absent() { assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err()); } +#[tokio::test] +async fn scanner_usage_floor_leadership_claim_recovers_legacy_backup_after_primary_decode_error() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await; + + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle::default(); + let mut persisted_epoch = 19; + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("leadership claim should persist after legacy backup recovery"); + let (_, claimed_epoch) = decode_scanner_cycle_state(&state).expect("leadership claim should decode"); + assert_eq!(claimed_epoch, 20); + assert_eq!(persisted_epoch, 20); + + let usage = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("legacy backup recovery should publish a fenced v2 usage primary"); + let usage = serde_json::from_slice::(&usage).expect("fenced v2 usage primary should decode"); + assert_eq!(usage.scanner_epoch, Some(20)); + assert_eq!(usage.scanner_cycle, Some(41)); +} + +#[tokio::test] +#[serial_test::serial] +async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_metadata_truncation() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let (temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await; + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.usage_snapshot_complete = false; + usage.scanner_cycle = Some(41); + let mut data = serde_json::to_vec(&usage).expect("legacy usage should encode"); + data.resize(data.len() + 16 * 1024, b' '); + let legacy_path = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + let backup_path = format!("{legacy_path}.bkp"); + for path in [legacy_path, backup_path.as_str()] { + save_config(store.clone(), path, data.clone()) + .await + .expect("legacy usage fixture should persist"); + } + let mut truncated_files = Vec::new(); + for disk_index in 0..4 { + let path = temp_dir + .path() + .join(format!("pool0/disk{disk_index}")) + .join(RUSTFS_META_BUCKET) + .join(legacy_path) + .join("xl.meta"); + let file = tokio::fs::OpenOptions::new() + .write(true) + .open(&path) + .await + .expect("legacy inline metadata should exist"); + assert!(file.metadata().await.expect("metadata should be readable").len() > 4096); + file.set_len(4096).await.expect("fixture should truncate at a page boundary"); + truncated_files.push(( + path.clone(), + tokio::fs::read(&path) + .await + .expect("truncated evidence should remain readable"), + )); + } + + let store = restart_scanner_cycle_store_from(&store).await; + let error = read_config_with_revision(store.clone(), legacy_path) + .await + .expect_err("truncated primary must fail in the real object reader"); + assert!( + error.to_string().contains("InlineData value out of range"), + "unexpected truncated-primary error: {error}" + ); + assert_eq!( + read_config_with_revision(store.clone(), &backup_path) + .await + .expect("backup should remain readable") + .0, + Some(data.clone()), + ); + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("intact legacy backup must recover startup despite truncated primary"); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 0 + } + ); + assert_eq!(state, PersistedUsageFloorStartup::Authoritative); + + let baseline = read_data_usage_persist_baseline(store.clone()) + .await + .expect("publication must also read the intact backup"); + assert_eq!(baseline.data.as_deref(), Some(data.as_slice())); + assert_eq!(baseline.revision, DataUsageCacheRevision::Missing); + fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false) + .await + .expect("legacy backup must be fenced into v2"); + let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fencing must publish a v2 usage primary"); + let fenced = serde_json::from_slice::(&fenced).expect("fenced v2 usage primary should decode"); + assert!( + fenced.usage_snapshot_complete, + "the fenced pre-marker baseline must become a complete v2 identity" + ); + + let store = restart_scanner_cycle_store_from(&store).await; + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("a restart after fencing must preserve the recovered floor"); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 7 + } + ); + assert_eq!(state, PersistedUsageFloorStartup::Authoritative); + let restarted = restart_scanner_cycle_store_from(&store).await; + assert_eq!( + persisted_usage_floor(restarted) + .await + .expect("fenced floor must survive another restart"), + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 7 + } + ); + for (path, bytes) in truncated_files { + assert_eq!(tokio::fs::read(path).await.expect("legacy evidence must not be removed"), bytes); + } + assert_eq!( + read_config(store, &backup_path) + .await + .expect("legacy backup must remain intact"), + data + ); + global_metrics().set_cycle(None).await; + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + #[tokio::test] async fn usage_bootstrap_pending_unblocks_first_leadership_claim() { let store = Arc::new(MemoryConfigStore::default()); diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index b9dd53271..af7c616ee 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -13,6 +13,7 @@ // limitations under the License. /// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup. use super::*; +use crate::data_usage_define::usage_floor_primary_read_error_allows_backup; use crate::storage_api::owner::ScannerPublicationCommitState; use std::collections::HashMap; use std::sync::atomic::AtomicBool; @@ -53,6 +54,30 @@ pub(super) struct DataUsagePersistBaseline { pub(super) revision: DataUsageCacheRevision, } +async fn read_usage_persist_candidate( + storeapi: Arc, + path: &str, +) -> Result<(Option>, DataUsageCacheRevision), EcstoreError> { + let primary = read_config_with_revision(storeapi.clone(), path).await; + if path != LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() { + return primary; + } + let Err(primary_error) = &primary else { + return primary; + }; + if !usage_floor_primary_read_error_allows_backup(primary_error) { + return primary; + } + let backup_path = format!("{path}.bkp"); + let backup = read_config_with_revision(storeapi, &backup_path).await?; + if backup.0.as_deref().is_some_and(|data| { + serde_json::from_slice::(data).is_ok_and(|usage| data_usage_info_has_persisted_baseline_identity(&usage)) + }) { + return Ok(backup); + } + primary +} + /// Read the bytes used as the baseline for a usage publication while keeping /// the v2 primary revision as the CAS fence. During an interrupted upgrade the /// primary can be valid JSON without a baseline identity; in that case a @@ -68,7 +93,7 @@ pub(super) async fn read_data_usage_persist_baseline( LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), ] { - let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?; + let (candidate, _) = read_usage_persist_candidate(storeapi.clone(), &path).await?; let Some(candidate) = candidate else { continue; }; @@ -107,7 +132,7 @@ pub(super) async fn read_data_usage_persist_baseline( LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), ] { - let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?; + let (candidate, _) = read_usage_persist_candidate(storeapi.clone(), &path).await?; let Some(candidate) = candidate else { continue; };