// Copyright 2026 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use super::*; use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk}; use crate::storage_api::scanner_io::ObjectIO; use crate::{DataUsageCacheSource, DataUsageScanPlanDigest}; use std::io::Cursor; use tokio::io::AsyncReadExt; const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin"; const STATIC_OBJECTS: u64 = 24; const MAX_CACHE_BYTES: u64 = 1024 * 1024; const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0); const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]); /// Real cache persistence codec and CAS calls, backed by two bounded local files. #[derive(Debug)] struct FixtureStore { root: tempfile::TempDir, reject_save: AtomicBool, } impl FixtureStore { fn new() -> Arc { Arc::new(Self { root: tempfile::tempdir().expect("checkpoint fixture storage directory"), reject_save: AtomicBool::new(false), }) } fn path(&self, object: &str) -> std::path::PathBuf { assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp"))); self.root .path() .join(if object.ends_with(".bkp") { "backup" } else { "main" }) } async fn strict_load(&self) -> DataUsageCache { let bytes = tokio::fs::read(self.root.path().join("main")) .await .expect("saved checkpoint fixture must exist"); decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root") } } #[async_trait::async_trait] impl ObjectIO for FixtureStore { type Error = crate::EcstoreError; type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec; type HeaderMap = http::HeaderMap; type ObjectOptions = crate::ScannerObjectOptions; type ObjectInfo = crate::ScannerObjectInfo; type GetObjectReader = crate::ScannerGetObjectReader; type PutObjectReader = crate::ScannerPutObjReader; async fn get_object_reader( &self, _bucket: &str, object: &str, _range: Option, _headers: Self::HeaderMap, _options: &Self::ObjectOptions, ) -> crate::EcstoreResult { let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| { if error.kind() == std::io::ErrorKind::NotFound { crate::EcstoreError::FileNotFound } else { crate::EcstoreError::from(error) } })?; assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES); Ok(crate::ScannerGetObjectReader { stream: Box::new(Cursor::new(bytes)), object_info: crate::ScannerObjectInfo { etag: Some("fixture".into()), ..Default::default() }, buffered_body: None, body_source: Default::default(), }) } async fn put_object( &self, _bucket: &str, object: &str, data: &mut Self::PutObjectReader, options: &Self::ObjectOptions, ) -> crate::EcstoreResult { if self.reject_save.load(Ordering::SeqCst) { return Err(crate::EcstoreError::PreconditionFailed); } let path = self.path(object); let exists = tokio::fs::try_exists(&path).await?; let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS"); if (exists && preconditions.if_none_match_value() == Some("*")) || (!exists && preconditions.if_match_value().is_some()) || (exists && preconditions.if_match_value() != Some("fixture")) { return Err(crate::EcstoreError::PreconditionFailed); } let mut bytes = Vec::new(); (&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?; assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES); tokio::fs::write(path, bytes).await?; Ok(crate::ScannerObjectInfo { etag: Some("fixture".into()), ..Default::default() }) } } #[async_trait::async_trait] impl crate::ScannerConfigObjectDelete for FixtureStore { async fn delete_config_object( &self, _bucket: &str, _object: &str, _options: crate::ScannerObjectOptions, ) -> crate::EcstoreResult { Err(crate::EcstoreError::NotImplemented) } async fn scanner_data_usage_publication_admission(&self) -> Option { Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) } } fn decode_fixture(bytes: &[u8]) -> Result { if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") { return Err("missing or oversized checkpoint fixture"); } let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?; if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() { return Err("checkpoint fixture has no valid bucket root"); } Ok(cache) } fn retained(cache: &DataUsageCache) -> u64 { assert!( !cache.root().is_some_and(|root| root.compacted), "a compacted bucket root cannot prove static-prefix coverage" ); cache .checked_flatten("bucket/static") .map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64")) } #[derive(Debug, PartialEq, Eq)] enum CoverageDiagnosis { Progress, NoNewWork, LostAtPrepare, LostAtReload, WalkWithoutRetention, } fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis { if reloaded < scanned { CoverageDiagnosis::LostAtReload } else if prepared < previous { CoverageDiagnosis::LostAtPrepare } else if walked > 0 && reloaded <= previous { CoverageDiagnosis::WalkWithoutRetention } else if reloaded > previous { CoverageDiagnosis::Progress } else { CoverageDiagnosis::NoNewWork } } #[test] fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() { assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress); assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention); assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare); assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload); assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork); } #[test] fn checkpoint_fixture_missing_and_corrupt_inputs_fail() { for bytes in [ vec![], vec![0xc1], DataUsageCache::default().marshal_msg().expect("empty cache encoding"), vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")], ] { assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root"); } } #[test] fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() { let mut cache = DataUsageCache::default(); cache.info.name = "bucket".to_string(); cache.replace("bucket", "", DataUsageEntry::default()); cache.replace("bucket/static", "bucket", DataUsageEntry::default()); for index in 0..4 { cache.replace( &format!("bucket/static/{index}"), "bucket/static", DataUsageEntry { objects: 1, ..Default::default() }, ); } cache.reduce_children_of(&hash_path("bucket/static"), 1, true); let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture"); let entry = decoded .find("bucket/static") .expect("compaction must retain the static subtree root"); assert!(entry.compacted); assert!(entry.children.is_empty()); assert_eq!( retained(&decoded), 4, "compaction retains aggregate coverage even when leaf keys are absent" ); } #[tokio::test] #[serial] async fn checkpoint_fixture_save_reload_resume() { run_checkpoint_fixture(false).await; } #[tokio::test] #[serial] async fn checkpoint_fixture_hot_digest_diagnostic() { run_checkpoint_fixture(true).await; } async fn run_checkpoint_fixture(change_digest: bool) { let (scanner, root) = build_test_scanner().await; let _guard = TestGuard { temp_dir: Some(root.clone()), }; for index in 0..STATIC_OBJECTS { write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await; } let store = FixtureStore::new(); let mut previous = 0; let mut visited = 0; for round in 0..3_u8 { write_test_object_metadata(&root, "bucket", "hot/current").await; let mut cache = DataUsageCache::default(); let revisions = cache .load_with_revisions(store.clone(), CACHE_NAME) .await .expect("load checkpoint revisions"); if round > 0 { assert_eq!(retained(&store.strict_load().await), previous); } let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round))); crate::scanner_io::current_cache_root_or_prepare_with_generation( &mut cache, "bucket", SOURCE, 11, 7, plan, crate::scanner_io::DataUsageCacheReuseOptions { require_source: true, tier_registry_generation: None, }, ); let prepared = retained(&cache); let parent = CancellationToken::new(); let budget = ScannerCycleBudget::new_with_progress_tracking( &parent, ScannerCycleBudgetConfig { max_objects: Some(4), ..Default::default() }, ); let outcome = scanner .local_disk .clone() .nsscanner_disk( budget.token(), budget.clone(), vec![scanner.local_disk.clone()], cache, None, HealScanMode::Normal, ) .await .expect("budgeted local disk scan returns partial cache"); let ScannerDiskScanOutcome::Partial(cache) = outcome else { panic!("budgeted fixture must remain partial") }; assert!(!cache.info.snapshot_complete, "partial must never publish a complete root"); assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects)); let scanned = retained(&cache); cache .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) .await .expect("persist partial checkpoint"); let mut loaded = DataUsageCache::default(); loaded .load(store.clone(), CACHE_NAME) .await .expect("reload persisted partial checkpoint"); let reloaded = retained(&loaded); assert_eq!(reloaded, retained(&store.strict_load().await)); assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage"); assert!(!loaded.info.snapshot_complete); visited += budget.entries_visited(); let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded); eprintln!( "checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}" ); if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() { assert_eq!( diagnosis, CoverageDiagnosis::Progress, "visited growth must produce durable static coverage" ); } crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await; previous = reloaded; } assert!(visited > 0, "fixture must exercise the directory walk"); assert!(previous > 0, "fixture must retain and enumerate static subtree entries"); let mut loaded = DataUsageCache::default(); let revisions = loaded .load_with_revisions(store.clone(), CACHE_NAME) .await .expect("load final checkpoint"); let before = tokio::fs::read(store.root.path().join("main")) .await .expect("read durable checkpoint bytes"); let epoch_error = loaded .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1) .await .expect_err("stale publication epoch must reject persistence"); assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED)); store.reject_save.store(true, Ordering::SeqCst); loaded.info.next_cycle += 1; loaded .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) .await .expect_err("injected save failure must not report durable progress"); assert_eq!( tokio::fs::read(store.root.path().join("main")) .await .expect("read unchanged checkpoint bytes"), before ); let parent = CancellationToken::new(); parent.cancel(); let budget = ScannerCycleBudget::new(&parent, Default::default()); let result = scanner .local_disk .clone() .nsscanner_disk( budget.token(), budget.clone(), vec![scanner.local_disk.clone()], loaded.clone(), None, HealScanMode::Normal, ) .await; assert!(result.is_err(), "pre-scan cancellation must not produce a complete root"); assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion"); let parent = CancellationToken::new(); let budget = ScannerCycleBudget::new(&parent, Default::default()); let result = scanner .local_disk .clone() .nsscanner_disk( budget.token(), budget, vec![scanner.local_disk.clone()], loaded, None, HealScanMode::Normal, ) .await .expect("unbounded scan must complete after durable partial progress"); let ScannerDiskScanOutcome::Complete(cache) = result else { panic!("unbounded fixture must produce a complete disk cache"); }; assert!(cache.info.snapshot_complete); assert!(cache.info.scan_checkpoint.is_none()); assert_eq!( cache.checked_flatten("bucket").expect("complete bucket root").objects, usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize") ); }