Compare commits

...

1 Commits

Author SHA1 Message Date
overtrue 7cf04018da fix(scanner): revisit compacted prefixes with active lifecycle rules 2026-11-21 11:12:02 +08:00
5 changed files with 98 additions and 16 deletions
-10
View File
@@ -1370,15 +1370,6 @@ jobs:
# the intermediate 4-object plateau requires 4*lc_interval < 5*debug_day, i.e.
# debug_day > 8. A smaller value lets the Days=5 rule fire inside the Days=1
# poll window so the count collapses straight to 2 and the test fails (#4764).
#
# RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 is also required. A compacted directory
# is only re-descended (and its objects re-evaluated for ILM) once every
# DATA_USAGE_UPDATE_DIR_CYCLES scanner cycles (default 16). At the accelerated
# RUSTFS_SCANNER_CYCLE=2 that is ~32s between evaluations, so a Days=1 object
# that becomes due at debug_day (10s) is not actually expired until the next
# ~32s boundary (~42s), landing just past the 4*lc_interval (40s) poll window
# and making the count stall at 6 (#4767). Forcing every-cycle re-descent
# evaluates ILM within ~2s of the due time, well inside the poll window.
s3-lifecycle-behavior-tests:
name: S3 Lifecycle Behavior Tests
# Also gated on e2e-tests, matching s3-implemented-tests: when the e2e smoke
@@ -1420,7 +1411,6 @@ jobs:
RUSTFS_ILM_PROCESS_TIME=1 \
RUSTFS_SCANNER_ENABLED=true \
RUSTFS_SCANNER_CYCLE=2 \
RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 \
RUSTFS_SCANNER_START_DELAY_SECS=0 \
RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s \
S3_PORT="${S3_PORT}" \
+5 -1
View File
@@ -2704,7 +2704,7 @@ impl FolderScanner {
}
}
FolderScanSource::Existing => {
// Usage sampling is not proof that a Deep check ran.
// An unchanged subtree can contain objects that have aged into a lifecycle rule.
if self.scan_mode != HealScanMode::Deep
&& !forward_sweep
&& !self
@@ -2715,6 +2715,10 @@ impl FolderScanner {
.is_some_and(|config| config.has_active_rules("", true))
&& !into.compacted
&& self.old_cache.is_compacted(&h)
&& !active_life_cycle.as_ref().is_some_and(|config| {
let (_, prefix) = path2_bucket_object_with_base_path(&self.root, &folder_item.name);
config.has_active_rules(&prefix)
})
{
let next_cycle = self.old_cache.info.next_cycle as u32;
if !h.mod_(next_cycle, data_usage_update_dir_cycles()) {
@@ -80,6 +80,98 @@ async fn disk_scan_withdraws_persisted_lifecycle_policy() {
}
}
#[tokio::test]
#[serial]
async fn compacted_prefix_expiration_between_usage_samples() {
let (_temp_dir, store) = setup_local_scanner_store(1, 4).await;
let bucket = "compacted-expiration";
let expired = "photos/vehicle/old.jpg";
let fresh = "photos/vehicle/new.jpg";
let unrelated = "archive/old.jpg";
let now = OffsetDateTime::now_utc();
let old = now - time::Duration::days(4);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
for (key, mod_time) in [(expired, old), (fresh, now), (unrelated, old)] {
let mut reader = ScannerPutObjReader::from_vec(vec![1]);
store
.put_object(
bucket,
key,
&mut reader,
&ScannerObjectOptions {
mod_time: Some(mod_time),
..Default::default()
},
)
.await
.expect("write dated object");
}
let disks = scanner_set_disk_inventory(&store.pools[0].disk_set[0]).await;
let scan = async |cache| {
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
let outcome = disks[0]
.clone()
.nsscanner_disk(
ctx,
budget.clone(),
disks.clone(),
cache,
None,
ScannerDiskScanOptions {
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
checkpoint_tx: None,
},
)
.await
.expect("scan bucket");
let ScannerDiskScanOutcome::Complete(cache) = outcome else { panic!("expected complete scan") };
(cache, budget.progress().0)
};
let mut cache = DataUsageCache::default();
cache.info.name = bucket.into();
let (mut cache, scanned) = scan(cache).await;
assert_eq!(scanned, 3);
let prefixes = ["photos", "archive"].map(|prefix| rustfs_data_usage::hash_path(&format!("{bucket}/{prefix}")));
assert!(prefixes.iter().all(|prefix| cache.is_compacted(prefix)));
let xml = br#"<LifecycleConfiguration><Rule><ID>photos</ID><Status>Enabled</Status><Filter><Prefix>photos/vehicle/</Prefix></Filter><Expiration><Days>2</Days></Expiration></Rule></LifecycleConfiguration>"#;
update_bucket_metadata(bucket, BUCKET_LIFECYCLE_CONFIG, xml.to_vec())
.await
.expect("configure expiration");
init_background_expiry(store.clone()).await;
let (_, scanned) = temp_env::async_with_vars([("RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES", None::<&str>)], async {
let interval = crate::scanner_folder::data_usage_update_dir_cycles();
let off_cycle = (1..=interval)
.find(|cycle| prefixes.iter().all(|prefix| !prefix.mod_(*cycle, interval)))
.expect("cycle that samples neither compacted prefix");
cache.info.next_cycle = u64::from(off_cycle);
scan(cache).await
})
.await;
assert_eq!(scanned, 2, "visit the matching subtree and reuse the unrelated prefix");
tokio::time::timeout(Duration::from_secs(10), async {
loop {
match store.get_object_info(bucket, expired, &ScannerObjectOptions::default()).await {
Err(Error::ObjectNotFound(_, _)) => break,
Ok(_) => tokio::time::sleep(Duration::from_millis(10)).await,
Err(err) => panic!("read expired object: {err}"),
}
}
})
.await
.expect("expiry worker must delete the old matching object");
for key in [fresh, unrelated] {
store
.get_object_info(bucket, key, &ScannerObjectOptions::default())
.await
.expect("retain non-expiring object");
}
}
// Lifecycle expiry goes through the process-global store, which the first
// store created in the process claims. Nextest runs this test in its own
// process; the lifecycle scenarios share the store that owns expiry processing.
+1 -1
View File
@@ -171,7 +171,7 @@ These have no persistent key and are read from the environment only.
| `RUSTFS_SCANNER_ALERT_COOLDOWN_SECS` | `86400` (`DEFAULT_SCANNER_ALERT_COOLDOWN_SECS`, `scanner_folder.rs`) | Per-(kind, bucket, object) cooldown between S3 excess-alert events; `0` emits every cycle. See [Scanner Excess Alerts](scanner-excess-alerts.md). |
| `RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS` | `60` (`DEFAULT_SCANNER_DEEP_VERIFY_COOLDOWN_SECS`, `scanner_folder.rs`) | Objects modified within this window are skipped by deep (bitrot) verification in the current cycle. |
| `RUSTFS_HEAL_OBJECT_SELECT_PROB` | `1024` (`DEFAULT_HEAL_OBJECT_SELECT_PROB`, `scanner_folder.rs`) | Sampling divisor for scanner-originated heal checks: roughly one object in N per cycle is selected for a low-priority heal check. |
| `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES` | `16` (`DATA_USAGE_UPDATE_DIR_CYCLES`, `scanner_folder.rs`) | Every N cycles a compacted directory is re-descended instead of reusing its cached usage. `1` forces re-descent every cycle (used by lifecycle e2e lanes). |
| `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES` | `16` (`DATA_USAGE_UPDATE_DIR_CYCLES`, `scanner_folder.rs`) | Revisits compacted directories every N cycles for usage accounting. `1` revisits them every cycle. A scanner walk also visits prefixes covered by active lifecycle rules between usage samples. |
| `RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS` | `86400` (`DEFAULT_FAILED_OBJECT_TTL_SECS`, `scanner_folder.rs`) | Retention of per-bucket failed-object entries in the usage cache. |
| `RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX` | `10000` (`DEFAULT_FAILED_OBJECTS_MAX`, `scanner_folder.rs`) | Cap on retained failed-object entries per bucket. |
@@ -20,10 +20,6 @@
# RUSTFS_ILM_DEBUG_DAY_SECS=10 (1 "day" = 10s, matches s3-tests lc_debug_interval)
# RUSTFS_SCANNER_ENABLED=true
# RUSTFS_SCANNER_CYCLE=2 (scan every 2s, well under a debug-day)
# RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 (re-descend compacted dirs every cycle so
# ILM is evaluated ~every 2s, not once per 16 cycles;
# without this a Days=1 object due at 10s is not
# expired until the next ~32s boundary — #4772)
# RUSTFS_SCANNER_START_DELAY_SECS=0
# RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s (multipart abort lane)
#