mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 11:56:38 +00:00
fix(ecstore): never cache fabricated bucket metadata as authoritative (#5307)
BucketMetadataSys::get_config lazily fabricated a default BucketMetadata (object-lock off) for any bucket whose .metadata.bin was ConfigNotFound and cached it in the map that the map-only, fail-closed metadata_sys::get() serves. The object-lock batch-delete gate (object_lock_delete_check_required, backlog#929 / #4297) treats that map as authoritative, so a metadata miss became a cached "no lock" answer: a versioning peek could poison the cache and let delete_objects skip the per-object retention/legal-hold stat. The same fabrication raced make_bucket (lost update overwriting freshly persisted lock-enabled metadata) and let the 15-minute refresh loop replace good cached metadata on a transient quorum dip. Production changes: - get_config caches only metadata actually read from disk; misses are recorded in a bounded negative cache (30s TTL, 10k entries, invalidated by set()) so repeated lookups for metadata-less names cost no extra namespace-lock + erasure-set fanout (reachable pre-auth via CORS preflight and per-key in DeleteObjects). - concurrent_load never lets a fabricated default REPLACE an existing map entry; startup insert-if-vacant behavior for legacy buckets is preserved. - delete_objects and new_ns_lock resolve dist-erasure, versioning, and the object-lock gate from the set's own instance context (backlog#1052) instead of the ambient facade, so a second in-process instance (or, in tests, another test's transient DistErasure window) cannot reroute locking onto an empty dist locker list or answer with the wrong instance's bucket state. Test-isolation changes (the bug that surfaced all of the above: the delete_objects lock-gating test failed deterministically when sharing a process with the lifecycle env tests): - The MinIO-migration test builds on an isolated InstanceContext instead of registering soon-deleted disks in the shared bootstrap registry. - The cached lifecycle env re-registers its disks on every use, surviving other serial tests' reset_local_disk_test_state. - Hermetic SetDisks helpers gain isolated-context variants pinned to plain erasure; tier-free non-serial test modules use them, guard-based SetupTypeGuard tests stay on the bootstrap context. - Three deterministic pin tests (nextest-safe) cover the caching contract, the delete gate resolution source, and the ns-lock resolution source. Verification: - cargo test -p rustfs-ecstore --lib -- --exact <4-test combo from the report> (previously failing, now green) - cargo test -p rustfs-ecstore --lib: 3169 passed / 0 failed across repeated runs; cargo fmt --check and cargo clippy --lib --tests clean - Adversarial validation (high-risk tier, all seven roles) run per AGENTS.md; all findings fixed or rebutted with evidence
This commit is contained in:
@@ -4596,12 +4596,19 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn make_test_set_disks(lockers: Vec<Arc<dyn LockClient>>) -> Arc<SetDisks> {
|
||||
make_test_set_disks_with_ctx(lockers, bootstrap_ctx()).await
|
||||
}
|
||||
|
||||
async fn make_test_set_disks_with_ctx(
|
||||
lockers: Vec<Arc<dyn LockClient>>,
|
||||
instance_ctx: Arc<InstanceContext>,
|
||||
) -> Arc<SetDisks> {
|
||||
let endpoints = vec![
|
||||
Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"),
|
||||
Endpoint::try_from("http://127.0.0.1:9001/data").expect("second endpoint should parse"),
|
||||
];
|
||||
|
||||
SetDisks::new(
|
||||
SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![None, None])),
|
||||
2,
|
||||
@@ -4611,10 +4618,53 @@ mod tests {
|
||||
endpoints,
|
||||
FormatV3::new(1, 2),
|
||||
lockers,
|
||||
instance_ctx,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Pins the dist-erasure resolution SOURCE for `new_ns_lock` (adversarial
|
||||
/// review): the lock strategy must come from the set's own instance
|
||||
/// context, never the ambient facade — otherwise another in-process
|
||||
/// instance (or a concurrent test's ambient DistErasure window) reroutes
|
||||
/// this set's locking onto the wrong strategy.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn new_ns_lock_resolves_dist_from_set_instance_context() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let locker: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
|
||||
|
||||
let dist_ctx = Arc::new(InstanceContext::new());
|
||||
dist_ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let dist_set = make_test_set_disks_with_ctx(vec![locker.clone()], dist_ctx).await;
|
||||
let dist_guard = dist_set
|
||||
.new_ns_lock("bucket", "object")
|
||||
.await
|
||||
.expect("namespace lock should be created")
|
||||
.get_read_lock(Duration::from_millis(500))
|
||||
.await
|
||||
.expect("dist read lock should succeed with one healthy locker");
|
||||
assert!(
|
||||
matches!(dist_guard, NamespaceLockGuard::Standard(_)),
|
||||
"a DistErasure instance context must select the distributed lock strategy"
|
||||
);
|
||||
drop(dist_guard);
|
||||
|
||||
let local_ctx = Arc::new(InstanceContext::new());
|
||||
local_ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let local_set = make_test_set_disks_with_ctx(vec![locker], local_ctx).await;
|
||||
let local_guard = local_set
|
||||
.new_ns_lock("bucket", "object")
|
||||
.await
|
||||
.expect("namespace lock should be created")
|
||||
.get_read_lock(Duration::from_millis(500))
|
||||
.await
|
||||
.expect("local read lock should succeed");
|
||||
assert!(
|
||||
matches!(local_guard, NamespaceLockGuard::Fast(_)),
|
||||
"a plain-erasure instance context must select the local lock strategy"
|
||||
);
|
||||
}
|
||||
|
||||
struct SetupTypeGuard {
|
||||
previous: SetupType,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user