gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 11:32:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ilm): harden tier transition failure boundaries (#5031)

Browse Source 
* fix(tier): fence generation-scoped operations

Refs rustfs/backlog#1354

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ilm): verify transition upload streams

Refs rustfs/backlog#1353

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(ecstore): expand transition fault matrix

Refs rustfs/backlog#1355

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-19 18:48:32 +08:00
committed by GitHub
parent 83d73b34f3
commit 21049401fa
25 changed files with 5808 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+3 -1
View File
@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
use crate::store::utils::is_reserved_or_invalid_bucket;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
LifecycleOps, gen_transition_objname, get_transitioned_object_reader_with_tier_manager, put_restore_opts,
},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::storageclass,
@@ -628,6 +628,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
mod read;
mod replication;
pub(crate) mod shard_source;
#[cfg(all(test, feature = "test-util"))]
mod transition_matrix_tests;
pub use ops::heal_walk::HealWalkVersion;
crates/ecstore/src/set_disk/ops/multipart.rs
+68
View File
@@ -1607,6 +1607,74 @@ mod tests {
.await
}
#[tokio::test]
#[serial(metadata_cache_invalidation_probe)]
async fn complete_multipart_generation_retires_cached_snapshot() {
use crate::storage_api_contracts::multipart::MultipartOperations as _;
use crate::storage_api_contracts::object::ObjectIO as _;
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-metadata-generation-bucket";
let object = "object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut initial_reader = PutObjReader::from_vec(b"old multipart body".to_vec());
set_disks
.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
.await
.expect("initial object should be written");
set_disks
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
.await
.expect("initial metadata should resolve");
let generation = set_disks
.get_object_metadata_cache_generation(bucket, object)
.expect("metadata cache generation should be active");
let retired_key = GetObjectMetadataCacheKey::new(bucket, object, generation);
assert!(set_disks.get_object_metadata_cache.get(&retired_key).await.is_some());
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let payload = vec![9u8; 4096];
let payload_len = i64::try_from(payload.len()).expect("test payload length should fit i64");
let mut part_reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(payload), payload_len, payload_len, None, None, false)
.expect("part hash reader should be created"),
);
let part = set_disks
.put_object_part(bucket, object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
.await
.expect("multipart part should be written");
let invalidations = MetadataCacheInvalidationProbe::install(bucket, object);
set_disks
.clone()
.complete_multipart_upload(
bucket,
object,
&upload.upload_id,
vec![CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&ObjectOptions::default(),
)
.await
.expect("multipart completion should succeed");
assert_eq!(
invalidations.count(),
2,
"multipart completion must invalidate before mutation and after commit"
);
set_disks.get_object_metadata_cache.run_pending_tasks().await;
assert!(set_disks.get_object_metadata_cache.get(&retired_key).await.is_none());
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn multipart_upload_read_lock_waits_for_upload_writer() {
crates/ecstore/src/set_disk/ops/object.rs
+1124 -62
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/set_disk/read.rs
+139 -5
View File
@@ -2540,20 +2540,27 @@ mod metadata_cache_tests {
}
#[tokio::test]
async fn get_object_metadata_cache_invalidation_removes_object_entry() {
async fn metadata_cache_per_key_invalidation_physically_reclaims_retired_generation() {
let set = new_metadata_cache_test_set().await;
let fi = valid_test_fileinfo("object");
let generation = set.get_object_metadata_cache_generation("bucket", "object");
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0)
let generation = set
.get_object_metadata_cache_generation("bucket", "object")
.expect("metadata cache generation should be active");
let retired_key = GetObjectMetadataCacheKey::new("bucket", "object", generation);
set.cache_get_object_fileinfo(("bucket", "object"), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0)
.await;
set.get_object_metadata_cache.run_pending_tasks().await;
assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_some());
assert_eq!(set.get_object_metadata_cache.entry_count(), 1);
set.invalidate_get_object_metadata_cache("bucket", "object").await;
set.get_object_metadata_cache.run_pending_tasks().await;
assert!(
set.cached_get_object_fileinfo("bucket", "object").await.is_none(),
"explicit invalidation must remove the cached object metadata"
set.get_object_metadata_cache.get(&retired_key).await.is_none(),
"per-key invalidation must physically remove the retired generation"
);
assert_eq!(set.get_object_metadata_cache.entry_count(), 0);
}
#[tokio::test]
@@ -2704,6 +2711,77 @@ mod metadata_cache_tests {
);
}
#[tokio::test]
async fn metadata_cache_generation_isolated_between_set_instances() {
let first = new_metadata_cache_test_set().await;
let second = new_metadata_cache_test_set().await;
let fi = valid_test_fileinfo("object");
let first_generation = first.get_object_metadata_cache_generation("bucket", "object");
let second_generation = second.get_object_metadata_cache_generation("bucket", "object");
first
.cache_get_object_fileinfo(("bucket", "object"), first_generation, &fi, std::slice::from_ref(&fi), &[], 0)
.await;
second
.cache_get_object_fileinfo(("bucket", "object"), second_generation, &fi, std::slice::from_ref(&fi), &[], 0)
.await;
first.invalidate_get_object_metadata_cache("bucket", "object").await;
assert!(first.cached_get_object_fileinfo("bucket", "object").await.is_none());
assert!(second.cached_get_object_fileinfo("bucket", "object").await.is_some());
assert_eq!(second.get_object_metadata_cache_generation("bucket", "object"), second_generation);
}
#[tokio::test]
async fn metadata_cache_cached_hash_collision_preserves_full_identity() {
let set = new_metadata_cache_test_set().await;
let generation = set
.get_object_metadata_cache_generation("bucket-a", "object-a")
.expect("metadata cache generation should be active");
let first_key = GetObjectMetadataCacheKey::new("bucket-a", "object-a", generation);
let second_key = GetObjectMetadataCacheKey {
bucket: Arc::from("bucket-b"),
object: Arc::from("object-b"),
generation: generation.value,
hash: generation.hash,
};
let first_fi = valid_test_fileinfo("object-a");
let second_fi = valid_test_fileinfo("object-b");
let entry = |fi: FileInfo| {
Arc::new(GetObjectMetadataCacheEntry {
created_at: Instant::now(),
parts_metadata: vec![fi.clone()],
fi,
online_disks: Vec::new(),
read_quorum: 0,
})
};
set.get_object_metadata_cache.insert(first_key.clone(), entry(first_fi)).await;
set.get_object_metadata_cache
.insert(second_key.clone(), entry(second_fi))
.await;
assert_eq!(
set.get_object_metadata_cache
.get(&first_key)
.await
.expect("first colliding entry should remain addressable")
.fi
.name,
"object-a"
);
assert_eq!(
set.get_object_metadata_cache
.get(&second_key)
.await
.expect("second colliding entry should remain addressable")
.fi
.name,
"object-b"
);
}
#[tokio::test]
async fn metadata_cache_generation_overflow_fails_closed() {
let set = new_metadata_cache_test_set().await;
@@ -2743,6 +2821,62 @@ mod metadata_cache_tests {
assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
}
#[tokio::test]
async fn metadata_cache_invalidate_all_physically_reclaims_retired_generations() {
let set = new_metadata_cache_test_set().await;
let mut retired_keys = Vec::new();
for object in ["object-a", "object-b", "object-c"] {
let fi = valid_test_fileinfo(object);
let generation = set
.get_object_metadata_cache_generation("bucket", object)
.expect("metadata cache generation should be active");
retired_keys.push(GetObjectMetadataCacheKey::new("bucket", object, generation));
set.cache_get_object_fileinfo(("bucket", object), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0)
.await;
}
set.get_object_metadata_cache.run_pending_tasks().await;
assert_eq!(set.get_object_metadata_cache.entry_count(), 3);
set.invalidate_all_get_object_metadata_cache();
set.get_object_metadata_cache.run_pending_tasks().await;
for key in retired_keys {
assert!(
set.get_object_metadata_cache.get(&key).await.is_none(),
"invalidate-all must physically remove every retired generation"
);
}
assert_eq!(set.get_object_metadata_cache.entry_count(), 0);
}
#[tokio::test]
async fn metadata_cache_invalidate_all_at_max_fails_closed_and_clears_entries() {
let set = new_metadata_cache_test_set().await;
let fi = valid_test_fileinfo("object");
let generation = set
.get_object_metadata_cache_generation("bucket", "object")
.expect("metadata cache generation should be active");
let retired_key = GetObjectMetadataCacheKey::new("bucket", "object", generation);
set.cache_get_object_fileinfo(("bucket", "object"), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0)
.await;
for fence in set.get_object_metadata_cache_generations.iter() {
fence.store(u64::MAX, Ordering::Release);
}
set.invalidate_all_get_object_metadata_cache();
set.get_object_metadata_cache.run_pending_tasks().await;
assert!(
set.get_object_metadata_cache_generations
.iter()
.all(|fence| fence.load(Ordering::Acquire) == u64::MAX),
"invalidate-all must not wrap a saturated fence"
);
assert_eq!(set.get_object_metadata_cache_generation("bucket", "object"), None);
assert!(set.get_object_metadata_cache.get(&retired_key).await.is_none());
assert_eq!(set.get_object_metadata_cache.entry_count(), 0);
}
#[tokio::test]
async fn get_object_metadata_cache_prunes_when_capacity_is_reached() {
// moka handles capacity eviction automatically via the configured max_capacity.
crates/ecstore/src/set_disk/transition_matrix_tests.rs
+104
View File
@@ -0,0 +1,104 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
use crate::ecstore_validation_blackbox::make_local_set_disks;
use crate::services::tier::test_util::register_mock_tier;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use tokio::io::AsyncReadExt;
async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
set_disks
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
.await
.expect("object metadata should resolve");
let generation = set_disks
.get_object_metadata_cache_generation(bucket, object)
.expect("metadata generation should be active");
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
assert!(
set_disks.get_object_metadata_cache.get(&key).await.is_some(),
"metadata read should publish the generation under test"
);
key
}
async fn assert_generation_reclaimed(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) {
set_disks.get_object_metadata_cache.run_pending_tasks().await;
assert!(
set_disks.get_object_metadata_cache.get(key).await.is_none(),
"metadata mutation must physically reclaim the prior generation"
);
}
#[tokio::test]
#[serial_test::serial]
async fn transition_and_restore_reclaim_prior_metadata_generations() {
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
let bucket = "transition-restore-generation-bucket";
let object = "object.bin";
let payload = vec![0x5au8; 1024 * 1024];
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
let original = set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("source object should be written");
let source_generation = prime_metadata_generation(&set_disks, bucket, object).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await;
let transition_opts = ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name,
etag: original.etag.clone().expect("source ETag should be present"),
..Default::default()
},
version_id: original.version_id.map(|version| version.to_string()),
mod_time: original.mod_time,
..Default::default()
};
set_disks
.transition_object(bucket, object, &transition_opts)
.await
.expect("transition should succeed");
assert_generation_reclaimed(&set_disks, &source_generation).await;
let transitioned_generation = prime_metadata_generation(&set_disks, bucket, object).await;
let mut restore_opts = ObjectOptions::default();
restore_opts.transition.restore_request.days = Some(1);
Arc::clone(&set_disks)
.restore_transitioned_object(bucket, object, &restore_opts)
.await
.expect("restore should succeed");
assert_generation_reclaimed(&set_disks, &transitioned_generation).await;
assert_eq!(backend.get_count().await, 1, "restore should read the remote candidate exactly once");
let mut restored = Vec::new();
set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("restored object should be readable")
.stream
.read_to_end(&mut restored)
.await
.expect("restored body should drain");
assert_eq!(restored, payload);
assert_eq!(backend.get_count().await, 1, "restored GET should use the local copy");
}