mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-08 06:13:14 +00:00
fix(config): fence persisted config updates and reloads (#5512)
* fix(config): fence persisted config updates and reloads * fix(ci): unblock config and e2e checks Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
cxymds
@@ -4714,6 +4714,7 @@ mod tests {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::object_api::BLOCK_SIZE_V2;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
|
||||
use crate::storage_api_contracts::{
|
||||
heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart,
|
||||
namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _,
|
||||
@@ -9565,6 +9566,15 @@ mod tests {
|
||||
make_local_bucket_test_set_disks_with_drive_count(2).await
|
||||
}
|
||||
|
||||
fn assert_exclusive_object_lock_held(set_disks: &SetDisks, bucket: &str, object: &str) {
|
||||
let lock = set_disks
|
||||
.local_lock_manager_for_test()
|
||||
.get_lock_info(&ObjectKey::new(bucket, object))
|
||||
.expect("object lock should be visible while rename is paused");
|
||||
assert!(matches!(lock.mode, rustfs_lock::LockMode::Exclusive));
|
||||
assert_eq!(lock.owner.as_ref(), set_disks.locker_owner.as_str());
|
||||
}
|
||||
|
||||
async fn make_local_bucket_test_set_disks_with_drive_count(drive_count: usize) -> Arc<SetDisks> {
|
||||
let format = FormatV3::new(1, drive_count);
|
||||
let mut endpoints = Vec::new();
|
||||
@@ -9599,7 +9609,9 @@ mod tests {
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let set_disks = SetDisks::new(
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
instance_ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set_disks = SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
drive_count,
|
||||
@@ -9609,6 +9621,7 @@ mod tests {
|
||||
endpoints,
|
||||
format,
|
||||
Vec::new(),
|
||||
instance_ctx,
|
||||
)
|
||||
.await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
@@ -10262,6 +10275,216 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_replace_holds_object_lock_through_rename() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "bucket-conditional-replace-fence";
|
||||
let object = "config/conditional-replace.json";
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let mut initial_reader = PutObjReader::from_vec(b"initial config".to_vec());
|
||||
let initial = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut initial_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("initial config should be written");
|
||||
let initial_etag = initial.etag.expect("initial config should have an ETag");
|
||||
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer_store = set_disks.clone();
|
||||
let expected_etag = initial_etag.clone();
|
||||
let writer = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(b"replacement config".to_vec());
|
||||
writer_store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
preserve_etag: Some("replacement-etag".to_string()),
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(expected_etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("conditional replace should reach the rename barrier");
|
||||
assert_exclusive_object_lock_held(&set_disks, bucket, object);
|
||||
barrier.release();
|
||||
writer
|
||||
.await
|
||||
.expect("conditional writer task should finish")
|
||||
.expect("matching conditional replace should commit");
|
||||
assert!(
|
||||
set_disks
|
||||
.local_lock_manager_for_test()
|
||||
.get_lock_info(&ObjectKey::new(bucket, object))
|
||||
.is_none(),
|
||||
"conditional replace should release the object lock after commit"
|
||||
);
|
||||
let contender = set_disks
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.expect("contender namespace lock should be created");
|
||||
let contender_guard = contender
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("contender should acquire after conditional replace commits");
|
||||
drop(contender_guard);
|
||||
|
||||
let mut stale_reader = PutObjReader::from_vec(b"stale config".to_vec());
|
||||
let err = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut stale_reader,
|
||||
&ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(initial_etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("the old ETag must fail after the fenced replacement commits");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "bucket-write-generation";
|
||||
let object = "config/write-generation.json";
|
||||
let body = b"identical config body".to_vec();
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let mut first_reader = PutObjReader::from_vec(body.clone());
|
||||
let first = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut first_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("first config body should be written");
|
||||
let mut second_reader = PutObjReader::from_vec(body);
|
||||
let second = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut second_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("identical config body should be rewritten");
|
||||
|
||||
assert_eq!(first.etag, second.etag, "content ETag should expose the ABA collision");
|
||||
assert_ne!(first.data_dir, second.data_dir, "each committed body write needs a unique generation");
|
||||
assert!(first.data_dir.is_some() && second.data_dir.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_create_holds_object_lock_through_rename() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "bucket-conditional-create-fence";
|
||||
let object = "config/conditional-create.json";
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer_store = set_disks.clone();
|
||||
let writer = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(b"created config".to_vec());
|
||||
writer_store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("conditional create should reach the rename barrier");
|
||||
assert_exclusive_object_lock_held(&set_disks, bucket, object);
|
||||
barrier.release();
|
||||
writer
|
||||
.await
|
||||
.expect("conditional writer task should finish")
|
||||
.expect("first conditional create should commit");
|
||||
assert!(
|
||||
set_disks
|
||||
.local_lock_manager_for_test()
|
||||
.get_lock_info(&ObjectKey::new(bucket, object))
|
||||
.is_none(),
|
||||
"conditional create should release the object lock after commit"
|
||||
);
|
||||
let contender = set_disks
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.expect("contender namespace lock should be created");
|
||||
let contender_guard = contender
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("contender should acquire after conditional create commits");
|
||||
drop(contender_guard);
|
||||
|
||||
let mut duplicate_reader = PutObjReader::from_vec(b"duplicate config".to_vec());
|
||||
let err = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut duplicate_reader,
|
||||
&ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a second create-only write must not replace the committed config");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_level_if_none_match_fails_closed_without_read_quorum() {
|
||||
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
|
||||
|
||||
Reference in New Issue
Block a user