mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 13:16:28 +00:00
fix(ecstore): harden tier reader and restore cleanup races (#5035)
* fix(tier): hold generation lease through readers Refs rustfs/backlog#1354 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(restore): fence failed cleanup by source identity Refs rustfs/backlog#1356 Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -3598,7 +3598,7 @@ mod metadata_mutation_generation_tests {
|
||||
mod transition_commit_failure_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions};
|
||||
use crate::disk::DiskAPI as _;
|
||||
use crate::services::tier::test_util::{MockWarmBackend, register_mock_tier};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
@@ -3813,6 +3813,89 @@ mod transition_commit_failure_tests {
|
||||
assert_eq!(old_backend.object_count().await, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn failed_restore_cleanup_does_not_overwrite_concurrent_unversioned_put() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "restore-cleanup-cas-bucket";
|
||||
let object = "object.bin";
|
||||
let original_payload = b"old transitioned body".repeat(1024);
|
||||
let replacement_payload = b"new visible unversioned body".repeat(1024);
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::from_vec(original_payload.clone());
|
||||
let original = set_disks
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
set_disks
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source object should transition before restore");
|
||||
|
||||
let get_barrier = backend.arm_failing_get_barrier().await;
|
||||
let restore_set = Arc::clone(&set_disks);
|
||||
let restore = tokio::spawn(async move {
|
||||
let mut opts = ObjectOptions::default();
|
||||
opts.transition.restore_request.days = Some(1);
|
||||
restore_set.restore_transitioned_object(bucket, object, &opts).await
|
||||
});
|
||||
get_barrier.wait_until_paused().await;
|
||||
|
||||
let mut replacement_reader = PutObjReader::from_vec(replacement_payload.clone());
|
||||
let replacement = set_disks
|
||||
.put_object(bucket, object, &mut replacement_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("concurrent unversioned PUT should commit while restore GET is paused");
|
||||
get_barrier.release();
|
||||
restore
|
||||
.await
|
||||
.expect("restore task should join")
|
||||
.expect_err("injected tier GET failure should surface");
|
||||
|
||||
let visible = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("replacement metadata should remain visible");
|
||||
assert_eq!(visible.etag, replacement.etag, "stale restore cleanup must not republish the old ETag");
|
||||
assert_ne!(
|
||||
visible.transitioned_object.status, TRANSITION_COMPLETE,
|
||||
"replacement object must not regain stale transition metadata"
|
||||
);
|
||||
let mut body = Vec::new();
|
||||
set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("replacement object should be readable")
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("replacement body should drain");
|
||||
assert_eq!(
|
||||
body, replacement_payload,
|
||||
"stale restore cleanup must not make the old remote body current again"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn legacy_reload_rejects_route_change_after_local_transition_commit() {
|
||||
|
||||
@@ -13,6 +13,34 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct RestoreCleanupIdentity {
|
||||
version_id: Option<Uuid>,
|
||||
data_dir: Option<Uuid>,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
size: i64,
|
||||
}
|
||||
|
||||
impl RestoreCleanupIdentity {
|
||||
fn from_object_info(obj_info: &ObjectInfo) -> Self {
|
||||
Self {
|
||||
version_id: obj_info.version_id,
|
||||
data_dir: obj_info.data_dir,
|
||||
mod_time: obj_info.mod_time,
|
||||
size: obj_info.size,
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_file_info(&self, fi: &FileInfo, expected_etag: &str) -> bool {
|
||||
self.version_id == fi.version_id
|
||||
&& self.data_dir == fi.data_dir
|
||||
&& self.mod_time == fi.mod_time
|
||||
&& self.size == fi.size
|
||||
&& expected_etag == get_raw_etag(&fi.metadata)
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub async fn update_restore_metadata(
|
||||
@@ -20,29 +48,46 @@ impl SetDisks {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
obj_info: &ObjectInfo,
|
||||
_opts: &ObjectOptions,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
let mut oi = obj_info.clone();
|
||||
oi.metadata_only = true;
|
||||
Arc::make_mut(&mut oi.user_defined).remove(X_AMZ_RESTORE.as_str());
|
||||
let version_id = oi.version_id.map(|v| v.to_string());
|
||||
let _obj = self
|
||||
.copy_object(
|
||||
bucket,
|
||||
object,
|
||||
bucket,
|
||||
object,
|
||||
&mut oi,
|
||||
&ObjectOptions {
|
||||
version_id: version_id.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
&ObjectOptions {
|
||||
version_id,
|
||||
..Default::default()
|
||||
},
|
||||
if obj_info.bucket.is_empty() || obj_info.name.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let expected = RestoreCleanupIdentity::from_object_info(obj_info);
|
||||
let expected_etag = obj_info
|
||||
.etag
|
||||
.clone()
|
||||
.unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
|
||||
let version_id = expected.version_id.map(|v| v.to_string());
|
||||
let lock_guard = if !opts.no_lock {
|
||||
Some(
|
||||
self.acquire_write_lock_diag("restore_cleanup_metadata", bucket, object)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let read_opts = ObjectOptions {
|
||||
version_id,
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let (mut fi, _, disks) = self
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?;
|
||||
if !expected.matches_file_info(&fi, &expected_etag) {
|
||||
return Ok(());
|
||||
}
|
||||
fi.metadata.remove(X_AMZ_RESTORE.as_str());
|
||||
fi.metadata.remove(AMZ_RESTORE_EXPIRY_DAYS);
|
||||
fi.metadata.remove(AMZ_RESTORE_REQUEST_DATE);
|
||||
if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(Error::other("restore cleanup lock lost before metadata update".to_string()));
|
||||
}
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
self.update_object_meta(bucket, object, fi, disks.as_slice()).await?;
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user