mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 16:37:07 +00:00
test(ecstore): wait for multipart rename tail epochs (#6723)
This commit is contained in:
@@ -205,16 +205,6 @@ retries = 2
|
|||||||
filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
|
filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
|
||||||
test-group = 'ecstore-serial-flaky'
|
test-group = 'ecstore-serial-flaky'
|
||||||
|
|
||||||
# QUARANTINE: OPEN rustfs#6711 — the multipart fencing test's epoch helper
|
|
||||||
# reads xl.meta back from EVERY disk, but a multipart commit only guarantees
|
|
||||||
# quorum-many disks have persisted; a lagging disk under CI load panics the
|
|
||||||
# read-back with "file not found" (observed on the rio-v2 leg of a
|
|
||||||
# nextest-config-only PR; same all-disk-materialization assumption as the
|
|
||||||
# relocated-pool fixture fixed by #6707).
|
|
||||||
[[profile.ci.overrides]]
|
|
||||||
filter = 'package(rustfs-ecstore) & test(object_transaction_fencing_persists_epoch_on_multipart_commit)'
|
|
||||||
retries = 2
|
|
||||||
|
|
||||||
# Serialize the 4-disk reliability / degraded-read e2e tests under the ci
|
# Serialize the 4-disk reliability / degraded-read e2e tests under the ci
|
||||||
# profile too (see the e2e-reliability test-group note near the top). Not a
|
# profile too (see the e2e-reliability test-group note near the top). Not a
|
||||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||||
|
|||||||
@@ -3695,11 +3695,17 @@ mod tests {
|
|||||||
|
|
||||||
async fn object_transaction_epochs(disks: &[DiskStore], bucket: &str, object: &str) -> Vec<Option<Uuid>> {
|
async fn object_transaction_epochs(disks: &[DiskStore], bucket: &str, object: &str) -> Vec<Option<Uuid>> {
|
||||||
let mut epochs = Vec::with_capacity(disks.len());
|
let mut epochs = Vec::with_capacity(disks.len());
|
||||||
|
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||||
for (disk_index, disk) in disks.iter().enumerate() {
|
for (disk_index, disk) in disks.iter().enumerate() {
|
||||||
let file_info = disk
|
let file_info = loop {
|
||||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
|
||||||
.await
|
Ok(file_info) => break file_info,
|
||||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist object metadata: {err}"));
|
Err(DiskError::FileNotFound) if tokio::time::Instant::now() < deadline => {
|
||||||
|
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||||
|
}
|
||||||
|
Err(err) => panic!("disk {disk_index} should persist object metadata: {err}"),
|
||||||
|
}
|
||||||
|
};
|
||||||
epochs.push(
|
epochs.push(
|
||||||
file_info
|
file_info
|
||||||
.object_transaction_epoch()
|
.object_transaction_epoch()
|
||||||
@@ -3762,22 +3768,34 @@ mod tests {
|
|||||||
let (upload_id, parts) =
|
let (upload_id, parts) =
|
||||||
stage_upload_with_create_opts(&set_disks, bucket, object, b"multipart fenced epoch", &ObjectOptions::default()).await;
|
stage_upload_with_create_opts(&set_disks, bucket, object, b"multipart fenced epoch", &ObjectOptions::default()).await;
|
||||||
|
|
||||||
temp_env::async_with_vars(
|
let epochs = temp_env::async_with_vars(
|
||||||
[
|
[
|
||||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
|
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
|
||||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
|
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
|
||||||
|
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
|
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||||
set_disks
|
set_disks
|
||||||
.clone()
|
.clone()
|
||||||
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
|
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
|
||||||
.await
|
.await
|
||||||
.expect("fenced multipart completion should commit with a live proof");
|
.expect("fenced multipart completion should commit with a live proof");
|
||||||
|
|
||||||
|
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||||
|
.await
|
||||||
|
.expect("multipart completion should leave one rename tail in flight after quorum ACK");
|
||||||
|
let disks = disk_stores.clone();
|
||||||
|
let mut epochs = tokio::spawn(async move { object_transaction_epochs(&disks, bucket, object).await });
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(100), &mut epochs).await.is_err(),
|
||||||
|
"epoch read-back should wait for the lagging rename tail"
|
||||||
|
);
|
||||||
|
rename_barrier.release();
|
||||||
|
epochs.await.expect("epoch read-back should finish after the rename tail")
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let epochs = object_transaction_epochs(&disk_stores, bucket, object).await;
|
|
||||||
let first = epochs[0].expect("fenced multipart completion should persist an epoch");
|
let first = epochs[0].expect("fenced multipart completion should persist an epoch");
|
||||||
assert!(!first.is_nil());
|
assert!(!first.is_nil());
|
||||||
assert!(epochs.into_iter().all(|epoch| epoch == Some(first)));
|
assert!(epochs.into_iter().all(|epoch| epoch == Some(first)));
|
||||||
|
|||||||
Reference in New Issue
Block a user