mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 20:46:11 +00:00
test(e2e): add distributed cluster regression coverage (#7158)
* test(e2e): add distributed 4x4 validation * test(e2e): prove operations overlap data movement --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
|
||||
unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("concdecom");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let baseline_client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let barrier = Arc::new(Barrier::new(17));
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..16 {
|
||||
let clients = clients.clone();
|
||||
let barrier = barrier.clone();
|
||||
let bucket = bucket.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let client = &clients[idx % clients.len()];
|
||||
let key = format!("live/{idx:02}.bin");
|
||||
let body = payload_for(&key, 8 * 1024);
|
||||
retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
|
||||
}));
|
||||
}
|
||||
|
||||
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
barrier.wait().await;
|
||||
|
||||
let mut live_objects = Vec::new();
|
||||
for handle in handles {
|
||||
live_objects.push(handle.await??);
|
||||
}
|
||||
let status = decommission_status_json(&dist.cluster).await?;
|
||||
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
|
||||
return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
|
||||
}
|
||||
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let checker = dist.client(2)?;
|
||||
assert_inventory(&checker, &bucket, &inventory).await?;
|
||||
for (key, body) in live_objects {
|
||||
retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user