mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-04 12:31:36 +00:00
fix(ci): isolate scanner deadline and expose test failure details (#8210)
* fix(ci): isolate scanner deadline fixture and expose readiness errors * test(ecstore): report unexpected capacity reservation errors
This commit is contained in:
@@ -22,6 +22,7 @@
|
||||
//! - Common test constants and utilities
|
||||
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
@@ -1577,6 +1578,7 @@ impl RustFSTestClusterEnvironment {
|
||||
/// retries up to 120 times with a 1-second interval between attempts.
|
||||
async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let client = self.create_s3_client(node_idx)?;
|
||||
let mut last_error = None;
|
||||
|
||||
for attempt in 0..120 {
|
||||
match client.list_buckets().send().await {
|
||||
@@ -1584,13 +1586,36 @@ impl RustFSTestClusterEnvironment {
|
||||
info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);
|
||||
return Ok(());
|
||||
}
|
||||
Err(_) => {
|
||||
Err(err) => {
|
||||
last_error = Some(err);
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(format!("Cluster node {} service failed to become ready", node_idx).into())
|
||||
let last_error = last_error.as_ref().map(|err| {
|
||||
// SDK Display reports only the category. Do not expose raw response bodies or headers.
|
||||
let service_code = err.as_service_error().and_then(|error| error.code()).filter(|code| {
|
||||
matches!(
|
||||
*code,
|
||||
"ServiceUnavailable"
|
||||
| "ServerNotInitialized"
|
||||
| "InternalError"
|
||||
| "AccessDenied"
|
||||
| "InvalidAccessKeyId"
|
||||
| "SignatureDoesNotMatch"
|
||||
)
|
||||
});
|
||||
format!(
|
||||
"{err}; http_status={:?}; service_code={service_code:?}",
|
||||
err.raw_response().map(|response| response.status().as_u16())
|
||||
)
|
||||
});
|
||||
Err(format!(
|
||||
"Cluster node {node_idx} service failed to become ready; last ListBuckets error={last_error:?}; capture_log_path={:?}",
|
||||
self.node_capture_log_paths[node_idx]
|
||||
)
|
||||
.into())
|
||||
}
|
||||
|
||||
/// Create an S3 client configured to communicate with a specific cluster node.
|
||||
|
||||
@@ -19542,7 +19542,10 @@ mod tests {
|
||||
.await
|
||||
.expect("second-node activation should not panic")
|
||||
.expect_err("the second reservation must observe and reject the committed first reservation");
|
||||
assert!(err.to_string().contains("requires 60 bytes, but 40 bytes are available"));
|
||||
assert!(
|
||||
err.to_string().contains("requires 60 bytes, but 40 bytes are available"),
|
||||
"the second reservation must report the committed capacity rejection, got {err:?}"
|
||||
);
|
||||
|
||||
let mut persisted = PoolMeta::default();
|
||||
persisted
|
||||
|
||||
@@ -636,8 +636,8 @@ async fn cycle_budget_fence_accepts_bootstrap_pending_usage_marker() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let lock = store
|
||||
let (_temp_dir, lock_store) = setup_scanner_cycle_store().await;
|
||||
let lock = lock_store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("scanner leader lock should be created");
|
||||
@@ -646,6 +646,17 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
.await
|
||||
.expect("scanner leader lock should be acquired");
|
||||
|
||||
// Keep the real guard, but isolate the fencing deadline from filesystem I/O.
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0))
|
||||
.expect("scanner cycle usage baseline should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("scanner cycle usage baseline should persist");
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 12,
|
||||
@@ -679,6 +690,8 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
.await;
|
||||
|
||||
assert!(guard.is_released());
|
||||
assert_eq!(leader_epoch, 2, "deadline handler should claim the next epoch");
|
||||
assert!(matches!(cycle_revision, DataUsageCacheRevision::Etag(_)));
|
||||
let persisted = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
|
||||
.await
|
||||
.expect("deadline handler should persist a fenced cursor");
|
||||
|
||||
Reference in New Issue
Block a user