fix(ci): isolate scanner deadline and expose test failure details (#8210)

* fix(ci): isolate scanner deadline fixture and expose readiness errors

* test(ecstore): report unexpected capacity reservation errors
This commit is contained in:
Chris
2026-09-29 07:16:17 +08:00
committed by GitHub
parent c9acf01fd0
commit c8f42b8dca
3 changed files with 46 additions and 5 deletions
+27 -2
View File
@@ -22,6 +22,7 @@
//! - Common test constants and utilities
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::{Client, Config};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use http::header::{CONTENT_TYPE, HOST};
@@ -1577,6 +1578,7 @@ impl RustFSTestClusterEnvironment {
/// retries up to 120 times with a 1-second interval between attempts.
async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let client = self.create_s3_client(node_idx)?;
let mut last_error = None;
for attempt in 0..120 {
match client.list_buckets().send().await {
@@ -1584,13 +1586,36 @@ impl RustFSTestClusterEnvironment {
info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);
return Ok(());
}
Err(_) => {
Err(err) => {
last_error = Some(err);
sleep(Duration::from_secs(1)).await;
}
}
}
Err(format!("Cluster node {} service failed to become ready", node_idx).into())
let last_error = last_error.as_ref().map(|err| {
// SDK Display reports only the category. Do not expose raw response bodies or headers.
let service_code = err.as_service_error().and_then(|error| error.code()).filter(|code| {
matches!(
*code,
"ServiceUnavailable"
| "ServerNotInitialized"
| "InternalError"
| "AccessDenied"
| "InvalidAccessKeyId"
| "SignatureDoesNotMatch"
)
});
format!(
"{err}; http_status={:?}; service_code={service_code:?}",
err.raw_response().map(|response| response.status().as_u16())
)
});
Err(format!(
"Cluster node {node_idx} service failed to become ready; last ListBuckets error={last_error:?}; capture_log_path={:?}",
self.node_capture_log_paths[node_idx]
)
.into())
}
/// Create an S3 client configured to communicate with a specific cluster node.
+4 -1
View File
@@ -19542,7 +19542,10 @@ mod tests {
.await
.expect("second-node activation should not panic")
.expect_err("the second reservation must observe and reject the committed first reservation");
assert!(err.to_string().contains("requires 60 bytes, but 40 bytes are available"));
assert!(
err.to_string().contains("requires 60 bytes, but 40 bytes are available"),
"the second reservation must report the committed capacity rejection, got {err:?}"
);
let mut persisted = PoolMeta::default();
persisted
+15 -2
View File
@@ -636,8 +636,8 @@ async fn cycle_budget_fence_accepts_bootstrap_pending_usage_marker() {
#[tokio::test]
async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
let (_temp_dir, store) = setup_scanner_cycle_store().await;
let lock = store
let (_temp_dir, lock_store) = setup_scanner_cycle_store().await;
let lock = lock_store
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
.await
.expect("scanner leader lock should be created");
@@ -646,6 +646,17 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
.await
.expect("scanner leader lock should be acquired");
// Keep the real guard, but isolate the fencing deadline from filesystem I/O.
let store = Arc::new(MemoryConfigStore::default());
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0))
.expect("scanner cycle usage baseline should encode"),
)
.await
.expect("scanner cycle usage baseline should persist");
let ctx = CancellationToken::new();
let mut cycle_info = CurrentCycle {
current: 12,
@@ -679,6 +690,8 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
.await;
assert!(guard.is_released());
assert_eq!(leader_epoch, 2, "deadline handler should claim the next epoch");
assert!(matches!(cycle_revision, DataUsageCacheRevision::Etag(_)));
let persisted = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
.await
.expect("deadline handler should persist a fenced cursor");