test(e2e): wait for heal peers after node rejoin

crates/config/src/constants/runtime.rs

@@ -57,13 +57,6 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
 pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
 pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 
-/// Dedicated blocking thread pool for fsync/fdatasync operations.
-/// When > 1, fsync operations are isolated from the main blocking pool to
-/// prevent device-bound fsync from starving read operations (pread/stat/open).
-/// Default 0 means auto (no isolation, use main runtime).
-pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
-pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
-
 // Dial9 Tokio Telemetry Default values
 pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
 pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";

crates/e2e_test/src/common.rs

@@ -30,7 +30,6 @@ use reqwest::StatusCode;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
-use serde_json;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::io::ErrorKind;
@@ -1584,156 +1583,6 @@ impl Drop for RustFSTestClusterEnvironment {
     }
 }
 
-/// Send a SigV4-signed HTTP request and return the raw `reqwest::Response`.
-///
-/// Unlike [`signed_s3_request`], this variant accepts `body: Option<Vec<u8>>`
-/// (binary-safe) and reorders parameters so that `access_key`/`secret_key`
-/// appear before the body — matching the convention used by the replication
-/// extension and object-lambda e2e suites.
-pub(crate) async fn signed_request(
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let client = local_http_client();
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-/// Like [`signed_request`], but uses a caller-supplied `reqwest::Client`
-/// instead of the shared [`local_http_client`].
-pub(crate) async fn signed_request_with_client(
-    client: &reqwest::Client,
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-/// Like [`signed_request`], but includes a `session_token` in the
-/// `x-amz-security-token` header and passes it to the SigV4 signer.
-pub(crate) async fn signed_request_with_session_token(
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    session_token: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if !session_token.is_empty() {
-        request = request.header("x-amz-security-token", session_token);
-    }
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(
-        request.body(Body::empty())?,
-        content_len,
-        access_key,
-        secret_key,
-        session_token,
-        "us-east-1",
-    );
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let client = local_http_client();
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-/// Create a new user via the admin API.
-pub(crate) async fn admin_create_user(
-    env: &RustFSTestEnvironment,
-    username: &str,
-    secret_key: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
-    let body = serde_json::json!({
-        "secretKey": secret_key,
-        "status": "enabled"
-    });
-    let response = signed_request(
-        http::Method::PUT,
-        &url,
-        &env.access_key,
-        &env.secret_key,
-        Some(body.to_string().into_bytes()),
-        Some("application/json"),
-    )
-    .await?;
-
-    if response.status() != reqwest::StatusCode::OK {
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        return Err(format!("create user failed: {status} {body}").into());
-    }
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -380,10 +380,24 @@ mod tests {
         cluster.start_node(1).await?;
 
         let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
-        let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
-        assert!(
-            !status_body.contains("MissingContentLength"),
-            "background heal status should not fail without an explicit Content-Length: {status_body}"
+        let mut recovered = serde_json::Value::Null;
+        for _ in 0..60 {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            assert!(
+                !status_body.contains("MissingContentLength"),
+                "background heal status should not fail without an explicit Content-Length: {status_body}"
+            );
+            recovered = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
+                break;
+            }
+            sleep(Duration::from_secs(1)).await;
+        }
+        assert_eq!(
+            recovered["clusterStatusComplete"],
+            serde_json::Value::Bool(true),
+            "cluster heal status should recover before root heal starts: {recovered}"
         );
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;

crates/e2e_test/src/kms/common.rs

@@ -40,7 +40,7 @@ use std::time::Duration;
 use tokio::fs;
 use tokio::net::TcpStream;
 use tokio::time::sleep;
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info};
 
 // KMS-specific constants
 pub const TEST_BUCKET: &str = "kms-test-bucket";
@@ -177,49 +177,6 @@ pub async fn get_kms_status(
     Ok(status)
 }
 
-/// Poll the KMS status endpoint until the backend reports ready or the timeout
-/// expires.  Replaces hard-coded `sleep(Duration::from_secs(3))` startup waits
-/// with an active readiness probe so tests start as soon as KMS is usable
-/// (typically < 1 s) instead of always waiting the full 3 s.
-///
-/// Uses exponential back-off starting at 200 ms (doubling each attempt, capped
-/// at 1 s) up to a total wall-clock budget of 5 s.
-pub async fn wait_for_kms_ready(
-    base_url: &str,
-    access_key: &str,
-    secret_key: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let total_deadline = Duration::from_secs(5);
-    let start = tokio::time::Instant::now();
-    let mut backoff = Duration::from_millis(200);
-    let max_backoff = Duration::from_secs(1);
-    let mut first_attempt = true;
-
-    loop {
-        if !first_attempt {
-            if start.elapsed() >= total_deadline {
-                return Err("KMS failed to become ready within 5 seconds".into());
-            }
-            sleep(backoff).await;
-            backoff = (backoff * 2).min(max_backoff);
-        }
-        first_attempt = false;
-
-        match get_kms_status(base_url, access_key, secret_key).await {
-            Ok(status) => {
-                info!("KMS is ready (status: {})", status);
-                return Ok(());
-            }
-            Err(e) => {
-                if start.elapsed() >= total_deadline {
-                    return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
-                }
-                warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
-            }
-        }
-    }
-}
-
 /// Create a default KMS key for testing and return the created key ID
 pub async fn create_default_key(
     base_url: &str,
@@ -904,13 +861,6 @@ impl LocalKMSTestEnvironment {
         Ok(default_key_id.to_string())
     }
 
-    /// Poll the KMS status endpoint until the backend reports ready.
-    ///
-    /// Prefer this over a fixed `sleep` after calling `start_rustfs_for_local_kms`.
-    pub async fn wait_for_kms_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        wait_for_kms_ready(&self.base_env.url, &self.base_env.access_key, &self.base_env.secret_key).await
-    }
-
     /// Configure Local KMS backend with a predefined default key
     pub async fn configure_local_kms(&self) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
         // Use a fixed, predictable default key ID

crates/e2e_test/src/object_lambda_test.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client, signed_request};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::primitives::ByteStream;
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
-use rustfs_signer::pre_sign_v4;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::{pre_sign_v4, sign_v4};
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use s3s::Body;
 use std::collections::HashMap;
@@ -226,6 +227,39 @@ async fn presigned_get_request(
     Ok(local_http_client().get(signed.uri().to_string()).send().await?)
 }
 
+async fn signed_request(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
 async fn configure_webhook_target(
     env: &RustFSTestEnvironment,
     target_name: &str,

crates/e2e_test/src/replication_extension_test.rs

@@ -13,9 +13,8 @@
 // limitations under the License.
 
 use crate::common::{
-    RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
-    local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
-    signed_request_with_session_token,
+    RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+    replication_fast_env, rustfs_binary_path,
 };
 use crate::fake_s3_target::{
     FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -36,7 +35,7 @@ use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
 use bytes::Bytes;
 use flate2::read::GzDecoder;
 use futures::{Stream, StreamExt};
-use http::header::CONTENT_ENCODING;
+use http::header::{CONTENT_ENCODING, CONTENT_TYPE, HOST};
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
 use hyper::server::conn::http1;
@@ -57,6 +56,9 @@ use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
 use s3s::header::X_AMZ_REPLICATION_STATUS;
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
@@ -385,6 +387,116 @@ struct ReplicationResetStatusTarget {
     object: String,
 }
 
+async fn signed_request(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+async fn signed_request_with_client(
+    client: &reqwest::Client,
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+async fn signed_request_with_session_token(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if !session_token.is_empty() {
+        request = request.header("x-amz-security-token", session_token);
+    }
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(
+        request.body(Body::empty())?,
+        content_len,
+        access_key,
+        secret_key,
+        session_token,
+        "us-east-1",
+    );
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
 fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
     let open = format!("<{tag}>");
     let close = format!("</{tag}>");
@@ -904,6 +1016,35 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
     Client::from_conf(config)
 }
 
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    let body = serde_json::json!({
+        "secretKey": secret_key,
+        "status": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("create user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
 async fn admin_add_canned_policy(
     env: &RustFSTestEnvironment,
     policy_name: &str,

crates/ecstore/src/disk/os.rs

@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
     #[cfg(unix)]
     {
         let dir = dir.as_ref().to_path_buf();
-        fsync_spawn_blocking(move || fsync_dir_std(dir)).await?
+        tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
     }
 
     #[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
     #[cfg(test)]
     let dir = group.dir.clone();
     let dir_file = group.dir_file.clone();
-    fsync_spawn_blocking(move || {
+    tokio::task::spawn_blocking(move || {
         #[cfg(test)]
         {
             if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,44 +1080,6 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
 static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
-
-/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
-/// configured with >1 threads, isolates device-bound fsync from the main
-/// blocking pool so reads (pread/stat/open) are not starved. `None` means
-/// fall back to the main runtime (zero behavior change).
-static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
-    let threads =
-        rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
-    if threads <= 1 {
-        return None;
-    }
-    let mut builder = tokio::runtime::Builder::new_multi_thread();
-    builder
-        .worker_threads(num_cpus::get().min(8))
-        .max_blocking_threads(threads)
-        .thread_name("rustfs-fsync")
-        .thread_stack_size(512 * 1024)
-        .enable_all();
-    match builder.build() {
-        Ok(rt) => {
-            tracing::info!(threads, "fsync dedicated blocking pool enabled");
-            Some(rt)
-        }
-        Err(err) => {
-            tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
-            None
-        }
-    }
-});
-
-/// Spawn a blocking task on the fsync-dedicated runtime if configured,
-/// otherwise fall back to the main tokio blocking pool.
-fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
-    match FSYNC_RUNTIME.as_ref() {
-        Some(rt) => rt.spawn_blocking(f),
-        None => tokio::task::spawn_blocking(f),
-    }
-}
 static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
     LazyLock::new(|| Mutex::new(HashMap::new()));
 type NamespaceMutationLock = AsyncMutex<()>;
@@ -1255,7 +1217,7 @@ where
     F: FnOnce() -> io::Result<T> + Send + 'static,
 {
     let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
-    let result = fsync_spawn_blocking(move || {
+    let result = tokio::task::spawn_blocking(move || {
         let _disk_permit = disk_permit;
         work()
     })
@@ -2184,7 +2146,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
         wait_started,
     );
     let disk_permit = admission.disk_permit.clone();
-    let result = fsync_spawn_blocking(move || {
+    let result = tokio::task::spawn_blocking(move || {
         let _lease = lease;
         let _disk_permit = disk_permit;
         operation()

crates/heal/src/heal/manager/tests.rs

@@ -110,10 +110,7 @@ impl HealStorageAPI for MockStorage {
         Ok(Vec::new())
     }
 
-    async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
-        if bucket == "panic" {
-            panic!("test-only panic payload must not escape the scheduler");
-        }
+    async fn get_bucket_info(&self, _bucket: &str) -> Result<Option<BucketInfo>> {
         Ok(None)
     }
 
@@ -1024,231 +1021,6 @@ async fn test_task_alias_is_removed_after_terminal_completion() {
     assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
 }
 
-#[tokio::test]
-#[serial_test::serial]
-async fn scheduler_panic_releases_active_slot_and_allows_same_target_readmission() {
-    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
-    let request = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
-    let task_id = request.id.clone();
-
-    assert_eq!(
-        manager
-            .submit_heal_request(request)
-            .await
-            .expect("panic request should be admitted"),
-        HealAdmissionResult::Accepted
-    );
-    let duplicate = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
-    let duplicate_id = duplicate.id.clone();
-    assert_eq!(
-        manager
-            .submit_heal_request(duplicate)
-            .await
-            .expect("same target should merge while active is queued"),
-        HealAdmissionResult::Merged
-    );
-    assert_eq!(manager.canonical_task_id(&duplicate_id).await, task_id);
-    process_manager_queue_once(&manager).await;
-
-    let status = tokio::time::timeout(Duration::from_secs(1), async {
-        loop {
-            if let Ok(status) = manager.get_task_status(&task_id).await
-                && matches!(status, HealTaskStatus::Failed { .. })
-            {
-                break status;
-            }
-            tokio::time::sleep(Duration::from_millis(10)).await;
-        }
-    })
-    .await
-    .expect("panic task should reach a terminal status");
-    assert_eq!(
-        status,
-        HealTaskStatus::Failed {
-            error: PANICKED_HEAL_TASK_ERROR.to_string()
-        }
-    );
-    assert_eq!(manager.get_active_task_count().await, 0);
-    assert_eq!(manager.get_queue_length().await, 0);
-    assert!(manager.retrying_heals.lock().await.is_empty());
-    assert!(manager.task_aliases.lock().await.is_empty());
-    assert!(manager.completed_heals.lock().await.contains_key(&task_id));
-    assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
-
-    let readmitted = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
-    assert_eq!(
-        manager
-            .submit_heal_request(readmitted)
-            .await
-            .expect("same target should be re-admitted after a panic"),
-        HealAdmissionResult::Accepted
-    );
-}
-
-#[tokio::test]
-#[serial_test::serial]
-async fn retry_child_panic_finishes_parent_once() {
-    clear_scheduler_panic();
-    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
-    let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
-    let task_id = request.id.clone();
-    assert_eq!(
-        manager
-            .submit_heal_request(request)
-            .await
-            .expect("retry request should be admitted"),
-        HealAdmissionResult::Accepted
-    );
-    arm_scheduler_panic(SchedulerPanicPoint::RetryChild, &task_id);
-    process_manager_queue_once(&manager).await;
-
-    let status = tokio::time::timeout(Duration::from_secs(1), async {
-        loop {
-            if let Ok(status) = manager.get_task_status(&task_id).await
-                && matches!(status, HealTaskStatus::Failed { .. })
-            {
-                break status;
-            }
-            tokio::time::sleep(Duration::from_millis(10)).await;
-        }
-    })
-    .await
-    .expect("retry child panic should finish the parent");
-    clear_scheduler_panic();
-    assert_eq!(
-        status,
-        HealTaskStatus::Failed {
-            error: PANICKED_HEAL_TASK_ERROR.to_string()
-        }
-    );
-    assert_eq!(manager.get_active_task_count().await, 0);
-    assert_eq!(manager.get_queue_length().await, 0);
-    assert!(manager.retrying_heals.lock().await.is_empty());
-    assert!(manager.task_aliases.lock().await.is_empty());
-    assert_eq!(manager.completed_heals.lock().await.len(), 1);
-    assert_eq!(manager.get_statistics().await.failed_tasks, 1);
-}
-
-#[tokio::test]
-#[serial_test::serial]
-async fn cleanup_panic_is_supervised() {
-    clear_scheduler_panic();
-    let notice_bucket = "cleanup-panic-mrf";
-    let notice_object = "object";
-    let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(notice_bucket);
-    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
-    let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
-    request.source = HealRequestSource::Admin;
-    let task_id = request.id.clone();
-    assert_eq!(
-        manager
-            .submit_heal_request(request)
-            .await
-            .expect("cleanup request should be admitted"),
-        HealAdmissionResult::Accepted
-    );
-    manager
-        .mrf_repair_notice_targets
-        .lock()
-        .expect("mrf repair notice registry poisoned")
-        .insert(
-            task_id.clone(),
-            vec![MrfRepairNoticeTarget {
-                bucket: Arc::from(notice_bucket),
-                object: Arc::from(notice_object),
-                version_id: None,
-            }],
-        );
-    arm_scheduler_panic(SchedulerPanicPoint::Cleanup, &task_id);
-    process_manager_queue_once(&manager).await;
-
-    let status = tokio::time::timeout(Duration::from_secs(1), async {
-        loop {
-            if let Ok(status) = manager.get_task_status(&task_id).await
-                && matches!(status, HealTaskStatus::Completed)
-            {
-                break status;
-            }
-            tokio::time::sleep(Duration::from_millis(10)).await;
-        }
-    })
-    .await
-    .expect("cleanup panic should leave a terminal status");
-    clear_scheduler_panic();
-    assert_eq!(status, HealTaskStatus::Completed);
-    assert_eq!(manager.get_active_task_count().await, 0);
-    assert!(manager.task_aliases.lock().await.is_empty());
-    assert_eq!(manager.completed_heals.lock().await.len(), 1);
-    assert_eq!(manager.get_statistics().await.successful_tasks, 1);
-    let events = rustfs_common::mrf_channel::take_mrf_repaired_events_for(notice_bucket);
-    assert_eq!(events.len(), 1, "cleanup panic must preserve successful MRF notice delivery");
-    assert_eq!(events[0].object.as_ref(), notice_object);
-}
-
-#[tokio::test]
-#[serial_test::serial]
-async fn cancelled_retry_child_panic_does_not_rearchive_failed_status() {
-    let manager = HealManager::new(Arc::new(MockStorage), None);
-    let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
-    let task_id = request.id.clone();
-    let retry_cancel_token = insert_retrying_request(&manager, request.clone()).await;
-
-    manager
-        .cancel_task(&task_id)
-        .await
-        .expect("retry cancellation should succeed");
-    assert!(retry_cancel_token.is_cancelled());
-
-    let state = PanicCleanupState {
-        active_heals: manager.active_heals.clone(),
-        heal_queue: manager.heal_queue.clone(),
-        completed_heals: manager.completed_heals.clone(),
-        task_aliases: manager.task_aliases.clone(),
-        retrying_heals: manager.retrying_heals.clone(),
-        mrf_repair_notice_targets: manager.mrf_repair_notice_targets.clone(),
-        replacement_recovery_anchors: manager.replacement_recovery_anchors.clone(),
-        statistics: manager.statistics.clone(),
-    };
-    finish_panicked_retry_child(task_id.clone(), request.heal_type, retry_cancel_token, state).await;
-
-    assert!(manager.retrying_heals.lock().await.is_empty());
-    assert!(manager.completed_heals.lock().await.is_empty());
-    assert_eq!(manager.get_statistics().await.failed_tasks, 0);
-}
-
-#[tokio::test]
-async fn active_cancel_wins_parent_panic_cleanup_without_completed_status() {
-    let manager = HealManager::new(Arc::new(MockStorage), None);
-    let request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
-    let task_id = request.id.clone();
-    let task = Arc::new(HealTask::from_request(request, Arc::new(MockStorage)));
-    manager.active_heals.lock().await.insert(task_id.clone(), task.clone());
-
-    manager
-        .cancel_task(&task_id)
-        .await
-        .expect("active task cancellation should win");
-    assert_eq!(task.get_status().await, HealTaskStatus::Cancelled);
-
-    let state = PanicCleanupState {
-        active_heals: manager.active_heals.clone(),
-        heal_queue: manager.heal_queue.clone(),
-        completed_heals: manager.completed_heals.clone(),
-        task_aliases: manager.task_aliases.clone(),
-        retrying_heals: manager.retrying_heals.clone(),
-        mrf_repair_notice_targets: manager.mrf_repair_notice_targets.clone(),
-        replacement_recovery_anchors: manager.replacement_recovery_anchors.clone(),
-        statistics: manager.statistics.clone(),
-    };
-    finish_panicked_heal_task(task, task_id, state).await;
-
-    assert!(manager.completed_heals.lock().await.is_empty());
-    assert_eq!(manager.get_statistics().await.failed_tasks, 0);
-}
-
 #[tokio::test]
 async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);

crates/madmin/src/trace.rs

@@ -12,16 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::{collections::HashMap, time::Duration};
+
+use jiff::Timestamp;
 use serde::{Deserialize, Serialize};
 
-/// Bitflag helper for service trace categories.
-///
-/// Each variant occupies a single bit so that a `TraceType` value can represent
-/// an arbitrary combination of categories via bitwise OR.
+use crate::heal_commands::HealResultItem;
+
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
 pub struct TraceType(u64);
 
 impl TraceType {
+    // Define some constants
     pub const OS: TraceType = TraceType(1 << 0);
     pub const STORAGE: TraceType = TraceType(1 << 1);
     pub const S3: TraceType = TraceType(1 << 2);
@@ -38,13 +40,15 @@ impl TraceType {
     pub const FTP: TraceType = TraceType(1 << 13);
     pub const ILM: TraceType = TraceType(1 << 14);
 
-    /// All trace categories combined. Must be updated when adding new variants.
+    // MetricsAll must be last.
     pub const ALL: TraceType = TraceType((1 << 15) - 1);
 
     pub fn new(t: u64) -> Self {
         Self(t)
     }
+}
 
+impl TraceType {
     pub fn contains(&self, x: &TraceType) -> bool {
         (self.0 & x.0) == x.0
     }
@@ -72,38 +76,140 @@ impl TraceType {
     }
 }
 
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceInfo {
+    #[serde(rename = "type")]
+    trace_type: u64,
+    #[serde(rename = "nodename")]
+    node_name: String,
+    #[serde(rename = "funcname")]
+    func_name: String,
+    #[serde(rename = "time")]
+    time: Timestamp,
+    #[serde(rename = "path")]
+    path: String,
+    #[serde(rename = "dur")]
+    duration: Duration,
+    #[serde(rename = "bytes", skip_serializing_if = "Option::is_none")]
+    bytes: Option<i64>,
+    #[serde(rename = "msg", skip_serializing_if = "Option::is_none")]
+    message: Option<String>,
+    #[serde(rename = "error", skip_serializing_if = "Option::is_none")]
+    error: Option<String>,
+    #[serde(rename = "custom", skip_serializing_if = "Option::is_none")]
+    custom: Option<HashMap<String, String>>,
+    #[serde(rename = "http", skip_serializing_if = "Option::is_none")]
+    http: Option<TraceHTTPStats>,
+    #[serde(rename = "healResult", skip_serializing_if = "Option::is_none")]
+    heal_result: Option<HealResultItem>,
+}
+
+impl TraceInfo {
+    pub fn mask(&self) -> u64 {
+        TraceType::new(self.trace_type).mask()
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceInfoLegacy {
+    trace_info: TraceInfo,
+    #[serde(rename = "request")]
+    req_info: Option<TraceRequestInfo>,
+    #[serde(rename = "response")]
+    resp_info: Option<TraceResponseInfo>,
+    #[serde(rename = "stats")]
+    call_stats: Option<TraceCallStats>,
+    #[serde(rename = "storageStats")]
+    storage_stats: Option<StorageStats>,
+    #[serde(rename = "osStats")]
+    os_stats: Option<OSStats>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct StorageStats {
+    path: String,
+    duration: Duration,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct OSStats {
+    path: String,
+    duration: Duration,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceHTTPStats {
+    req_info: TraceRequestInfo,
+    resp_info: TraceResponseInfo,
+    call_stats: TraceCallStats,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceCallStats {
+    input_bytes: i32,
+    output_bytes: i32,
+    latency: Duration,
+    time_to_first_byte: Duration,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceRequestInfo {
+    time: Timestamp,
+    proto: String,
+    method: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    path: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    raw_query: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    headers: Option<HashMap<String, String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    body: Option<Vec<u8>>,
+    client: String,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceResponseInfo {
+    time: Timestamp,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    headers: Option<HashMap<String, String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    body: Option<Vec<u8>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    status_code: Option<i32>,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    fn trace_type_contains_and_overlaps() {
-        let mut combined = TraceType::default();
-        combined.merge(&TraceType::S3);
-        combined.merge(&TraceType::HEALING);
+    fn trace_timestamps_serialize_as_rfc3339_utc() {
+        let timestamp = Timestamp::constant(1_700_000_000, 123_456_000);
+        let trace = TraceInfo {
+            time: timestamp,
+            http: Some(TraceHTTPStats {
+                req_info: TraceRequestInfo {
+                    time: timestamp,
+                    ..Default::default()
+                },
+                resp_info: TraceResponseInfo {
+                    time: timestamp,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
 
-        assert!(combined.contains(&TraceType::S3));
-        assert!(combined.contains(&TraceType::HEALING));
-        assert!(!combined.contains(&TraceType::SCANNER));
-        assert!(combined.overlaps(&TraceType::S3));
-        assert!(combined.overlaps(&TraceType::HEALING));
-        assert!(!combined.overlaps(&TraceType::SCANNER));
-    }
-
-    #[test]
-    fn trace_type_set_if() {
-        let mut tt = TraceType::default();
-        tt.set_if(true, &TraceType::OS);
-        tt.set_if(false, &TraceType::S3);
-        assert!(tt.contains(&TraceType::OS));
-        assert!(!tt.contains(&TraceType::S3));
-    }
-
-    #[test]
-    fn trace_type_single_type() {
-        assert!(TraceType::S3.single_type());
-        let mut combined = TraceType::S3;
-        combined.merge(&TraceType::HEALING);
-        assert!(!combined.single_type());
+        let value = serde_json::to_value(trace).expect("trace should serialize");
+        assert_eq!(value["time"], "2023-11-14T22:13:20.123456Z");
+        assert_eq!(value["http"]["req_info"]["time"], "2023-11-14T22:13:20.123456Z");
+        assert_eq!(value["http"]["resp_info"]["time"], "2023-11-14T22:13:20.123456Z");
+        let trace: TraceInfo = serde_json::from_value(value).expect("trace should deserialize");
+        assert_eq!(trace.time, timestamp);
+        let http = trace.http.expect("http trace should deserialize");
+        assert_eq!(http.req_info.time, timestamp);
+        assert_eq!(http.resp_info.time, timestamp);
     }
 }

scripts/check_test_wiring.py

@@ -206,13 +206,6 @@ def check_runner_selection(root: Path) -> list[str]:
     return errors
 
 
-def check_s3_tests_runner(root: Path) -> list[str]:
-    runner = (root / "scripts/s3-tests/run.sh").read_text()
-    if "--showlocals" in runner:
-        return ["scripts/s3-tests/run.sh: pytest failure diagnostics must not dump local values"]
-    return []
-
-
 def profile_selection(root: Path, profile: str) -> str:
     if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
         raise ValueError(f"invalid e2e profile name: {profile}")
@@ -279,7 +272,6 @@ def validate(root: Path) -> list[str]:
     errors.extend(check_e2e_modules(root))
     errors.extend(check_fuzz_targets(root))
     errors.extend(check_runner_selection(root))
-    errors.extend(check_s3_tests_runner(root))
     errors.extend(check_profile_definitions(root))
     return errors
 
@@ -349,23 +341,6 @@ class SelfTests(unittest.TestCase):
             )
             self.assertEqual(len(check_fuzz_targets(root)), 1)
 
-    def test_s3_runner_rejects_unbounded_failure_locals(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            runner = root / "scripts/s3-tests/run.sh"
-            runner.parent.mkdir(parents=True)
-            runner.write_text("tox -- -vv -ra --tb=long\n")
-            self.assertEqual(check_s3_tests_runner(root), [])
-            runner.write_text("tox -- -vv -ra --showlocals --tb=long\n")
-            self.assertEqual(len(check_s3_tests_runner(root)), 1)
-            with (
-                mock.patch(__name__ + ".check_e2e_modules", return_value=[]),
-                mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
-                mock.patch(__name__ + ".check_runner_selection", return_value=[]),
-                mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
-            ):
-                self.assertEqual(len(validate(root)), 1)
-
     def test_profile_listing_enforces_selection(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -436,7 +411,7 @@ def main() -> int:
         for error in errors:
             print(f"ERROR: {error}", file=sys.stderr)
         return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
     return 0
 
 

scripts/s3-tests/run.sh

@@ -1028,11 +1028,10 @@ else
 fi
 
 # Run tests from s3tests/functional
-# Failure locals can contain multi-MiB request bodies; keep tracebacks without expanding local values.
 set +e
 S3TEST_CONF="${CONF_OUTPUT_PATH}" \
     tox -- \
-    -vv -ra --tb=long \
+    -vv -ra --showlocals --tb=long \
     --maxfail="${MAXFAIL}" \
     --timeout="${TEST_TIMEOUT}" \
     --junitxml="${ARTIFACTS_DIR}/junit.xml" \