Merge branch 'main' into cxymds/fix-1925-heal-state-fence

chore(madmin): remove dead trace structs, keep TraceType bitflag helper only

TraceInfo, TraceInfoLegacy, TraceHTTPStats, TraceCallStats, TraceRequestInfo,
TraceResponseInfo, StorageStats, and OSStats are unreferenced outside trace.rs.
Trim to TraceType + its bitflag operations which are actively used by
service_commands.rs and profile_admin.rs.

-139 lines (215 -> 76 lines)

crates/config/src/constants/runtime.rs

@@ -57,6 +57,13 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
 pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
 pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 
+/// Dedicated blocking thread pool for fsync/fdatasync operations.
+/// When > 1, fsync operations are isolated from the main blocking pool to
+/// prevent device-bound fsync from starving read operations (pread/stat/open).
+/// Default 0 means auto (no isolation, use main runtime).
+pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
+pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
+
 // Dial9 Tokio Telemetry Default values
 pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
 pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";

crates/e2e_test/src/common.rs

@@ -30,6 +30,7 @@ use reqwest::StatusCode;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
+use serde_json;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::io::ErrorKind;
@@ -1583,6 +1584,156 @@ impl Drop for RustFSTestClusterEnvironment {
     }
 }
 
+/// Send a SigV4-signed HTTP request and return the raw `reqwest::Response`.
+///
+/// Unlike [`signed_s3_request`], this variant accepts `body: Option<Vec<u8>>`
+/// (binary-safe) and reorders parameters so that `access_key`/`secret_key`
+/// appear before the body — matching the convention used by the replication
+/// extension and object-lambda e2e suites.
+pub(crate) async fn signed_request(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+/// Like [`signed_request`], but uses a caller-supplied `reqwest::Client`
+/// instead of the shared [`local_http_client`].
+pub(crate) async fn signed_request_with_client(
+    client: &reqwest::Client,
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+/// Like [`signed_request`], but includes a `session_token` in the
+/// `x-amz-security-token` header and passes it to the SigV4 signer.
+pub(crate) async fn signed_request_with_session_token(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if !session_token.is_empty() {
+        request = request.header("x-amz-security-token", session_token);
+    }
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(
+        request.body(Body::empty())?,
+        content_len,
+        access_key,
+        secret_key,
+        session_token,
+        "us-east-1",
+    );
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+/// Create a new user via the admin API.
+pub(crate) async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    let body = serde_json::json!({
+        "secretKey": secret_key,
+        "status": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != reqwest::StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("create user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/e2e_test/src/kms/common.rs

@@ -40,7 +40,7 @@ use std::time::Duration;
 use tokio::fs;
 use tokio::net::TcpStream;
 use tokio::time::sleep;
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, warn};
 
 // KMS-specific constants
 pub const TEST_BUCKET: &str = "kms-test-bucket";
@@ -177,6 +177,49 @@ pub async fn get_kms_status(
     Ok(status)
 }
 
+/// Poll the KMS status endpoint until the backend reports ready or the timeout
+/// expires.  Replaces hard-coded `sleep(Duration::from_secs(3))` startup waits
+/// with an active readiness probe so tests start as soon as KMS is usable
+/// (typically < 1 s) instead of always waiting the full 3 s.
+///
+/// Uses exponential back-off starting at 200 ms (doubling each attempt, capped
+/// at 1 s) up to a total wall-clock budget of 5 s.
+pub async fn wait_for_kms_ready(
+    base_url: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let total_deadline = Duration::from_secs(5);
+    let start = tokio::time::Instant::now();
+    let mut backoff = Duration::from_millis(200);
+    let max_backoff = Duration::from_secs(1);
+    let mut first_attempt = true;
+
+    loop {
+        if !first_attempt {
+            if start.elapsed() >= total_deadline {
+                return Err("KMS failed to become ready within 5 seconds".into());
+            }
+            sleep(backoff).await;
+            backoff = (backoff * 2).min(max_backoff);
+        }
+        first_attempt = false;
+
+        match get_kms_status(base_url, access_key, secret_key).await {
+            Ok(status) => {
+                info!("KMS is ready (status: {})", status);
+                return Ok(());
+            }
+            Err(e) => {
+                if start.elapsed() >= total_deadline {
+                    return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
+                }
+                warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
+            }
+        }
+    }
+}
+
 /// Create a default KMS key for testing and return the created key ID
 pub async fn create_default_key(
     base_url: &str,
@@ -861,6 +904,13 @@ impl LocalKMSTestEnvironment {
         Ok(default_key_id.to_string())
     }
 
+    /// Poll the KMS status endpoint until the backend reports ready.
+    ///
+    /// Prefer this over a fixed `sleep` after calling `start_rustfs_for_local_kms`.
+    pub async fn wait_for_kms_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        wait_for_kms_ready(&self.base_env.url, &self.base_env.access_key, &self.base_env.secret_key).await
+    }
+
     /// Configure Local KMS backend with a predefined default key
     pub async fn configure_local_kms(&self) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
         // Use a fixed, predictable default key ID

crates/e2e_test/src/object_lambda_test.rs

@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client, signed_request};
 use aws_sdk_s3::primitives::ByteStream;
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::{pre_sign_v4, sign_v4};
+use rustfs_signer::pre_sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use s3s::Body;
 use std::collections::HashMap;
@@ -227,39 +226,6 @@ async fn presigned_get_request(
     Ok(local_http_client().get(signed.uri().to_string()).send().await?)
 }
 
-async fn signed_request(
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let client = local_http_client();
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
 async fn configure_webhook_target(
     env: &RustFSTestEnvironment,
     target_name: &str,

crates/e2e_test/src/replication_extension_test.rs

@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
-    replication_fast_env, rustfs_binary_path,
+    RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
+    local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
+    signed_request_with_session_token,
 };
 use crate::fake_s3_target::{
     FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -35,7 +36,7 @@ use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
 use bytes::Bytes;
 use flate2::read::GzDecoder;
 use futures::{Stream, StreamExt};
-use http::header::{CONTENT_ENCODING, CONTENT_TYPE, HOST};
+use http::header::CONTENT_ENCODING;
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
 use hyper::server::conn::http1;
@@ -56,9 +57,6 @@ use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
 use s3s::header::X_AMZ_REPLICATION_STATUS;
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
@@ -387,116 +385,6 @@ struct ReplicationResetStatusTarget {
     object: String,
 }
 
-async fn signed_request(
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let client = local_http_client();
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-async fn signed_request_with_client(
-    client: &reqwest::Client,
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-async fn signed_request_with_session_token(
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    session_token: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if !session_token.is_empty() {
-        request = request.header("x-amz-security-token", session_token);
-    }
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(
-        request.body(Body::empty())?,
-        content_len,
-        access_key,
-        secret_key,
-        session_token,
-        "us-east-1",
-    );
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let client = local_http_client();
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
 fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
     let open = format!("<{tag}>");
     let close = format!("</{tag}>");
@@ -1016,35 +904,6 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
     Client::from_conf(config)
 }
 
-async fn admin_create_user(
-    env: &RustFSTestEnvironment,
-    username: &str,
-    secret_key: &str,
-) -> Result<(), Box<dyn Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
-    let body = serde_json::json!({
-        "secretKey": secret_key,
-        "status": "enabled"
-    });
-    let response = signed_request(
-        http::Method::PUT,
-        &url,
-        &env.access_key,
-        &env.secret_key,
-        Some(body.to_string().into_bytes()),
-        Some("application/json"),
-    )
-    .await?;
-
-    if response.status() != StatusCode::OK {
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        return Err(format!("create user failed: {status} {body}").into());
-    }
-
-    Ok(())
-}
-
 async fn admin_add_canned_policy(
     env: &RustFSTestEnvironment,
     policy_name: &str,

crates/ecstore/src/core/pools.rs

@@ -1675,7 +1675,7 @@ impl PoolMeta {
         self.load_no_lock(pool).await
     }
 
-    async fn load_no_lock<S>(&mut self, pool: Arc<S>) -> Result<()>
+    pub(crate) async fn load_no_lock<S>(&mut self, pool: Arc<S>) -> Result<()>
     where
         S: EcstoreObjectIO,
     {

crates/ecstore/src/disk/os.rs

@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
     #[cfg(unix)]
     {
         let dir = dir.as_ref().to_path_buf();
-        tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
+        fsync_spawn_blocking(move || fsync_dir_std(dir)).await?
     }
 
     #[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
     #[cfg(test)]
     let dir = group.dir.clone();
     let dir_file = group.dir_file.clone();
-    tokio::task::spawn_blocking(move || {
+    fsync_spawn_blocking(move || {
         #[cfg(test)]
         {
             if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,6 +1080,44 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
 static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+
+/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
+/// configured with >1 threads, isolates device-bound fsync from the main
+/// blocking pool so reads (pread/stat/open) are not starved. `None` means
+/// fall back to the main runtime (zero behavior change).
+static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
+    let threads =
+        rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
+    if threads <= 1 {
+        return None;
+    }
+    let mut builder = tokio::runtime::Builder::new_multi_thread();
+    builder
+        .worker_threads(num_cpus::get().min(8))
+        .max_blocking_threads(threads)
+        .thread_name("rustfs-fsync")
+        .thread_stack_size(512 * 1024)
+        .enable_all();
+    match builder.build() {
+        Ok(rt) => {
+            tracing::info!(threads, "fsync dedicated blocking pool enabled");
+            Some(rt)
+        }
+        Err(err) => {
+            tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
+            None
+        }
+    }
+});
+
+/// Spawn a blocking task on the fsync-dedicated runtime if configured,
+/// otherwise fall back to the main tokio blocking pool.
+fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
+    match FSYNC_RUNTIME.as_ref() {
+        Some(rt) => rt.spawn_blocking(f),
+        None => tokio::task::spawn_blocking(f),
+    }
+}
 static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
     LazyLock::new(|| Mutex::new(HashMap::new()));
 type NamespaceMutationLock = AsyncMutex<()>;
@@ -1217,7 +1255,7 @@ where
     F: FnOnce() -> io::Result<T> + Send + 'static,
 {
     let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
-    let result = tokio::task::spawn_blocking(move || {
+    let result = fsync_spawn_blocking(move || {
         let _disk_permit = disk_permit;
         work()
     })
@@ -2146,7 +2184,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
         wait_started,
     );
     let disk_permit = admission.disk_permit.clone();
-    let result = tokio::task::spawn_blocking(move || {
+    let result = fsync_spawn_blocking(move || {
         let _lease = lease;
         let _disk_permit = disk_permit;
         operation()

crates/ecstore/src/store/heal.rs

@@ -13,7 +13,12 @@
 // limitations under the License.
 
 use super::*;
+use crate::core::pools::POOL_META_NAME;
+use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::heal::HealOperations as _;
+use crate::storage_api_contracts::namespace::NamespaceLocking as _;
+use rustfs_lock::NamespaceLockGuard;
 use tracing::trace;
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -30,7 +35,119 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
     )
 }
 
+#[derive(Debug, Clone, Copy)]
+enum HealFormatPoolSkip {
+    Completed,
+    Retryable,
+}
+
+fn classify_heal_format_pool(
+    pool_idx: usize,
+    pool_cmd_line: &str,
+    pool_meta: &PoolMeta,
+    rebalance_meta: Option<&RebalanceMeta>,
+) -> Option<HealFormatPoolSkip> {
+    let Some(pool) = pool_meta.pools.get(pool_idx) else {
+        return Some(HealFormatPoolSkip::Retryable);
+    };
+
+    if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
+        return Some(HealFormatPoolSkip::Retryable);
+    }
+
+    if let Some(decommission) = pool.decommission.as_ref() {
+        if decommission.complete {
+            return Some(HealFormatPoolSkip::Completed);
+        }
+        if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
+            return Some(HealFormatPoolSkip::Retryable);
+        }
+    }
+
+    if let Some(meta) = rebalance_meta {
+        let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
+            return Some(HealFormatPoolSkip::Retryable);
+        };
+        if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
+            return Some(HealFormatPoolSkip::Retryable);
+        }
+    }
+
+    None
+}
+
+fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
+    match skip {
+        HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
+        HealFormatPoolSkip::Retryable => StorageError::SlowDown,
+    }
+}
+
+fn heal_format_fence_lost_error() -> Error {
+    StorageError::SlowDown
+}
+
 impl ECStore {
+    async fn acquire_heal_format_fence(
+        &self,
+    ) -> Result<(NamespaceLockGuard, NamespaceLockGuard, PoolMeta, Option<RebalanceMeta>)> {
+        let metadata_pool = self
+            .pools
+            .first()
+            .cloned()
+            .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
+
+        // Metadata fence order is part of the decommission/rebalance protocol:
+        // pool.bin must always be acquired before rebalance.bin.
+        let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
+        let pool_guard = pool_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
+        let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+
+        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        let mut pool_meta = PoolMeta::default();
+        pool_meta.load_no_lock(metadata_pool.clone()).await?;
+        if pool_meta.pools.len() != self.pools.len()
+            || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
+                pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
+            })
+        {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        let mut rebalance_meta = RebalanceMeta::new();
+        let rebalance_meta = match rebalance_meta
+            .load_with_opts(
+                metadata_pool,
+                ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+        {
+            Ok(()) => Some(rebalance_meta),
+            Err(Error::ConfigNotFound) => None,
+            Err(err) => return Err(err),
+        };
+
+        if rebalance_meta
+            .as_ref()
+            .is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
+        {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        Ok((pool_guard, rebalance_guard, pool_meta, rebalance_meta))
+    }
+
     fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
         match opts.pool {
             Some(pool_idx) => Ok(vec![
@@ -52,8 +169,24 @@ impl ECStore {
         };
 
         let mut count_no_heal = 0;
+        let mut count_completed = 0;
         let mut first_error = None;
-        for pool in self.pools.iter() {
+        for (pool_idx, pool) in self.pools.iter().enumerate() {
+            let (pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
+            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+                first_error.get_or_insert(heal_format_fence_lost_error());
+                break;
+            }
+            if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
+            {
+                if matches!(skip, HealFormatPoolSkip::Completed) {
+                    count_completed += 1;
+                } else {
+                    first_error.get_or_insert(heal_format_pool_skip_error(skip));
+                }
+                continue;
+            }
+
             let (mut result, err) = pool.heal_format(dry_run).await?;
             if let Some(err) = err {
                 match err {
@@ -69,11 +202,18 @@ impl ECStore {
             r.set_count += result.set_count;
             r.before.drives.append(&mut result.before.drives);
             r.after.drives.append(&mut result.after.drives);
+
+            // Sets::heal_format cannot observe this guard before each disk write;
+            // fail closed after the call if the lease was lost during format IO.
+            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+                first_error.get_or_insert(heal_format_fence_lost_error());
+                break;
+            }
         }
         if let Some(err) = first_error {
             return Ok((r, Some(err)));
         }
-        if count_no_heal == self.pools.len() {
+        if count_no_heal + count_completed == self.pools.len() {
             info!(
                 event = EVENT_HEAL_FORMAT_COMPLETED,
                 component = LOG_COMPONENT_ECSTORE,
@@ -300,6 +440,7 @@ mod tests {
     use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
     use crate::disk::{DiskOption, format::FormatV3, new_disk};
     use crate::layout::endpoints::{Endpoints, PoolEndpoints};
+    use crate::services::rebalance::{RebalanceInfo, RebalanceStats};
     use crate::store::init_format::{load_format_erasure, save_format_file};
 
     async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
@@ -347,6 +488,164 @@ mod tests {
         }
     }
 
+    fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
+        PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(info),
+            }],
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn heal_format_pool_state_barriers_are_classified() {
+        let active = pool_meta_with_decommission(PoolDecommissionInfo {
+            start_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        });
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &active, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        for info in [
+            PoolDecommissionInfo {
+                failed: true,
+                ..Default::default()
+            },
+            PoolDecommissionInfo {
+                canceled: true,
+                ..Default::default()
+            },
+        ] {
+            assert!(matches!(
+                classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
+                Some(HealFormatPoolSkip::Retryable)
+            ));
+        }
+
+        let completed = pool_meta_with_decommission(PoolDecommissionInfo {
+            complete: true,
+            ..Default::default()
+        });
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &completed, None),
+            Some(HealFormatPoolSkip::Completed)
+        ));
+    }
+
+    #[test]
+    fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
+        let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
+        let rebalance = RebalanceMeta {
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let stopping = RebalanceMeta {
+            pool_stats: vec![RebalanceStats {
+                info: RebalanceInfo {
+                    stopping: true,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-new", &identity, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let identity_without_decommission = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        assert!(matches!(
+            classify_heal_format_pool(0, "", &identity_meta, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let stopped = RebalanceMeta {
+            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Stopped,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
+
+        let stopping_after_stop = RebalanceMeta {
+            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    stopping: true,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+    }
+
+    #[test]
+    fn skipped_heal_format_pool_is_never_reported_as_success() {
+        assert!(matches!(
+            heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
+            StorageError::SlowDown
+        ));
+        assert!(matches!(
+            heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
+            StorageError::NoHealRequired
+        ));
+    }
+
     #[tokio::test]
     async fn heal_object_pool_scope_selects_only_requested_pool() {
         let store = minimal_heal_store().await;
@@ -615,6 +914,18 @@ mod tests {
             bucket_fence_registry: std::sync::Arc::default(),
         };
 
+        let err = store
+            .handle_heal_format(false)
+            .await
+            .expect_err("missing pool metadata must fail closed before format writes");
+        assert!(matches!(err, StorageError::SlowDown));
+
+        let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
+        pool_meta
+            .save(store.pools.clone())
+            .await
+            .expect("pool metadata should be persisted before format heal");
+
         let (result, err) = store
             .handle_heal_format(false)
             .await
@@ -628,5 +939,22 @@ mod tests {
             .await
             .expect("the later pool should be healed despite the first pool error");
         assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
+
+        let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
+        for status in &mut completed_meta.pools {
+            status.decommission = Some(PoolDecommissionInfo {
+                complete: true,
+                ..Default::default()
+            });
+        }
+        completed_meta
+            .save(store.pools.clone())
+            .await
+            .expect("completed pool metadata should be persisted");
+        let (_, err) = store
+            .handle_heal_format(false)
+            .await
+            .expect("completed pools should be reported as a no-op");
+        assert!(matches!(err, Some(StorageError::NoHealRequired)));
     }
 }

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -231,6 +231,10 @@ impl HealTask {
                             "Heal erasure set format repair skipped because no format heal was required"
                         );
                     } else {
+                        let error = e;
+                        if error.is_recoverable_heal() {
+                            return Err(error);
+                        }
                         error!(
                             target: "rustfs::heal::task",
                             event = EVENT_HEAL_ERASURE_SET_RESULT,
@@ -239,7 +243,7 @@ impl HealTask {
                             task_id = %self.id,
                             set_disk_id,
                             result = "format_failed",
-                            error = %e,
+                            error = %error,
                             "Heal erasure set failed"
                         );
                         {
@@ -247,7 +251,7 @@ impl HealTask {
                             progress.update_progress(4, 4, 0, 0);
                         }
                         return Err(Error::TaskExecutionFailed {
-                            message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
+                            message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
                         });
                     }
                 } else {
@@ -284,6 +288,9 @@ impl HealTask {
             Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
             Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
             Err(e) => {
+                if e.is_recoverable_heal() {
+                    return Err(e);
+                }
                 error!(
                     target: "rustfs::heal::task",
                     event = EVENT_HEAL_ERASURE_SET_RESULT,

crates/heal/src/heal/task/tests.rs

@@ -547,6 +547,7 @@ struct MockStorage {
     heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
     heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
     format_no_heal_required: Mutex<bool>,
+    format_error: Mutex<Option<Error>>,
     global_format_calls: Mutex<u32>,
     replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
     replacement_targets_ready: Mutex<bool>,
@@ -867,6 +868,9 @@ impl HealStorageAPI for MockStorage {
 
     async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         *self.global_format_calls.lock().unwrap() += 1;
+        if let Some(error) = self.format_error.lock().unwrap().take() {
+            return Err(error);
+        }
         let no_heal_required = *self.format_no_heal_required.lock().unwrap();
         if no_heal_required {
             Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired))))
@@ -2052,6 +2056,30 @@ async fn test_erasure_set_heal_continues_after_format_no_heal_required() {
     );
 }
 
+#[tokio::test]
+async fn erasure_set_format_slowdown_is_propagated() {
+    let storage = Arc::new(MockStorage {
+        format_error: Mutex::new(Some(Error::Storage(EcstoreError::SlowDown))),
+        ..Default::default()
+    });
+    let request = HealRequest::new(
+        HealType::ErasureSet {
+            buckets: Vec::new(),
+            set_disk_id: "pool_0_set_0".to_string(),
+        },
+        HealOptions::default(),
+        HealPriority::Normal,
+    );
+    let task = HealTask::from_request(request, storage);
+
+    let error = task
+        .execute()
+        .await
+        .expect_err("format SlowDown must remain recoverable for the task manager");
+
+    assert!(matches!(error, Error::Storage(EcstoreError::SlowDown)));
+}
+
 #[tokio::test]
 async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
     let temp = TempDir::new().expect("temporary directory should be created");

crates/madmin/src/trace.rs

@@ -12,18 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::{collections::HashMap, time::Duration};
-
-use jiff::Timestamp;
 use serde::{Deserialize, Serialize};
 
-use crate::heal_commands::HealResultItem;
-
+/// Bitflag helper for service trace categories.
+///
+/// Each variant occupies a single bit so that a `TraceType` value can represent
+/// an arbitrary combination of categories via bitwise OR.
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
 pub struct TraceType(u64);
 
 impl TraceType {
-    // Define some constants
     pub const OS: TraceType = TraceType(1 << 0);
     pub const STORAGE: TraceType = TraceType(1 << 1);
     pub const S3: TraceType = TraceType(1 << 2);
@@ -40,15 +38,13 @@ impl TraceType {
     pub const FTP: TraceType = TraceType(1 << 13);
     pub const ILM: TraceType = TraceType(1 << 14);
 
-    // MetricsAll must be last.
+    /// All trace categories combined. Must be updated when adding new variants.
     pub const ALL: TraceType = TraceType((1 << 15) - 1);
 
     pub fn new(t: u64) -> Self {
         Self(t)
     }
-}
 
-impl TraceType {
     pub fn contains(&self, x: &TraceType) -> bool {
         (self.0 & x.0) == x.0
     }
@@ -76,140 +72,38 @@ impl TraceType {
     }
 }
 
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct TraceInfo {
-    #[serde(rename = "type")]
-    trace_type: u64,
-    #[serde(rename = "nodename")]
-    node_name: String,
-    #[serde(rename = "funcname")]
-    func_name: String,
-    #[serde(rename = "time")]
-    time: Timestamp,
-    #[serde(rename = "path")]
-    path: String,
-    #[serde(rename = "dur")]
-    duration: Duration,
-    #[serde(rename = "bytes", skip_serializing_if = "Option::is_none")]
-    bytes: Option<i64>,
-    #[serde(rename = "msg", skip_serializing_if = "Option::is_none")]
-    message: Option<String>,
-    #[serde(rename = "error", skip_serializing_if = "Option::is_none")]
-    error: Option<String>,
-    #[serde(rename = "custom", skip_serializing_if = "Option::is_none")]
-    custom: Option<HashMap<String, String>>,
-    #[serde(rename = "http", skip_serializing_if = "Option::is_none")]
-    http: Option<TraceHTTPStats>,
-    #[serde(rename = "healResult", skip_serializing_if = "Option::is_none")]
-    heal_result: Option<HealResultItem>,
-}
-
-impl TraceInfo {
-    pub fn mask(&self) -> u64 {
-        TraceType::new(self.trace_type).mask()
-    }
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct TraceInfoLegacy {
-    trace_info: TraceInfo,
-    #[serde(rename = "request")]
-    req_info: Option<TraceRequestInfo>,
-    #[serde(rename = "response")]
-    resp_info: Option<TraceResponseInfo>,
-    #[serde(rename = "stats")]
-    call_stats: Option<TraceCallStats>,
-    #[serde(rename = "storageStats")]
-    storage_stats: Option<StorageStats>,
-    #[serde(rename = "osStats")]
-    os_stats: Option<OSStats>,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct StorageStats {
-    path: String,
-    duration: Duration,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct OSStats {
-    path: String,
-    duration: Duration,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct TraceHTTPStats {
-    req_info: TraceRequestInfo,
-    resp_info: TraceResponseInfo,
-    call_stats: TraceCallStats,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct TraceCallStats {
-    input_bytes: i32,
-    output_bytes: i32,
-    latency: Duration,
-    time_to_first_byte: Duration,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct TraceRequestInfo {
-    time: Timestamp,
-    proto: String,
-    method: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    path: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    raw_query: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    headers: Option<HashMap<String, String>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    body: Option<Vec<u8>>,
-    client: String,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct TraceResponseInfo {
-    time: Timestamp,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    headers: Option<HashMap<String, String>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    body: Option<Vec<u8>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    status_code: Option<i32>,
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    fn trace_timestamps_serialize_as_rfc3339_utc() {
-        let timestamp = Timestamp::constant(1_700_000_000, 123_456_000);
-        let trace = TraceInfo {
-            time: timestamp,
-            http: Some(TraceHTTPStats {
-                req_info: TraceRequestInfo {
-                    time: timestamp,
-                    ..Default::default()
-                },
-                resp_info: TraceResponseInfo {
-                    time: timestamp,
-                    ..Default::default()
-                },
-                ..Default::default()
-            }),
-            ..Default::default()
-        };
+    fn trace_type_contains_and_overlaps() {
+        let mut combined = TraceType::default();
+        combined.merge(&TraceType::S3);
+        combined.merge(&TraceType::HEALING);
 
-        let value = serde_json::to_value(trace).expect("trace should serialize");
-        assert_eq!(value["time"], "2023-11-14T22:13:20.123456Z");
-        assert_eq!(value["http"]["req_info"]["time"], "2023-11-14T22:13:20.123456Z");
-        assert_eq!(value["http"]["resp_info"]["time"], "2023-11-14T22:13:20.123456Z");
-        let trace: TraceInfo = serde_json::from_value(value).expect("trace should deserialize");
-        assert_eq!(trace.time, timestamp);
-        let http = trace.http.expect("http trace should deserialize");
-        assert_eq!(http.req_info.time, timestamp);
-        assert_eq!(http.resp_info.time, timestamp);
+        assert!(combined.contains(&TraceType::S3));
+        assert!(combined.contains(&TraceType::HEALING));
+        assert!(!combined.contains(&TraceType::SCANNER));
+        assert!(combined.overlaps(&TraceType::S3));
+        assert!(combined.overlaps(&TraceType::HEALING));
+        assert!(!combined.overlaps(&TraceType::SCANNER));
+    }
+
+    #[test]
+    fn trace_type_set_if() {
+        let mut tt = TraceType::default();
+        tt.set_if(true, &TraceType::OS);
+        tt.set_if(false, &TraceType::S3);
+        assert!(tt.contains(&TraceType::OS));
+        assert!(!tt.contains(&TraceType::S3));
+    }
+
+    #[test]
+    fn trace_type_single_type() {
+        assert!(TraceType::S3.single_type());
+        let mut combined = TraceType::S3;
+        combined.merge(&TraceType::HEALING);
+        assert!(!combined.single_type());
     }
 }

crates/test-utils/src/lib.rs

@@ -245,6 +245,18 @@ impl TestECStoreEnvBuilder {
             .await
             .expect("build test ECStore");
 
+        // The production bootstrap only persists pool.bin from the elected
+        // first cluster node.  Test stores intentionally have no cluster
+        // election, but heal-format still requires that durable fence before
+        // it can write any disk format.  Materialize the validated topology
+        // here so the shared fixture models a ready single-node store.
+        let mut pool_meta = ecstore.pool_meta.read().await.clone();
+        pool_meta.dont_save = false;
+        pool_meta
+            .save(ecstore.pools.clone())
+            .await
+            .expect("persist test pool metadata");
+
         if self.init_bucket_metadata {
             let buckets_list = ecstore
                 .list_bucket(&BucketOptions {

scripts/check_test_wiring.py

@@ -206,6 +206,13 @@ def check_runner_selection(root: Path) -> list[str]:
     return errors
 
 
+def check_s3_tests_runner(root: Path) -> list[str]:
+    runner = (root / "scripts/s3-tests/run.sh").read_text()
+    if "--showlocals" in runner:
+        return ["scripts/s3-tests/run.sh: pytest failure diagnostics must not dump local values"]
+    return []
+
+
 def profile_selection(root: Path, profile: str) -> str:
     if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
         raise ValueError(f"invalid e2e profile name: {profile}")
@@ -272,6 +279,7 @@ def validate(root: Path) -> list[str]:
     errors.extend(check_e2e_modules(root))
     errors.extend(check_fuzz_targets(root))
     errors.extend(check_runner_selection(root))
+    errors.extend(check_s3_tests_runner(root))
     errors.extend(check_profile_definitions(root))
     return errors
 
@@ -341,6 +349,23 @@ class SelfTests(unittest.TestCase):
             )
             self.assertEqual(len(check_fuzz_targets(root)), 1)
 
+    def test_s3_runner_rejects_unbounded_failure_locals(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            runner = root / "scripts/s3-tests/run.sh"
+            runner.parent.mkdir(parents=True)
+            runner.write_text("tox -- -vv -ra --tb=long\n")
+            self.assertEqual(check_s3_tests_runner(root), [])
+            runner.write_text("tox -- -vv -ra --showlocals --tb=long\n")
+            self.assertEqual(len(check_s3_tests_runner(root)), 1)
+            with (
+                mock.patch(__name__ + ".check_e2e_modules", return_value=[]),
+                mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
+                mock.patch(__name__ + ".check_runner_selection", return_value=[]),
+                mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
+            ):
+                self.assertEqual(len(validate(root)), 1)
+
     def test_profile_listing_enforces_selection(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -411,7 +436,7 @@ def main() -> int:
         for error in errors:
             print(f"ERROR: {error}", file=sys.stderr)
         return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
     return 0
 
 

scripts/s3-tests/run.sh

@@ -1028,10 +1028,11 @@ else
 fi
 
 # Run tests from s3tests/functional
+# Failure locals can contain multi-MiB request bodies; keep tracebacks without expanding local values.
 set +e
 S3TEST_CONF="${CONF_OUTPUT_PATH}" \
     tox -- \
-    -vv -ra --showlocals --tb=long \
+    -vv -ra --tb=long \
     --maxfail="${MAXFAIL}" \
     --timeout="${TEST_TIMEOUT}" \
     --junitxml="${ARTIFACTS_DIR}/junit.xml" \