Merge branch 'main' into cxymds/fix-1928-scheduler-panic

crates/config/src/constants/runtime.rs

@@ -57,6 +57,13 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
 pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
 pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 
+/// Dedicated blocking thread pool for fsync/fdatasync operations.
+/// When > 1, fsync operations are isolated from the main blocking pool to
+/// prevent device-bound fsync from starving read operations (pread/stat/open).
+/// Default 0 means auto (no isolation, use main runtime).
+pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
+pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
+
 // Dial9 Tokio Telemetry Default values
 pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
 pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";

crates/data-usage/src/data_usage.rs

@@ -585,12 +585,9 @@ impl VersionsHistogram {
     }
 }
 
-/// Replication statistics for a single target.
-///
-/// Renamed from `ReplicationStats`; serde field names are preserved
-/// byte-identically to maintain wire compatibility with existing snapshots.
-#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct ReplicationTargetUsage {
+/// Replication statistics for a single target
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct ReplicationStats {
     pub pending_size: u64,
     pub replicated_size: u64,
     pub failed_size: u64,
@@ -603,7 +600,7 @@ pub struct ReplicationTargetUsage {
     pub replicated_count: u64,
 }
 
-impl ReplicationTargetUsage {
+impl ReplicationStats {
     pub fn is_empty(&self) -> bool {
         let Self {
             pending_size,
@@ -639,7 +636,7 @@ impl ReplicationTargetUsage {
 /// Replication statistics for all targets
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct ReplicationAllStats {
-    pub targets: HashMap<String, ReplicationTargetUsage>,
+    pub targets: HashMap<String, ReplicationStats>,
     pub replica_size: u64,
     pub replica_count: u64,
 }
@@ -652,7 +649,7 @@ impl ReplicationAllStats {
             targets,
         } = self;
 
-        *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty)
+        *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
     }
 
     #[deprecated(note = "use is_empty instead")]
@@ -2469,7 +2466,7 @@ mod tests {
 
     #[test]
     fn replication_stats_empty_checks_every_field() {
-        type SetField = fn(&mut ReplicationTargetUsage);
+        type SetField = fn(&mut ReplicationStats);
 
         let cases: [(&str, SetField); 10] = [
             ("pending_size", |stats| stats.pending_size = 1),
@@ -2484,9 +2481,9 @@ mod tests {
             ("replicated_count", |stats| stats.replicated_count = 1),
         ];
 
-        assert!(ReplicationTargetUsage::default().is_empty());
+        assert!(ReplicationStats::default().is_empty());
         for (field, set_nonzero) in cases {
-            let mut stats = ReplicationTargetUsage::default();
+            let mut stats = ReplicationStats::default();
             set_nonzero(&mut stats);
             assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
         }
@@ -2517,17 +2514,17 @@ mod tests {
         }
 
         let empty_targets = ReplicationAllStats {
-            targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]),
+            targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
             ..Default::default()
         };
         assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
 
         let stats = ReplicationAllStats {
             targets: HashMap::from([
-                ("arn:test:empty".to_string(), ReplicationTargetUsage::default()),
+                ("arn:test:empty".to_string(), ReplicationStats::default()),
                 (
                     "arn:test:non-empty".to_string(),
-                    ReplicationTargetUsage {
+                    ReplicationStats {
                         pending_count: 1,
                         ..Default::default()
                     },
@@ -2568,7 +2565,7 @@ mod tests {
                 replication_stats: Some(ReplicationAllStats {
                     targets: HashMap::from([(
                         "arn:test:pending".to_string(),
-                        ReplicationTargetUsage {
+                        ReplicationStats {
                             pending_count: 1,
                             ..Default::default()
                         },
@@ -2717,7 +2714,7 @@ mod tests {
                 targets: HashMap::from([
                     (
                         "arn:self-only".to_string(),
-                        ReplicationTargetUsage {
+                        ReplicationStats {
                             pending_size: 7,
                             pending_count: 1,
                             ..Default::default()
@@ -2725,7 +2722,7 @@ mod tests {
                     ),
                     (
                         "arn:shared".to_string(),
-                        ReplicationTargetUsage {
+                        ReplicationStats {
                             failed_size: 3,
                             failed_count: 1,
                             missed_threshold_size: 2,
@@ -2744,7 +2741,7 @@ mod tests {
                 targets: HashMap::from([
                     (
                         "arn:shared".to_string(),
-                        ReplicationTargetUsage {
+                        ReplicationStats {
                             failed_size: 5,
                             failed_count: 2,
                             after_threshold_size: 4,
@@ -2754,7 +2751,7 @@ mod tests {
                     ),
                     (
                         "arn:other-only".to_string(),
-                        ReplicationTargetUsage {
+                        ReplicationStats {
                             replicated_size: 11,
                             replicated_count: 3,
                             ..Default::default()
@@ -2996,9 +2993,7 @@ mod tests {
     fn replication_target_deserialization_preserves_large_historical_maps() {
         let mut stats = ReplicationAllStats::default();
         for index in 0..=1024 {
-            stats
-                .targets
-                .insert(format!("target-{index}"), ReplicationTargetUsage::default());
+            stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
         }
         let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
         let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
@@ -3007,47 +3002,6 @@ mod tests {
         assert_eq!(decoded.targets.len(), stats.targets.len());
     }
 
-    /// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back
-    /// must produce the exact same value.  This guards against accidental serde
-    /// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage`
-    /// rename.  Wire-level field names are the serialized Rust field identifiers,
-    /// which must remain byte-identical.
-    #[test]
-    fn replication_target_usage_rmp_round_trip() {
-        let original = ReplicationTargetUsage {
-            pending_size: 100,
-            replicated_size: 2_000,
-            failed_size: 50,
-            failed_count: 3,
-            pending_count: 7,
-            missed_threshold_size: 11,
-            after_threshold_size: 22,
-            missed_threshold_count: 1,
-            after_threshold_count: 2,
-            replicated_count: 99,
-        };
-
-        let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack");
-        let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack");
-        assert_eq!(original, decoded, "round-trip through rmp must preserve every field");
-
-        // Also verify that encoding as an unnamed sequence and then decoding
-        // with named fields produces the correct mapping (this catches reordering).
-        let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning");
-        // Spot-check that known field names appear in the named encoding.
-        let named_str = String::from_utf8_lossy(&named_buf);
-        assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename");
-        assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename");
-        assert!(
-            named_str.contains("missed_threshold_size"),
-            "field 'missed_threshold_size' must survive the rename"
-        );
-        assert!(
-            named_str.contains("after_threshold_count"),
-            "field 'after_threshold_count' must survive the rename"
-        );
-    }
-
     #[test]
     fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
         let mut entry = DataUsageEntry {

crates/ecstore/src/disk/os.rs

@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
     #[cfg(unix)]
     {
         let dir = dir.as_ref().to_path_buf();
-        tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
+        fsync_spawn_blocking(move || fsync_dir_std(dir)).await?
     }
 
     #[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
     #[cfg(test)]
     let dir = group.dir.clone();
     let dir_file = group.dir_file.clone();
-    tokio::task::spawn_blocking(move || {
+    fsync_spawn_blocking(move || {
         #[cfg(test)]
         {
             if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,6 +1080,44 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
 static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+
+/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
+/// configured with >1 threads, isolates device-bound fsync from the main
+/// blocking pool so reads (pread/stat/open) are not starved. `None` means
+/// fall back to the main runtime (zero behavior change).
+static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
+    let threads =
+        rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
+    if threads <= 1 {
+        return None;
+    }
+    let mut builder = tokio::runtime::Builder::new_multi_thread();
+    builder
+        .worker_threads(num_cpus::get().min(8))
+        .max_blocking_threads(threads)
+        .thread_name("rustfs-fsync")
+        .thread_stack_size(512 * 1024)
+        .enable_all();
+    match builder.build() {
+        Ok(rt) => {
+            tracing::info!(threads, "fsync dedicated blocking pool enabled");
+            Some(rt)
+        }
+        Err(err) => {
+            tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
+            None
+        }
+    }
+});
+
+/// Spawn a blocking task on the fsync-dedicated runtime if configured,
+/// otherwise fall back to the main tokio blocking pool.
+fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
+    match FSYNC_RUNTIME.as_ref() {
+        Some(rt) => rt.spawn_blocking(f),
+        None => tokio::task::spawn_blocking(f),
+    }
+}
 static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
     LazyLock::new(|| Mutex::new(HashMap::new()));
 type NamespaceMutationLock = AsyncMutex<()>;
@@ -1217,7 +1255,7 @@ where
     F: FnOnce() -> io::Result<T> + Send + 'static,
 {
     let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
-    let result = tokio::task::spawn_blocking(move || {
+    let result = fsync_spawn_blocking(move || {
         let _disk_permit = disk_permit;
         work()
     })
@@ -2146,7 +2184,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
         wait_started,
     );
     let disk_permit = admission.disk_permit.clone();
-    let result = tokio::task::spawn_blocking(move || {
+    let result = fsync_spawn_blocking(move || {
         let _lease = lease;
         let _disk_permit = disk_permit;
         operation()

crates/heal/src/heal/manager/tests.rs

@@ -110,7 +110,10 @@ impl HealStorageAPI for MockStorage {
         Ok(Vec::new())
     }
 
-    async fn get_bucket_info(&self, _bucket: &str) -> Result<Option<BucketInfo>> {
+    async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
+        if bucket == "panic" {
+            panic!("test-only panic payload must not escape the scheduler");
+        }
         Ok(None)
     }
 
@@ -1021,6 +1024,231 @@ async fn test_task_alias_is_removed_after_terminal_completion() {
     assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
 }
 
+#[tokio::test]
+#[serial_test::serial]
+async fn scheduler_panic_releases_active_slot_and_allows_same_target_readmission() {
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new(storage, None);
+    let request = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
+    let task_id = request.id.clone();
+
+    assert_eq!(
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("panic request should be admitted"),
+        HealAdmissionResult::Accepted
+    );
+    let duplicate = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
+    let duplicate_id = duplicate.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request(duplicate)
+            .await
+            .expect("same target should merge while active is queued"),
+        HealAdmissionResult::Merged
+    );
+    assert_eq!(manager.canonical_task_id(&duplicate_id).await, task_id);
+    process_manager_queue_once(&manager).await;
+
+    let status = tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if let Ok(status) = manager.get_task_status(&task_id).await
+                && matches!(status, HealTaskStatus::Failed { .. })
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("panic task should reach a terminal status");
+    assert_eq!(
+        status,
+        HealTaskStatus::Failed {
+            error: PANICKED_HEAL_TASK_ERROR.to_string()
+        }
+    );
+    assert_eq!(manager.get_active_task_count().await, 0);
+    assert_eq!(manager.get_queue_length().await, 0);
+    assert!(manager.retrying_heals.lock().await.is_empty());
+    assert!(manager.task_aliases.lock().await.is_empty());
+    assert!(manager.completed_heals.lock().await.contains_key(&task_id));
+    assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
+
+    let readmitted = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
+    assert_eq!(
+        manager
+            .submit_heal_request(readmitted)
+            .await
+            .expect("same target should be re-admitted after a panic"),
+        HealAdmissionResult::Accepted
+    );
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn retry_child_panic_finishes_parent_once() {
+    clear_scheduler_panic();
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new(storage, None);
+    let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
+    let task_id = request.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("retry request should be admitted"),
+        HealAdmissionResult::Accepted
+    );
+    arm_scheduler_panic(SchedulerPanicPoint::RetryChild, &task_id);
+    process_manager_queue_once(&manager).await;
+
+    let status = tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if let Ok(status) = manager.get_task_status(&task_id).await
+                && matches!(status, HealTaskStatus::Failed { .. })
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("retry child panic should finish the parent");
+    clear_scheduler_panic();
+    assert_eq!(
+        status,
+        HealTaskStatus::Failed {
+            error: PANICKED_HEAL_TASK_ERROR.to_string()
+        }
+    );
+    assert_eq!(manager.get_active_task_count().await, 0);
+    assert_eq!(manager.get_queue_length().await, 0);
+    assert!(manager.retrying_heals.lock().await.is_empty());
+    assert!(manager.task_aliases.lock().await.is_empty());
+    assert_eq!(manager.completed_heals.lock().await.len(), 1);
+    assert_eq!(manager.get_statistics().await.failed_tasks, 1);
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn cleanup_panic_is_supervised() {
+    clear_scheduler_panic();
+    let notice_bucket = "cleanup-panic-mrf";
+    let notice_object = "object";
+    let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(notice_bucket);
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new(storage, None);
+    let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
+    request.source = HealRequestSource::Admin;
+    let task_id = request.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("cleanup request should be admitted"),
+        HealAdmissionResult::Accepted
+    );
+    manager
+        .mrf_repair_notice_targets
+        .lock()
+        .expect("mrf repair notice registry poisoned")
+        .insert(
+            task_id.clone(),
+            vec![MrfRepairNoticeTarget {
+                bucket: Arc::from(notice_bucket),
+                object: Arc::from(notice_object),
+                version_id: None,
+            }],
+        );
+    arm_scheduler_panic(SchedulerPanicPoint::Cleanup, &task_id);
+    process_manager_queue_once(&manager).await;
+
+    let status = tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if let Ok(status) = manager.get_task_status(&task_id).await
+                && matches!(status, HealTaskStatus::Completed)
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("cleanup panic should leave a terminal status");
+    clear_scheduler_panic();
+    assert_eq!(status, HealTaskStatus::Completed);
+    assert_eq!(manager.get_active_task_count().await, 0);
+    assert!(manager.task_aliases.lock().await.is_empty());
+    assert_eq!(manager.completed_heals.lock().await.len(), 1);
+    assert_eq!(manager.get_statistics().await.successful_tasks, 1);
+    let events = rustfs_common::mrf_channel::take_mrf_repaired_events_for(notice_bucket);
+    assert_eq!(events.len(), 1, "cleanup panic must preserve successful MRF notice delivery");
+    assert_eq!(events[0].object.as_ref(), notice_object);
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn cancelled_retry_child_panic_does_not_rearchive_failed_status() {
+    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
+    let task_id = request.id.clone();
+    let retry_cancel_token = insert_retrying_request(&manager, request.clone()).await;
+
+    manager
+        .cancel_task(&task_id)
+        .await
+        .expect("retry cancellation should succeed");
+    assert!(retry_cancel_token.is_cancelled());
+
+    let state = PanicCleanupState {
+        active_heals: manager.active_heals.clone(),
+        heal_queue: manager.heal_queue.clone(),
+        completed_heals: manager.completed_heals.clone(),
+        task_aliases: manager.task_aliases.clone(),
+        retrying_heals: manager.retrying_heals.clone(),
+        mrf_repair_notice_targets: manager.mrf_repair_notice_targets.clone(),
+        replacement_recovery_anchors: manager.replacement_recovery_anchors.clone(),
+        statistics: manager.statistics.clone(),
+    };
+    finish_panicked_retry_child(task_id.clone(), request.heal_type, retry_cancel_token, state).await;
+
+    assert!(manager.retrying_heals.lock().await.is_empty());
+    assert!(manager.completed_heals.lock().await.is_empty());
+    assert_eq!(manager.get_statistics().await.failed_tasks, 0);
+}
+
+#[tokio::test]
+async fn active_cancel_wins_parent_panic_cleanup_without_completed_status() {
+    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
+    let task_id = request.id.clone();
+    let task = Arc::new(HealTask::from_request(request, Arc::new(MockStorage)));
+    manager.active_heals.lock().await.insert(task_id.clone(), task.clone());
+
+    manager
+        .cancel_task(&task_id)
+        .await
+        .expect("active task cancellation should win");
+    assert_eq!(task.get_status().await, HealTaskStatus::Cancelled);
+
+    let state = PanicCleanupState {
+        active_heals: manager.active_heals.clone(),
+        heal_queue: manager.heal_queue.clone(),
+        completed_heals: manager.completed_heals.clone(),
+        task_aliases: manager.task_aliases.clone(),
+        retrying_heals: manager.retrying_heals.clone(),
+        mrf_repair_notice_targets: manager.mrf_repair_notice_targets.clone(),
+        replacement_recovery_anchors: manager.replacement_recovery_anchors.clone(),
+        statistics: manager.statistics.clone(),
+    };
+    finish_panicked_heal_task(task, task_id, state).await;
+
+    assert!(manager.completed_heals.lock().await.is_empty());
+    assert_eq!(manager.get_statistics().await.failed_tasks, 0);
+}
+
 #[tokio::test]
 async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);

crates/scanner/src/data_usage_define/tests.rs

@@ -16,7 +16,7 @@ use super::persistence::DataUsageCacheLoadAttempt;
 use super::*;
 use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
 use crate::{ScannerGetObjectReader, ScannerPutObjReader};
-use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
+use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
 use serde_json::Value;
 use std::io::Cursor;
 use std::pin::Pin;
@@ -1636,7 +1636,7 @@ fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
             replication_stats: Some(ReplicationAllStats {
                 targets: HashMap::from([(
                     "arn:test:threshold".to_string(),
-                    ReplicationTargetUsage {
+                    ReplicationStats {
                         after_threshold_count: 1,
                         ..Default::default()
                     },

crates/scanner/src/scanner_io/publish_gate_tests.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use super::*;
-use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
+use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
 
 const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
 
@@ -271,7 +271,7 @@ fn completed_data_usage_info_flattens_nested_bucket_entries() {
         replication_stats: Some(ReplicationAllStats {
             targets: HashMap::from([(
                 "arn:target".to_string(),
-                ReplicationTargetUsage {
+                ReplicationStats {
                     replicated_size: 2048,
                     replicated_count: 2,
                     ..Default::default()

scripts/check_test_wiring.py

@@ -206,6 +206,13 @@ def check_runner_selection(root: Path) -> list[str]:
     return errors
 
 
+def check_s3_tests_runner(root: Path) -> list[str]:
+    runner = (root / "scripts/s3-tests/run.sh").read_text()
+    if "--showlocals" in runner:
+        return ["scripts/s3-tests/run.sh: pytest failure diagnostics must not dump local values"]
+    return []
+
+
 def profile_selection(root: Path, profile: str) -> str:
     if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
         raise ValueError(f"invalid e2e profile name: {profile}")
@@ -272,6 +279,7 @@ def validate(root: Path) -> list[str]:
     errors.extend(check_e2e_modules(root))
     errors.extend(check_fuzz_targets(root))
     errors.extend(check_runner_selection(root))
+    errors.extend(check_s3_tests_runner(root))
     errors.extend(check_profile_definitions(root))
     return errors
 
@@ -341,6 +349,23 @@ class SelfTests(unittest.TestCase):
             )
             self.assertEqual(len(check_fuzz_targets(root)), 1)
 
+    def test_s3_runner_rejects_unbounded_failure_locals(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            runner = root / "scripts/s3-tests/run.sh"
+            runner.parent.mkdir(parents=True)
+            runner.write_text("tox -- -vv -ra --tb=long\n")
+            self.assertEqual(check_s3_tests_runner(root), [])
+            runner.write_text("tox -- -vv -ra --showlocals --tb=long\n")
+            self.assertEqual(len(check_s3_tests_runner(root)), 1)
+            with (
+                mock.patch(__name__ + ".check_e2e_modules", return_value=[]),
+                mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
+                mock.patch(__name__ + ".check_runner_selection", return_value=[]),
+                mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
+            ):
+                self.assertEqual(len(validate(root)), 1)
+
     def test_profile_listing_enforces_selection(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -411,7 +436,7 @@ def main() -> int:
         for error in errors:
             print(f"ERROR: {error}", file=sys.stderr)
         return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
     return 0
 
 

scripts/s3-tests/run.sh

@@ -1028,10 +1028,11 @@ else
 fi
 
 # Run tests from s3tests/functional
+# Failure locals can contain multi-MiB request bodies; keep tracebacks without expanding local values.
 set +e
 S3TEST_CONF="${CONF_OUTPUT_PATH}" \
     tox -- \
-    -vv -ra --showlocals --tb=long \
+    -vv -ra --tb=long \
     --maxfail="${MAXFAIL}" \
     --timeout="${TEST_TIMEOUT}" \
     --junitxml="${ARTIFACTS_DIR}/junit.xml" \