fix: prevent metadata lock reentry and stabilize CI fixtures (#8257)

* fix(test): establish a writable previous-release upgrade baseline

* fix(ci): reserve capacity for durable admin fixtures

* fix(ci): group durable IAM state fixtures by resource needs

* ci: run E2E doctests with the E2E dependency graph

* fix(ci): separate fixture startup from transport deadlines

* fix(ci): bound pagination after seeding and revisit restored copies

* fix(ci): make filesystem fixture timing deterministic

* fix(ecstore): avoid metadata lock reentry during internal mutations

* test: align recovery fixtures with durable ownership contracts

rustfs/src/admin/handlers/site_replication.rs

@@ -8878,6 +8878,8 @@ mod tests {
     }
 
     /// Publish a ready IAM app context so `apply_iam_item` gets past its IAM guard.
+    /// Tests using this real storage fixture keep the `durable_iam_state_` prefix so
+    /// nextest reserves capacity while their internal races retain production deadlines.
     async fn publish_ready_iam_context() {
         use crate::admin::runtime_sources::{AppContext, publish_test_app_context};
         use rustfs_iam::store::{Store as _, object::IAM_CONFIG_PREFIX};
@@ -9077,7 +9079,7 @@ mod tests {
     /// every gated item type.
     #[tokio::test]
     #[serial]
-    async fn apply_iam_item_applies_delayed_in_order_updates_through_the_receiver() {
+    async fn durable_iam_state_apply_iam_item_applies_delayed_in_order_updates_through_the_receiver() {
         publish_ready_iam_context().await;
         seed_two_peer_state_for_iam_apply().await;
         let iam = current_iam_handle().expect("test IAM");
@@ -9158,7 +9160,7 @@ mod tests {
 
     #[tokio::test]
     #[serial]
-    async fn apply_group_member_add_preserves_a_disabled_group_status() {
+    async fn durable_iam_state_apply_group_member_add_preserves_a_disabled_group_status() {
         publish_ready_iam_context().await;
         seed_two_peer_state_for_iam_apply().await;
         let iam = current_iam_handle().expect("test IAM");
@@ -9193,7 +9195,7 @@ mod tests {
     /// membership-only guard above exists to prevent.
     #[tokio::test]
     #[serial]
-    async fn apply_group_snapshot_propagates_a_disabled_status_with_members() {
+    async fn durable_iam_state_apply_group_snapshot_propagates_a_disabled_status_with_members() {
         publish_ready_iam_context().await;
         seed_two_peer_state_for_iam_apply().await;
         let iam = current_iam_handle().expect("test IAM");
@@ -9228,7 +9230,7 @@ mod tests {
     /// the write of one cannot interleave with the other's.
     #[tokio::test]
     #[serial]
-    async fn apply_iam_item_serializes_a_concurrent_older_grant_and_newer_revoke() {
+    async fn durable_iam_state_apply_iam_item_serializes_a_concurrent_older_grant_and_newer_revoke() {
         publish_ready_iam_context().await;
         seed_two_peer_state_for_iam_apply().await;
         let t1 = OffsetDateTime::now_utc() - time::Duration::hours(2);
@@ -9267,7 +9269,7 @@ mod tests {
     /// applied the delete; a genuinely newer create still lands.
     #[tokio::test]
     #[serial]
-    async fn apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete() {
+    async fn durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete() {
         publish_ready_iam_context().await;
         seed_two_peer_state_for_iam_apply().await;
         let iam = current_iam_handle().expect("test IAM");
@@ -9312,7 +9314,7 @@ mod tests {
     /// then switched off, and carries the source stamp.
     #[tokio::test]
     #[serial]
-    async fn apply_iam_item_creates_a_replicated_service_account_with_its_status() {
+    async fn durable_iam_state_apply_iam_item_creates_a_replicated_service_account_with_its_status() {
         publish_ready_iam_context().await;
         seed_two_peer_state_for_iam_apply().await;
         let iam = current_iam_handle().expect("test IAM");
@@ -9341,7 +9343,7 @@ mod tests {
 
     #[tokio::test]
     #[serial]
-    async fn apply_iam_item_accepts_minio_sts_account_item_type() {
+    async fn durable_iam_state_apply_iam_item_accepts_minio_sts_account_item_type() {
         publish_ready_iam_context().await;
 
         // MinIO madmin-go sends `SRIAMItemSTSAcc = "sts-account"`. The bogus session token
@@ -9363,7 +9365,7 @@ mod tests {
 
     #[tokio::test]
     #[serial]
-    async fn apply_iam_item_still_accepts_legacy_sts_credential_item_type() {
+    async fn durable_iam_state_apply_iam_item_still_accepts_legacy_sts_credential_item_type() {
         publish_ready_iam_context().await;
 
         // Older RustFS peers emit `sts-credential`; the alias stays accepted permanently
@@ -15400,7 +15402,7 @@ mod tests {
     /// would delete the retry queue and every other field along with it.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     #[serial]
-    async fn test_missed_pending_clear_must_not_rewrite_the_state_object() {
+    async fn durable_iam_state_test_missed_pending_clear_must_not_rewrite_the_state_object() {
         publish_ready_iam_context().await;
 
         // One peer, no pending records: exactly the shape the persist helper's clear
@@ -15461,7 +15463,7 @@ mod tests {
     /// write before A finishes.
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
-    async fn test_peer_join_admission_serializes_iam_apply_against_a_newer_join() {
+    async fn durable_iam_state_test_peer_join_admission_serializes_iam_apply_against_a_newer_join() {
         publish_ready_iam_context().await;
 
         // Whole-second timestamps so the RFC3339 round trip through the state
@@ -15565,7 +15567,7 @@ mod tests {
     /// admission: the assertion on the empty IAM log turns red.
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
-    async fn test_peer_join_admission_serializes_across_separate_nodes() {
+    async fn durable_iam_state_test_peer_join_admission_serializes_across_separate_nodes() {
         publish_ready_iam_context().await;
 
         let now = OffsetDateTime::now_utc().replace_nanosecond(0).expect("truncate nanos");
@@ -15662,7 +15664,7 @@ mod tests {
     /// acked rotation is cleared in the same transaction that reports true.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     #[serial]
-    async fn test_finalize_pending_rotation_three_way_contract() {
+    async fn durable_iam_state_test_finalize_pending_rotation_three_way_contract() {
         publish_ready_iam_context().await;
 
         let local_peer = PeerInfo {
@@ -15744,7 +15746,7 @@ mod tests {
     /// update.
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
-    async fn test_state_object_lock_serializes_writers_from_separate_nodes() {
+    async fn durable_iam_state_test_state_object_lock_serializes_writers_from_separate_nodes() {
         publish_ready_iam_context().await;
 
         let seed = SiteReplicationState {
@@ -15796,7 +15798,7 @@ mod tests {
     /// leave the receiver unable to tell which edit is newer.
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
-    async fn test_peer_edit_generations_are_unique_across_nodes() {
+    async fn durable_iam_state_test_peer_edit_generations_are_unique_across_nodes() {
         publish_ready_iam_context().await;
         // A configured site: `persist_site_replication_state_no_lock` clears
         // the object once a site drops below two peers, and a cleared object
@@ -15847,7 +15849,7 @@ mod tests {
     /// accepts the restarted counter instead of fencing it.
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
-    async fn test_recreated_state_object_allocates_over_the_previous_lifetimes_mark() {
+    async fn durable_iam_state_test_recreated_state_object_allocates_over_the_previous_lifetimes_mark() {
         publish_ready_iam_context().await;
         let seed = || SiteReplicationState {
             peers: ["site-a", "site-b"]
@@ -15912,7 +15914,7 @@ mod tests {
     /// (the red-light commit pinned the exact interleaving).
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
-    async fn test_retry_event_persist_must_not_wipe_concurrent_locked_rmw() {
+    async fn durable_iam_state_test_retry_event_persist_must_not_wipe_concurrent_locked_rmw() {
         publish_ready_iam_context().await;
 
         const ROUNDS: usize = 8;

rustfs/tests/connect_heartbeat.rs

@@ -41,6 +41,9 @@ use tokio::sync::watch;
 use tokio_rustls::TlsAcceptor;
 use tokio_util::sync::CancellationToken;
 
+// Allow durable state and real TLS setup independently of transport and cancellation deadlines.
+const STATUS_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(10);
+
 const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70";
 const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81";
 const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92";
@@ -191,9 +194,23 @@ impl Reply {
 struct TestServer {
     endpoint: String,
     seen: Arc<Mutex<Vec<Value>>>,
+    paths: Arc<Mutex<Vec<String>>>,
     task: tokio::task::JoinHandle<()>,
 }
 
+impl TestServer {
+    fn request_diagnostics(&self) -> String {
+        match self.paths.try_lock() {
+            Ok(paths) => format!(
+                "request_count={} request_paths={paths:?} server_task_finished={}",
+                paths.len(),
+                self.task.is_finished()
+            ),
+            Err(error) => format!("request_paths_unavailable={error} server_task_finished={}", self.task.is_finished()),
+        }
+    }
+}
+
 impl Drop for TestServer {
     fn drop(&mut self) {
         self.task.abort();
@@ -207,16 +224,20 @@ async fn server(pki: &TestPki, replies: Vec<Reply>) -> TestServer {
     let replies = Arc::new(Mutex::new(VecDeque::from(replies)));
     let seen = Arc::new(Mutex::new(Vec::new()));
     let captured = seen.clone();
+    let paths = Arc::new(Mutex::new(Vec::new()));
+    let captured_paths = paths.clone();
     let task = tokio::spawn(async move {
         while let Ok((stream, _)) = listener.accept().await {
             let acceptor = acceptor.clone();
             let replies = replies.clone();
             let seen = captured.clone();
+            let paths = captured_paths.clone();
             tokio::spawn(async move {
                 let Ok(stream) = acceptor.accept(stream).await else { return };
                 let service = service_fn(move |request: Request<hyper::body::Incoming>| {
                     let replies = replies.clone();
                     let seen = seen.clone();
+                    let paths = paths.clone();
                     async move {
                         let path = request.uri().path().to_owned();
                         assert!(
@@ -227,6 +248,7 @@ async fn server(pki: &TestPki, replies: Vec<Reply>) -> TestServer {
                         seen.lock()
                             .expect("seen lock")
                             .push(serde_json::from_slice(&body).expect("request JSON"));
+                        paths.lock().expect("paths lock").push(path.clone());
                         let reply = if path.ends_with("/diagnosticExecutionReceipts") {
                             Reply {
                                 status: StatusCode::OK,
@@ -266,6 +288,7 @@ async fn server(pki: &TestPki, replies: Vec<Reply>) -> TestServer {
     TestServer {
         endpoint: format!("https://localhost:{}/agent/", address.port()),
         seen,
+        paths,
         task,
     }
 }
@@ -314,23 +337,33 @@ fn summary() -> CoarseNodeSummary {
 }
 
 async fn wait_for(
+    server: &TestServer,
     status: &mut watch::Receiver<HeartbeatStatus>,
     predicate: impl Fn(&HeartbeatStatus) -> bool,
 ) -> HeartbeatStatus {
-    tokio::time::timeout(Duration::from_secs(3), async {
+    tokio::time::timeout(STATUS_OBSERVATION_TIMEOUT, async {
         loop {
             let current = status.borrow_and_update().clone();
             if predicate(&current) {
                 return current;
             }
-            status
-                .changed()
-                .await
-                .unwrap_or_else(|error| panic!("heartbeat status channel closed: {error}; last status: {:?}", *status.borrow()));
+            status.changed().await.unwrap_or_else(|error| {
+                panic!(
+                    "heartbeat status channel closed: {error}; last status: {:?}; {}",
+                    *status.borrow(),
+                    server.request_diagnostics()
+                )
+            });
         }
     })
     .await
-    .unwrap_or_else(|error| panic!("heartbeat status timeout: {error}; last status: {:?}", *status.borrow()))
+    .unwrap_or_else(|error| {
+        panic!(
+            "heartbeat status timeout: {error}; last status: {:?}; {}",
+            *status.borrow(),
+            server.request_diagnostics()
+        )
+    })
 }
 
 async fn assert_credential_failure(config: HeartbeatConfig, server: &TestServer, expected: &str) {
@@ -340,7 +373,7 @@ async fn assert_credential_failure(config: HeartbeatConfig, server: &TestServer,
         .expect("configured runtime");
     let mut status = runtime.status();
     assert!(matches!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
+        wait_for(server, &mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
         HeartbeatStatus::Failed { reason } if reason.contains(expected)
     ));
     assert!(server.seen.lock().expect("seen lock").is_empty());
@@ -436,7 +469,7 @@ async fn corrupt_persisted_state_is_rejected_before_network_delivery() {
     let mut status = runtime.status();
 
     assert!(matches!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
+        wait_for(&server, &mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
         HeartbeatStatus::Failed { reason } if reason.contains("violates the protocol invariants")
     ));
     assert!(server.seen.lock().expect("seen lock").is_empty());
@@ -505,7 +538,7 @@ async fn sends_only_l0_fields_and_accepts_additive_response_fields() {
     let mut status = runtime.status();
 
     assert_eq!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+        wait_for(&server, &mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
         HeartbeatStatus::Online {
             server_time: "2038-01-19T03:14:07Z".to_owned()
         }
@@ -599,7 +632,7 @@ async fn restart_replays_a_pending_heartbeat_from_before_environment_collection(
         .expect("configured runtime");
     let mut status = runtime.status();
     assert!(matches!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+        wait_for(&server, &mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
         HeartbeatStatus::Online { .. }
     ));
     runtime.shutdown().await;
@@ -620,7 +653,7 @@ async fn restart_replays_pending_request_then_advances_sequence() {
         .expect("start runtime")
         .expect("configured runtime");
     let mut status = runtime.status();
-    wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await;
+    wait_for(&first_server, &mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await;
     runtime.shutdown().await;
     let first = first_server.seen.lock().expect("seen lock")[0].clone();
     drop(first_server);
@@ -730,7 +763,7 @@ async fn restart_replays_exact_legacy_capabilities_with_and_without_jobs() {
             .expect("configured runtime");
         let mut status = runtime.status();
         assert!(matches!(
-            wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+            wait_for(&server, &mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
             HeartbeatStatus::Online { .. }
         ));
         runtime.shutdown().await;
@@ -770,7 +803,7 @@ async fn legacy_compatibility_does_not_accept_changed_capabilities() {
             let mut status = runtime.status();
             assert!(
                 matches!(
-                    wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
+                    wait_for(&server, &mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
                     HeartbeatStatus::Failed { reason } if reason.contains("violates the protocol invariants")
                 ),
                 "accepted altered persisted capabilities: {mutation}, jobs={job_capable}, service_memory={service_memory}"
@@ -859,7 +892,7 @@ async fn retry_after_is_respected_with_the_local_upper_bound() {
         .expect("configured runtime");
     let mut status = runtime.status();
     assert_eq!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await,
+        wait_for(&server, &mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await,
         HeartbeatStatus::BackingOff {
             delay: Duration::from_millis(80)
         }
@@ -887,7 +920,7 @@ async fn disconnects_use_exponential_backoff_with_a_cap() {
     let mut status = runtime.status();
     for delay in [20, 40, 80] {
         assert_eq!(
-            wait_for(&mut status, |status| {
+            wait_for(&server, &mut status, |status| {
                 matches!(status, HeartbeatStatus::BackingOff { delay: observed } if *observed == Duration::from_millis(delay))
             })
             .await,
@@ -912,7 +945,11 @@ async fn revoked_credential_stops_and_exposes_local_status() {
         .expect("configured runtime");
     let mut status = runtime.status();
     assert_eq!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::AuthenticationStopped { .. })).await,
+        wait_for(&server, &mut status, |status| matches!(
+            status,
+            HeartbeatStatus::AuthenticationStopped { .. }
+        ))
+        .await,
         HeartbeatStatus::AuthenticationStopped {
             status: 401,
             reason: Some("CREDENTIAL_REVOKED".to_owned())

rustfs/tests/connect_perf_object.rs

@@ -38,6 +38,9 @@ use zeroize::Zeroizing;
 mod common;
 
 static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+// Real storage setup shares the measurement window. Verify functionality with
+// an integration budget; deadline behavior is checked against a stalled peer.
+const REAL_SERVER_MEASUREMENT_BUDGET: Duration = Duration::from_secs(10);
 
 fn now() -> i64 {
     SystemTime::now().duration_since(UNIX_EPOCH).expect("current time").as_secs() as i64
@@ -305,6 +308,56 @@ async fn deadline_and_in_flight_cancellation_stop_a_stalled_probe() {
     assert_eq!(measurement.target.reason_code, ObjectTargetReasonCode::Cancelled);
 }
 
+#[tokio::test]
+async fn s3_request_deadline_stops_a_peer_that_never_responds() {
+    use tokio::io::AsyncReadExt as _;
+
+    let _guard = TEST_LOCK.lock().await;
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("response listener");
+    let address = listener.local_addr().expect("listener address");
+    let (received, request_received) = tokio::sync::oneshot::channel();
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("object connection");
+        let mut request = [0_u8; 4096];
+        assert!(socket.read(&mut request).await.expect("request headers") > 0);
+        received.send(()).expect("report received request");
+        std::future::pending::<()>().await;
+        drop(socket);
+    });
+    let probe = S3ObjectProbe::new(
+        &format!("http://{address}"),
+        None,
+        None,
+        Zeroizing::new("access".to_owned()),
+        Zeroizing::new("secret".to_owned()),
+        Zeroizing::new(String::new()),
+        Duration::from_secs(60),
+    )
+    .expect("object probe");
+    let mut request = request(ObjectOperation::GetObject);
+    request.duration = Duration::from_secs(30);
+    let measurement = tokio::spawn(async move { measure_object(&request, &probe, &CancellationToken::new()).await });
+
+    // Complete real TCP setup before pausing time. The 30s operation deadline
+    // must fire independently of the client's longer 60s transport timeout.
+    tokio::time::timeout(Duration::from_secs(10), request_received)
+        .await
+        .expect("peer received the request")
+        .expect("peer notification");
+    tokio::time::pause();
+    tokio::time::advance(Duration::from_secs(30)).await;
+    let measurement = tokio::time::timeout(Duration::from_secs(1), measurement)
+        .await
+        .expect("operation deadline must stop the request")
+        .expect("measurement task")
+        .expect("typed timeout");
+    assert_eq!(measurement.result.outcome(), ObjectOutcome::Failed);
+    assert_eq!(measurement.target.reason_code, ObjectTargetReasonCode::TimedOut);
+    assert_eq!(measurement.target.completed_operations, 0);
+    server.abort();
+    let _ = server.await;
+}
+
 #[tokio::test]
 async fn only_one_object_collector_can_run_at_a_time() {
     let _guard = TEST_LOCK.lock().await;
@@ -519,6 +572,7 @@ fn real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put() {
 
 async fn real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put_body() {
     let _guard = TEST_LOCK.lock().await;
+    let startup = std::time::Instant::now();
     let port = match find_available_port() {
         Ok(port) => port,
         Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
@@ -531,6 +585,7 @@ async fn real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put_bod
         .build()
         .await
         .expect("start embedded server");
+    eprintln!("embedded object fixture ready after {:?}", startup.elapsed());
     let probe = S3ObjectProbe::new(
         &server.endpoint(),
         None,
@@ -538,13 +593,13 @@ async fn real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put_bod
         Zeroizing::new(server.access_key().to_owned()),
         Zeroizing::new(server.secret_key().to_owned()),
         Zeroizing::new(String::new()),
-        Duration::from_secs(2),
+        REAL_SERVER_MEASUREMENT_BUDGET,
     )
     .expect("object probe");
 
     for operation in [ObjectOperation::GetObject, ObjectOperation::PutObject] {
         let mut request = request(operation);
-        request.duration = Duration::from_secs(2);
+        request.duration = REAL_SERVER_MEASUREMENT_BUDGET;
         if operation == ObjectOperation::PutObject {
             request.artifact_uid = "019e3ae0-0000-7000-8000-000000000016".to_owned();
         }
@@ -604,7 +659,7 @@ async fn real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put_bod
             "--traffic-bytes",
             "65536",
             "--duration-millis",
-            "1000",
+            &REAL_SERVER_MEASUREMENT_BUDGET.as_millis().to_string(),
             "--acknowledge-l1",
         ]);
     let result = tokio::task::spawn_blocking(move || command.output())

rustfs/tests/connect_registration.rs

@@ -59,6 +59,9 @@ const EXPECTED_CERTIFICATE_LIFETIME_SECONDS: i64 = 86_400;
 const EXPECTED_ROTATION_THRESHOLD_SECONDS: i64 = 120;
 #[cfg(not(feature = "connect-e2e-short-credentials"))]
 const EXPECTED_ROTATION_THRESHOLD_SECONDS: i64 = 8 * 60 * 60;
+// Allow durable state and real TLS setup independently of transport and cancellation deadlines.
+const STATUS_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(10);
+
 const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70";
 const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81";
 const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92";
@@ -209,6 +212,19 @@ struct TestServer {
     task: tokio::task::JoinHandle<()>,
 }
 
+impl TestServer {
+    fn request_diagnostics(&self) -> String {
+        match self.paths.try_lock() {
+            Ok(paths) => format!(
+                "request_count={} request_paths={paths:?} server_task_finished={}",
+                paths.len(),
+                self.task.is_finished()
+            ),
+            Err(error) => format!("request_paths_unavailable={error} server_task_finished={}", self.task.is_finished()),
+        }
+    }
+}
+
 impl Drop for TestServer {
     fn drop(&mut self) {
         self.task.abort();
@@ -678,7 +694,7 @@ async fn explicit_proxy_carries_registration_rotation_and_heartbeat_with_mtls()
         .expect("configured heartbeat runtime");
     let mut status = runtime.status();
     assert!(matches!(
-        wait_for_heartbeat_status(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+        wait_for_heartbeat_status(&server, &mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
         HeartbeatStatus::Online { .. }
     ));
     runtime.shutdown().await;
@@ -872,22 +888,33 @@ async fn wait_for_requests(server: &TestServer, count: usize) {
 }
 
 async fn wait_for_heartbeat_status(
+    server: &TestServer,
     status: &mut watch::Receiver<HeartbeatStatus>,
     predicate: impl Fn(&HeartbeatStatus) -> bool,
 ) -> HeartbeatStatus {
-    tokio::time::timeout(Duration::from_secs(3), async {
+    tokio::time::timeout(STATUS_OBSERVATION_TIMEOUT, async {
         loop {
             let current = status.borrow_and_update().clone();
             if predicate(&current) {
                 return current;
             }
             status.changed().await.unwrap_or_else(|error| {
-                panic!("heartbeat status channel: {error}; last status: {:?}", *status.borrow());
+                panic!(
+                    "heartbeat status channel: {error}; last status: {:?}; {}",
+                    *status.borrow(),
+                    server.request_diagnostics()
+                );
             });
         }
     })
     .await
-    .unwrap_or_else(|error| panic!("heartbeat status: {error}; last status: {:?}", *status.borrow()))
+    .unwrap_or_else(|error| {
+        panic!(
+            "heartbeat status: {error}; last status: {:?}; {}",
+            *status.borrow(),
+            server.request_diagnostics()
+        )
+    })
 }
 
 fn rotation_response(pki: &TestPki, identity: &rustfs::connect::DeviceIdentity, serial: u8) -> (Value, Value) {
@@ -1301,7 +1328,25 @@ async fn heartbeat_runtime_respects_rotation_retry_after_without_blocking_heartb
         .expect("configured runtime");
     let mut status = runtime.status();
 
-    wait_for_heartbeat_status(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await;
+    // Start the progress bound after the durable rotation claim reaches the server.
+    // Heartbeats must still proceed before the five-second rotation Retry-After expires.
+    wait_for_requests(&server, 1).await;
+    assert!(
+        server.paths.lock().expect("paths lock")[0].ends_with(":rotateCredential"),
+        "rotation request must precede heartbeat progress"
+    );
+    tokio::time::timeout(
+        Duration::from_secs(3),
+        wait_for_heartbeat_status(&server, &mut status, |status| matches!(status, HeartbeatStatus::Online { .. })),
+    )
+    .await
+    .unwrap_or_else(|error| {
+        panic!(
+            "heartbeat progress blocked after rotation: {error}; last status: {:?}; {}",
+            *status.borrow(),
+            server.request_diagnostics()
+        )
+    });
     wait_for_requests(&server, 4).await;
     runtime.shutdown().await;
 
@@ -1353,7 +1398,7 @@ async fn heartbeat_runtime_skips_only_valid_pending_reenrollment() {
     let mut status = runtime.status();
 
     assert!(matches!(
-        wait_for_heartbeat_status(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+        wait_for_heartbeat_status(&server, &mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
         HeartbeatStatus::Online { .. }
     ));
     runtime.shutdown().await;
@@ -1393,7 +1438,8 @@ async fn heartbeat_runtime_skips_only_valid_pending_reenrollment() {
         .expect("configured runtime");
         let mut corrupted_status = corrupted_runtime.status();
         assert_eq!(
-            wait_for_heartbeat_status(&mut corrupted_status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
+            wait_for_heartbeat_status(&server, &mut corrupted_status, |status| matches!(status, HeartbeatStatus::Failed { .. }))
+                .await,
             HeartbeatStatus::Failed {
                 reason: HeartbeatError::StateConflict.to_string(),
             }
@@ -1462,7 +1508,7 @@ async fn heartbeat_retries_with_the_new_credential_after_concurrent_rotation() {
         .expect("concurrent rotation")
         .expect("rotation due");
     assert!(matches!(
-        wait_for_heartbeat_status(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+        wait_for_heartbeat_status(&server, &mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
         HeartbeatStatus::Online { .. }
     ));
     runtime.shutdown().await;
@@ -1547,7 +1593,7 @@ async fn inventory_retries_with_the_new_credential_after_concurrent_rotation() {
         .expect("concurrent rotation")
         .expect("rotation due");
     assert!(matches!(
-        tokio::time::timeout(Duration::from_secs(3), async {
+        tokio::time::timeout(STATUS_OBSERVATION_TIMEOUT, async {
             loop {
                 let current = status.borrow_and_update().clone();
                 if matches!(current, InventoryStatus::BackingOff { .. }) {
@@ -1642,20 +1688,37 @@ async fn inventory_first_recovers_a_saved_reenrollment_before_telemetry() {
     .expect("start inventory runtime")
     .expect("configured inventory runtime");
     let mut inventory_status = inventory.status();
+    let startup_diagnostics = || {
+        format!(
+            "{}; pending_registration={} staged_identity={} completed_registration={}",
+            telemetry.request_diagnostics(),
+            temp.path().join("credential/registration.pending.json").exists(),
+            temp.path().join("identity/device.key.next").exists(),
+            temp.path().join("credential/registration.completed.json").exists()
+        )
+    };
     assert!(matches!(
-        tokio::time::timeout(Duration::from_secs(3), async {
+        tokio::time::timeout(STATUS_OBSERVATION_TIMEOUT, async {
             loop {
                 let current = inventory_status.borrow_and_update().clone();
                 if matches!(current, InventoryStatus::Online { .. }) {
                     break current;
                 }
                 inventory_status.changed().await.unwrap_or_else(|error| {
-                    panic!("inventory status channel: {error}; last status: {:?}", *inventory_status.borrow());
+                    panic!(
+                        "inventory status channel: {error}; last status: {:?}; {}",
+                        *inventory_status.borrow(),
+                        startup_diagnostics()
+                    );
                 });
             }
         })
         .await
-        .unwrap_or_else(|error| panic!("inventory online status: {error}; last status: {:?}", *inventory_status.borrow())),
+        .unwrap_or_else(|error| panic!(
+            "inventory online status: {error}; last status: {:?}; {}",
+            *inventory_status.borrow(),
+            startup_diagnostics()
+        )),
         InventoryStatus::Online { .. }
     ));
 
@@ -1684,7 +1747,11 @@ async fn inventory_first_recovers_a_saved_reenrollment_before_telemetry() {
     .expect("configured heartbeat runtime");
     let mut heartbeat_status = heartbeat.status();
     assert!(matches!(
-        wait_for_heartbeat_status(&mut heartbeat_status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+        wait_for_heartbeat_status(&telemetry, &mut heartbeat_status, |status| matches!(
+            status,
+            HeartbeatStatus::Online { .. }
+        ))
+        .await,
         HeartbeatStatus::Online { .. }
     ));
     heartbeat.shutdown().await;
@@ -1747,7 +1814,11 @@ async fn heartbeat_runtime_stops_when_rotation_reports_revocation() {
     let mut status = runtime.status();
 
     assert_eq!(
-        wait_for_heartbeat_status(&mut status, |status| matches!(status, HeartbeatStatus::AuthenticationStopped { .. })).await,
+        wait_for_heartbeat_status(&server, &mut status, |status| matches!(
+            status,
+            HeartbeatStatus::AuthenticationStopped { .. }
+        ))
+        .await,
         HeartbeatStatus::AuthenticationStopped {
             status: 401,
             reason: Some("DEVICE_REVOKED".to_owned()),