fix: repair integration checks and IPC test readiness

Wait for the RPC/locks test sockets to pass the existing owner and mode
checks inside the original 5s readiness window and 10ms poll interval.

Use the existing admin S3 carrier for the new recovery and quota error
calls. Regenerate the test-count baseline for main's 118 existing tests
across seven crates. Pair the private pagination helper's raw-scan inputs
and remove the redundant Copy clone from the retirement event.

Keep the original assertions, responses, authorization, locks and storage
operations. The pagination body and raw-scan expressions at all five calls
are byte-identical; reinserting only the Copy clone restores the entire
base heal file. All ignored baseline values and s3s counts stay unchanged.

Validation base: c47432e353
Validation tree: f081084bc5
Patch SHA-256: 2dcf0028691bfb2772edfb38e262829b04f4151832dcce823c9dd9f7434b52de
Trace production-prefix SHA-256: fd994f218a21429d62866f4efb8eee1ee57b58f4a51865aa35d05a54ef1ccb45

All commands used LANG=en_US.UTF-8, CARGO_BUILD_JOBS=1,
CARGO_INCREMENTAL=0 and CARGO_NET_OFFLINE=true in one exclusive target.
Every gate retained the same tree and six source hashes before and after.
54 Cargo.toml files parse without duplicate keys.

cargo clippy -p rustfs --all-targets -- -D warnings
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 12m 50s
Exit: 0. Wall time: 770.855s.

The original 19 quick-check steps:
shellcheck --version && actionlint -> exit 0
cargo fmt --all --check -> exit 0
./scripts/check_unsafe_code_allowances.sh -> exit 0
./scripts/check_layer_dependencies.sh -> exit 0
./scripts/check_architecture_migration_rules.sh -> exit 0
./scripts/check_logging_guardrails.sh -> exit 0
./scripts/check_error_other_format_ratchet.sh -> exit 0
./scripts/check_no_tokio_io_uring.sh -> exit 0
./scripts/check_extension_schema_boundaries.sh -> exit 0
./scripts/check_body_cache_whitelist.sh -> exit 0
./scripts/check_s3s_footprint.sh -> exit 0
./scripts/check_test_count_ratchet.sh -> exit 0
./scripts/check_fips_wording.sh -> exit 0
./scripts/check_embedded_secrets.sh -> exit 0
make script-tests -> exit 0
python3 ./scripts/test_object_generation_protocol_model.py -> exit 0
python3 ./scripts/check_test_wiring.py -> exit 0
./scripts/check_no_planning_docs.sh -> exit 0
./scripts/check_uring_lane_lib_only.sh -> exit 0
Wall time: 286.337s.

cargo nextest run --profile ci -p rustfs --no-fail-fast
     Summary [ 217.017s] 5382 tests run: 5382 passed (4 leaky), 40 skipped
Exit: 0. Wall time: 1324.414s.

cargo nextest run --profile ci -p rustfs-s3-types -p rustfs-protocols --features rustfs-protocols/swift --no-fail-fast
     Summary [   2.189s] 543 tests run: 543 passed, 0 skipped
Exit: 0. Wall time: 123.192s.

cargo nextest run --profile ci -p rustfs-ecstore list_objects_paginate --no-fail-fast
     Summary [   0.170s] 8 tests run: 8 passed, 5921 skipped
Exit: 0. Wall time: 261.324s.

cargo nextest run --profile ci -p rustfs-heal durable_mrf_anchor_retires_only_after_exact_bucket_generation_is_deleted --no-fail-fast
     Summary [   0.805s] 1 test run: 1 passed, 835 skipped
Exit: 0. Wall time: 199.801s.

Baseline regeneration:
./scripts/check_test_count_ratchet.sh --update
wrote .config/test-count-baseline.txt: 52 crates, 21340 tests, 164 ignored
Test-only increases: ecstore48, heal3, iam14, io-metrics2, obs3, policy14,
rustfs34. Existing ignored values and all other rows are byte-identical.
s3s footprint OK: files importing s3s is 195 (baseline files=195)
s3s footprint OK: s3s s3_error! lines is 1584 (baseline s3_error_lines=1584)
s3s footprint OK: ecstore files referencing s3s is 36 (baseline ecstore_files=36)

Historical runtime red controls on f622 (same retained two assertions):
cargo nextest run --profile ci -p rustfs --lib -E 'test(=connect::diagnostics::trace_runtime::tests::local_top_rpc_reads_service_http_completions_and_signs_offline) | test(=connect::diagnostics::trace_runtime::tests::local_top_locks_reads_service_manager_and_signs_offline)' --no-fail-fast
Original path readiness with a temporary 500ms bind/seal gap:
     Summary [   0.863s] 2 tests run: 0 passed, 2 failed, 5113 skipped
Exit: 100.
Sealed readiness in that same gap:
     Summary [   5.621s] 2 tests run: 2 passed, 5113 skipped
Exit: 0.
Compiled M1: replace the helper predicate with path.exists():
     Summary [   0.870s] 2 tests run: 0 passed, 2 failed, 5113 skipped
Exit: 100.
Restore the sealed predicate byte-for-byte:
     Summary [   5.640s] 2 tests run: 2 passed, 5113 skipped
Exit: 0.
Both reds observe mode755 with owner_matches=true and fail at the original
RPC/locks unwraps with StateSecurity. M1 failure locations: trace_runtime.rs
2547:29 and 2713:29. No new assertions were introduced.

cargo nextest run --profile ci -p rustfs --lib -E 'test(connect::diagnostics::trace_runtime::tests::)' --no-fail-fast
     Summary [  20.748s] 28 tests run: 28 passed (1 leaky), 5087 skipped
Exit: 0. This security sweep is historical; the full final unit gate above
covers the actual c474 candidate.
The first disk-stopped compile ran no tests and is excluded from red credit.
All temporary seal instrumentation is removed from the final source.

Final unit cleanup markers: 4 leaky, 40 original skips (exit0). The ld
__eh_frame linker warning also appears verbatim in the original f622 run
and the earlier c474 run; there are no new source warnings.

.config/test-count-baseline.txt

@@ -10,7 +10,7 @@
 # intentionally removed. Regenerate with:
 #   scripts/check_test_count_ratchet.sh --update
 e2e_test	890	18
-rustfs	5498	62
+rustfs	5532	62
 rustfs-audit	70	0
 rustfs-checksums	26	0
 rustfs-common	23	0
@@ -20,14 +20,14 @@ rustfs-connect-symbol-catalog	6	0
 rustfs-credentials	24	0
 rustfs-crypto	67	0
 rustfs-data-usage	54	0
-rustfs-ecstore	6222	1
+rustfs-ecstore	6270	1
 rustfs-extension-schema	23	0
 rustfs-filemeta	306	0
-rustfs-heal	836	3
+rustfs-heal	839	3
 rustfs-heal-contracts	6	0
-rustfs-iam	357	0
+rustfs-iam	371	0
 rustfs-io-core	49	0
-rustfs-io-metrics	123	0
+rustfs-io-metrics	125	0
 rustfs-keystone	29	0
 rustfs-kms	715	25
 rustfs-license	2	0
@@ -38,8 +38,8 @@ rustfs-madmin	209	0
 rustfs-notify	136	0
 rustfs-object-capacity	92	0
 rustfs-object-data-cache	140	0
-rustfs-obs	362	0
-rustfs-policy	663	0
+rustfs-obs	365	0
+rustfs-policy	677	0
 rustfs-protocols	736	1
 rustfs-protos	52	0
 rustfs-replication	200	0

crates/ecstore/src/store/list_objects.rs

@@ -2368,8 +2368,7 @@ fn list_objects_paginate_versions<'a>(
     disk_has_more: bool,
     cache_id: Option<&str>,
     marker: Option<&str>,
-    last_scanned_key: Option<&str>,
-    raw_keys: impl Iterator<Item = &'a str>,
+    (last_scanned_key, raw_keys): (Option<&str>, impl Iterator<Item = &'a str>),
 ) -> (Vec<ObjectInfo>, Vec<String>, bool, Option<String>, Option<String>) {
     filter_versions_common_prefixes_after_marker(&mut get_objects, delimiter.as_deref(), marker);
     let prefix_at_page_boundary =
@@ -4331,8 +4330,7 @@ impl ECStore {
             disk_has_more,
             next_cache_id.as_deref(),
             opts.marker.as_deref(),
-            last_scanned_key.as_deref(),
-            entries.entries().iter().map(|entry| entry.name.as_str()),
+            (last_scanned_key.as_deref(), entries.entries().iter().map(|entry| entry.name.as_str())),
         );
 
         Ok(ListObjectVersionsInfo {
@@ -5759,8 +5757,7 @@ impl Sets {
             disk_has_more,
             next_cache_id.as_deref(),
             opts.marker.as_deref(),
-            last_scanned_key.as_deref(),
-            entries.entries().iter().map(|entry| entry.name.as_str()),
+            (last_scanned_key.as_deref(), entries.entries().iter().map(|entry| entry.name.as_str())),
         );
 
         Ok(ListObjectVersionsInfo {
@@ -6573,8 +6570,7 @@ impl SetDisks {
             disk_has_more,
             next_cache_id.as_deref(),
             opts.marker.as_deref(),
-            last_scanned_key.as_deref(),
-            entries.entries().iter().map(|entry| entry.name.as_str()),
+            (last_scanned_key.as_deref(), entries.entries().iter().map(|entry| entry.name.as_str())),
         );
 
         Ok(ListObjectVersionsInfo {
@@ -8853,8 +8849,7 @@ mod test {
                 disk_has_more,
                 None,
                 marker.as_deref(),
-                last_scanned.as_deref(),
-                window.iter().map(String::as_str),
+                (last_scanned.as_deref(), window.iter().map(String::as_str)),
             );
 
             assert!(
@@ -8932,8 +8927,7 @@ mod test {
                 disk_has_more,
                 None,
                 marker.as_deref(),
-                last_scanned.as_deref(),
-                window.iter().map(String::as_str),
+                (last_scanned.as_deref(), window.iter().map(String::as_str)),
             );
 
             assert_eq!(objects.len() + prefixes.len(), 1, "a page must respect max-keys");

crates/heal/src/heal/mrf_queue/partial_write.rs

@@ -392,7 +392,7 @@ impl PartialWrites {
                     object: anchor.object.clone(),
                     version_id: anchor.version_id,
                     scope: anchor.scope,
-                    delete_marker_purge: anchor.delete_marker_purge.clone(),
+                    delete_marker_purge: anchor.delete_marker_purge,
                     lease: Some(anchor.lease),
                     bucket_incarnation_id: anchor.bucket_incarnation_id,
                     disposition: MrfVerifiedRepairDisposition::AuthoritativelyAbsent,

rustfs/src/admin/handlers/bucket_recovery.rs

@@ -17,12 +17,13 @@
 use crate::admin::auth::authorize_admin_request;
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::admin::runtime_sources::current_object_store_handle;
+use crate::admin::storage_api::s3::{Body, S3Error, S3Request, S3Response, S3Result};
 use crate::server::ADMIN_PREFIX;
 use crate::site_replication::{site_replication_enabled, with_site_replication_bucket_mutation_lock};
 use hyper::{Method, StatusCode};
 use matchit::Params;
 use rustfs_policy::policy::action::{Action, AdminAction};
-use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
+use rustfs_s3_types::s3_error;
 use serde::{Deserialize, Serialize};
 use tracing::info;
 use uuid::Uuid;
@@ -76,10 +77,10 @@ impl Operation for RecoverOrphanedBucketHandler {
         let request: RecoverOrphanedBucketRequest = serde_json::from_slice(&body)
             .map_err(|_| s3_error!(InvalidRequest, "expected JSON with a non-nil expectedIncarnationId"))?;
         if request.expected_incarnation_id.is_nil() {
-            return Err(s3_error!(InvalidRequest, "expectedIncarnationId must be non-nil"));
+            return Err(s3_error!(InvalidRequest, "expectedIncarnationId must be non-nil").into());
         }
         let Some(store) = current_object_store_handle() else {
-            return Err(s3_error!(InternalError, "object store is not initialized"));
+            return Err(s3_error!(InternalError, "object store is not initialized").into());
         };
 
         let operation_bucket = bucket.to_owned();
@@ -94,18 +95,19 @@ impl Operation for RecoverOrphanedBucketHandler {
                         return Err(s3_error!(
                             OperationAborted,
                             "orphaned bucket recovery is local; reconcile every site before recreating this bucket"
-                        ));
+                        )
+                        .into());
                     }
                     operation_store
                         .recover_orphaned_bucket(&operation_bucket, expected_incarnation)
                         .await
-                        .map_err(|error| s3s::S3Error::from(crate::error::ApiError::from(error)))
+                        .map_err(|error| S3Error::from(crate::error::ApiError::from(error)))
                 })
                 .await??;
-                Ok::<(), s3s::S3Error>(())
+                Ok::<(), S3Error>(())
             })
             .await
-            .map_err(|error| s3s::S3Error::from(crate::error::ApiError::from(error)))?;
+            .map_err(|error| S3Error::from(crate::error::ApiError::from(error)))?;
         completed?;
 
         info!(

rustfs/src/admin/handlers/quota.rs

@@ -322,7 +322,9 @@ impl Operation for SetBucketQuotaHandler {
                 Some(limit) => BucketQuota::new_sharded(limit),
                 None => BucketQuota::new(None),
             },
-            Some(_) => return Err(s3_error!(InvalidArgument, "unsupported quota reservation protocol")),
+            Some(_) => {
+                return Err(rustfs_s3_types::s3_error!(InvalidArgument, "unsupported quota reservation protocol").into());
+            }
         };
 
         let metadata_sys_lock = bucket_metadata_from_context()

rustfs/src/connect/diagnostics/trace_runtime.rs

@@ -1901,6 +1901,20 @@ mod tests {
     use super::{LocalTraceCaptureError, request_local_trace_capture, spawn_local_trace_capture_runtime};
     use crate::connect::{TelemetryOperation, TelemetryProducerError, TraceRecordCompletion, TraceRecordLimits};
 
+    async fn wait_for_sealed_socket(state: &std::path::Path) -> Result<(), tokio::time::error::Elapsed> {
+        let socket = state.join(super::SOCKET_FILE);
+        tokio::time::timeout(Duration::from_secs(5), async {
+            // Binding publishes the socket path before its private permissions are set.
+            while super::private_state_owner(state)
+                .and_then(|owner| super::socket_identity(&socket, owner))
+                .is_err()
+            {
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+    }
+
     async fn wait_for_subscriber() {
         tokio::time::timeout(Duration::from_secs(1), async {
             while telemetry_trace_subscriber_count() == 0 {
@@ -2519,12 +2533,7 @@ mod tests {
             .stdin(std::process::Stdio::piped())
             .spawn()
             .unwrap();
-        let ready = tokio::time::timeout(Duration::from_secs(5), async {
-            while !state.path().join(super::SOCKET_FILE).exists() {
-                tokio::time::sleep(Duration::from_millis(10)).await;
-            }
-        })
-        .await;
+        let ready = wait_for_sealed_socket(state.path()).await;
         let result = if ready.is_ok() {
             super::request_local_top_rpc(state.path(), request.clone(), &CancellationToken::new()).await
         } else {
@@ -2694,12 +2703,7 @@ mod tests {
             .stdin(std::process::Stdio::piped())
             .spawn()
             .unwrap();
-        let ready = tokio::time::timeout(Duration::from_secs(5), async {
-            while !state.path().join(super::SOCKET_FILE).exists() {
-                tokio::time::sleep(Duration::from_millis(10)).await;
-            }
-        })
-        .await;
+        let ready = wait_for_sealed_socket(state.path()).await;
         let result = if ready.is_ok() {
             super::request_local_top_locks(state.path(), request.clone(), &CancellationToken::new()).await
         } else {