Merge remote-tracking branch 'upstream-https/main' into fix/ant-storage-layer-batch

# Conflicts:
#	crates/ecstore/src/disk/local/delete_recovery.rs

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
-sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
+sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
+sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
-sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
+sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
+sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
-sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
+sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
+sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
+sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf

.config/nextest.toml

@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^conditional_delete_objects_test::/)

.docker/README.md

@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
 | File | Description | Usage |
 | :--- | :--- | :--- |
 | **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
-| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
-| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
-| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
 
 ### Docker Compose Configurations
 

.docker/observability/otel-collector-config.yaml

@@ -28,15 +28,6 @@ processors:
     check_interval: 1s
     limit_mib: 1024
     spike_limit_mib: 256
-  transform/metrics:
-    error_mode: ignore
-    metric_statements:
-      # RustFS process metrics already expose process_pid as a metric label.
-      # Remove only the duplicate datapoint label and keep the shared resource
-      # process.pid available to sibling metrics and lock histograms.
-      - context: datapoint
-        statements:
-          - delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
   transform/logs:
     error_mode: ignore
     log_statements:
@@ -121,7 +112,7 @@ service:
       exporters: [ otlp/tempo, otlp/jaeger ]
     metrics:
       receivers: [ otlp ]
-      processors: [ transform/metrics, memory_limiter, batch ]
+      processors: [ batch ]
       exporters: [ prometheus ]
     logs:
       receivers: [ otlp ]

.github/workflows/cla.yml

@@ -95,7 +95,7 @@ jobs:
 
       - name: Run CLA Bot
         if: github.event_name != 'merge_group'
-        uses: overtrue/cla-bot@f054da99d5777e47e1af9933737b9c0c5584be8d # v0.1.0
+        uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
         with:
           github-token: ${{ github.token }}
           registry-token: ${{ steps.registry-token.outputs.token }}

Cargo.lock

@@ -10090,7 +10090,6 @@ version = "1.0.2"
 dependencies = [
  "bytes",
  "hotpath",
- "metrics",
  "rustfs-io-metrics",
  "thiserror 2.0.21",
  "tokio",

Dockerfile

@@ -16,7 +16,6 @@ FROM alpine:3.24.2 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
-ARG RC_RELEASE=latest
 
 RUN apk add --no-cache ca-certificates curl jq unzip
 WORKDIR /build
@@ -70,9 +69,6 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
-COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
-RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
-
 
 FROM alpine:3.24.2
 
@@ -99,18 +95,14 @@ RUN apk upgrade --no-cache && \
     ca-certificates \
     coreutils \
     curl \
-    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY --from=build /build/rc /usr/bin/rc
-COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-COPY bootstrap/ /etc/rustfs/bootstrap/
+COPY entrypoint.sh /entrypoint.sh
 
-RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
-    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /entrypoint.sh
 
 RUN addgroup -g 10001 -S rustfs && \
     adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -132,6 +124,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.decommission-local

@@ -12,10 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.99.1-trixie
-
-ARG TARGETARCH
-ARG RC_RELEASE=latest
+FROM rust:1.97.1-trixie
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \
@@ -24,7 +21,6 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
-    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -40,7 +36,6 @@ COPY . .
 RUN ./scripts/static.sh
 RUN cargo run --bin gproto
 RUN cargo build --release --locked --bin rustfs
-RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
 
 RUN set -eux; \
     groupadd -g 10001 rustfs; \
@@ -55,12 +50,8 @@ RUN set -eux; \
     chown -R rustfs:rustfs /data /logs /app; \
     chmod 0750 /data /logs
 
-COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-COPY bootstrap/ /etc/rustfs/bootstrap/
-RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
-    install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
-    chmod +x /etc/rustfs/entrypoint.sh && \
-    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
+COPY entrypoint.sh /entrypoint.sh
+RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
 
 ENV RUSTFS_VOLUMES="/data" \
     RUST_LOG="warn" \
@@ -72,5 +63,5 @@ ENV RUSTFS_VOLUMES="/data" \
 
 EXPOSE 9000 9001
 
-ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

Dockerfile.glibc

@@ -16,7 +16,6 @@ FROM ubuntu:26.04 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
-ARG RC_RELEASE=latest
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
@@ -77,9 +76,6 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
-COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
-RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
-
 FROM ubuntu:26.04
 
 ARG RELEASE=latest
@@ -103,18 +99,14 @@ RUN apt-get update && apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
-    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY --from=build /build/rc /usr/bin/rc
-COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-COPY bootstrap/ /etc/rustfs/bootstrap/
+COPY entrypoint.sh /entrypoint.sh
 
-RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
-    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /entrypoint.sh
 
 RUN groupadd -g 10001 rustfs && \
     useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -139,6 +131,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.source

@@ -28,19 +28,17 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES=""
-ARG RC_RELEASE=latest
 
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.99.1-trixie AS builder
+FROM rust:1.97.1-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES
-ARG RC_RELEASE
 
 # Debug: print platforms
 RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -54,7 +52,6 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
-    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -169,8 +166,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
-RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
-
 # -----------------------------
 # Development stage (keeps toolchain)
 # -----------------------------
@@ -204,13 +199,10 @@ ENV RUSTFS_VOLUMES="/data" \
 # We rely on cargo run to build and run
 EXPOSE 9000 9001
 
-COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-COPY bootstrap/ /etc/rustfs/bootstrap/
-RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
-    chmod +x /etc/rustfs/entrypoint.sh && \
-    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
 
-ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]
 CMD ["cargo", "run", "--bin", "rustfs", "--"]
 
 # -----------------------------
@@ -235,7 +227,6 @@ RUN set -eux; \
     apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
-    jq \
     tzdata \
     coreutils; \
     rm -rf /var/lib/apt/lists/*
@@ -257,11 +248,8 @@ RUN set -eux; \
 
 # Copy the freshly built binary and the entrypoint
 COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
-COPY --from=builder /usr/local/bin/rc /usr/bin/rc
-COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-COPY bootstrap/ /etc/rustfs/bootstrap/
-RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
-    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /entrypoint.sh
 
 # Default environment (override in docker run/compose as needed)
 ENV RUSTFS_VOLUMES="/data" \
@@ -275,5 +263,5 @@ EXPOSE 9000
 VOLUME ["/data"]
 
 # Keep root here; entrypoint will drop privileges using chroot --userspec
-ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

bootstrap/README.md

@@ -1,21 +0,0 @@
-# RustFS image bootstrap
-
-The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
-
-To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
-
-Optional per-bucket files use the bucket name as the filename without an extension:
-
-| Directory | File format | Applied operation |
-| --- | --- | --- |
-| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
-| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
-| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
-| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
-| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
-
-Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
-
-When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
-
-RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.

bootstrap/buckets.json

@@ -1 +0,0 @@
-[]

bootstrap/cors/.gitkeep

@@ -1 +0,0 @@
-

bootstrap/lifecycle/.gitkeep

@@ -1 +0,0 @@
-

bootstrap/notifications/.gitkeep

@@ -1 +0,0 @@
-

bootstrap/policies/.gitkeep

@@ -1 +0,0 @@
-

bootstrap/public-access-block.json

@@ -1,6 +0,0 @@
-{
-  "block_public_acls": true,
-  "ignore_public_acls": true,
-  "block_public_policy": true,
-  "restrict_public_buckets": true
-}

bootstrap/versioning/.gitkeep

@@ -1 +0,0 @@
-

crates/config/src/constants/tls.rs

@@ -175,17 +175,12 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
 pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
 
 /// Environment variable for the S3 request-body inter-chunk read timeout
-/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
+/// (seconds). Default: 300. Set to 0 to disable.
 ///
 /// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
 /// while more bytes are still expected. It resets on every chunk, so it does not
 /// penalize slow-but-progressing uploads — it only fires when a peer sends a
 /// partial body and then goes silent *without* closing the connection (no EOF).
-/// Independently of this knob, `PutObject` and `UploadPart` require at least
-/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
-/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
-/// that would otherwise pin upload admission indefinitely, and it stays in
-/// force when this timeout is set to 0.
 /// A reverse proxy or CDN that forwards a truncated body this way would
 /// otherwise make RustFS wait forever for bytes that never arrive; with this
 /// timeout the stalled read is aborted and logged with the received/expected

crates/e2e_test/src/create_bucket_region_test.rs

@@ -502,163 +502,10 @@ mod tests {
                 assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
                     "<Code>AccessDenied</Code>"
                 } else {
-                    "<Code>BucketAlreadyOwnedByYou</Code>"
+                    "<Code>BucketAlreadyExists</Code>"
                 }));
             }
         }
         Ok(())
     }
-
-    /// A deployment owns a single bucket namespace, so an IAM user that recreates
-    /// an existing bucket is the owner, not another account. In the default region
-    /// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
-    /// semantics), including buckets created by root and by another IAM user.
-    #[tokio::test]
-    async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
-    -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
-        let root = env.create_s3_client();
-
-        admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
-        admin_attach_user_policy_via(
-            AdminTransport::Signed,
-            &env.url,
-            &env.access_key,
-            &env.secret_key,
-            "readwrite",
-            "duplicate-iam",
-        )
-        .await?;
-        let iam = Client::from_conf(build_test_s3_config(
-            &env.url,
-            "duplicate-iam",
-            "duplicate-iam-secret",
-            None,
-            "duplicate-create",
-        ));
-
-        root.create_bucket()
-            .bucket("duplicate-root-owned")
-            .send()
-            .await
-            .expect("root creates the first bucket");
-        iam.create_bucket()
-            .bucket("duplicate-root-owned")
-            .send()
-            .await
-            .expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
-
-        iam.create_bucket()
-            .bucket("duplicate-iam-owned")
-            .send()
-            .await
-            .expect("the IAM user creates its own bucket");
-        iam.create_bucket()
-            .bucket("duplicate-iam-owned")
-            .send()
-            .await
-            .expect("the same IAM user recreating its bucket gets 200 OK");
-        root.create_bucket()
-            .bucket("duplicate-iam-owned")
-            .send()
-            .await
-            .expect("root recreating an IAM-created bucket gets 200 OK");
-
-        Ok(())
-    }
-
-    /// Outside the default region the same ownership answer becomes
-    /// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
-    /// lack `s3:CreateBucket` still fail authorization before the duplicate check.
-    #[tokio::test]
-    async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
-    {
-        init_logging();
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
-        let region = "eu-west-1";
-        let root = Client::from_conf(
-            env.create_s3_client()
-                .config()
-                .to_builder()
-                .region(Region::new(region))
-                .build(),
-        );
-
-        admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
-        admin_attach_user_policy_via(
-            AdminTransport::Signed,
-            &env.url,
-            &env.access_key,
-            &env.secret_key,
-            "readwrite",
-            "duplicate-eu-iam",
-        )
-        .await?;
-        let iam = Client::from_conf(
-            build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
-                .to_builder()
-                .region(Region::new(region))
-                .build(),
-        );
-
-        admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
-        admin_attach_user_policy_via(
-            AdminTransport::Signed,
-            &env.url,
-            &env.access_key,
-            &env.secret_key,
-            "readonly",
-            "duplicate-eu-readonly",
-        )
-        .await?;
-        let readonly = Client::from_conf(
-            build_test_s3_config(
-                &env.url,
-                "duplicate-eu-readonly",
-                "duplicate-eu-readonly-secret",
-                None,
-                "duplicate-create",
-            )
-            .to_builder()
-            .region(Region::new(region))
-            .build(),
-        );
-
-        for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
-            client
-                .create_bucket()
-                .bucket(bucket)
-                .send()
-                .await
-                .unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
-            // Both the creator and the other authenticated identity own the namespace.
-            for creator in [&root, &iam] {
-                let error = creator
-                    .create_bucket()
-                    .bucket(bucket)
-                    .send()
-                    .await
-                    .expect_err("a duplicate create outside us-east-1 must report ownership");
-                assert_eq!(
-                    error.as_service_error().and_then(ProvideErrorMetadata::code),
-                    Some("BucketAlreadyOwnedByYou"),
-                    "{bucket}"
-                );
-                assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
-            }
-        }
-
-        let denied = readonly
-            .create_bucket()
-            .bucket("duplicate-eu-root")
-            .send()
-            .await
-            .expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
-        assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
-
-        Ok(())
-    }
 }

crates/e2e_test/src/distributed/scanner_heal_topology_fixture.rs

@@ -25,8 +25,7 @@
 
 use super::harness::{
     DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
-    payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
-    unique_bucket, wait_until,
+    payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
 };
 use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
 use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
@@ -48,7 +47,6 @@ use tokio::time::{Instant, sleep};
 use uuid::Uuid;
 
 pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
-pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
 pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
@@ -1168,21 +1166,8 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
     .await
 }
 
-pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
-    let path = "/rustfs/admin/v3/rebalance/stop";
-    let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
-    if status.is_success() {
-        return Ok(response);
-    }
-    // The run can finish between the active observation and the stop request.
-    // Only completion of that same run makes a not-started response acceptable.
-    if status == StatusCode::NOT_FOUND
-        && response.contains("<Code>NoSuchResource</Code>")
-        && rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
-    {
-        return Ok(response);
-    }
-    Err(format!("POST {path} failed: {status} {response}").into())
+pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
+    cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
 }
 
 pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
@@ -1600,129 +1585,6 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
 mod tests {
     use super::*;
 
-    const REBALANCE_NOT_STARTED: &str =
-        "<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
-
-    async fn mock_rebalance_stop(
-        stop_status: StatusCode,
-        stop_body: &str,
-        run_status: Value,
-    ) -> TestResult<(TestResult<String>, Vec<String>)> {
-        use bytes::Bytes;
-        use http_body_util::Full;
-        use hyper::service::service_fn;
-        use hyper_util::rt::TokioIo;
-        use tokio::net::TcpListener;
-        use tokio::sync::mpsc;
-        use tokio::task::JoinSet;
-
-        let listener = TcpListener::bind("127.0.0.1:0").await?;
-        let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
-        cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
-        let stop_body = stop_body.to_string();
-        let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
-        let mut server = JoinSet::new();
-        server.spawn(async move {
-            loop {
-                let (stream, _) = match listener.accept().await {
-                    Ok(connection) => connection,
-                    Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
-                };
-                let stop_body = stop_body.clone();
-                let run_status = run_status.clone();
-                let requests_tx = requests_tx.clone();
-                let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
-                    let route = format!("{} {}", request.method(), request.uri().path());
-                    requests_tx.send(route.clone()).expect("record mock admin request");
-                    let (status, body) = match route.as_str() {
-                        "POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
-                        "GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
-                        _ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
-                    };
-                    async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
-                });
-                hyper::server::conn::http1::Builder::new()
-                    .keep_alive(false)
-                    .serve_connection(TokioIo::new(stream), handler)
-                    .await?;
-            }
-        });
-        let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
-        server.abort_all();
-        let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
-        Ok((outcome, requests))
-    }
-
-    fn completed_rebalance() -> Value {
-        serde_json::json!({
-            "id": "run-1",
-            "pools": [
-                { "id": 0, "status": "Completed", "progress": { "versions": 113 } },
-                { "id": 1, "status": "None" }
-            ]
-        })
-    }
-
-    #[tokio::test]
-    async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
-        let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
-        assert_eq!(outcome?, "");
-        assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
-        let (outcome, requests) =
-            mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
-        outcome?;
-        assert_eq!(
-            requests,
-            [
-                "POST /rustfs/admin/v3/rebalance/stop",
-                "GET /rustfs/admin/v3/rebalance/status"
-            ]
-        );
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
-        let complete = completed_rebalance();
-        let mut invalid = vec![Value::Null];
-        for (pointer, value) in [
-            ("/id", serde_json::json!("run-2")),
-            ("/pools/0/status", serde_json::json!("Started")),
-            ("/pools/0/status", serde_json::json!("Failed")),
-            ("/pools/0/status", serde_json::json!("Stopped")),
-            ("/pools/0/progress/versions", serde_json::json!(0)),
-        ] {
-            let mut status = complete.clone();
-            *status.pointer_mut(pointer).expect("completion fixture field") = value;
-            invalid.push(status);
-        }
-        let mut pending = complete.clone();
-        pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
-        invalid.push(pending);
-        let mut failed = complete.clone();
-        failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
-        invalid.push(failed);
-        for status in invalid {
-            let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
-            assert!(outcome.is_err(), "unproven completion accepted: {status}");
-        }
-        for (code, body) in [
-            (StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
-            (StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
-            (StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
-        ] {
-            let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
-            assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
-            assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
-        }
-        Ok(())
-    }
-
     #[test]
     fn version_ids_normalize_null_spellings() {
         for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {

crates/e2e_test/src/distributed/scanner_heal_topology_test.rs

@@ -28,7 +28,7 @@ use super::harness::{
     wait_for_rebalance_active, wait_for_rebalance_complete,
 };
 use super::scanner_heal_topology_fixture::{
-    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
+    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
     STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
     assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
     assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
@@ -274,7 +274,6 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
 #[tokio::test]
 async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
     init_logging();
-    require_test_hooks();
     let mut run = TopologyRun::start("rebalance").await?;
     let manifest = seed_manifest(&run).await?;
     assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
@@ -289,52 +288,23 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         .map(|entry| (entry.key.clone(), entry.sha256.clone()))
         .collect();
 
-    // Keep an admitted entry on the source until stop cancels admission. Start
-    // propagation may await that entry, so observe and stop before awaiting ACK.
-    let first_key = manifest.plain_key("external", 0)?;
-    run.movement_barrier
-        .arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
-    let observe_first = async {
-        let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
-            let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
-            let status = rebalance_status_json(&run.dist.cluster).await?;
-            let observed_id = status
-                .get("id")
-                .and_then(Value::as_str)
-                .ok_or("rebalance status omitted id")?;
-            wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
-            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
-        })
-        .await??;
-        let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
-        let workload = run_live_workload(LiveWorkload {
-            clients: run.dist.clients()?,
-            bucket: scratch.clone(),
-            prefix: "live".to_string(),
-            writers: LIVE_WRITERS,
-            rounds: LIVE_ROUNDS,
-            baseline_bucket: manifest.plain_bucket.clone(),
-            baseline: baseline.clone(),
-            pause: Duration::from_millis(100),
-        });
-        let stop = async {
-            sleep(Duration::from_secs(2)).await;
-            stop_rebalance(&run.dist.cluster, &observed_id).await?;
-            let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
-            if outcome != "stopped" {
-                return Err("rebalance completed before stop; mid-run stop was not exercised".into());
-            }
-            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
-        };
-        let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
-        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
-    };
-    let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
-        tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
-    if first_id != observed_id {
-        return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
-    }
-    run.movement_barrier.disarm()?;
+    let first_id = start_rebalance(&run.dist.cluster).await?;
+    wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+    let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
+    let workload = tokio::spawn(run_live_workload(LiveWorkload {
+        clients: run.dist.clients()?,
+        bucket: scratch.clone(),
+        prefix: "live".to_string(),
+        writers: LIVE_WRITERS,
+        rounds: LIVE_ROUNDS,
+        baseline_bucket: manifest.plain_bucket.clone(),
+        baseline: baseline.clone(),
+        pause: Duration::from_millis(100),
+    }));
+    sleep(Duration::from_secs(2)).await;
+    stop_rebalance(&run.dist.cluster).await?;
+    let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
+    let ops = workload.await??;
     let live_state = expected_live_state(&ops);
 
     for node_idx in 0..run.dist.cluster.nodes.len() {
@@ -363,40 +333,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
     verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
     verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
 
-    // A short second run can finish before the start response reaches us.
-    // Hold one remaining source entry until its active state is observed.
-    let mut second_key = None;
-    for entry in manifest.plain_entries() {
-        if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
-            second_key = Some(entry.key.clone());
-            break;
-        }
-    }
-    let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
-    let second_barrier = run.movement_barrier.next_phase();
-    second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
-    let observe_second = async {
-        let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
-        let status = rebalance_status_json(&run.dist.cluster).await?;
-        let observed_id = status
-            .get("id")
-            .and_then(Value::as_str)
-            .ok_or("rebalance status omitted id")?;
-        if observed_id == first_id {
-            return Err("the second rebalance reused the stopped run id".into());
-        }
-        wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
-        second_barrier.disarm()?;
-        second_barrier.release_movement()?;
-        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
-    };
-    // Worker propagation can hold the start response until movement drains.
-    let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
-        tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
-    })?;
-    if second_id != observed_id {
-        return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
-    }
+    let second_id = start_rebalance(&run.dist.cluster).await?;
+    wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
     wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
     verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
     verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
@@ -413,8 +351,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         &json!({
             "gate": "G-REB",
             "manifest": manifest_summary(&manifest),
-            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
-            "second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
+            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
+            "second_run": { "id": second_id, "versions_off_seed_pool": moved },
             "live_ops": ops,
             "probe_placement_during_rebalance": during_placement,
             "process_durability_strict_nodes": strict_nodes,

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -33,7 +33,7 @@ mod tests {
     use sha2::{Digest, Sha256};
     use std::collections::HashSet;
     use std::error::Error;
-    use std::io::{Read, Seek, SeekFrom, Write};
+    use std::io::{Read, Write};
     use std::net::SocketAddr;
     use std::path::{Path, PathBuf};
     use std::process::Command;
@@ -1654,89 +1654,6 @@ mod tests {
         TargetEndpointBlackhole,
     }
 
-    async fn release_heal_commit_barrier_after_shutdown(
-        log: &Path,
-        log_offset: u64,
-        barrier: &Path,
-    ) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
-        // Observe within the existing graceful process-stop budget.
-        timeout(Duration::from_secs(45), async {
-            loop {
-                let mut file = std::fs::File::open(log)?;
-                file.seek(SeekFrom::Start(log_offset))?;
-                let mut tail = String::new();
-                file.read_to_string(&mut tail)?;
-                if let Some(event) = tail
-                    .lines()
-                    .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
-                    .find(|event| {
-                        event["event"] == "server_shutdown_state"
-                            && event["component"] == "main"
-                            && event["subsystem"] == "startup"
-                            && event["state"] == "stopping"
-                    })
-                {
-                    // Local heal has stopped and HTTP shutdown has been signalled.
-                    // Let admitted peer RPCs finish; keeping this synthetic barrier
-                    // until process exit would make the fixture block its own drain.
-                    std::fs::remove_file(barrier)?;
-                    return Ok(event);
-                }
-                sleep(Duration::from_millis(10)).await;
-            }
-        })
-        .await?
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
-        let directory = tempfile::tempdir()?;
-        let log = directory.path().join("server.log");
-        let barrier = directory.path().join("barrier");
-        let event = serde_json::json!({
-            "event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
-        });
-        std::fs::write(&log, format!("{event}\n"))?;
-        std::fs::write(&barrier, "bucket/prefix/")?;
-        let log_offset = log.metadata()?.len();
-        let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
-        writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
-        writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
-
-        let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
-        tokio::pin!(release);
-        assert!(
-            futures::poll!(release.as_mut()).is_pending(),
-            "old or unrelated events must not release the barrier"
-        );
-        assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
-        tokio::time::advance(Duration::from_secs(6)).await;
-        assert!(
-            futures::poll!(release.as_mut()).is_pending(),
-            "background shutdown may take longer than five seconds within the process-stop budget"
-        );
-        writeln!(writer, "{event}")?;
-        tokio::time::advance(Duration::from_millis(10)).await;
-        assert_eq!(release.await?, event, "retain the actual new shutdown witness");
-        assert!(!barrier.exists(), "shutdown must release the synthetic hold");
-        Ok(())
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
-        let directory = tempfile::tempdir()?;
-        let log = directory.path().join("server.log");
-        let barrier = directory.path().join("barrier");
-        std::fs::write(&log, "")?;
-        std::fs::write(&barrier, "bucket/prefix/")?;
-        let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
-            .await
-            .expect_err("missing shutdown evidence must fail the fixture");
-        assert!(error.is::<tokio::time::error::Elapsed>());
-        assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
-        Ok(())
-    }
-
     async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
         let server_binary = rustfs_binary_path();
         let evidence_case = match scenario {
@@ -1874,7 +1791,6 @@ mod tests {
         let mut created_online_objects = Vec::with_capacity(online_object_count);
         let mut expected_manifests = Vec::with_capacity(online_object_count);
         let mut unclean_shutdown_marker_observed = None;
-        let mut shutdown_barrier_release = None;
         let mut attempt_count = 0usize;
         let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
         while expected_manifests.len() < online_object_count {
@@ -2457,21 +2373,7 @@ mod tests {
                     | InterruptionScenario::BackgroundTargetRestartEc84MultiSet
                     | InterruptionScenario::BackgroundCoordinatorRestart
             ) {
-                if interruption_node == 1
-                    && let Some(barrier) = &commit_barrier
-                {
-                    let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
-                    let log_offset = target_log.metadata()?.len();
-                    let (stopped, released) = tokio::join!(
-                        cluster.stop_node_gracefully(interruption_node),
-                        release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
-                    );
-                    // Await both branches even on error so the child is reaped.
-                    stopped?;
-                    shutdown_barrier_release = Some(released?);
-                } else {
-                    cluster.stop_node_gracefully(interruption_node).await?;
-                }
+                cluster.stop_node_gracefully(interruption_node).await?;
             } else {
                 cluster.stop_node(interruption_node)?;
             }
@@ -2513,9 +2415,7 @@ mod tests {
                     }
                 }
             }
-            if shutdown_barrier_release.is_none()
-                && let Some(barrier) = &commit_barrier
-            {
+            if let Some(barrier) = &commit_barrier {
                 std::fs::remove_file(barrier)?;
             }
             cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2846,7 +2746,6 @@ mod tests {
                 "all_peers_bound_to_generation_window": true,
                 "pid_before": target_pid, "pid_after": restarted_pid,
                 "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
-                "graceful_barrier_release": shutdown_barrier_release,
                 "objects": evidence_objects, "node_listings": node_listings,
             });
             if let Some(outage_write) = outage_write_diagnostic {

crates/e2e_test/src/lib.rs

@@ -283,9 +283,6 @@ mod ssec_copy_test;
 #[cfg(test)]
 mod multipart_storage_class_test;
 
-#[cfg(test)]
-mod multipart_upload_id_test;
-
 #[cfg(test)]
 mod storage_class_capability_test;
 

crates/e2e_test/src/multipart_upload_id_test.rs

@@ -1,175 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Multipart upload ID regression tests.
-//!
-//! Every operation that addresses an existing multipart upload must report an
-//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
-//! the ID is malformed or is a well-formed ID of an aborted upload.
-
-#[cfg(test)]
-mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
-    use aws_sdk_s3::Client;
-    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
-    use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
-
-    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-
-    const BUCKET: &str = "multipart-upload-id";
-    const KEY: &str = "target.bin";
-    const COPY_SOURCE_KEY: &str = "copy-source.bin";
-    // RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
-    // the object layer rejects this ID as malformed instead of looking it up.
-    const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
-    const PART_NUMBER: i32 = 1;
-    const PART_BODY: &[u8] = b"multipart upload id regression part";
-    const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
-    const NO_SUCH_UPLOAD_STATUS: u16 = 404;
-    const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
-    const OBJECT_NOT_FOUND_STATUS: u16 = 404;
-
-    /// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
-    fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
-    where
-        E: ProvideErrorMetadata + std::fmt::Debug,
-    {
-        let Err(error) = result else {
-            return Some(format!("{operation}: succeeded"));
-        };
-        let status = error.raw_response().map(|response| response.status().as_u16());
-        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
-        (status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
-            .then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
-    }
-
-    async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
-    {
-        init_logging();
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(Vec::new()).await?;
-        env.create_test_bucket(BUCKET).await?;
-        let client = env.create_s3_client();
-        client
-            .put_object()
-            .bucket(BUCKET)
-            .key(COPY_SOURCE_KEY)
-            .body(ByteStream::from_static(PART_BODY))
-            .send()
-            .await?;
-        Ok((env, client))
-    }
-
-    /// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
-    async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
-        let upload_part = client
-            .upload_part()
-            .bucket(BUCKET)
-            .key(KEY)
-            .upload_id(upload_id)
-            .part_number(PART_NUMBER)
-            .body(ByteStream::from_static(PART_BODY))
-            .send()
-            .await;
-        let upload_part_copy = client
-            .upload_part_copy()
-            .bucket(BUCKET)
-            .key(KEY)
-            .upload_id(upload_id)
-            .part_number(PART_NUMBER)
-            .copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
-            .send()
-            .await;
-        let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
-        let complete = client
-            .complete_multipart_upload()
-            .bucket(BUCKET)
-            .key(KEY)
-            .upload_id(upload_id)
-            .multipart_upload(
-                CompletedMultipartUpload::builder()
-                    .parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
-                    .build(),
-            )
-            .send()
-            .await;
-        let abort = client
-            .abort_multipart_upload()
-            .bucket(BUCKET)
-            .key(KEY)
-            .upload_id(upload_id)
-            .send()
-            .await;
-
-        let mismatches: Vec<String> = [
-            no_such_upload_mismatch("UploadPart", upload_part),
-            no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
-            no_such_upload_mismatch("ListParts", list_parts),
-            no_such_upload_mismatch("CompleteMultipartUpload", complete),
-            no_such_upload_mismatch("AbortMultipartUpload", abort),
-        ]
-        .into_iter()
-        .flatten()
-        .collect();
-        assert!(
-            mismatches.is_empty(),
-            "upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
-            mismatches.join("\n")
-        );
-    }
-
-    /// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
-    async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
-        let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
-        assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
-        let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
-        let status = head
-            .as_ref()
-            .err()
-            .and_then(|error| error.raw_response())
-            .map(|response| response.status().as_u16());
-        assert_eq!(
-            status,
-            Some(OBJECT_NOT_FOUND_STATUS),
-            "rejected operations must not publish {KEY}: {head:?}"
-        );
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
-        let (_env, client) = start_server_with_copy_source().await?;
-
-        assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
-        assert_no_upload_or_object_published(&client).await
-    }
-
-    #[tokio::test]
-    async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
-        let (_env, client) = start_server_with_copy_source().await?;
-        let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
-        let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
-        client
-            .abort_multipart_upload()
-            .bucket(BUCKET)
-            .key(KEY)
-            .upload_id(upload_id)
-            .send()
-            .await?;
-
-        assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
-        assert_no_upload_or_object_published(&client).await
-    }
-}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -360,26 +360,12 @@ impl OpaMock {
 
     pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
         let payload = self.next_request().await?;
-        Self::assert_bucket_tags(&payload, action, bucket, department);
-        Ok(())
-    }
-
-    async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
-        let payload = self.next_request().await?;
-        Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
-        assert_eq!(payload["input"]["resource"]["object"], object);
-        assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
-        Ok(())
-    }
-
-    fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
         let input = &payload["input"];
-        assert_eq!(input["identity"]["account"], "opabuckettags");
-        assert_eq!(input["identity"]["is_owner"], false);
         assert_eq!(input["action"], action);
         assert_eq!(input["resource"]["bucket"], bucket);
         let expected = department.map(|value| serde_json::json!([value]));
         assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
+        Ok(())
     }
 
     async fn wait_for_validation(&mut self) -> TestResult {
@@ -734,7 +720,7 @@ async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, buck
 async fn test_bucket_tags_opa_contract() -> TestResult {
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{
-        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
+        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
     };
 
     init_logging();
@@ -786,7 +772,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .await?;
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    let head = client
+    client
         .head_object()
         .bucket(source)
         .key("report")
@@ -794,8 +780,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .send()
         .await?;
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    assert_eq!(head.tag_count(), Some(1));
-    opa.expect_versioned_tag_permission(source, "report", version).await?;
     client.list_objects_v2().bucket(source).send().await?;
     opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
     client.list_object_versions().bucket(source).send().await?;
@@ -805,65 +789,28 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     admin.create_bucket().bucket(destination).send().await?;
     set_department(&admin, destination, "engineering").await?;
     let copy_source = format!("{source}/report?versionId={version}");
-    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
-        let error = client
-            .copy_object()
-            .bucket(destination)
-            .key("copied")
-            .copy_source(&copy_source)
-            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
-            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
-            .send()
-            .await
-            .expect_err("source tags must not authorize a different destination");
-        assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
-        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-        if directive == TaggingDirective::Copy {
-            opa.expect_versioned_tag_permission(source, "report", version).await?;
-        }
-        opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
-            .await?;
-        let missing = admin
-            .head_object()
-            .bucket(destination)
-            .key("copied")
-            .send()
-            .await
-            .expect_err("denied COPY and REPLACE must leave no destination object");
-        assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
-    }
+    let error = client
+        .copy_object()
+        .bucket(destination)
+        .key("copied")
+        .copy_source(&copy_source)
+        .send()
+        .await
+        .expect_err("source tags must not authorize a different destination");
+    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
+        .await?;
     set_department(&admin, destination, "finance").await?;
-    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
-        client
-            .copy_object()
-            .bucket(destination)
-            .key("copied")
-            .copy_source(&copy_source)
-            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
-            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
-            .send()
-            .await?;
-        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-        // The access hook checks preliminary tags, then the copy use case checks
-        // the exact source reader's tags before publishing the destination.
-        for _ in 0..2 {
-            if directive == TaggingDirective::Copy {
-                opa.expect_versioned_tag_permission(source, "report", version).await?;
-            }
-            opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
-        }
-        let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
-        assert_eq!(tags.tag_set().len(), 1);
-        assert_eq!(tags.tag_set()[0].key(), "department");
-        assert_eq!(
-            tags.tag_set()[0].value(),
-            if directive == TaggingDirective::Copy {
-                "engineering"
-            } else {
-                "legal"
-            }
-        );
-    }
+    client
+        .copy_object()
+        .bucket(destination)
+        .key("copied")
+        .copy_source(&copy_source)
+        .send()
+        .await?;
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
 
     let upload = client
         .create_multipart_upload()
@@ -939,14 +886,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
         .await?;
-    let missing = admin
-        .head_object()
-        .bucket(destination)
-        .key("denied-source")
-        .send()
-        .await
-        .expect_err("a denied source must leave no destination object");
-    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
     set_department(&client, source, "finance")
         .await
         .expect_err("requested tags must not authorize their own mutation");
@@ -985,6 +924,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
         .await?;
+    // Duplicate keys are refused on write; the ambiguous record below is seeded
+    // offline the way an upgraded deployment may still hold it on disk.
     let invalid_write = admin
         .put_bucket_tagging()
         .bucket(destination)
@@ -997,7 +938,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .send()
         .await
         .expect_err("new duplicate tags must be rejected before publication");
-    assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
     assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
 
     set_department(&admin, source, "finance").await?;

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -48,7 +48,6 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
 const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
 const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
-const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -955,8 +954,7 @@ async fn exercise_mixed_cluster(
     phase: &str,
     current_node: usize,
     previous_node: usize,
-    previous_has_delete_recovery: bool,
-) -> Result<Option<String>, BoxError> {
+) -> TestResult {
     let clients = cluster.create_all_clients()?;
     wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
     wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
@@ -1011,30 +1009,6 @@ async fn exercise_mixed_cluster(
         vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
     );
 
-    // 1.0.4 already implements the delete decision protocol. Use a separate
-    // old-writer key so the original seed still spans the entire rollout.
-    if previous_has_delete_recovery {
-        let key = format!(".upgrade-delete/{phase}/previous-writer");
-        previous_client
-            .put_object()
-            .bucket(MIXED_BUCKET)
-            .key(&key)
-            .body(ByteStream::from_static(UPGRADE_READINESS_BODY))
-            .send()
-            .await?;
-        for client in &clients {
-            assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
-        }
-        upgrade_probe_client(current_client)
-            .delete_object()
-            .bucket(MIXED_BUCKET)
-            .key(&key)
-            .send()
-            .await?;
-        assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
-        return Ok(Some(key));
-    }
-
     // Every online participant must understand the delete decision protocol.
     // An upgrade phase can serve reads/writes while intentionally refusing a
     // modern DELETE; rejecting it must leave the old writer's seed intact.
@@ -1055,31 +1029,6 @@ async fn exercise_mixed_cluster(
         );
     }
 
-    Ok(None)
-}
-
-async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
-    for (node, client) in clients.iter().enumerate() {
-        for key in keys {
-            let error = upgrade_probe_client(client)
-                .get_object()
-                .bucket(MIXED_BUCKET)
-                .key(key)
-                .send()
-                .await
-                .expect_err("a successful mixed-version DELETE must remain visible through every reader");
-            assert_eq!(
-                error.raw_response().map(|response| response.status().as_u16()),
-                Some(404),
-                "node {node}: {key}"
-            );
-            assert_eq!(
-                error.as_service_error().and_then(ProvideErrorMetadata::code),
-                Some("NoSuchKey"),
-                "node {node}: {key}"
-            );
-        }
-    }
     Ok(())
 }
 
@@ -2291,9 +2240,6 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
 async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
     init_logging();
     let previous_binary = source_binary()?;
-    // Bind the expected protocol to the verified release source, never to the
-    // DELETE result itself. The older CI source pins retain their 503 contract.
-    let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
     let evidence_context = g09_evidence_context(&previous_binary).await?;
     let current_binary = rustfs_binary_path();
     let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2314,10 +2260,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
 
     cluster.stop_node(0)?;
     cluster.start_node_from_binary(0, &current_binary).await?;
-    let mut mixed_deleted_keys = Vec::new();
-    if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
-        mixed_deleted_keys.push(key);
-    }
+    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
     // One controlled out-of-policy attempt proves fail-closed behavior below
     // native write quorum; positive MPU writers remain paused while mixed.
     assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
@@ -2325,9 +2268,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
         cluster.stop_node(node_idx)?;
         cluster.start_node_from_binary(node_idx, &current_binary).await?;
-        if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
-            mixed_deleted_keys.push(key);
-        }
+        exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
     }
 
     cluster.stop_node(3)?;
@@ -2337,7 +2278,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
     exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
     verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
-    assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
 
     for (node_idx, client) in clients.iter().enumerate() {
         assert_eq!(
@@ -2386,7 +2326,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     cluster.start_with_binary(&previous_binary).await?;
     prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
     let old_clients = cluster.create_all_clients()?;
-    assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
     assert_resumed_multipart(&old_clients, &completed_pending).await?;
     for phase in ["homogeneous-previous", "homogeneous-current"] {
         verify_load_bodies(&old_clients, phase, "multipart").await?;
@@ -2451,8 +2390,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                 "phases": phases,
                 "objects_per_phase": objects_per_phase,
                 "mixed_load": "plain-put",
-                "mixed_delete_protocol_supported": previous_has_delete_recovery,
-                "mixed_deleted_keys": mixed_deleted_keys,
                 "homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
                 "multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
                 "assertions": [
@@ -2485,11 +2422,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                     "16x16 multipart load runs in homogeneous previous, current and rollback phases",
                     "one-current-node Complete returns typed 503 and preserves actual source authority and parts",
                     "the same legacy upload completes and replays once fully upgraded without adding a version",
-                    if previous_has_delete_recovery {
-                        "mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
-                    } else {
-                        "modern DELETE fails closed and preserves every byte while old peers remain"
-                    }
+                    "modern DELETE fails closed and preserves every byte while old peers remain"
                 ],
             }),
         )?;

crates/ecstore/src/api/mod.rs

@@ -479,9 +479,9 @@ pub mod disk {
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
-        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
-        NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
-        PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
+        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
+        POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
+        PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
         QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
         ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
         UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,

crates/ecstore/src/bucket/quota/reservation.rs

@@ -1151,41 +1151,12 @@ pub(crate) async fn cleanup_retired_bucket_ledger(store: Arc<ECStore>, bucket: &
     let object = ledger_object(bucket);
     let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
     let guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?;
-    let ledger = match load_ledger_locked(Arc::clone(&store), &object).await {
-        Ok(ledger) if ledger.bucket_incarnation == incarnation => Some(ledger.bucket_incarnation),
-        Ok(ledger) if crate::bucket::retirement::is_retired(store.clone(), bucket, ledger.bucket_incarnation).await? => {
-            Some(ledger.bucket_incarnation)
-        }
+    match load_ledger_locked(Arc::clone(&store), &object).await {
+        Ok(ledger) if ledger.bucket_incarnation == incarnation => {}
+        Ok(ledger) if crate::bucket::retirement::is_retired(store.clone(), bucket, ledger.bucket_incarnation).await? => {}
         Ok(_) => return Err(StorageError::PreconditionFailed),
-        Err(StorageError::ConfigNotFound) => None,
+        Err(StorageError::ConfigNotFound) => return Ok(()),
         Err(error) => return Err(error),
-    };
-    // The reconciliation throttle object of the retiring incarnation goes first,
-    // so a crash after the ledger delete cannot leave it behind; the requested
-    // incarnation is cleaned even when its ledger is already gone.
-    for retired in [Some(incarnation), ledger.filter(|retired| *retired != incarnation)]
-        .into_iter()
-        .flatten()
-    {
-        let control = reconcile_control_object(&object, retired);
-        match store
-            .delete_object(
-                RUSTFS_META_BUCKET,
-                &control,
-                ObjectOptions {
-                    no_lock: true,
-                    ..Default::default()
-                },
-            )
-            .await
-        {
-            Ok(_) => {}
-            Err(error) if is_err_object_not_found(&error) => {}
-            Err(error) => return Err(error),
-        }
-    }
-    if ledger.is_none() {
-        return Ok(());
     }
     if guard.is_lock_lost() {
         return Err(StorageError::NamespaceLockQuorumUnavailable {
@@ -1208,13 +1179,6 @@ pub(crate) async fn cleanup_retired_bucket_ledger(store: Arc<ECStore>, bucket: &
     }
 }
 
-/// The reconciliation throttle object of one bucket incarnation. One object per
-/// incarnation: a quota revision change replaces the throttle instead of
-/// leaving the previous revision's object behind.
-fn reconcile_control_object(ledger_object: &str, incarnation: Uuid) -> String {
-    format!("{ledger_object}.reconcile/{incarnation}")
-}
-
 async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_object: &str) -> Result<()> {
     let now = now_unix();
     let (candidates, reconcile_required, next_cursor) = {
@@ -1377,7 +1341,12 @@ impl ReconcileJob {
     }
 
     fn control_object(&self) -> String {
-        reconcile_control_object(&ledger_object(&self.bucket), self.bucket_incarnation)
+        format!(
+            "{}.reconcile/{}/{}",
+            ledger_object(&self.bucket),
+            self.bucket_incarnation,
+            self.quota_revision.unix_timestamp_nanos()
+        )
     }
 
     async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
@@ -1406,14 +1375,12 @@ impl ReconcileJob {
         match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
             Ok(bytes) => {
                 let control: ReconcileControl = serde_json::from_slice(&bytes)?;
-                if control.bucket_incarnation != self.bucket_incarnation {
+                if control.bucket_incarnation != self.bucket_incarnation
+                    || control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
+                {
                     return Err(StorageError::PartMissingOrCorrupt);
                 }
-                // A previous quota revision's throttle does not apply: the new
-                // revision needs its own scan and overwrites the object below.
-                if control.quota_revision_unix_nanos == self.quota_revision.unix_timestamp_nanos()
-                    && control.next_attempt_at > now
-                {
+                if control.next_attempt_at > now {
                     return Ok(());
                 }
             }
@@ -1429,15 +1396,8 @@ impl ReconcileJob {
         // Persist the throttle before work, including bootstrap and failed scans.
         self.save_control(&control_object, &control, &guard).await?;
         let result = self.scan_and_publish(&guard).await;
-        // The throttle outlives a successful scan on purpose: denied requests
-        // keep scheduling the job, and they must not rescan a full bucket more
-        // than once per interval. A scan refused because the bucket generation
-        // or its quota no longer matches writes nothing back: the retirement
-        // cleanup may already have removed this object.
-        if !matches!(result, Err(StorageError::PreconditionFailed)) {
-            control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
-            self.save_control(&control_object, &control, &guard).await?;
-        }
+        control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
+        self.save_control(&control_object, &control, &guard).await?;
         result
     }
 
@@ -2446,12 +2406,9 @@ mod tests {
         let object = job.control_object();
         let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
         let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
-        // No control object exists before the first scan took the owner lock,
-        // or after a retirement cleanup; there is nothing to expire then.
-        let Ok(bytes) = read_config_no_lock(Arc::clone(&job.store), &object).await else {
-            return;
-        };
-        let mut control: ReconcileControl = serde_json::from_slice(&bytes).expect("control JSON");
+        let mut control: ReconcileControl =
+            serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
+                .expect("control JSON");
         control.next_attempt_at = 0;
         fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
             .await
@@ -2822,21 +2779,6 @@ mod tests {
         crate::config::com::save_config(Arc::clone(&store), &ledger_path, serde_json::to_vec(&stale).expect("encode ledger"))
             .await
             .expect("persist crash-left reservation fixture");
-        // A throttled reconciliation of the old generation left its control object behind.
-        let stale_control =
-            ReconcileJob::new(Arc::clone(&store), bucket.clone(), old_incarnation, OffsetDateTime::now_utc()).control_object();
-        let control = ReconcileControl {
-            bucket_incarnation: old_incarnation,
-            quota_revision_unix_nanos: 0,
-            next_attempt_at: i64::MAX,
-        };
-        crate::config::com::save_config(
-            Arc::clone(&store),
-            &stale_control,
-            serde_json::to_vec(&control).expect("encode control"),
-        )
-        .await
-        .expect("persist stale reconciliation control");
 
         assert!(matches!(
             load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
@@ -2856,29 +2798,6 @@ mod tests {
             load_ledger_locked(Arc::clone(&store), &ledger_path).await,
             Err(StorageError::ConfigNotFound)
         ));
-        assert!(
-            matches!(
-                read_config_no_lock(Arc::clone(&store), &stale_control).await,
-                Err(StorageError::ConfigNotFound)
-            ),
-            "the retired generation's reconciliation control object goes with its ledger"
-        );
-        // A control object written back after the ledger was already gone (a scan
-        // that was mid-flight during retirement) is removed by the next cleanup.
-        crate::config::com::save_config(
-            Arc::clone(&store),
-            &stale_control,
-            serde_json::to_vec(&control).expect("encode control"),
-        )
-        .await
-        .expect("persist late reconciliation control");
-        cleanup_retired_bucket_ledger(Arc::clone(&store), &bucket, old_incarnation)
-            .await
-            .expect("cleanup without a ledger still removes the generation's control object");
-        assert!(matches!(
-            read_config_no_lock(Arc::clone(&store), &stale_control).await,
-            Err(StorageError::ConfigNotFound)
-        ));
         let replacement =
             load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
                 .await

crates/ecstore/src/cache_value/metacache_set.rs

@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
 use tokio::task::JoinSet;
 use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
-use tracing::{Instrument, debug, error, warn};
+use tracing::{debug, error, warn};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -371,15 +371,10 @@ async fn list_path_raw_inner(
     let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
 
     let cancel_rx = CancellationToken::new();
-    let walk_options = Arc::new(walk_dir_options(&opts));
 
     for (disk_idx, disk) in opts.disks.iter().enumerate() {
         let opdisk = disk.clone();
-        let walk_options = walk_options.clone();
-        #[cfg(test)]
-        let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
-        #[cfg(test)]
-        let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
+        let opts_clone = opts.clone();
         let fallback_claim_tracker = fallback_claim_tracker.clone();
         let fds_clone = fds.clone();
         #[cfg(test)]
@@ -389,17 +384,17 @@ async fn list_path_raw_inner(
         let disk_io_context_clone = disk_io_context.clone();
         let (rd, wr) = tokio::io::duplex(64 * 1024);
         readers.push(MetacacheReader::new(rd));
-        let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
+        jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
             let mut wr = PublishedBytesWriter::new(wr);
             #[cfg(test)]
-            if let Some(observer) = &test_disk_io_context_observed {
+            if let Some(observer) = &opts_clone.test_disk_io_context_observed {
                 observer.store(
                     crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
                     std::sync::atomic::Ordering::Release,
                 );
             }
             #[cfg(test)]
-            let test_primary_error = if let Some(behavior) = test_reader_behavior {
+            let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
                 match behavior {
                     TestReaderBehavior::Eof => return Ok(()),
                     TestReaderBehavior::Entries(entries) => {
@@ -456,6 +451,8 @@ async fn list_path_raw_inner(
                 None
             };
 
+            let wakl_opts = walk_dir_options(&opts_clone);
+
             let mut need_fallback = false;
             let mut last_err = None;
             #[cfg(test)]
@@ -470,7 +467,7 @@ async fn list_path_raw_inner(
 
             if !need_fallback && let Some(disk) = opdisk {
                 let primary_walk_started = std::time::Instant::now();
-                match disk.walk_dir((*walk_options).clone(), &mut wr).await {
+                match disk.walk_dir(wakl_opts, &mut wr).await {
                     Ok(_res) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_primary",
@@ -494,8 +491,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %walk_options.bucket,
-                                path = %walk_options.base_dir,
+                                bucket = %opts_clone.bucket,
+                                path = %opts_clone.path,
                                 disk_index = disk_idx,
                                 state = "walk_dir_missing_path",
                                 error = ?err,
@@ -506,8 +503,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %walk_options.bucket,
-                                path = %walk_options.base_dir,
+                                bucket = %opts_clone.bucket,
+                                path = %opts_clone.path,
                                 disk_index = disk_idx,
                                 state = "walk_dir_failed",
                                 error = ?err,
@@ -609,8 +606,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %walk_options.bucket,
-                            path = %walk_options.base_dir,
+                            bucket = %opts_clone.bucket,
+                            path = %opts_clone.path,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing_for_path",
                             error = ?err,
@@ -621,8 +618,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %walk_options.bucket,
-                            path = %walk_options.base_dir,
+                            bucket = %opts_clone.bucket,
+                            path = %opts_clone.path,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing",
                             error = ?err,
@@ -637,7 +634,7 @@ async fn list_path_raw_inner(
                 }
 
                 let fallback_walk_started = std::time::Instant::now();
-                match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
+                match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
                     Ok(_r) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_fallback",
@@ -663,8 +660,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %walk_options.bucket,
-                                path = %walk_options.base_dir,
+                                bucket = %opts_clone.bucket,
+                                path = %opts_clone.path,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_missing_path",
                                 error = ?err,
@@ -675,8 +672,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %walk_options.bucket,
-                                path = %walk_options.base_dir,
+                                bucket = %opts_clone.bucket,
+                                path = %opts_clone.path,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_failed",
                                 error = ?err,
@@ -698,13 +695,12 @@ async fn list_path_raw_inner(
 
             // warn!("list_path_raw: while need_fallback done");
             Ok(())
-        });
-        jobs.spawn(producer.in_current_span());
+        }));
     }
 
     let revjob_rx = rx.clone();
     let mut revjobs = JoinSet::new();
-    let merge = async move {
+    revjobs.spawn(async move {
         // Consumer-side peek timeout, and a caveat worth understanding
         // (rustfs/backlog#1217).
         //
@@ -1063,8 +1059,7 @@ async fn list_path_raw_inner(
             }
         }
         Ok(ListPathRawOutcome { limit_reached, scan_cut })
-    };
-    revjobs.spawn(merge.in_current_span());
+    });
 
     let merge_started = std::time::Instant::now();
     let Some(revjob) = revjobs.join_next().await else {
@@ -1237,8 +1232,8 @@ mod tests {
             forward_to: Some("pending/0099".to_owned()),
             ..Default::default()
         };
-        // Both walks are built once from the shared options; the per-job clone
-        // must carry every field that the walk derives from the request.
+        // Primary and fallback readers both construct their walk from the
+        // per-job clone, rather than directly from the request options.
         let job = request.clone();
         let walk = walk_dir_options(&job);
         assert_eq!(walk.hidden_entry_budget, 100);
@@ -1296,163 +1291,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn list_path_raw_fanout_retains_only_linear_disk_references() {
-        let root = tempfile::tempdir().expect("create disk root");
-        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
-            .expect("parse disk endpoint");
-        let local = Arc::new(
-            crate::disk::local::LocalDisk::new(&endpoint, false)
-                .await
-                .expect("open local disk"),
-        );
-        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
-        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
-        let baseline = Arc::strong_count(&disk);
-
-        for primary_count in [1, 4, 16] {
-            for fallback_count in [0, 2] {
-                let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
-                let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
-                let behaviors = entered
-                    .iter()
-                    .zip(&stopped)
-                    .map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
-                        entered: entered.clone(),
-                        stopped: stopped.clone(),
-                    })
-                    .collect();
-                let cancel = CancellationToken::new();
-                let listing = tokio::spawn(list_path_raw(
-                    cancel.clone(),
-                    ListPathRawOptions {
-                        disks: vec![Some(disk.clone()); primary_count],
-                        fallback_disks: vec![Some(disk.clone()); fallback_count],
-                        min_disks: primary_count,
-                        test_reader_behaviors: behaviors,
-                        peek_timeout: Some(Duration::from_secs(60)),
-                        ..Default::default()
-                    },
-                ));
-                for signal in &entered {
-                    timeout(Duration::from_secs(2), signal.notified())
-                        .await
-                        .expect("every producer must enter the real fanout before counting references");
-                }
-
-                // The merge owns the configured primary/fallback slots, each producer
-                // owns its primary, and the shared fallback queue owns each fallback.
-                let expected = baseline + 2 * (primary_count + fallback_count);
-                let retained = Arc::strong_count(&disk);
-                cancel.cancel();
-                timeout(Duration::from_secs(2), listing)
-                    .await
-                    .expect("cancellation must stop the listing")
-                    .expect("listing task must join")
-                    .expect("external cancellation must finish cleanly");
-                for signal in &stopped {
-                    timeout(Duration::from_secs(2), signal.notified())
-                        .await
-                        .expect("cancellation must drop every producer");
-                }
-                assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
-                assert_eq!(
-                    retained, expected,
-                    "fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
-                );
-            }
-        }
-    }
-
-    #[tokio::test(flavor = "current_thread")]
-    async fn list_path_raw_producer_and_merge_preserve_request_span() {
-        use tracing_subscriber::{Layer, layer::SubscriberExt};
-
-        struct ProducerSpan(Arc<AtomicBool>);
-        #[derive(Default)]
-        struct StateField(bool);
-        impl tracing::field::Visit for StateField {
-            fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
-
-            fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
-                self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
-            }
-        }
-        impl<S> Layer<S> for ProducerSpan
-        where
-            S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
-        {
-            fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
-                let mut state = StateField::default();
-                event.record(&mut state);
-                if state.0
-                    && context
-                        .lookup_current()
-                        .is_some_and(|span| span.name() == "flink_list_request")
-                {
-                    self.0.store(true, std::sync::atomic::Ordering::Release);
-                }
-            }
-        }
-
-        // The subscriber is thread-local; keep the spawned tasks on this runtime thread.
-        let producer_observed = Arc::new(AtomicBool::new(false));
-        let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
-        let _subscriber = tracing::subscriber::set_default(subscriber);
-        let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
-        let root = tempfile::tempdir().expect("create disk root");
-        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
-            .expect("parse disk endpoint");
-        let local = Arc::new(
-            crate::disk::local::LocalDisk::new(&endpoint, false)
-                .await
-                .expect("open local disk"),
-        );
-        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
-        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
-        let error = list_path_raw(
-            CancellationToken::new(),
-            ListPathRawOptions {
-                disks: vec![Some(disk)],
-                bucket: "missing-bucket".to_owned(),
-                min_disks: 1,
-                ..Default::default()
-            },
-        )
-        .instrument(request.clone())
-        .await
-        .expect_err("missing bucket must fail the real disk producer");
-        assert_eq!(error, DiskError::VolumeNotFound);
-        assert!(
-            producer_observed.load(std::sync::atomic::Ordering::Acquire),
-            "disk error must retain the request span"
-        );
-
-        let merge_observed = Arc::new(AtomicBool::new(false));
-        let merge_sink = merge_observed.clone();
-        let request_id = request.id();
-        list_path_raw(
-            CancellationToken::new(),
-            ListPathRawOptions {
-                disks: vec![None],
-                min_disks: 1,
-                test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
-                agreed: Some(Box::new(move |_| {
-                    merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
-                    Box::pin(async { Ok(()) })
-                })),
-                ..Default::default()
-            },
-        )
-        .instrument(request)
-        .await
-        .expect("merge should deliver the test entry");
-        assert!(
-            merge_observed.load(std::sync::atomic::Ordering::Acquire),
-            "merge callback must retain the request span"
-        );
-    }
-
     #[tokio::test]
     async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
         let root = tempfile::tempdir().expect("local disk root should be created");

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -4356,8 +4356,7 @@ impl DiskAPI for RemoteDisk {
         object: &str,
         max_entries: u32,
     ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
-        // Older peers reject this variant. Do not retry through the unbounded protocol;
-        // they are reported as unsupported participants (rolling-upgrade contract).
+        // Older peers reject this variant. Do not retry through the unbounded protocol.
         match self
             .recover_delete_transaction(
                 volume,
@@ -4375,9 +4374,7 @@ impl DiskAPI for RemoteDisk {
                     || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
                     .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
             {
-                // No paginated fallback through the unbounded protocol; the peer
-                // is an unsupported participant until it is upgraded.
-                Err(Error::UnsupportedDisk)
+                Err(Error::DeleteRecoveryWorkLimit)
             }
             Err(error) => Err(error),
         }
@@ -4434,9 +4431,7 @@ impl DiskAPI for RemoteDisk {
                     || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
                     .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
             {
-                // No paginated fallback through the unbounded protocol; the peer
-                // is an unsupported participant until it is upgraded.
-                Err(Error::UnsupportedDisk)
+                Err(Error::DeleteRecoveryWorkLimit)
             }
             Err(error) => Err(error),
         }
@@ -4466,9 +4461,7 @@ impl DiskAPI for RemoteDisk {
                     || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
                     .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
             {
-                // No paginated fallback through the unbounded protocol; the peer
-                // is an unsupported participant until it is upgraded.
-                Err(Error::UnsupportedDisk)
+                Err(Error::DeleteRecoveryWorkLimit)
             }
             Err(error) => Err(error),
         }
@@ -5066,7 +5059,7 @@ mod tests {
                 .discover_delete_transactions_bounded("bucket", "prefix", 4096)
                 .await
                 .expect_err("old peer cannot prove a bounded inventory"),
-            DiskError::UnsupportedDisk
+            DiskError::DeleteRecoveryWorkLimit
         );
         let transaction = Uuid::new_v4();
         assert_eq!(
@@ -5074,7 +5067,7 @@ mod tests {
                 .inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
                 .await
                 .expect_err("old peer cannot inspect with a budget"),
-            DiskError::UnsupportedDisk
+            DiskError::DeleteRecoveryWorkLimit
         );
         assert_eq!(
             remote
@@ -5091,7 +5084,7 @@ mod tests {
                 )
                 .await
                 .expect_err("old peer cannot settle with a budget"),
-            DiskError::UnsupportedDisk
+            DiskError::DeleteRecoveryWorkLimit
         );
         assert_eq!(
             *peer.peer.unsupported_methods.lock().expect("observed methods"),
@@ -5991,7 +5984,6 @@ mod tests {
         object_read_all_disks: Arc<StdMutex<Vec<String>>>,
         format_data: Bytes,
         read_all_data: Bytes,
-        fail_format_reads: Arc<AtomicBool>,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
         bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
@@ -6018,7 +6010,6 @@ mod tests {
                 object_read_all_disks: Arc::default(),
                 format_data,
                 read_all_data,
-                fail_format_reads: Arc::default(),
                 next_format_read_gate: Arc::default(),
                 bounded_discovery: Arc::default(),
                 unsupported_methods: Arc::default(),
@@ -6193,9 +6184,6 @@ mod tests {
                                         gate.entered.notify_one();
                                         gate.release.cancelled().await;
                                     }
-                                    if peer.fail_format_reads.load(Ordering::Acquire) {
-                                        return Err(Status::unavailable("format read temporarily unavailable"));
-                                    }
                                     peer.format_data.clone()
                                 } else {
                                     peer.object_read_all_disks
@@ -9161,124 +9149,6 @@ mod tests {
         peer.stop().await;
     }
 
-    #[tokio::test]
-    #[serial(remote_disk_recovery_probe)]
-    async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
-        temp_env::async_with_vars(
-            [
-                (crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
-                (rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
-            ],
-            async {
-                runtime_sources::ensure_test_rpc_secret();
-                let mut reference = crate::layout::format::FormatV3::new(1, 1);
-                reference.erasure.this = reference.erasure.sets[0][0];
-                let peer = TestGrpcPeer::spawn(
-                    Bytes::from(reference.to_json().expect("format should serialize")),
-                    Bytes::from_static(b"recovered-peer-data"),
-                )
-                .await
-                .expect("authenticated regression peer is required");
-                let set = missing_read_set(&peer, reference).await;
-                let disk = crate::disk::new_disk(
-                    &set.set_endpoints[0],
-                    &DiskOption {
-                        cleanup: false,
-                        health_check: true,
-                    },
-                )
-                .await
-                .expect("registered remote disk should construct");
-                let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
-                    panic!("fixture must create a remote disk");
-                };
-                assert!(remote.health_check);
-                remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
-                set.disks.write().await[0] = Some(Arc::clone(&disk));
-
-                peer.peer.fail_format_reads.store(true, Ordering::Release);
-                set.connect_disks().await;
-                peer.peer.fail_format_reads.store(false, Ordering::Release);
-                let cancelled_after_failure = remote.cancel_token.is_cancelled();
-                assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
-                assert!(
-                    Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
-                    "a failed reconnect must not publish its unvalidated candidate"
-                );
-                remote.spawn_recovery_monitor_if_needed();
-                let recovery = time::timeout(Duration::from_secs(5), async {
-                    while !disk.is_online().await {
-                        tokio::task::yield_now().await;
-                    }
-                    disk.read_all("bucket", "object").await
-                })
-                .await;
-                disk.close().await.expect("stop the registered disk's monitoring");
-                peer.stop().await;
-                assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
-                assert_eq!(
-                    recovery
-                        .expect("failed reconnect must leave the registered disk able to recover without another sweep")
-                        .expect("the recovered registered disk must serve real I/O"),
-                    Bytes::from_static(b"recovered-peer-data")
-                );
-            },
-        )
-        .await;
-    }
-
-    #[tokio::test]
-    #[serial(remote_disk_recovery_probe)]
-    async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
-        runtime_sources::ensure_test_rpc_secret();
-        let mut reference = crate::layout::format::FormatV3::new(1, 1);
-        reference.erasure.this = reference.erasure.sets[0][0];
-        let peer = TestGrpcPeer::spawn(
-            Bytes::from(reference.to_json().expect("format should serialize")),
-            Bytes::from_static(b"replacement-peer-data"),
-        )
-        .await
-        .expect("authenticated regression peer is required");
-        let set = missing_read_set(&peer, reference).await;
-        let disk = crate::disk::new_disk(
-            &set.set_endpoints[0],
-            &DiskOption {
-                cleanup: false,
-                health_check: true,
-            },
-        )
-        .await
-        .expect("registered remote disk should construct");
-        let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
-            panic!("fixture must create a remote disk");
-        };
-        remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
-        set.disks.write().await[0] = Some(Arc::clone(&disk));
-        let gate = peer.peer.pause_next_format_read();
-        let pending_set = Arc::clone(&set);
-        let pending = tokio::spawn(async move { pending_set.connect_disks().await });
-        time::timeout(Duration::from_secs(5), gate.entered.notified())
-            .await
-            .expect("replacement must reach the format validation barrier");
-        let cancelled_before_publication = remote.cancel_token.is_cancelled();
-        gate.release.cancel();
-        pending.await.expect("validated replacement must finish publication");
-        let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
-        let readback = replacement.read_all("bucket", "object").await;
-        replacement.close().await.expect("stop replacement monitoring");
-        peer.stop().await;
-        assert!(
-            !cancelled_before_publication,
-            "the registered monitor must survive pending format validation"
-        );
-        assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
-        assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
-        assert_eq!(
-            readback.expect("replacement must serve real I/O"),
-            Bytes::from_static(b"replacement-peer-data")
-        );
-    }
-
     #[tokio::test]
     #[serial(remote_disk_recovery_probe)]
     async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -18,7 +18,6 @@ use crate::cluster::rpc::client::{
 use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
 use async_trait::async_trait;
 use bytes::Bytes;
-use futures::{FutureExt, future::BoxFuture};
 use rustfs_lock::{
     LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
@@ -33,17 +32,16 @@ use rustfs_protos::{
 };
 use std::collections::HashMap;
 use std::future::Future;
-use std::panic::AssertUnwindSafe;
 use std::pin::Pin;
 use std::sync::{Mutex, OnceLock};
 use std::time::Duration;
+use tokio::task::JoinHandle;
 use tokio::time::{Instant, timeout};
 use tonic::service::interceptor::InterceptedService;
 use tonic::{Request, Response};
 use tracing::{debug, info, warn};
 
 const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
-const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
 
@@ -51,38 +49,18 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
     set_tonic_rolling_mutation_body_digest(request)
 }
 
-/// Work to run if an RPC completes after its caller stops waiting.
+/// Work to run if an RPC that already timed out for its caller completes later.
 type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
 type LateFailure = LateCompletion<()>;
-type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
 
-/// Own admission before connecting as well as while an RPC is in flight.
-struct LockRequestGuard<'a> {
-    client: &'a RemoteClient,
-    op: &'static str,
+struct RpcCleanup<T> {
+    late: LateCompletion<T>,
+    late_failure: LateFailure,
     admission: Option<LockRequestAdmission>,
 }
 
-impl LockRequestGuard<'_> {
-    fn into_admission(mut self) -> Option<LockRequestAdmission> {
-        self.admission.take()
-    }
-}
-
-impl Drop for LockRequestGuard<'_> {
-    fn drop(&mut self) {
-        self.client.release_lock_request(self.op, self.admission.take());
-    }
-}
-
-struct RpcCleanup<'a, T> {
-    late: LateCompletion<T>,
-    late_failure: LateFailure,
-    admission: Option<LockRequestGuard<'a>>,
-}
-
-impl<'a, T> RpcCleanup<'a, T> {
-    fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
+impl<T> RpcCleanup<T> {
+    fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
         Self {
             late,
             late_failure,
@@ -91,25 +69,6 @@ impl<'a, T> RpcCleanup<'a, T> {
     }
 }
 
-/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
-struct InFlightRpc<'a, T: Send + 'static> {
-    client: &'a RemoteClient,
-    op: &'static str,
-    resource_summary: &'a str,
-    future: Option<RpcFuture<T>>,
-    cleanup: Option<RpcCleanup<'a, T>>,
-}
-
-impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
-    fn drop(&mut self) {
-        if let Some(future) = self.future.take()
-            && let Some(cleanup) = self.cleanup.take()
-        {
-            self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
-        }
-    }
-}
-
 /// The liveness window is this many RPC deadlines: a peer that completed a
 /// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
 const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
@@ -178,9 +137,6 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
     let mut peers = lock_peer_channel_health()
         .lock()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
-    if let Some(health) = peers.get_mut(addr) {
-        return update(health);
-    }
     update(peers.entry(addr.to_string()).or_default())
 }
 
@@ -202,15 +158,6 @@ fn reset_lock_peer_health_for_test(addr: &str) {
         .remove(addr);
 }
 
-#[cfg(test)]
-type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
-
-#[cfg(test)]
-fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
-    static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
-    PAUSES.get_or_init(Mutex::default)
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum EvictionTrigger {
     /// The caller's deadline expired while the stream was still open.
@@ -559,14 +506,6 @@ impl RemoteClient {
         admission
     }
 
-    fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
-        self.admit_lock_request(op).map(|admission| LockRequestGuard {
-            client: self,
-            op,
-            admission: Some(admission),
-        })
-    }
-
     fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
         if is_request_breaker_operation(op) && admission.is_some() {
             with_lock_peer_health(&self.addr, |health| {
@@ -648,17 +587,6 @@ impl RemoteClient {
         trigger: EvictionTrigger,
         deadline: Duration,
     ) {
-        #[cfg(test)]
-        let pause = eviction_pauses_for_test()
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner)
-            .remove(&self.addr);
-        #[cfg(test)]
-        if let Some((entered, resume)) = pause {
-            let _ = entered.send(());
-            let _ = resume.await;
-        }
-
         let now = Instant::now();
         let cooldown = Self::eviction_cooldown();
         let liveness_window = Self::liveness_window(deadline);
@@ -689,26 +617,22 @@ impl RemoteClient {
         );
     }
 
-    /// Keep an abandoned RPC running instead of cancelling its stream.
+    /// Keep a timed-out RPC running instead of cancelling its stream.
     ///
     /// Dropping the future sends `RST_STREAM`; under load those resets pile up
     /// in the server's pending-accept queue until it answers `GOAWAY
     /// too_many_resets` and kills every stream on the connection. A detached
     /// stream ends on its own within the internode RPC timeout, the number per
     /// peer is bounded, and a lock granted after its caller gave up is released.
-    fn detach_rpc<T: Send + 'static>(
+    fn detach_timed_out_rpc<T: Send + 'static>(
         &self,
         op: &'static str,
         resource_summary: &str,
-        future: RpcFuture<T>,
-        cleanup: RpcCleanup<'_, T>,
+        handle: JoinHandle<std::result::Result<T, tonic::Status>>,
+        late: LateCompletion<T>,
+        late_failure: LateFailure,
+        admission: Option<LockRequestAdmission>,
     ) {
-        let RpcCleanup {
-            late,
-            late_failure,
-            admission,
-        } = cleanup;
-        let admission = admission.and_then(LockRequestGuard::into_admission);
         let limit = Self::detached_rpc_limit();
         let admitted = with_lock_peer_health(&self.addr, |health| {
             if health.detached_rpcs >= limit {
@@ -719,29 +643,25 @@ impl RemoteClient {
             }
         });
         if !admitted {
-            drop(future);
+            handle.abort();
             self.release_lock_request(op, admission);
             if let Some(late_failure) = late_failure {
                 drop(tokio::spawn(late_failure(())));
             }
             rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
             debug!(
-                event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                state = "detached_budget_exhausted",
                 addr = %self.addr,
                 op,
                 resource_summary,
                 limit,
-                "Abandoned remote lock RPC cancelled"
+                "Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
             );
             return;
         }
         rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
         let addr = self.addr.clone();
         tokio::spawn(async move {
-            let outcome = future.await;
+            let outcome = handle.await;
             with_lock_peer_health(&addr, |health| {
                 health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
                 if is_request_breaker_operation(op) && admission.is_some() {
@@ -768,31 +688,19 @@ impl RemoteClient {
                     }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
                     debug!(
-                        event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                        state = "late_rpc_failed",
                         addr = %addr,
                         op,
                         tonic_code = ?status.code(),
                         tonic_message = status.message(),
-                        "Abandoned remote lock RPC failed"
+                        "Detached remote lock RPC failed after its caller timed out"
                     );
                 }
-                Err(_) => {
+                Err(join_error) => {
                     if let Some(late_failure) = late_failure {
                         late_failure(()).await;
                     }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
-                    debug!(
-                        event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                        state = "late_rpc_panicked",
-                        addr = %addr,
-                        op,
-                        "Abandoned remote lock RPC panicked"
-                    );
+                    debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
                 }
             }
         });
@@ -880,28 +788,16 @@ impl RemoteClient {
         self.release_late_release();
         rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
         if outcome == "released" {
-            debug!(
-                event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                state = "late_grants_released",
-                addr = %self.addr,
-                count = original_count,
-                "Released remote locks granted after their caller stopped waiting"
-            );
+            debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
         } else {
             let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
             if log_warning {
                 warn!(
-                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                    state = "late_grants_unreleased",
                     addr = %self.addr,
                     count = original_count,
                     outcome,
                     suppressed_timeouts,
-                    "Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
+                    "Could not release every remote lock granted after its caller timed out; the server lease will expire it"
                 );
             } else {
                 debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
@@ -915,47 +811,25 @@ impl RemoteClient {
         resource_summary: &str,
         deadline: Duration,
         future: Fut,
-        cleanup: RpcCleanup<'_, T>,
+        cleanup: RpcCleanup<T>,
     ) -> std::result::Result<T, LockError>
     where
         Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
         T: Send + 'static,
     {
-        let mut rpc = InFlightRpc {
-            client: self,
-            op,
-            resource_summary,
-            future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
-            cleanup: Some(cleanup),
-        };
-        let outcome = match rpc.future.as_mut() {
-            Some(future) => tokio::select! {
-                // A response becoming ready on the deadline tick must not turn an
-                // expired caller budget into a transport error and channel eviction.
-                biased;
-                _ = tokio::time::sleep(deadline) => None,
-                outcome = future => Some(outcome),
-            },
-            None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
-        };
-        if outcome.is_some() {
-            rpc.future.take();
-        }
-        let Some(RpcCleanup {
-            late: _,
+        let RpcCleanup {
+            late,
             late_failure,
             admission,
-        }) = rpc.cleanup.as_mut()
-        else {
-            return Err(LockError::internal("remote lock RPC has no cleanup owner"));
-        };
-        let token = admission.as_ref().and_then(|guard| guard.admission);
-        let result = match outcome {
-            Some(Ok(Ok(response))) => {
-                self.record_rpc_success(op, token);
+        } = cleanup;
+        let mut handle = tokio::spawn(future);
+        match timeout(deadline, &mut handle).await {
+            Ok(Ok(Ok(response))) => {
+                self.record_rpc_success(op, admission);
+                self.release_lock_request(op, admission);
                 Ok(response)
             }
-            Some(Ok(Err(err))) => {
+            Ok(Ok(Err(err))) => {
                 let reason = err.to_string();
                 if err.code() == tonic::Code::DeadlineExceeded {
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
@@ -988,11 +862,12 @@ impl RemoteClient {
                             "Suppressed repeated remote lock RPC deadline"
                         );
                     }
-                    if let Some(late_failure) = late_failure.take() {
-                        drop(tokio::spawn(late_failure(())));
-                    }
                     self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
                         .await;
+                    self.release_lock_request(op, admission);
+                    if let Some(late_failure) = late_failure {
+                        drop(tokio::spawn(late_failure(())));
+                    }
                     return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
                 }
                 // Only evict (and re-dial) the cached channel when the failure is a genuine
@@ -1053,32 +928,30 @@ impl RemoteClient {
                     );
                 }
                 if transport_failure {
-                    // Ambiguous-grant cleanup owns its work before channel eviction can suspend.
-                    if let Some(late_failure) = late_failure.take() {
-                        drop(tokio::spawn(late_failure(())));
-                    }
                     self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
                         .await;
                 }
+                self.release_lock_request(op, admission);
+                if transport_failure && let Some(late_failure) = late_failure {
+                    drop(tokio::spawn(late_failure(())));
+                }
                 Err(LockError::internal(format!("{op} RPC failed: {reason}")))
             }
-            Some(Err(_)) => {
+            Ok(Err(join_error)) => {
                 warn!(
-                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                    state = "rpc_panicked",
                     addr = %self.addr,
                     op,
                     resource_summary,
-                    "Remote lock RPC panicked"
+                    error = %join_error,
+                    "Remote lock RPC task ended abnormally"
                 );
-                if let Some(late_failure) = late_failure.take() {
+                self.release_lock_request(op, admission);
+                if let Some(late_failure) = late_failure {
                     drop(tokio::spawn(late_failure(())));
                 }
-                Err(LockError::internal(format!("{op} RPC future panicked")))
+                Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
             }
-            None => {
+            Err(_) => {
                 let reason = format!("RPC timed out after {deadline:?}");
                 rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
                 self.record_request_timeout(op, deadline);
@@ -1110,17 +983,12 @@ impl RemoteClient {
                         "Suppressed repeated remote lock RPC timeout"
                     );
                 }
-                // Continue draining the abandoned stream while channel eviction waits.
-                drop(rpc);
                 self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
                     .await;
-                return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
+                self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
+                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
             }
-        };
-        if rpc.future.is_none() {
-            rpc.cleanup.take();
         }
-        result
     }
 
     fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
@@ -1211,7 +1079,7 @@ impl RemoteClient {
 #[async_trait]
 impl LockClient for RemoteClient {
     async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
-        let admission = match self.admit_lock_request_guard("lock") {
+        let admission = match self.admit_lock_request("lock") {
             Some(admission) => admission,
             None => return Ok(Self::rpc_backoff_failure_response(request)),
         };
@@ -1222,6 +1090,7 @@ impl LockClient for RemoteClient {
             Ok(client) => client,
             Err(err) => {
                 self.record_request_timeout("lock", rpc_timeout);
+                self.release_lock_request("lock", Some(admission));
                 return Err(err);
             }
         };
@@ -1229,12 +1098,14 @@ impl LockClient for RemoteClient {
         let args = match serde_json::to_string(&bounded_request) {
             Ok(args) => args,
             Err(err) => {
+                self.release_lock_request("lock", Some(admission));
                 return Err(LockError::internal(format!("Failed to serialize request: {err}")));
             }
         };
         let mut req = Request::new(GenerallyLockRequest { args });
         req.set_timeout(rpc_timeout);
         if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
+            self.release_lock_request("lock", Some(admission));
             return Err(err.into());
         }
         let late = self.late_release_hook(request.lock_id.clone());
@@ -1276,7 +1147,7 @@ impl LockClient for RemoteClient {
             return Ok(Vec::new());
         }
 
-        let admission = match self.admit_lock_request_guard("lock_batch") {
+        let admission = match self.admit_lock_request("lock_batch") {
             Some(admission) => admission,
             None => return Ok(Self::rpc_backoff_failure_batch(requests)),
         };
@@ -1290,6 +1161,7 @@ impl LockClient for RemoteClient {
             Ok(client) => client,
             Err(err) => {
                 self.record_request_timeout("lock_batch", rpc_timeout);
+                self.release_lock_request("lock_batch", Some(admission));
                 return Err(err);
             }
         };
@@ -1303,12 +1175,14 @@ impl LockClient for RemoteClient {
         {
             Ok(args) => args,
             Err(err) => {
+                self.release_lock_request("lock_batch", Some(admission));
                 return Err(err);
             }
         };
         let mut req = Request::new(BatchGenerallyLockRequest { args });
         req.set_timeout(rpc_timeout);
         if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
+            self.release_lock_request("lock_batch", Some(admission));
             return Err(err.into());
         }
         let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
@@ -1836,351 +1710,6 @@ mod tests {
         assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
     }
 
-    #[tokio::test]
-    async fn test_remote_client_cancelled_connection_releases_request_admission() {
-        ensure_test_rpc_secret();
-        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
-        let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
-        let client = RemoteClient::new(addr.clone());
-        let request = test_lock_request(Duration::from_secs(30));
-
-        for batch in [false, true] {
-            reset_lock_peer_health_for_test(&addr);
-            with_lock_peer_health(&addr, |health| {
-                health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
-            });
-            let mut acquisition = Box::pin(async {
-                if batch {
-                    client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
-                } else {
-                    client.acquire_lock(&request).await.map(|_| ())
-                }
-            });
-            std::future::poll_fn(|cx| {
-                assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
-                std::task::Poll::Ready(())
-            })
-            .await;
-            let pending = lock_peer_health_for_test(&addr);
-            assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
-            assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
-
-            drop(acquisition);
-            let cancelled = lock_peer_health_for_test(&addr);
-            assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
-            assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
-            let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
-            client.release_lock_request("lock", Some(probe));
-        }
-        reset_lock_peer_health_for_test(&addr);
-    }
-
-    #[tokio::test]
-    async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
-        let addr = "http://cancelled-rpc-late-grant";
-        reset_lock_peer_health_for_test(addr);
-        let client = RemoteClient::new(addr.to_string());
-        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-        let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
-        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
-        let caller = tokio::spawn(async move {
-            let late: LateCompletion<bool> = Some(Box::new(move |granted| {
-                Box::pin(async move {
-                    released_tx.send(granted).expect("late grant observer remains alive");
-                })
-            }));
-            client
-                .execute_rpc(
-                    "lock",
-                    "bucket/object",
-                    Duration::from_secs(30),
-                    async move {
-                        started_tx.send(()).expect("caller observes RPC dispatch");
-                        Ok(grant_rx.await.expect("test decides when peer grants lock"))
-                    },
-                    RpcCleanup::new(late, None, None),
-                )
-                .await
-        });
-        timeout(Duration::from_secs(1), started_rx)
-            .await
-            .expect("RPC must dispatch")
-            .expect("dispatch signal");
-        caller.abort();
-        assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
-        assert_eq!(
-            lock_peer_health_for_test(addr).detached_rpcs,
-            1,
-            "cancelled RPC uses bounded detached ownership"
-        );
-        grant_tx.send(true).expect("cancellation must preserve the running RPC");
-        assert!(
-            timeout(Duration::from_secs(1), released_rx)
-                .await
-                .expect("late grant must be cleaned up")
-                .expect("cleanup result")
-        );
-        assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
-        reset_lock_peer_health_for_test(addr);
-    }
-
-    #[tokio::test]
-    async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
-        let addr = "http://cancelled-rpc-late-failure";
-        reset_lock_peer_health_for_test(addr);
-        let client = RemoteClient::new(addr.to_string());
-        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-        let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
-        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
-        let caller = tokio::spawn(async move {
-            let late_failure: LateFailure = Some(Box::new(move |()| {
-                Box::pin(async move {
-                    released_tx.send(()).expect("ambiguous cleanup observer remains alive");
-                })
-            }));
-            client
-                .execute_rpc(
-                    "lock_batch",
-                    "bucket/object",
-                    Duration::from_secs(30),
-                    async move {
-                        started_tx.send(()).expect("caller observes RPC dispatch");
-                        failure_rx.await.expect("test decides when transport fails");
-                        Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
-                    },
-                    RpcCleanup::new(None, late_failure, None),
-                )
-                .await
-        });
-        timeout(Duration::from_secs(1), started_rx)
-            .await
-            .expect("RPC must dispatch")
-            .expect("dispatch signal");
-        caller.abort();
-        assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
-        failure_tx.send(()).expect("cancellation must preserve the running RPC");
-        timeout(Duration::from_secs(1), released_rx)
-            .await
-            .expect("ambiguous grant must be cleaned up")
-            .expect("cleanup signal");
-        assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
-        reset_lock_peer_health_for_test(addr);
-    }
-
-    #[tokio::test]
-    async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
-        for (label, status) in [
-            ("transport", tonic::Status::cancelled("ambiguous transport failure")),
-            ("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
-        ] {
-            let addr = format!("http://cancelled-completed-{label}");
-            reset_lock_peer_health_for_test(&addr);
-            let client = RemoteClient::new(addr.clone());
-            let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
-            let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
-            eviction_pauses_for_test()
-                .lock()
-                .expect("eviction pause registry")
-                .insert(addr.clone(), (evicting_tx, resume_rx));
-            let (released_tx, released_rx) = tokio::sync::oneshot::channel();
-            let caller = tokio::spawn(async move {
-                let late_failure: LateFailure = Some(Box::new(move |()| {
-                    Box::pin(async move {
-                        released_tx
-                            .send(())
-                            .expect("completed-failure cleanup observer remains alive");
-                    })
-                }));
-                client
-                    .execute_rpc(
-                        "lock",
-                        "bucket/object",
-                        Duration::from_secs(30),
-                        async move { Err::<(), _>(status) },
-                        RpcCleanup::new(None, late_failure, None),
-                    )
-                    .await
-            });
-            timeout(Duration::from_secs(1), evicting_rx)
-                .await
-                .expect("completed error must reach channel eviction")
-                .expect("eviction signal");
-            timeout(Duration::from_secs(1), released_rx)
-                .await
-                .expect("cleanup must transfer before channel eviction suspends")
-                .expect("cleanup signal");
-            caller.abort();
-            assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
-            assert_eq!(
-                lock_peer_health_for_test(&addr).detached_rpcs,
-                0,
-                "completed RPC cannot consume a detached slot"
-            );
-            reset_lock_peer_health_for_test(&addr);
-        }
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
-        let addr = "http://same-tick-rpc-deadline";
-        reset_lock_peer_health_for_test(addr);
-        with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
-        let client = RemoteClient::new(addr.to_string());
-        let budget = Duration::from_millis(50);
-        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
-        let late_failure: LateFailure = Some(Box::new(move |()| {
-            Box::pin(async move {
-                released_tx.send(()).expect("same-tick cleanup observer remains alive");
-            })
-        }));
-        let result = client
-            .execute_rpc(
-                "lock",
-                "bucket/object",
-                budget,
-                async move {
-                    tokio::time::sleep(budget).await;
-                    Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
-                },
-                RpcCleanup::new(None, late_failure, None),
-            )
-            .await;
-        assert!(
-            matches!(result, Err(LockError::Timeout { .. })),
-            "expired caller budget must retain timeout classification"
-        );
-        let health = lock_peer_health_for_test(addr);
-        assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
-        assert!(
-            health.last_eviction.is_none(),
-            "a recently serving peer keeps its channel after a caller timeout"
-        );
-        timeout(Duration::from_secs(1), released_rx)
-            .await
-            .expect("same-tick ambiguous completion must trigger cleanup")
-            .expect("cleanup signal");
-        assert_eq!(
-            lock_peer_health_for_test(addr).detached_rpcs,
-            0,
-            "same-tick completion returns detached admission"
-        );
-        reset_lock_peer_health_for_test(addr);
-    }
-
-    #[tokio::test]
-    async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
-        let addr = "http://timeout-late-grant-during-eviction";
-        reset_lock_peer_health_for_test(addr);
-        let client = RemoteClient::new(addr.to_string());
-        let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
-        let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
-        eviction_pauses_for_test()
-            .lock()
-            .expect("eviction pause registry")
-            .insert(addr.to_string(), (evicting_tx, resume_rx));
-        let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
-        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
-        let caller = tokio::spawn(async move {
-            let late: LateCompletion<bool> = Some(Box::new(move |granted| {
-                Box::pin(async move {
-                    released_tx.send(granted).expect("late grant observer remains alive");
-                })
-            }));
-            client
-                .execute_rpc(
-                    "lock",
-                    "bucket/object",
-                    Duration::from_millis(25),
-                    async move { Ok(grant_rx.await.expect("test controls the late grant")) },
-                    RpcCleanup::new(late, None, None),
-                )
-                .await
-        });
-        timeout(Duration::from_secs(1), evicting_rx)
-            .await
-            .expect("timed-out RPC must reach channel eviction")
-            .expect("eviction signal");
-        assert_eq!(
-            lock_peer_health_for_test(addr).detached_rpcs,
-            1,
-            "eviction waits with a detached RPC owner"
-        );
-        grant_tx
-            .send(true)
-            .expect("timed-out stream remains available for a late grant");
-        assert!(
-            timeout(Duration::from_secs(1), released_rx)
-                .await
-                .expect("late grant must be cleaned up before eviction resumes")
-                .expect("late cleanup result")
-        );
-        assert!(!caller.is_finished(), "channel eviction is still paused");
-        assert_eq!(
-            lock_peer_health_for_test(addr).detached_rpcs,
-            0,
-            "late completion returns detached admission"
-        );
-        resume_tx.send(()).expect("resume channel eviction");
-        let result = caller.await.expect("timed-out caller completes without panicking");
-        assert!(
-            matches!(result, Err(LockError::Timeout { .. })),
-            "caller preserves its RPC timeout result"
-        );
-        reset_lock_peer_health_for_test(addr);
-    }
-
-    #[tokio::test]
-    async fn test_remote_client_success_polls_rpc_on_callers_task() {
-        tokio::spawn(async {
-            let client = RemoteClient::new("http://direct-poll-success".to_string());
-            let caller_id = tokio::task::id();
-            let rpc_id = client
-                .execute_rpc(
-                    "ping",
-                    "health-lock-online",
-                    Duration::from_secs(1),
-                    async { Ok(tokio::task::id()) },
-                    RpcCleanup::new(None, None, None),
-                )
-                .await
-                .expect("successful RPC must return its result");
-            assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
-        })
-        .await
-        .expect("caller task must complete");
-    }
-
-    #[tokio::test]
-    async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
-        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
-        let caller = tokio::spawn(async move {
-            let client = RemoteClient::new("http://direct-poll-panic".to_string());
-            let late_failure: LateFailure = Some(Box::new(move |()| {
-                Box::pin(async move {
-                    released_tx.send(()).expect("panic cleanup observer remains alive");
-                })
-            }));
-            client
-                .execute_rpc::<(), _>(
-                    "lock",
-                    "bucket/object",
-                    Duration::from_secs(1),
-                    async { panic!("test peer RPC panic") },
-                    RpcCleanup::new(None, late_failure, None),
-                )
-                .await
-        });
-        let result = caller.await.expect("RPC panic must not escape to its caller");
-        assert!(
-            matches!(result, Err(LockError::Internal { .. })),
-            "RPC panic remains an internal lock failure"
-        );
-        timeout(Duration::from_secs(1), released_rx)
-            .await
-            .expect("panic must trigger ambiguous-grant cleanup")
-            .expect("cleanup signal");
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -217,264 +217,6 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
         "all current heads, identities and complete backups must remain unchanged"
     );
     assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
-    state
-        .ensure_write_safe("ordinary authority temporarily unavailable")
-        .expect("missing ordinary authority alone must not install a permanent corruption latch");
-}
-
-async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
-    let fixture = fixture(3).await;
-    let PoolMetaReplica::Valid {
-        previous: Some(previous),
-        committed: false,
-        ..
-    } = decode_pool_meta_replica(fixture.pending.clone())
-    else {
-        panic!("fixture must retain its acknowledged predecessor");
-    };
-    let physical = disks(&fixture.store).await;
-    let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
-    for (index, disk) in physical[0].iter().enumerate() {
-        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
-            .await
-            .expect("one pool must retain an ordinary-readable PREPARE");
-    }
-    (fixture, *previous)
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
-    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
-    let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-    assert!(
-        before[1..]
-            .iter()
-            .all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
-    );
-    assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
-    let mut state = clean_state(&fixture);
-    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
-    assert!(
-        repair(&fixture.store, &mut state, &fence)
-            .await
-            .expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
-    );
-    state
-        .ensure_write_safe("first commit rollback")
-        .expect("rollback must finish its transaction");
-    assert_ne!(
-        previous.canonical, fixture.canonical,
-        "the interrupted COMMIT must not become authoritative"
-    );
-    fixture.canonical = previous.canonical;
-    fixture.revision = previous.revision;
-    assert_committed(&fixture).await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
-    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
-    let pause = Arc::new(NativeApplyPause::default());
-    let reached = pause.reached.notified();
-    tokio::pin!(reached);
-    reached.as_mut().enable();
-    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
-    let mut state = clean_state(&fixture);
-    let recovery = tokio::spawn({
-        let store = Arc::clone(&fixture.store);
-        let pause = Arc::clone(&pause);
-        async move {
-            AFTER_NATIVE_POOL_APPLY
-                .scope(pause, async move { repair(&store, &mut state, &fence).await })
-                .await
-        }
-    });
-    tokio::time::timeout(std::time::Duration::from_secs(10), reached)
-        .await
-        .expect("real rollback should pause after one pool is durably restored");
-    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-    assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
-    assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
-        if canonical == &previous.canonical));
-    assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
-    recovery.abort();
-    assert!(
-        recovery
-            .await
-            .expect_err("the paused repair must be cancelled")
-            .is_cancelled()
-    );
-    let mut state = clean_state(&fixture);
-    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
-    assert!(
-        repair(&fixture.store, &mut state, &fence)
-            .await
-            .expect("restart must resume from the retained ordinary PREPARE")
-    );
-    fixture.canonical = previous.canonical;
-    fixture.revision = previous.revision;
-    assert_committed(&fixture).await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
-    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
-    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
-    let inspected = inspect(&fixture.store.pools, &fence)
-        .await
-        .expect("physical rollback inventory");
-    let replacements = plan(
-        &inspected,
-        &fixture.canonical,
-        fixture.revision,
-        &fixture.identity,
-        Some(&previous.canonical),
-    )
-    .await
-    .expect("the ordinary PREPARE must authorize its predecessor");
-    for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
-        for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
-            let snapshot = &inspected[pool].snapshots[index];
-            assert_eq!(
-                inspected[pool].disks[index]
-                    .pool_meta_native_recovery(Request::Apply {
-                        object: PoolMetaNativeObject::PoolBin,
-                        expected_disk_id: snapshot.disk_id,
-                        expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
-                        expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
-                        replacement: replacement.clone(),
-                    })
-                    .await
-                    .expect("partial rollback must use the real conditional publication and sync path"),
-                Response::Applied
-            );
-        }
-    }
-    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-    assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
-    assert!(partial[1..].iter().all(|read| matches!(&read.replica,
-        PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
-    let mut state = clean_state(&fixture);
-    assert!(
-        repair(&fixture.store, &mut state, &fence)
-            .await
-            .expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
-    );
-    fixture.canonical = previous.canonical;
-    fixture.revision = previous.revision;
-    assert_committed(&fixture).await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
-    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
-    let mut state = clean_state(&fixture);
-    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
-    assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
-    let revision = PoolMetaRevision {
-        transaction_id: Some(Uuid::new_v4()),
-        ..fixture.revision
-    };
-    let mut meta = previous.meta.clone();
-    meta.pools[0].last_update += Duration::seconds(1);
-    let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
-    let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
-    let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
-    let split = split(&pending, &canonical, modified).await;
-    let commit = replicas(&canonical, modified).await;
-    let prepared = replicas(&pending, modified).await;
-    let directory = Uuid::new_v4();
-    let retained = snapshots(&fixture.store).await;
-    for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
-        for (disk_index, disk) in pool.iter().enumerate() {
-            assert!(
-                !retained[pool_index][disk_index].backups.is_empty(),
-                "the abandoned PREPARE must remain as real on-disk evidence"
-            );
-            let object = object_path(disk);
-            fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
-                .await
-                .expect("install later partial COMMIT without clearing old backups");
-            fs::create_dir(object.join(directory.to_string()))
-                .await
-                .expect("later backup directory");
-            fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
-                .await
-                .expect("later PREPARE backup");
-        }
-    }
-    assert!(
-        repair(&fixture.store, &mut state, &fence)
-            .await
-            .expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
-    );
-    fixture.canonical = canonical;
-    fixture.revision = revision;
-    assert_committed(&fixture).await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
-    let (fixture, _) = interrupted_first_commit_fixture().await;
-    let physical = disks(&fixture.store).await;
-    let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
-    let directory = Uuid::new_v4();
-    let object = object_path(&physical[0][0]);
-    fs::create_dir(object.join(directory.to_string()))
-        .await
-        .expect("retained COMMIT directory");
-    fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
-        .await
-        .expect("retain evidence that the interrupted transaction may already have committed");
-    let before = snapshots(&fixture.store).await;
-    let mut state = clean_state(&fixture);
-    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
-    assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
-    assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
-    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
-    let (fixture, previous) = interrupted_first_commit_fixture().await;
-    let pending = encode_pool_meta_v3_envelope(
-        &previous.meta,
-        PoolMetaRevision {
-            transaction_id: Some(Uuid::new_v4()),
-            ..fixture.revision
-        },
-        false,
-        Some(&previous),
-    )
-    .expect("competing PREPARE must remain structurally valid");
-    let physical = disks(&fixture.store).await;
-    let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
-    for (index, disk) in physical[1].iter().enumerate() {
-        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
-            .await
-            .expect("second pool must expose the competing transaction through ordinary reads");
-    }
-    let before = snapshots(&fixture.store).await;
-    let mut state = clean_state(&fixture);
-    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
-    let error = repair(&fixture.store, &mut state, &fence)
-        .await
-        .expect_err("two distinct prepared transactions must not authorize a guessed rollback");
-    assert_eq!(
-        error
-            .pool_metadata_failure()
-            .expect("transaction conflict must be typed")
-            .kind,
-        crate::error::PoolMetadataFailure::RecoveryRequired
-    );
-    assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
-    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
 }
 
 struct IdentityEpochFork {
@@ -726,7 +468,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
     let inspected = inspect(&fixture.store.pools, &fence)
         .await
         .expect("real native snapshots should be obtained");
-    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
+    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
         .await
         .expect("physical recovery plan should validate");
     let snapshot = &inspected[0].snapshots[0];

crates/ecstore/src/core/pool_meta_native_recovery.rs

@@ -30,7 +30,6 @@ pub(super) struct NativeApplyPause {
 #[cfg(test)]
 tokio::task_local! {
     pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
-    pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
 }
 
 struct InspectedPool {
@@ -169,7 +168,6 @@ async fn plan(
     canonical: &[u8],
     revision: PoolMetaRevision,
     identity: &[u8],
-    rollback: Option<&[u8]>,
 ) -> Result<Vec<Vec<Bytes>>> {
     let mut pending = Vec::<Vec<u8>>::new();
     let mut predecessors = Vec::<Vec<u8>>::new();
@@ -225,83 +223,23 @@ async fn plan(
             }
         }
     }
-    // The last ordinary PREPARE can itself become split during rollback. In
-    // that case another pool already exposes the restored ordinary COMMIT.
-    // Its authority permits continuing the rollback, never promoting a child
-    // transaction reconstructed from disk evidence.
-    let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
-    if rollback.is_none() {
-        let mut outstanding = Vec::new();
-        for pool in inspected {
-            for (index, snapshot) in pool.snapshots.iter().enumerate() {
-                let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
-                if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
-                    outstanding.push((index, raw));
-                }
-            }
-        }
-        for payload in &decoded {
-            if let PoolMetaReplica::Valid {
+    let mut obsolete = Vec::<Vec<u8>>::new();
+    for payload in &decoded {
+        match decode_pool_meta_replica(payload.clone()) {
+            PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
                 committed: false,
                 previous: Some(previous),
                 ..
-            } = decode_pool_meta_replica(payload.clone())
-                && previous.revision == revision
-                && previous.canonical == canonical
-            {
-                let mut matches = true;
-                for (index, raw) in &outstanding {
-                    matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
-                        || codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
+            } if candidate_revision == revision && candidate == canonical => {
+                if !pending.contains(payload) {
+                    pending.push(payload.clone());
                 }
-                if matches {
-                    if !outstanding.is_empty()
-                        && continuation.as_ref().is_some_and(|(selected, selected_revision)| {
-                            selected != &candidate || *selected_revision != candidate_revision
-                        })
-                    {
-                        return Err(known_validation(invalid(), "native_transaction_validation"));
-                    }
-                    continuation.get_or_insert((candidate, candidate_revision));
+                if !predecessors.contains(&previous.canonical) {
+                    predecessors.push(previous.canonical);
                 }
             }
-        }
-    }
-    let (canonical, revision, rollback) = match &continuation {
-        Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
-        None => (canonical, revision, rollback),
-    };
-    for payload in &decoded {
-        if let PoolMetaReplica::Valid {
-            canonical: candidate,
-            revision: candidate_revision,
-            committed: false,
-            previous: Some(previous),
-            ..
-        } = decode_pool_meta_replica(payload.clone())
-            && candidate_revision == revision
-            && candidate == canonical
-        {
-            pending.push(payload.clone());
-            if !predecessors.contains(&previous.canonical) {
-                predecessors.push(previous.canonical);
-            }
-        }
-    }
-    if pending.len() > 1 {
-        return Err(known_validation(
-            Error::other("native V3 prepare records disagree on the predecessor"),
-            "native_transaction_validation",
-        ));
-    }
-    let mut obsolete = Vec::<Vec<u8>>::new();
-    for payload in &decoded {
-        if pending.contains(payload) {
-            continue;
-        }
-        match decode_pool_meta_replica(payload.clone()) {
             PoolMetaReplica::Valid { revision: older, .. }
                 if older.is_generation_protocol()
                     && older.cluster_id == revision.cluster_id
@@ -313,23 +251,6 @@ async fn plan(
                 // fully reconstructed and bound into the native CAS.
                 obsolete.push(payload.clone());
             }
-            PoolMetaReplica::Valid {
-                revision: abandoned,
-                committed: false,
-                previous: Some(previous),
-                ..
-            } if abandoned.is_generation_protocol()
-                && abandoned.cluster_id == revision.cluster_id
-                && abandoned.epoch == revision.epoch
-                && abandoned.generation == revision.generation
-                && abandoned.transaction_id != revision.transaction_id
-                && predecessors.contains(&previous.canonical) =>
-            {
-                // A rollback retains the failed PREPARE as evidence. A later
-                // transaction may reuse that generation, but only a backup of
-                // an uncommitted sibling with the same predecessor is obsolete.
-                obsolete.push(payload.clone());
-            }
             PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
@@ -351,7 +272,13 @@ async fn plan(
             _ => {}
         }
     }
-    if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
+    if pending.len() > 1 {
+        return Err(known_validation(
+            Error::other("native V3 prepare records disagree on the predecessor"),
+            "native_transaction_validation",
+        ));
+    }
+    if pending.is_empty() {
         return Err(invalid());
     }
     for payload in &decoded {
@@ -367,25 +294,13 @@ async fn plan(
         for (index, snapshot) in pool.snapshots.iter().enumerate() {
             let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
             let mut matched = false;
-            for payload in std::iter::once(canonical)
-                .chain(pending.iter().map(Vec::as_slice))
-                .chain(rollback)
-            {
+            for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
                 matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
             }
             if !matched {
                 return Err(invalid());
             }
             for backup in &snapshot.backups {
-                if rollback.is_some()
-                    && codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
-                        .await
-                        .is_ok()
-                {
-                    // A retained COMMIT might have been authoritative before
-                    // this interruption. Never roll it back from PREPARE alone.
-                    return Err(invalid());
-                }
                 let mut matched = false;
                 for payload in std::iter::once(canonical).chain(
                     pending
@@ -408,7 +323,7 @@ async fn plan(
     let mut replacements = Vec::with_capacity(inspected.len());
     for pool in inspected {
         replacements.push(
-            codec::encode_inline_pool_meta_replicas(&current(pool)?, rollback.unwrap_or(canonical), modified)
+            codec::encode_inline_pool_meta_replicas(&current(pool)?, canonical, modified)
                 .await
                 .map_err(|error| codec_validation(error, "native_metadata_validation"))?,
         );
@@ -423,12 +338,10 @@ pub(super) async fn repair(
 ) -> Result<bool> {
     fence.ensure_held()?;
     let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
-    let unreadable = reads
+    if !reads
         .iter()
-        .enumerate()
-        .filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
-        .collect::<Vec<_>>();
-    if unreadable.is_empty() {
+        .any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
+    {
         return Ok(false);
     }
     state.ensure_write_safe("native pool metadata recovery")?;
@@ -447,9 +360,8 @@ pub(super) async fn repair(
     {
         return Err(invalid());
     }
-    // Ordinary reads grant authority: a COMMIT selects its own state, while a
-    // PREPARE can select only its acknowledged predecessor. Reconstruction
-    // below proves integrity and never promotes an interrupted transaction.
+    // Only an ordinary, successfully read COMMIT grants authority. Rebuilding
+    // two erasure shards below proves integrity, never a commit decision.
     let ordinary_commits = reads
         .iter()
         .filter_map(|read| match &read.replica {
@@ -463,22 +375,7 @@ pub(super) async fn repair(
             _ => None,
         })
         .collect::<Vec<_>>();
-    let ordinary_prepares = reads
-        .iter()
-        .filter_map(|read| match &read.replica {
-            PoolMetaReplica::Valid {
-                canonical,
-                revision,
-                committed: false,
-                previous: Some(previous),
-                ..
-            } if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
-                Some((canonical.clone(), *revision, previous.canonical.clone()))
-            }
-            _ => None,
-        })
-        .collect::<Vec<_>>();
-    if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
+    if ordinary_commits.is_empty() {
         return Err(invalid());
     }
     let selection = select_pool_meta_replica_reads(reads)
@@ -498,20 +395,10 @@ pub(super) async fn repair(
         return Err(invalid());
     }
     let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
-    let mut rollback = None;
-    for (pending, revision, previous) in &ordinary_prepares {
-        if previous == canonical {
-            if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
-                return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
-            }
-            rollback = Some((pending, *revision));
-        }
-    }
     if !selection.revision.is_generation_protocol()
-        || (rollback.is_none()
-            && !ordinary_commits
-                .iter()
-                .any(|(raw, revision)| raw == canonical && *revision == selection.revision))
+        || !ordinary_commits
+            .iter()
+            .any(|(raw, revision)| raw == canonical && *revision == selection.revision)
     {
         return Err(invalid());
     }
@@ -558,18 +445,13 @@ pub(super) async fn repair(
         .await
         .map_err(|error| latch_validation(state, error))?;
     let inspected = inspect(&store.pools, fence).await?;
-    let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
-    let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
+    let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
         .await
         .map_err(|error| latch_validation(state, error))?;
     fence.ensure_held()?;
     let mut arm = state.arm_transaction();
     arm.phase = Some("native_repair_apply");
-    let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
-    // Preserve ordinary authority until all unreadable pools are repaired.
-    // Cancellation can then resume from a PREPARE or a restored COMMIT.
-    apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
-    for (_, (pool, replacements)) in apply_order {
+    for (pool, replacements) in inspected.iter().zip(replacements) {
         for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
             fence.ensure_held()?;
             let request = Request::Apply {
@@ -584,11 +466,6 @@ pub(super) async fn repair(
                 _ => return Err(invalid()),
             }
         }
-        #[cfg(test)]
-        if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
-            pause.reached.notify_one();
-            pause.resume.notified().await;
-        }
     }
     #[cfg(test)]
     if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
@@ -750,7 +627,7 @@ mod tests {
     #[tokio::test]
     async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
         let f = fixture().await;
-        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
+        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
         assert_eq!(repaired.len(), 4);
         for pool in repaired {
             assert_eq!(
@@ -771,7 +648,7 @@ mod tests {
         let mut f = fixture().await;
         let wrong = replicas(b"different identity", f.modified).await;
         f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
     }
 
     #[tokio::test]
@@ -794,10 +671,10 @@ mod tests {
         .unwrap();
         let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
         f.pools[0].snapshots[0].current = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
         f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
         f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
     }
 
     #[tokio::test]
@@ -840,17 +717,17 @@ mod tests {
             }
         }
         assert!(older_pending != f.pending);
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
     }
 
     #[tokio::test]
     async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
         let mut f = fixture().await;
         let before = f.pools[1].snapshots[2].current.take();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
         f.pools[1].snapshots[2].current = before;
         f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
     }
 }
 

crates/ecstore/src/core/pools.rs

@@ -8380,9 +8380,8 @@ where
         })
     });
     let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
-    let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
-    let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
-    if matches!(&result, Err(Error::PreconditionFailed)) {
+    let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
+    if !may_have_mutated {
         record_pool_meta_stale_write_rejection(phase);
     }
     let mut source = None;
@@ -8393,8 +8392,6 @@ where
             source = Some(Arc::clone(&original));
             if matches!(original.as_ref(), Error::PreconditionFailed) {
                 Err(Error::PreconditionFailed)
-            } else if write_not_started {
-                Err(Error::other(original))
             } else {
                 Err(Error::other(pool_metadata_error(
                     crate::error::PoolMetadataFailure::TransactionUnknown,
@@ -30829,154 +30826,6 @@ mod pools_tests {
         );
     }
 
-    #[tokio::test]
-    async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
-        let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
-        pool.disks.write().await[2..].fill(None);
-        let state = super::PoolMetaWriteState::default();
-        let data =
-            super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
-        let mut transaction = state.arm_transaction();
-        save_pool_meta_object_cas(
-            pool.clone(),
-            POOL_META_IDENTITY_NAME,
-            data.clone(),
-            &PoolMetaCasToken::Missing,
-            &PoolMetaPersistenceFence::OfflineRecovery,
-            "identity_cas",
-            &mut transaction,
-        )
-        .await
-        .expect_err("two of four disks cannot satisfy the identity write quorum");
-        drop(transaction);
-        for dir in &dirs {
-            assert!(
-                !dir.path()
-                    .join(crate::disk::RUSTFS_META_BUCKET)
-                    .join(POOL_META_IDENTITY_NAME)
-                    .exists(),
-                "the rejected write must not create an identity replica"
-            );
-        }
-        state
-            .ensure_write_safe("retry after disks return")
-            .expect("a rejected inline write must not poison the startup transaction");
-
-        let ordinary_error = crate::config::com::save_config_with_opts(
-            pool.clone(),
-            "ordinary-config",
-            data.clone(),
-            &ObjectOptions {
-                max_parity: true,
-                no_lock: true,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect_err("ordinary writes must still fail without quorum");
-        assert!(
-            matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
-            "ordinary callers must retain their existing quorum error"
-        );
-
-        *pool.disks.write().await = disks.into_iter().map(Some).collect();
-        let mut retry = state.arm_transaction();
-        save_pool_meta_object_cas(
-            pool.clone(),
-            POOL_META_IDENTITY_NAME,
-            data.clone(),
-            &PoolMetaCasToken::Missing,
-            &PoolMetaPersistenceFence::OfflineRecovery,
-            "identity_cas",
-            &mut retry,
-        )
-        .await
-        .expect("the same transaction must succeed when the disks return");
-        retry.disarm();
-        drop(retry);
-        assert_eq!(
-            crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
-                .await
-                .expect("read back the committed identity"),
-            data
-        );
-        state
-            .ensure_write_safe("after successful retry")
-            .expect("retry must leave writes enabled");
-    }
-
-    #[tokio::test]
-    async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
-        let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
-        let state = super::PoolMetaWriteState::default();
-        let data =
-            super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
-        let mut transaction = state.arm_transaction();
-        let written = save_pool_meta_object_cas(
-            pool.clone(),
-            POOL_META_IDENTITY_NAME,
-            data.clone(),
-            &PoolMetaCasToken::Missing,
-            &PoolMetaPersistenceFence::OfflineRecovery,
-            "prepare_cas",
-            &mut transaction,
-        )
-        .await
-        .expect("the earlier phase must really commit");
-        pool.disks.write().await[2..].fill(None);
-        save_pool_meta_object_cas(
-            pool.clone(),
-            POOL_META_IDENTITY_NAME,
-            data.clone(),
-            &PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
-            &PoolMetaPersistenceFence::OfflineRecovery,
-            "identity_cas",
-            &mut transaction,
-        )
-        .await
-        .expect_err("the next phase must fail without a write quorum");
-        drop(transaction);
-        *pool.disks.write().await = disks.into_iter().map(Some).collect();
-        assert_eq!(
-            crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
-                .await
-                .expect("the earlier commit must survive the rejected write"),
-            data
-        );
-        let error = state
-            .ensure_write_safe("after a committed phase")
-            .expect_err("a later rejection cannot discharge the earlier write");
-        assert_eq!(
-            error.pool_metadata_failure().expect("retain typed uncertainty").kind,
-            crate::error::PoolMetadataFailure::TransactionUnknown
-        );
-        assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
-    }
-
-    #[tokio::test]
-    async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
-        let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
-        pool.disks.write().await[2..].fill(None);
-        let state = super::PoolMetaWriteState::default();
-        let mut transaction = state.arm_transaction();
-        save_pool_meta_object_cas(
-            pool,
-            POOL_META_IDENTITY_NAME,
-            vec![0x51; 2 * 1024 * 1024],
-            &PoolMetaCasToken::Missing,
-            &PoolMetaPersistenceFence::OfflineRecovery,
-            "identity_cas",
-            &mut transaction,
-        )
-        .await
-        .expect_err("the noninline write must fail without quorum");
-        drop(transaction);
-        assert!(
-            state.ensure_write_safe("after writer setup").is_err(),
-            "a failure after disk writers were created must remain fail-closed"
-        );
-    }
-
     #[derive(Clone, Copy, Debug, PartialEq, Eq)]
     enum IdentityRepairFault {
         Healthy,

crates/ecstore/src/data_movement/e2e_barrier.rs

@@ -31,8 +31,6 @@ use tokio::fs;
 use tokio::time::Instant;
 
 pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
-/// Rebalance holds its admission fence; no version of this entry has moved.
-pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 /// Every target part is staged and the target version is not yet published.
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 /// The target commit is durable and the source preflight matched; the source

crates/ecstore/src/disk/disk_store.rs

@@ -3841,7 +3841,7 @@ mod tests {
         time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
         assert!(
-            !logs.contents().contains("disk_operation_slow") && !logs.contents().contains("WARN"),
+            !logs.contents().contains("disk_operation_slow"),
             "normal disk latency should not warn: {}",
             logs.contents()
         );

crates/ecstore/src/disk/local.rs

@@ -185,14 +185,13 @@ const DELETE_RESIDUE_PROBE_LIMIT: i32 = 8;
 /// large residue tree, which then hides one level at a time instead.
 const DELETE_RESIDUE_PROBE_READ_BUDGET: usize = 32;
 
-/// What one bounded directory read found under a prefix.
+/// One directory read owed by the delete-residue probe.
 enum PrefixListingProbe {
     Visible,
     Empty,
     Cut(Vec<MetaCacheEntry>),
 }
 
-/// One directory read owed by the delete-residue probe.
 enum ResidueProbeStep {
     /// Read a bounded batch of `dir` and probe what it holds.
     Sample(String),
@@ -8136,6 +8135,8 @@ impl LocalDisk {
             Self::check_startup_writable(&disk.publication_root, &disk.root.join(directory)).await?;
         }
 
+        disk.multipart_index.start();
+
         let (exit_tx, exit_rx) = tokio::sync::broadcast::channel(1);
         disk.exit_signal = Some(exit_tx);
 
@@ -8146,7 +8147,6 @@ impl LocalDisk {
             publication_root,
             #[cfg(target_os = "linux")]
             disk.mount_lease.clone(),
-            disk.multipart_index.clone(),
             exit_rx,
         ));
         debug!(
@@ -8165,19 +8165,8 @@ impl LocalDisk {
         root: PathBuf,
         publication_root: os::PublicationRoot,
         #[cfg(target_os = "linux")] mount_lease: Arc<std::fs::File>,
-        multipart_index: Arc<multipart_index::IndexOwner>,
         mut exit_rx: tokio::sync::broadcast::Receiver<()>,
     ) {
-        // The multipart index bootstrap rides this task instead of its own:
-        // the disk lifecycle accounts for exactly this background task, and
-        // stopping the disk abandons the scan (a reconnect resumes it).
-        if let Some(bootstrap) = multipart_index.start() {
-            tokio::select! {
-                _ = bootstrap => {}
-                _ = exit_rx.recv() => return,
-            }
-        }
-        drop(multipart_index);
         let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL;
         let mut interval = interval_at(start_at, DELETED_OBJECTS_CLEANUP_INTERVAL);
         interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
@@ -16469,7 +16458,7 @@ mod test {
                 assert!(slots[0][0][1].is_some());
                 drop(slots);
                 drop(entries);
-                let (disks, errors) = crate::store::init_format::init_disks(&ctx, &endpoints, &options).await;
+                let (disks, errors) = crate::store::init_format::init_disks(&endpoints, &options).await;
                 assert!(disks[0].is_none());
                 assert!(disks[1].is_some());
                 assert_eq!(errors[0], Some(DiskError::FileAccessDenied));
@@ -26576,7 +26565,6 @@ mod test {
             root.clone(),
             disk.publication_root.clone(),
             disk.mount_lease.clone(),
-            disk.multipart_index.clone(),
             exited,
         ));
         tokio::task::yield_now().await;

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -41,11 +41,6 @@ struct DiscoverySession {
     touched: Instant,
 }
 
-/// Plain (non-transaction) children of an object-scoped discovery root cost one
-/// budget entry per this many names: enough to bound a hostile prefix without
-/// making a real folder prefix undeletable.
-const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
-
 impl DiscoverySession {
     /// Consume one bounded step, returning ownership only when traversal remains.
     fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
@@ -66,7 +61,6 @@ impl DiscoverySession {
         let mut transactions = BTreeSet::new();
         let mut seen_candidates = HashSet::new();
         let mut visited = 0u32;
-        let mut plain_children = 0u32;
         while !self.frames.is_empty()
             && transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
             && visited < max_entries
@@ -82,6 +76,7 @@ impl DiscoverySession {
                     continue;
                 }
             };
+            visited += 1;
             let name = entry
                 .file_name()
                 .into_string()
@@ -89,19 +84,6 @@ impl DiscoverySession {
             if !safe_path(&name, false) || name.contains('/') {
                 return Err(std::io::Error::from(ErrorKind::InvalidData));
             }
-            // An object-scoped scan only ever descends into transaction (UUID)
-            // directories at the root. Plain child keys of a prefix directory are
-            // skipped by name without a stat; they are charged at one entry per
-            // readdir batch so the work stays bounded while deleting a missing key
-            // that names a prefix with thousands of children can still complete.
-            if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
-                plain_children += 1;
-                if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
-                    visited += 1;
-                }
-                continue;
-            }
-            visited += 1;
             let relative = if parent.is_empty() {
                 name.clone()
             } else {
@@ -1404,10 +1386,7 @@ impl LocalDisk {
         let lease = os::acquire_namespace_mutation_lease_with_owner(&root, Some(owner)).await;
         let volume = volume.to_owned();
         let object = object.to_owned();
-        // Same lane as the unbounded inspection: a bulk delete inspects one
-        // transaction per key on every drive, and the one-worker foreground
-        // namespace lane would serialize those behind every other mutation.
-        os::run_blocking_namespace_operation_with_sync(lease, true, move || {
+        os::run_blocking_namespace_operation(lease, move || {
             let budget = crate::object_api::DeleteRecoveryWorkBudget::new(max_entries);
             Ok(
                 inspection_with_markers_bounded(&root, &volume, &object, transaction, Some(&budget)).map(|(inspection, _)| {
@@ -6214,9 +6193,8 @@ mod tests {
         let (_temp, disk, _, _) = fixture().await;
         let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
         fs::create_dir_all(&path).await.expect("physical prefix");
-        // Transaction-shaped children are the entries a bounded probe pays for.
-        for _ in 0..64 {
-            fs::create_dir(path.join(Uuid::new_v4().to_string()))
+        for index in 0..64 {
+            fs::create_dir(path.join(format!("child-{index}")))
                 .await
                 .expect("legitimate child");
         }

crates/ecstore/src/disk/local/multipart_index.rs

@@ -1,9 +1,7 @@
 // Copyright 2026 RustFS Team
 // SPDX-License-Identifier: Apache-2.0
 
-//! Bucket-to-staging-root candidates: one marker per staging root, recorded
-//! before the root is published and pruned once the root is gone. Source
-//! metadata stays authoritative.
+//! Append-only bucket-to-staging-root candidates. Source metadata stays authoritative.
 
 use super::{LocalDisk, read_metadata_only_std};
 use crate::disk::{
@@ -14,7 +12,6 @@ use crate::disk::{
 };
 use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
 use sha2::{Digest, Sha256};
-use std::future::Future;
 use std::{
     collections::{BTreeSet, HashMap},
     io,
@@ -29,18 +26,13 @@ use uuid::Uuid;
 
 const INDEX: &str = ".rustfs.sys/multipart-index-v1";
 const REBUILD_DIRENTS_PER_JOB: usize = 64;
-/// A marker is recorded before its staging root is published. A marker younger
-/// than this without a root is an upload in flight, never a leftover to prune.
-const MULTIPART_INDEX_PRUNE_GRACE: std::time::Duration = std::time::Duration::from_secs(300);
 type DirectoryEntries = Box<dyn Iterator<Item = io::Result<String>> + Send>;
 type PhysicalRoot = (u64, [u8; 16]);
 static OWNERS: LazyLock<parking_lot::Mutex<HashMap<PhysicalRoot, Weak<IndexOwner>>>> =
     LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
 
 pub(super) struct IndexOwner {
-    // Only the path: holding the disk's pinned root handle here would keep the
-    // mount alive past the last disk owner, which the disk lifecycle forbids.
-    root_path: Arc<PathBuf>,
+    root: os::PublicationRoot,
     physical: PhysicalRoot,
     #[cfg(unix)]
     _identity_pin: std::fs::File,
@@ -52,9 +44,6 @@ pub(super) struct IndexOwner {
     // No persisted readiness is trusted: an old binary may have written while
     // this process was stopped. Same-process handles share the continuous owner.
     initialized: OnceLock<Arc<IndexBootstrap>>,
-    // Set while some task drives the bootstrap scan; cleared by that driver's
-    // drop so a cancelled scan can be resumed by the next caller.
-    driving: AtomicBool,
     disabled: AtomicBool,
     #[cfg(test)]
     rebuild_head_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
@@ -120,33 +109,6 @@ impl IndexDirectory {
             std::fs::File::open(self.path.join(name))
         }
     }
-    fn modified(&self, name: &str) -> io::Result<std::time::SystemTime> {
-        #[cfg(unix)]
-        {
-            self.open_file(name)?.metadata()?.modified()
-        }
-        #[cfg(not(unix))]
-        {
-            std::fs::metadata(self.path.join(name))?.modified()
-        }
-    }
-    /// Whether the marker is still inside the publication grace period. A
-    /// marker dated in the future (clock stepped back) is never a leftover.
-    fn marker_is_young(&self, name: &str) -> bool {
-        self.modified(name)
-            .ok()
-            .is_some_and(|modified| modified.elapsed().map_or(true, |age| age < MULTIPART_INDEX_PRUNE_GRACE))
-    }
-    fn remove_file(&self, name: &str) -> io::Result<()> {
-        #[cfg(unix)]
-        {
-            Ok(rustix::fs::unlinkat(&self.directory, name, rustix::fs::AtFlags::empty())?)
-        }
-        #[cfg(not(unix))]
-        {
-            std::fs::remove_file(self.path.join(name))
-        }
-    }
 }
 
 /// Resume the same pinned iterators, rather than reopening and rescanning a
@@ -270,7 +232,7 @@ impl IndexOwner {
             pin
         };
         let owner = Arc::new(Self {
-            root_path: Arc::new(root.path().to_path_buf()),
+            root: root.clone(),
             physical: key,
             #[cfg(unix)]
             _identity_pin: identity_pin,
@@ -278,7 +240,6 @@ impl IndexOwner {
             epoch: Uuid::new_v4(),
             blocking_queue: os::register_disk_blocking_queue(logical_root, base),
             initialized: OnceLock::new(),
-            driving: AtomicBool::new(false),
             disabled: AtomicBool::new(false),
             #[cfg(test)]
             rebuild_head_pause: parking_lot::Mutex::new(None),
@@ -299,46 +260,31 @@ impl IndexOwner {
         self.blocking_queue.clone()
     }
 
-    /// Hand out the one owned bootstrap scan. The caller drives the returned
-    /// future on a task it already owns (the disk's cleanup loop), so no task is
-    /// spawned per disk and dropping the future releases this owner; the next
-    /// `start` resumes an unfinished scan. Until EOF, callers retain complete
-    /// legacy discovery; a failed scan disables only this optimization.
-    pub(super) fn start(self: &Arc<Self>) -> Option<impl Future<Output = ()> + Send + 'static> {
-        let bootstrap = self
-            .initialized
-            .get_or_init(|| {
-                Arc::new(IndexBootstrap {
-                    result: AtomicU8::new(0),
-                    completed: Notify::new(),
-                })
-            })
-            .clone();
-        if bootstrap.result.load(Ordering::Acquire) != 0 || self.driving.swap(true, Ordering::AcqRel) {
-            return None;
-        }
-        struct Driver(Arc<IndexOwner>);
-        impl Drop for Driver {
-            fn drop(&mut self) {
-                self.0.driving.store(false, Ordering::Release);
-            }
-        }
-        let driver = Driver(self.clone());
-        Some(async move {
-            let success = driver.0.rebuild().await.is_ok();
-            // A completion waiter may drop the final disk handle and reopen it.
-            // Release this boot owner before publishing completion to that waiter.
-            drop(driver);
-            bootstrap.result.store(if success { 1 } else { 2 }, Ordering::Release);
-            bootstrap.completed.notify_waiters();
-        })
+    /// Schedule one owned scan without delaying disk admission. Until EOF, callers
+    /// retain complete legacy discovery; a failed scan disables only this optimization.
+    pub(super) fn start(self: &Arc<Self>) {
+        self.initialized.get_or_init(|| {
+            let bootstrap = Arc::new(IndexBootstrap {
+                result: AtomicU8::new(0),
+                completed: Notify::new(),
+            });
+            let completion = bootstrap.clone();
+            let owner = self.clone();
+            tokio::spawn(async move {
+                let success = owner.rebuild().await.is_ok();
+                // A completion waiter may drop the final disk handle and reopen it.
+                // Release this boot owner before publishing completion to that waiter.
+                drop(owner);
+                completion.result.store(if success { 1 } else { 2 }, Ordering::Release);
+                completion.completed.notify_waiters();
+            });
+            bootstrap
+        });
     }
 
     #[cfg(any(test, feature = "test-util"))]
     pub(super) async fn initialize(self: &Arc<Self>) {
-        if let Some(scan) = self.start() {
-            scan.await;
-        }
+        self.start();
         let Some(bootstrap) = self.initialized.get() else {
             return;
         };
@@ -388,49 +334,33 @@ impl IndexOwner {
                 directories.push(std::fs::File::from(directory));
             }
             let parent = directories.last().ok_or_else(|| io::Error::other("index parent missing"))?;
-            // The index is boot-local: every start rescans the staging area
-            // before serving it, so a marker lost to a crash is recreated before
-            // it can be consulted. Creation is enough; no fsync of the marker or
-            // its ancestors. An existing marker is touched so pruning sees the
-            // new upload's age, not the previous one's.
-            match openat(
+            let marker = std::fs::File::from(openat(
                 parent,
                 source_root,
-                OFlags::CREATE | OFlags::EXCL | OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+                OFlags::CREATE | OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
                 Mode::RUSR | Mode::WUSR,
-            ) {
-                Ok(marker) => {
-                    if !std::fs::File::from(marker).metadata()?.is_file() {
-                        return Err(io::Error::other("index marker is not a file"));
-                    }
-                }
-                Err(rustix::io::Errno::EXIST) => {
-                    // Best effort: a filesystem that refuses to set times keeps
-                    // the previous upload's age, which only shortens the prune
-                    // grace for this key; index completeness does not depend on it.
-                    if let Ok(marker) =
-                        openat(parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
-                    {
-                        let _ = std::fs::File::from(marker).set_modified(std::time::SystemTime::now());
-                    }
-                }
-                Err(error) => return Err(error.into()),
+            )?);
+            if !marker.metadata()?.is_file() {
+                return Err(io::Error::other("index marker is not a file"));
+            }
+            marker.sync_all()?;
+            for directory in directories.iter().rev() {
+                directory.sync_all()?;
             }
         }
         #[cfg(not(unix))]
         {
-            let root = os::PublicationRoot::new(&self.root_path)?;
             let directory = self.base.join(INDEX).join(sha(bucket));
             let marker = directory.join(source_root);
-            let _parents = os::mkdir_all_below_existing_base_std(&directory, &self.base, &root)?;
-            os::replace_multipart_source_metadata(&marker, b"1", &self.base, &root)?;
-            os::sync_multipart_owned_record_parents(&self.base.join(".rustfs.sys"), &directory, true, &self.base, &root)?;
+            let _parents = os::mkdir_all_below_existing_base_std(&directory, &self.base, &self.root)?;
+            os::replace_multipart_source_metadata(&marker, b"1", &self.base, &self.root)?;
+            os::sync_multipart_owned_record_parents(&self.base.join(".rustfs.sys"), &directory, true, &self.base, &self.root)?;
         }
         self.ensure_current_root()
     }
 
     fn ensure_current_root(&self) -> io::Result<()> {
-        let current = os::PublicationRoot::new(&self.root_path)?.physical_identity()?;
+        let current = os::PublicationRoot::new(self.root.path())?.physical_identity()?;
         if current != self.physical {
             self.disabled.store(true, Ordering::Release);
             return Err(io::Error::other("multipart index physical root changed"));
@@ -459,8 +389,7 @@ impl IndexOwner {
         #[cfg(not(unix))]
         {
             let path = self.base.join(relative);
-            let root = os::PublicationRoot::new(&self.root_path)?;
-            let guard = os::pin_multipart_source_directory(&path, &self.base, &root)?;
+            let guard = os::pin_multipart_source_directory(&path, &self.base, &self.root)?;
             Ok(IndexDirectory { path, _guard: guard })
         }
     }
@@ -572,7 +501,7 @@ impl IndexOwner {
         let owner = self.clone();
         let bucket = bucket.to_owned();
         let marker = marker.map(str::to_owned);
-        let (page, stale) = crate::disk::io_admission::spawn_blocking_with_disk_io_permit(None, move || {
+        crate::disk::io_admission::spawn_blocking_with_disk_io_permit(None, move || {
             owner.ensure_current_root()?;
             let directory = match owner.directory(&Path::new(INDEX).join(sha(&bucket))) {
                 Ok(directory) => Some(directory),
@@ -598,83 +527,17 @@ impl IndexOwner {
             if more {
                 roots.pop_last();
             }
-            // The marker advances past every examined name, live or stale, so a
-            // pruned name never stalls pagination.
             let next_marker = more.then(|| roots.last().cloned()).flatten();
-            // Completion and abort remove the staging root, not the marker, so
-            // markers outlive their uploads. Drop the ones whose staging root is
-            // gone, keeping the index proportional to live uploads rather than
-            // to every key ever multipart-initiated on this drive.
-            let mut live = Vec::with_capacity(roots.len());
-            let mut stale = Vec::new();
-            for root in roots {
-                match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
-                    Ok(_) => live.push(root),
-                    Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {
-                        // The marker precedes its staging root; a young one is an
-                        // upload in flight and stays a candidate.
-                        if directory.as_ref().is_some_and(|index| index.marker_is_young(&root)) {
-                            live.push(root);
-                        } else {
-                            stale.push(root);
-                        }
-                    }
-                    Err(error) => return Err(DiskError::from(error)),
-                }
-            }
-            Ok((
-                MultipartIndexPage {
-                    bucket,
-                    disk_id,
-                    epoch: owner.epoch,
-                    roots: live,
-                    next_marker,
-                },
-                stale,
-            ))
+            Ok(MultipartIndexPage {
+                bucket,
+                disk_id,
+                epoch: owner.epoch,
+                roots: roots.into_iter().collect(),
+                next_marker,
+            })
         })
         .await
-        .map_err(DiskError::from)??;
-        for root in stale {
-            self.prune_marker(&page.bucket, &root).await?;
-        }
-        Ok(page)
-    }
-
-    /// Remove one marker whose staging root was gone when a page examined it.
-    /// The marker's own namespace lease serializes this with `record_info`; the
-    /// staging root and the marker's age are re-checked under that lease, so an
-    /// upload recorded in between keeps its marker.
-    async fn prune_marker(self: &Arc<Self>, bucket: &str, root: &str) -> Result<()> {
-        let owner = self.clone();
-        let bucket_directory = Path::new(INDEX).join(sha(bucket));
-        let marker = owner.base.join(&bucket_directory).join(root);
-        let root = root.to_owned();
-        let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
-        // Durable-mutation lane: a page may prune up to 256 markers, which must
-        // not queue behind every other mutation on the one-worker foreground lane.
-        os::run_blocking_namespace_operation_with_sync(lease, true, move || {
-            owner.ensure_current_root()?;
-            match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
-                Ok(_) => return Ok(()),
-                Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {}
-                Err(error) => return Err(error),
-            }
-            let directory = match owner.directory(&bucket_directory) {
-                Ok(directory) => directory,
-                Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
-                Err(error) => return Err(error),
-            };
-            if directory.marker_is_young(&root) {
-                return Ok(());
-            }
-            match directory.remove_file(&root) {
-                Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error),
-                _ => Ok(()),
-            }
-        })
-        .await?;
-        Ok(())
+        .map_err(DiskError::from)?
     }
 }
 
@@ -861,8 +724,6 @@ mod tests {
         std::fs::create_dir_all(&directory).expect("seed ready candidates");
         for i in 0..(MULTIPART_INDEX_PAGE_SIZE + 2) {
             std::fs::write(directory.join(format!("{i:064x}")), b"1").expect("candidate marker");
-            std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(format!("{i:064x}")))
-                .expect("live staging root");
         }
         // A foreign bucket's unreadable layout must not enter this page's walk.
         std::fs::write(root.path().join(INDEX).join(sha("foreign")), b"not a directory").expect("foreign layout");
@@ -883,99 +744,6 @@ mod tests {
         assert!(second.next_marker.is_none());
         assert!(first.roots.last() < second.roots.first());
     }
-    #[tokio::test]
-    async fn multipart_index_pages_prune_markers_whose_staging_root_is_gone() {
-        let root = tempfile::tempdir().expect("disk root");
-        let disk = open(root.path()).await;
-        let directory = root.path().join(INDEX).join(sha("bucket"));
-        std::fs::create_dir_all(&directory).expect("marker directory");
-        let live = format!("{:064x}", 1);
-        let finished = format!("{:064x}", 2);
-        let in_flight = format!("{:064x}", 3);
-        let clock_stepped = format!("{:064x}", 4);
-        for name in [&live, &finished, &in_flight, &clock_stepped] {
-            std::fs::write(directory.join(name), b"1").expect("marker");
-        }
-        // Completion and abort remove the staging root and leave the marker.
-        std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&live)).expect("live staging root");
-        set_marker_modified(&directory.join(&finished), std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
-        // A marker dated after a backwards clock step is never a leftover.
-        set_marker_modified(
-            &directory.join(&clock_stepped),
-            std::time::SystemTime::now() + MULTIPART_INDEX_PRUNE_GRACE * 2,
-        );
-        let page = disk.multipart_index.page("bucket", None, Uuid::new_v4()).await.expect("page");
-        // The in-flight marker has no staging root yet but is young: a candidate, not a leftover.
-        assert_eq!(page.roots, vec![live.clone(), in_flight.clone(), clock_stepped.clone()]);
-        assert!(page.next_marker.is_none());
-        assert!(directory.join(&live).exists(), "a live root keeps its marker");
-        assert!(directory.join(&in_flight).exists(), "a marker inside the grace period is kept");
-        assert!(directory.join(&clock_stepped).exists(), "a future-dated marker is kept");
-        assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned");
-        // The next page no longer pays for the pruned marker.
-        let page = disk
-            .multipart_index
-            .page("bucket", None, Uuid::new_v4())
-            .await
-            .expect("page after pruning");
-        assert_eq!(page.roots, vec![live, in_flight, clock_stepped]);
-    }
-
-    /// Move a marker's modification time outside the publication grace period.
-    fn age_marker(marker: &Path) {
-        set_marker_modified(marker, std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
-    }
-
-    fn set_marker_modified(marker: &Path, modified: std::time::SystemTime) {
-        std::fs::File::options()
-            .write(true)
-            .open(marker)
-            .expect("marker")
-            .set_modified(modified)
-            .expect("set marker time");
-    }
-
-    #[tokio::test]
-    async fn multipart_index_pruned_page_keeps_a_valid_cursor() {
-        let root = tempfile::tempdir().expect("disk root");
-        let disk = open(root.path()).await;
-        let directory = root.path().join(INDEX).join(sha("bucket"));
-        std::fs::create_dir_all(&directory).expect("marker directory");
-        let stale = format!("{:064x}", 7);
-        for i in 0..(MULTIPART_INDEX_PAGE_SIZE + 2) {
-            let name = format!("{i:064x}");
-            std::fs::write(directory.join(&name), b"1").expect("marker");
-            if name != stale {
-                std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
-            }
-        }
-        age_marker(&directory.join(&stale));
-        let disk_id = Uuid::new_v4();
-        let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
-        first
-            .validate("bucket", None, disk_id)
-            .expect("a pruned page still validates");
-        assert_eq!(first.roots.len(), MULTIPART_INDEX_PAGE_SIZE - 1, "the stale marker is not a root");
-        assert!(!first.roots.contains(&stale));
-        let cursor = first.next_marker.clone().expect("more markers follow");
-        assert_eq!(
-            cursor,
-            format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1),
-            "the cursor is the last examined name"
-        );
-        assert!(!directory.join(&stale).exists(), "the stale marker is pruned");
-        let second = disk
-            .multipart_index
-            .page("bucket", Some(&cursor), disk_id)
-            .await
-            .expect("second page");
-        second
-            .validate("bucket", Some(&cursor), disk_id)
-            .expect("continuation validates");
-        assert_eq!(second.roots.len(), 2);
-        assert!(second.next_marker.is_none());
-    }
-
     #[cfg(unix)]
     #[tokio::test]
     async fn multipart_index_aliases_share_readiness_and_root_replacement_cannot_return_empty() {

crates/ecstore/src/disk/mod.rs

@@ -53,10 +53,6 @@ pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
 pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
 pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
 pub const BUCKET_META_PREFIX: &str = "buckets";
-/// Directory under `BUCKET_META_PREFIX` that holds drive-replacement heal
-/// intents, seals and recovery proofs. It shares the bucket-metadata root with
-/// a user bucket of the same name, so bucket deletion must leave it in place.
-pub const HEAL_REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
 pub const FORMAT_CONFIG_FILE: &str = "format.json";
 /// Per-disk marker present while an erasure-set heal is rebuilding this disk.
 /// `LocalDisk::disk_info` reports `healing = true` while the file exists.
@@ -1765,16 +1761,14 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         Err(DiskError::MethodNotAllowed)
     }
     /// Bounded foreground inventory owns all traversal state until return or cancellation.
-    /// An implementation without the bounded protocol is an unsupported participant
-    /// (the same verdict a remote peer without it receives); callers must never fall
-    /// back to paginated discovery.
+    /// Unsupported peers must fail closed; callers must never fall back to paginated discovery.
     async fn discover_delete_transactions_bounded(
         &self,
         _volume: &str,
         _object: &str,
         _max_entries: u32,
     ) -> Result<BoundedDeleteRecoveryDiscovery> {
-        Err(DiskError::UnsupportedDisk)
+        Err(DiskError::DeleteRecoveryWorkLimit)
     }
     async fn inspect_delete_transaction_bounded(
         &self,
@@ -1783,7 +1777,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         _transaction: Uuid,
         _max_entries: u32,
     ) -> Result<BoundedDeleteRecoveryInspection> {
-        Err(DiskError::UnsupportedDisk)
+        Err(DiskError::DeleteRecoveryWorkLimit)
     }
     async fn settle_delete_transaction_bounded(
         &self,
@@ -1791,7 +1785,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         _request: BoundedDeleteRecoverySettlement,
         _namespace_owner: Option<Arc<dyn Send + Sync>>,
     ) -> Result<BoundedDeleteRecoverySettlementReply> {
-        Err(DiskError::UnsupportedDisk)
+        Err(DiskError::DeleteRecoveryWorkLimit)
     }
     /// A read-only capability check; unsupported implementations must fail closed.
     async fn supports_guarded_delete_versions(&self) -> Result<()> {

crates/ecstore/src/disk/multipart_index.rs

@@ -9,11 +9,7 @@ use uuid::Uuid;
 pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
 pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
 
-pub(crate) const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
-/// Pages one listing accepts from one disk. Pruned pages may carry few roots, so
-/// the cursor alone no longer bounds a peer's answer; a disk that keeps
-/// continuing past this many pages contributes no coverage.
-pub(crate) const MULTIPART_INDEX_MAX_PAGES_PER_DISK: usize = 4096;
+pub const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
 
 /// A bounded page from one disk's continuously maintained, boot-local index.
 /// `epoch` changes whenever coverage must be rebuilt; callers must not combine
@@ -131,13 +127,10 @@ impl MultipartIndexPage {
                 .iter()
                 .any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
             || self.roots.windows(2).any(|pair| pair[0] >= pair[1])
-            // The cursor names the last examined marker; pruning may leave fewer
-            // live roots than examined names, so it only has to bound them.
-            || self.next_marker.as_ref().is_some_and(|next| {
-                !valid_root(next)
-                    || marker.is_some_and(|marker| next.as_str() <= marker)
-                    || self.roots.last().is_some_and(|last| last > next)
-            })
+            || self
+                .next_marker
+                .as_ref()
+                .is_some_and(|marker| self.roots.last() != Some(marker) || self.roots.len() != MULTIPART_INDEX_PAGE_SIZE)
         {
             return Err(crate::disk::error::DiskError::FileCorrupt);
         }
@@ -172,12 +165,7 @@ where
     let mut epoch = None;
     let mut marker = None;
     let mut roots = Vec::new();
-    let mut pages = 0_usize;
     loop {
-        if pages >= MULTIPART_INDEX_MAX_PAGES_PER_DISK {
-            return Err(crate::disk::error::DiskError::FileCorrupt);
-        }
-        pages += 1;
         let page = fetch(marker.clone()).await?;
         page.validate(bucket, marker.as_deref(), identity)?;
         if epoch.is_some_and(|epoch| epoch != page.epoch) {
@@ -219,12 +207,8 @@ mod tests {
             "old challenge cannot replay"
         );
         assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
-        reply.page.next_marker = Some("9".repeat(64));
-        assert!(reply.validate(&request, id).is_err(), "a cursor below the last root would re-list it");
         reply.page.next_marker = Some("a".repeat(64));
-        reply
-            .validate(&request, id)
-            .expect("a pruned short page continues from its last examined name");
+        assert!(reply.validate(&request, id).is_err(), "short page cannot claim a continuation");
         reply.page.next_marker = None;
         reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
         assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
@@ -256,64 +240,6 @@ mod tests {
         page.epoch = Uuid::new_v4();
         assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
     }
-    #[test]
-    fn multipart_index_page_cursor_bounds_roots_without_requiring_a_full_page() {
-        let disk = Uuid::new_v4();
-        let page = |roots: Vec<String>, next_marker: Option<String>| MultipartIndexPage {
-            bucket: "bucket".into(),
-            disk_id: disk,
-            epoch: Uuid::new_v4(),
-            roots,
-            next_marker,
-        };
-        // Pruning leaves fewer live roots than examined names; the cursor still advances.
-        page(vec![format!("{:064x}", 1)], Some(format!("{:064x}", 5)))
-            .validate("bucket", None, disk)
-            .expect("short page with a cursor");
-        page(vec![], Some(format!("{:064x}", 5)))
-            .validate("bucket", Some(&format!("{:064x}", 2)), disk)
-            .expect("all examined names pruned");
-        assert!(
-            page(vec![format!("{:064x}", 6)], Some(format!("{:064x}", 5)))
-                .validate("bucket", None, disk)
-                .is_err(),
-            "a root beyond the cursor would be re-listed"
-        );
-        assert!(
-            page(vec![], Some(format!("{:064x}", 2)))
-                .validate("bucket", Some(&format!("{:064x}", 2)), disk)
-                .is_err(),
-            "the cursor must advance past the request marker"
-        );
-        assert!(
-            page(vec![], Some("not-a-sha".to_owned()))
-                .validate("bucket", None, disk)
-                .is_err(),
-            "the cursor is a marker name"
-        );
-    }
-
-    #[tokio::test]
-    async fn multipart_index_page_collection_bounds_the_pages_one_disk_may_answer() {
-        let id = Uuid::new_v4();
-        let epoch = Uuid::new_v4();
-        let mut served = 0_usize;
-        let result = collect_pages("bucket", id, |marker| {
-            served += 1;
-            let next = marker.map_or(0_u64, |marker| u64::from_str_radix(&marker[48..], 16).expect("hex cursor") + 1);
-            std::future::ready(Ok(MultipartIndexPage {
-                bucket: "bucket".into(),
-                disk_id: id,
-                epoch,
-                roots: vec![],
-                next_marker: Some(format!("{next:064x}")),
-            }))
-        })
-        .await;
-        assert!(matches!(result, Err(crate::disk::error::DiskError::FileCorrupt)), "{result:?}");
-        assert_eq!(served, MULTIPART_INDEX_MAX_PAGES_PER_DISK, "the peer is cut off at the page cap");
-    }
-
     #[tokio::test]
     async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
         let id = Uuid::new_v4();
@@ -342,17 +268,7 @@ mod tests {
                 std::future::ready(Ok(page))
             })
             .await;
-            // A replaced disk fails the page's identity check; a restarted
-            // epoch fails the cross-page continuity check.
-            let expected_corrupt = replace_disk;
-            assert!(
-                match &result {
-                    Err(crate::disk::error::DiskError::FileCorrupt) => expected_corrupt,
-                    Err(crate::disk::error::DiskError::DiskNotFound) => !expected_corrupt,
-                    _ => false,
-                },
-                "changed physical source cannot contribute partial coverage: {result:?}"
-            );
+            assert!(result.is_err(), "changed physical source cannot contribute partial coverage");
             assert!(replies.is_empty(), "exercise the continuation response");
         }
     }

crates/ecstore/src/error/mod.rs

@@ -1710,8 +1710,8 @@ mod tests {
         // covers every variant and picks up newly appended ones automatically.
         let all_variants: Vec<DiskError> = (1u32..).map_while(DiskError::from_u32).collect();
         assert!(
-            all_variants.len() >= 46,
-            "DiskError variant enumeration shrank: got {}, expected at least 46",
+            all_variants.len() >= 42,
+            "DiskError variant enumeration shrank: got {}, expected at least 42",
             all_variants.len()
         );
 

crates/ecstore/src/object_api/types.rs

@@ -951,16 +951,15 @@ pub enum ShardIntegrityWriteMode {
 
 /// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
 /// Credits cover directory entries (including a minimum charge for an empty probe) and
-/// transaction inspections. Each bounded call is admitted while credits remain and is
-/// charged for the entries it visited when it completes.
+/// transaction inspections. Failed/cancelled probes keep their worst-case reservation.
 #[derive(Debug)]
 pub struct DeleteRecoveryWorkBudget {
     remaining: std::sync::atomic::AtomicU32,
 }
 
-/// Entries one disk may visit per recovery step (discovery plus inspection).
-/// Every disk of every set a request fans out to is admitted up to this much
-/// and charged on completion, so a budget must cover that many disks.
+/// Entries one disk may reserve up front per recovery step (discovery plus
+/// inspection). Every disk of every set a request fans out to reserves this
+/// much before any refund arrives, so a budget must cover that many disks.
 const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
 
 impl Default for DeleteRecoveryWorkBudget {
@@ -972,7 +971,7 @@ impl Default for DeleteRecoveryWorkBudget {
 }
 
 impl DeleteRecoveryWorkBudget {
-    /// A per-request budget sized for `disks` concurrently admitted recovery steps.
+    /// A per-request budget sized for `disks` concurrent up-front reservations.
     pub(crate) fn for_disks(disks: usize) -> Self {
         let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
         let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
@@ -1000,24 +999,8 @@ impl DeleteRecoveryWorkBudget {
         self.remaining.load(std::sync::atomic::Ordering::Acquire)
     }
 
-    /// Admit one bounded disk call: refuse once the budget is spent, otherwise
-    /// cap the call at `maximum` or what remains. The work actually performed is
-    /// charged on completion with `charge`, so concurrent calls never exhaust
-    /// the budget by reservation alone (a bulk delete fans out dozens of
-    /// inspections per key); each call is still bounded by its own cap.
-    pub(crate) fn admit(&self, maximum: u32) -> crate::disk::error::Result<u32> {
-        match self.remaining() {
-            0 => Err(crate::disk::error::DiskError::DeleteRecoveryWorkLimit),
-            remaining => Ok(remaining.min(maximum)),
-        }
-    }
-
-    /// Record the entries a bounded call visited (at least one per call).
-    pub(crate) fn charge(&self, visited: u32) {
-        use std::sync::atomic::Ordering;
-        let _ = self.remaining.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
-            Some(remaining.saturating_sub(visited.max(1)))
-        });
+    pub(crate) fn refund(&self, unused: u32) {
+        self.remaining.fetch_add(unused, std::sync::atomic::Ordering::Release);
     }
 }
 

crates/ecstore/src/services/rebalance/entry.rs

@@ -283,17 +283,6 @@ impl ECStore {
             arrived.notify_one();
             release.notified().await;
         }
-        #[cfg(feature = "e2e-test-hooks")]
-        tokio::select! {
-            // Stop must drain this pause before taking the activation writer.
-            _ = cancel.cancelled() => return Err(Error::OperationCanceled),
-            result = data_movement::e2e_barrier::wait(
-                "rebalance",
-                bucket.as_str(),
-                entry.name.as_str(),
-                data_movement::e2e_barrier::STAGE_AFTER_ENTRY_ADMISSION,
-            ) => result.map_err(Error::from)?,
-        }
         let lock_lost_signal = run_guard.lock_lost_signal();
         #[cfg(test)]
         let _run_signal_test_fence = lock_lost_signal

crates/ecstore/src/set_disk/mod.rs

@@ -5018,11 +5018,6 @@ impl DiskHealthEntry {
 #[error("committed system metadata write requires repair by its owning writer")]
 pub(crate) struct SystemMetadataRepairRequired;
 
-/// An inline metadata write was rejected before staging or commit began.
-#[derive(Debug, thiserror::Error)]
-#[error("system metadata write rejected before staging: {0}")]
-pub(crate) struct SystemMetadataWriteNotStarted(#[source] pub(crate) Error);
-
 impl SetDisks {
     pub(in crate::set_disk) async fn persist_delete_marker_purge(
         &self,

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -237,14 +237,14 @@ async fn inspect(
     join_all(disks.iter().map(|disk| async move {
         let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
         let state = if let Some(budget) = budget {
-            let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
+            let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
             let reply = disk
-                .inspect_delete_transaction_bounded(bucket, object, transaction, admitted)
+                .inspect_delete_transaction_bounded(bucket, object, transaction, reserved)
                 .await?;
-            if reply.visited > admitted {
+            if reply.visited > reserved {
                 return Err(DiskError::FileCorrupt);
             }
-            budget.charge(reply.visited);
+            budget.refund(reserved - reply.visited.max(1));
             reply.inspection
         } else {
             disk.inspect_delete_transaction(bucket, object, transaction).await?
@@ -261,9 +261,9 @@ async fn discover_bounded(
     object: &str,
     budget: &crate::object_api::DeleteRecoveryWorkBudget,
 ) -> Result<Vec<DeleteTransactionId>> {
-    let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
-    let page = disk.discover_delete_transactions_bounded(bucket, object, admitted).await?;
-    if page.visited > admitted
+    let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
+    let page = disk.discover_delete_transactions_bounded(bucket, object, reserved).await?;
+    if page.visited > reserved
         || page.transactions.len() > 256
         || page
             .transactions
@@ -273,7 +273,7 @@ async fn discover_bounded(
     {
         return Err(DiskError::FileCorrupt);
     }
-    budget.charge(page.visited);
+    budget.refund(reserved - page.visited.max(1));
     if !page.complete {
         return Err(DiskError::DeleteRecoveryWorkLimit);
     }
@@ -329,7 +329,7 @@ async fn settle_outcome(
         return Err(DiskError::other("delete recovery namespace fence was lost"));
     }
     let outcome = if let Some(budget) = scope.budget {
-        let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
+        let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
         let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
         let reply = disk
             .settle_delete_transaction_bounded(
@@ -339,17 +339,17 @@ async fn settle_outcome(
                     transaction: state.id.transaction,
                     expected_fingerprint: state.fingerprint,
                     action,
-                    max_entries: admitted,
+                    max_entries: reserved,
                 },
                 scope.namespace_owner.cloned(),
             )
             .await?;
-        if reply.visited > admitted
+        if reply.visited > reserved
             || (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
         {
             return Err(DiskError::FileCorrupt);
         }
-        budget.charge(reply.visited);
+        budget.refund(reserved - reply.visited.max(1));
         reply.outcome
     } else {
         DeleteRecoverySettlementOutcome::Inspected(Box::new(
@@ -1610,7 +1610,7 @@ mod tests {
             &outcomes[1],
             Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
         ));
-        assert!(matches!(outcomes[3], Err(crate::disk::error::DiskError::DeleteRecoveryWorkLimit)));
+        assert!(outcomes[3].is_err());
         assert!(has_receipt_quorum(&outcomes, 3));
         assert!(!has_receipt_quorum(&outcomes, 4));
         outcomes[2] = Err(DiskError::FileAccessDenied);
@@ -1629,8 +1629,8 @@ mod tests {
         for dir in &dirs {
             let prefix = dir.path().join(bucket).join("prefix");
             std::fs::create_dir_all(&prefix).expect("physical prefix");
-            for _ in 0..64 {
-                std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("legitimate child");
+            for index in 0..64 {
+                std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
             }
         }
         let error = store
@@ -1693,11 +1693,10 @@ mod tests {
                     .any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
                 "repeated actions must not each obtain a new allowance: {distinct_versions}"
             );
-            assert!(dirs.iter().all(|dir| {
-                std::fs::read_dir(dir.path().join(bucket).join("prefix"))
-                    .map(|entries| entries.count() == 64)
-                    .unwrap_or(false)
-            }));
+            assert!(
+                dirs.iter()
+                    .all(|dir| dir.path().join(bucket).join("prefix/child-63").is_dir())
+            );
         }
         let error = store
             .delete_object(bucket, "prefix", ObjectOptions::default())
@@ -1780,8 +1779,8 @@ mod tests {
             for key in &keys {
                 let prefix = dir.path().join(bucket).join(key);
                 std::fs::create_dir_all(&prefix).expect("physical prefix");
-                for _ in 0..8 {
-                    std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("child");
+                for child in 0..8 {
+                    std::fs::create_dir(prefix.join(format!("child-{child}"))).expect("child");
                 }
             }
         }
@@ -1810,11 +1809,10 @@ mod tests {
             "all pools and sets must share the finite allowance"
         );
         assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
-        assert!(dirs.iter().all(|dir| {
-            std::fs::read_dir(dir.path().join(bucket).join(&keys[0]))
-                .map(|entries| entries.count() == 8)
-                .unwrap_or(false)
-        }));
+        assert!(
+            dirs.iter()
+                .all(|dir| dir.path().join(bucket).join(&keys[0]).join("child-7").is_dir())
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/set_disk/ops/locking.rs

@@ -458,10 +458,7 @@ impl SetDisks {
                         return None;
                     }
 
-                    // A failed remote reconnect must retain its existing recovery monitor.
-                    if disk.is_local() {
-                        let _ = disk.close().await;
-                    }
+                    let _ = disk.close().await;
                 }
 
                 let endpoint = self.set_endpoints.get(i)?;
@@ -583,26 +580,19 @@ impl SetDisks {
         let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
 
         if !new_disk.is_local() {
-            let displaced = {
-                let mut disks = self.disks.write().await;
-                let slot = &mut disks[disk_idx];
-                let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
-                    (None, None) => true,
-                    (Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
-                    _ => false,
-                };
-                // A GET probe or another reconnect may have filled this slot while
-                // format I/O was pending. Only the successful publisher owns the
-                // displaced monitor; a failed or superseded probe must retain it.
-                if unchanged {
-                    new_disk.enable_health_check();
-                    slot.replace(new_disk)
-                } else {
-                    None
-                }
+            let mut disks = self.disks.write().await;
+            let slot = &mut disks[disk_idx];
+            let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
+                (None, None) => true,
+                (Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
+                _ => false,
             };
-            if let Some(disk) = displaced {
-                let _ = disk.close().await;
+            // A GET probe or another reconnect may have filled this slot while
+            // format I/O was pending. Do not replace its handle or start a
+            // second health monitor for an unpublished handle.
+            if unchanged {
+                new_disk.enable_health_check();
+                *slot = Some(new_disk);
             }
             return false;
         }

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -1989,19 +1989,12 @@ impl SetDisks {
                 roots.insert(Self::get_multipart_sha_dir(bucket, &encode_dir_object(prefix)));
             }
             let mut candidates = Vec::new();
-            let mut discoveries = futures::stream::iter(roots.into_iter().map(|root| {
-                let disks = disks.clone();
-                async move {
-                    self.discover_multipart_upload_paths_on_disks(disks, bucket, prefix, &root, false)
-                        .await
-                }
-            }))
-            .buffer_unordered(MULTIPART_LIST_IO_CONCURRENCY);
-            while let Some(discovered) = discoveries.next().await {
-                let (_, paths, _, _) = discovered?;
+            for root in roots {
+                let (_, paths, _, _) = self
+                    .discover_multipart_upload_paths_on_disks(disks.clone(), bucket, prefix, &root, false)
+                    .await?;
                 candidates.extend(paths);
             }
-            drop(discoveries);
             (disks, candidates, quorum, false)
         } else {
             self.discover_multipart_upload_paths(bucket, prefix, &root, false).await?

crates/ecstore/src/set_disk/ops/object.rs

@@ -4527,16 +4527,7 @@ impl SetDisks {
                     "Set disk write quorum unavailable"
                 );
                 if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
-                    let error = to_object_err(write_err.into(), vec![bucket, object]);
-                    if direct_inline_commit
-                        && is_meta_bucketname(bucket)
-                        && opts.write_completion == crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair
-                    {
-                        // Inline writer setup only checks disk availability;
-                        // no shard or metadata write has been dispatched yet.
-                        return Err(Error::other(crate::set_disk::SystemMetadataWriteNotStarted(error)));
-                    }
-                    return Err(error);
+                    return Err(to_object_err(write_err.into(), vec![bucket, object]));
                 }
 
                 return Err(Error::other(format!("not enough disks to write: {errors:?}")));

crates/ecstore/src/store/bucket.rs

@@ -150,7 +150,7 @@ fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> Vec<String> {
 
     let mut prefixes = vec![table_bucket_catalog_metadata_prefix(bucket)];
     let root = format!("{BUCKET_META_PREFIX}/{bucket}");
-    if bucket != crate::disk::HEAL_REPLACEMENT_RECOVERY_DIR {
+    if bucket != "ahm-replacement" {
         prefixes.push(root);
         return prefixes;
     }

crates/ecstore/src/store/init.rs

@@ -540,7 +540,6 @@ impl ECStore {
             // failures can still spawn recovery probes for peers that come up
             // after this node.
             let (mut disks, errs) = init_format::init_disks(
-                &instance_ctx,
                 &pool_eps.endpoints,
                 &DiskOption {
                     cleanup: true,
@@ -21013,7 +21012,6 @@ mod tests {
         );
     }
 
-    #[cfg(feature = "test-util")]
     async fn seed_retained_null_version(
         store: &Arc<crate::store::ECStore>,
         bucket: &str,
@@ -21071,7 +21069,6 @@ mod tests {
         }
     }
 
-    #[cfg(feature = "test-util")]
     async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
         let mut reader = store
             .get_object_reader(
@@ -21095,10 +21092,6 @@ mod tests {
         assert_eq!(body, b"protected null version");
     }
 
-    /// A versionless delete removes the null version, so the Object Lock check
-    /// must judge that version even when a UUID version or delete marker is
-    /// the key's latest entry (rustfs/backlog#3017).
-    #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn null_version_object_lock_delete_checks_actual_target() {
@@ -21236,11 +21229,6 @@ mod tests {
         }
     }
 
-    /// A versionless PUT or multipart completion replaces the null version in
-    /// place, so the commit-time WORM gate must judge that version even when a
-    /// UUID version or delete marker is the key's latest entry
-    /// (rustfs/backlog#3019).
-    #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {

crates/ecstore/src/store/init_format.rs

@@ -44,32 +44,12 @@ const LEGACY_FORMAT_READ_CONCURRENCY: usize = 16;
 const LEGACY_FORMAT_BASE_MAX_BYTES: usize = 16 * 1024;
 const LEGACY_FORMAT_MAX_BYTES_PER_DISK: usize = 64;
 
-pub async fn init_disks(
-    instance_ctx: &InstanceContext,
-    eps: &Endpoints,
-    opt: &DiskOption,
-) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
+pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
     let mut futures = Vec::with_capacity(eps.as_ref().len());
-    let local_disk_map = instance_ctx.local_disk_map();
-    let local_disks = local_disk_map.read().await;
 
     for ep in eps.as_ref().iter() {
-        // Foundation disks may already serve RPC writes. Opening them again
-        // with startup cleanup can move live staging out from under a writer.
-        let registered = if ep.is_local {
-            local_disks.get(&ep.to_string()).cloned().flatten()
-        } else {
-            None
-        };
-        futures.push(async move {
-            match registered {
-                Some(disk) if disk.endpoint() == *ep => Ok(disk),
-                Some(_) => Err(DiskError::InconsistentDisk),
-                None => new_disk(ep, opt).await,
-            }
-        });
+        futures.push(new_disk(ep, opt));
     }
-    drop(local_disks);
 
     let mut res = Vec::with_capacity(eps.as_ref().len());
     let mut errors = Vec::with_capacity(eps.as_ref().len());
@@ -1050,7 +1030,6 @@ mod tests {
         }
 
         let (disks, errors) = init_disks(
-            &InstanceContext::new(),
             &Endpoints::from(endpoints),
             &DiskOption {
                 cleanup: false,
@@ -1070,91 +1049,6 @@ mod tests {
         (temp_dir, disks)
     }
 
-    #[tokio::test]
-    async fn init_disks_preserves_live_staging_on_registered_local_disks() {
-        let (temp_dir, disks) = local_disks(1).await;
-        let registered = disks[0].as_ref().expect("registered disk should exist");
-        let instance_ctx = Arc::new(InstanceContext::new());
-        record_local_disks(&instance_ctx, vec![registered.clone()]).await;
-        let staging = temp_dir.path().join("disk-0/.rustfs.sys/tmp/in-flight/xl.meta");
-        tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
-            .await
-            .expect("create staging directory");
-        tokio::fs::write(&staging, b"in-flight pool metadata")
-            .await
-            .expect("stage metadata through the registered disk");
-
-        let (initialized, errors) = init_disks(
-            &instance_ctx,
-            &Endpoints::from(vec![registered.endpoint()]),
-            &DiskOption {
-                cleanup: true,
-                health_check: false,
-            },
-        )
-        .await;
-
-        assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
-        assert_eq!(
-            tokio::fs::read(&staging)
-                .await
-                .expect("startup must preserve live RPC staging"),
-            b"in-flight pool metadata"
-        );
-        assert!(Arc::ptr_eq(registered, initialized[0].as_ref().expect("initialized disk should exist")));
-    }
-
-    #[tokio::test]
-    async fn init_disks_cleans_stale_staging_without_a_registered_disk() {
-        let temp_dir = tempfile::tempdir().expect("create disk root");
-        let endpoint =
-            Endpoint::try_from(temp_dir.path().to_str().expect("disk root should be UTF-8")).expect("endpoint should parse");
-        let staging = temp_dir.path().join(".rustfs.sys/tmp/stale/xl.meta");
-        tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
-            .await
-            .expect("create stale staging directory");
-        tokio::fs::write(&staging, b"stale pool metadata")
-            .await
-            .expect("write stale staging");
-
-        let (disks, errors) = init_disks(
-            &InstanceContext::new(),
-            &Endpoints::from(vec![endpoint]),
-            &DiskOption {
-                cleanup: true,
-                health_check: false,
-            },
-        )
-        .await;
-
-        assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
-        assert!(disks[0].is_some());
-        assert!(!staging.exists(), "a fresh disk open must still clear stale staging");
-    }
-
-    #[tokio::test]
-    async fn init_disks_rejects_registered_disk_topology_mismatch() {
-        let (_temp_dir, disks) = local_disks(1).await;
-        let registered = disks[0].as_ref().expect("registered disk should exist");
-        let instance_ctx = Arc::new(InstanceContext::new());
-        record_local_disks(&instance_ctx, vec![registered.clone()]).await;
-        let mut endpoint = registered.endpoint();
-        endpoint.set_pool_index(1);
-
-        let (initialized, errors) = init_disks(
-            &instance_ctx,
-            &Endpoints::from(vec![endpoint]),
-            &DiskOption {
-                cleanup: true,
-                health_check: false,
-            },
-        )
-        .await;
-
-        assert!(initialized[0].is_none(), "a registered disk must not be reopened in a different slot");
-        assert_eq!(errors[0], Some(DiskError::InconsistentDisk));
-    }
-
     async fn format_bytes(disks: &[Option<DiskStore>]) -> Vec<Option<Vec<u8>>> {
         let mut snapshots = Vec::with_capacity(disks.len());
         for disk in disks {
@@ -1864,7 +1758,6 @@ mod tests {
             disk.set_disk_id(None).await.expect("test disk ID should be reset");
         }
         let (prewarm_disks, prewarm_errors) = init_disks(
-            &InstanceContext::new(),
             &prewarm_endpoints,
             &DiskOption {
                 cleanup: false,

crates/ecstore/src/store/list_objects.rs

@@ -69,6 +69,9 @@ use tracing::{Instrument, debug, error, info, warn};
 use uuid::Uuid;
 
 const MAX_OBJECT_LIST: i32 = 1000;
+// const MAX_DELETE_LIST: i32 = 1000;
+// const MAX_UPLOADS_LIST: i32 = 10000;
+// const MAX_PARTS_LIST: i32 = 10000;
 
 const METACACHE_SHARE_PREFIX: bool = false;
 

crates/ecstore/src/store/mod.rs

@@ -3293,7 +3293,6 @@ mod tests {
         init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
 
         let (mut disks, errs) = init_disks(
-            &crate::runtime::global::current_ctx(),
             &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
             &DiskOption {
                 cleanup: true,

crates/heal/src/heal/erasure_healer.rs

@@ -498,10 +498,7 @@ impl ErasureSetHealer {
             .execute_heal_with_resume(buckets, set_disk_id, &resume_manager, &checkpoint_manager)
             .await;
 
-        // A cancelled or timed-out pass keeps its control classification: the
-        // namespace check runs only for completed or storage-failed passes so a
-        // concurrent bucket deletion cannot reclassify a stop as a failure.
-        if self.replacement_task_id.is_some() && !matches!(result, Err(Error::TaskCancelled | Error::TaskTimeout)) {
+        if self.replacement_task_id.is_some() {
             resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
         }
         result?;
@@ -3602,85 +3599,6 @@ mod resume_loop_tests {
         drop(checkpoint);
     }
 
-    #[tokio::test]
-    async fn replacement_cancellation_keeps_its_classification_when_the_namespace_changes() {
-        let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
-        let replacement_task_id = ResumeUtils::generate_task_id();
-        ResumeManager::new_replacement_intent(
-            env.healer.disk.clone(),
-            replacement_task_id.clone(),
-            "pool_0_set_0".to_string(),
-            vec!["b".to_string()],
-            vec!["replacement-a".to_string()],
-            vec![crate::heal::resume::ReplacementTargetIdentity {
-                endpoint: "replacement-a".to_string(),
-                canonical_path: "/mnt/replacement-a".to_string(),
-                physical_device_ids: vec!["device-a".to_string()],
-                filesystem_identity: "1:2:3".to_string(),
-            }],
-        )
-        .await
-        .expect("replacement intent should persist");
-        env.storage.set_page(
-            None,
-            Page {
-                items: vec![item("retry", None, false)],
-                next: None,
-                truncated: false,
-            },
-        );
-        env.storage.set_outcome("retry", None, HealOutcome::Transient);
-        let healer = ErasureSetHealer::new(
-            env.storage.clone(),
-            Arc::new(RwLock::new(HealProgress::new())),
-            CancellationToken::new(),
-            env.healer.disk.clone(),
-            HealOpts::default(),
-            HealRequestSource::AutoHeal,
-        )
-        .with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
-        let cancel = healer.cancel_token.clone();
-        let observe = async {
-            tokio::time::timeout(std::time::Duration::from_secs(5), async {
-                loop {
-                    let notified = env.storage.heal_started.notified();
-                    if !env.storage.calls().is_empty() {
-                        break;
-                    }
-                    notified.await;
-                }
-            })
-            .await
-            .expect("the replacement object repair must start");
-            // The only bucket is recreated while the operator cancels the pass.
-            env.storage
-                .incarnations
-                .lock()
-                .expect("incarnations")
-                .insert("b".to_string(), uuid::Uuid::from_u128(7));
-            cancel.cancel();
-        };
-
-        let buckets = ["b".to_string()];
-        let (result, ()) = tokio::join!(healer.heal_erasure_set(&buckets, "pool_0_set_0"), observe);
-
-        assert!(
-            matches!(result, Err(Error::TaskCancelled)),
-            "a cancelled pass keeps its control classification: {result:?}"
-        );
-        let manager = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
-            .await
-            .expect("cancelled replacement intent must remain");
-        assert!(
-            matches!(
-                manager.verify_replacement_bucket_pass(env.storage.as_ref()).await,
-                Err(Error::ReplacementNamespaceChanged)
-            ),
-            "the fixture must have changed the recorded bucket identity"
-        );
-        assert!(!manager.get_state().await.completed);
-    }
-
     #[tokio::test]
     async fn replacement_pool_metadata_readback_failure_schedules_retry() {
         let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;

crates/heal/src/heal/resume.rs

@@ -70,9 +70,7 @@ const RESUME_PROGRESS_FILE: &str = "ahm_progress.json";
 pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
 pub(super) const RESUME_CHECKPOINT_BLOCKED_FILE: &str = "ahm_checkpoint.blocked";
 const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json";
-// Owned by ecstore: bucket deletion must keep this directory when a user
-// bucket carries the same name, so both sides read one definition.
-const REPLACEMENT_RECOVERY_DIR: &str = super::storage_api::owner::ECSTORE_HEAL_REPLACEMENT_RECOVERY_DIR;
+const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
 const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal";
 const LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE: &str = "ahm_replacement_recovery.json";
 const REPLACEMENT_RECOVERY_CONFLICT_PREFIX: &str = "replacement recovery conflict:";
@@ -355,12 +353,6 @@ pub struct ResumeState {
     /// Last proven retired identity per name, retained for returning stale volumes.
     #[serde(default)]
     pub replacement_retired_bucket_incarnations: std::collections::BTreeMap<String, Uuid>,
-    /// Positional bucket list of the current pass; the companion checkpoint's
-    /// bucket index refers to it. Buckets created mid-pass are appended so the
-    /// index keeps its meaning. Empty in snapshots written before additions
-    /// were admitted, where the order is the sorted identity map.
-    #[serde(default)]
-    pub replacement_bucket_pass_order: Vec<String>,
     /// Reapply the checkpoint rewind after any interruption between the two writes.
     #[serde(default)]
     pub replacement_bucket_pass_reset_pending: bool,
@@ -475,7 +467,6 @@ impl ResumeState {
             replacement_buckets: Vec::new(),
             replacement_bucket_incarnations: None,
             replacement_retired_bucket_incarnations: std::collections::BTreeMap::new(),
-            replacement_bucket_pass_order: Vec::new(),
             replacement_bucket_pass_reset_pending: false,
             replacement_generation: None,
             replacement_phase: ReplacementPhase::None,
@@ -596,18 +587,6 @@ impl ResumeState {
         self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
     }
 
-    /// Buckets of the current replacement pass in checkpoint order.
-    pub fn replacement_bucket_pass_buckets(&self) -> Vec<String> {
-        if self.replacement_bucket_pass_order.is_empty() {
-            self.replacement_bucket_incarnations
-                .as_ref()
-                .map(|identities| identities.keys().cloned().collect())
-                .unwrap_or_default()
-        } else {
-            self.replacement_bucket_pass_order.clone()
-        }
-    }
-
     pub fn complete_bucket(&mut self, bucket: &str) {
         if !self.completed_buckets.contains(&bucket.to_string()) {
             self.completed_buckets.push(bucket.to_string());
@@ -1028,20 +1007,7 @@ impl ResumeManager {
                     .is_some_and(|identities| identities.values().all(|incarnation| !incarnation.is_nil())),
                 _ => false,
             };
-            // A persisted pass order must name exactly the recorded identities;
-            // otherwise the checkpoint's bucket index would address the wrong bucket.
-            let valid_pass_order = state.replacement_bucket_pass_order.is_empty()
-                || state.replacement_bucket_incarnations.as_ref().is_some_and(|identities| {
-                    let ordered = state
-                        .replacement_bucket_pass_order
-                        .iter()
-                        .collect::<std::collections::BTreeSet<_>>();
-                    ordered.len() == state.replacement_bucket_pass_order.len()
-                        && ordered.len() == identities.len()
-                        && ordered.iter().all(|bucket| identities.contains_key(*bucket))
-                });
             if !valid_execution_protocol
-                || !valid_pass_order
                 || state.replacement_retired_bucket_incarnations.values().any(Uuid::is_nil)
                 || state.replacement_predecessor.as_deref()
                     != state.replacement_lineage.last().map(|link| link.predecessor.as_str())

crates/heal/src/heal/resume/bucket_pass.rs

@@ -61,49 +61,39 @@ impl ResumeManager {
     pub(crate) async fn prepare_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<Vec<String>> {
         let previous = self.get_state().await;
         let current = namespace_snapshot(storage, &previous).await?;
-        // Recorded identities that are gone or replaced. Only a durable deletion
-        // receipt discharges one; absence or a different identity alone does not.
-        let mut retired = BTreeMap::new();
         if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
             for (bucket, expected) in identities {
-                if current.get(bucket) != Some(expected) {
-                    if !storage.mrf_bucket_incarnation_retired(bucket, *expected).await? {
-                        return Err(Error::ReplacementNamespaceChanged);
-                    }
-                    retired.insert(bucket.clone(), *expected);
+                if current.get(bucket) != Some(expected) && !storage.mrf_bucket_incarnation_retired(bucket, *expected).await? {
+                    // Absence or a different current identity alone is not a deletion receipt.
+                    return Err(Error::ReplacementNamespaceChanged);
                 }
             }
         }
-        match previous.replacement_bucket_incarnations.as_ref() {
-            // The first pass of this generation, or a recorded bucket was deleted
-            // (and possibly recreated): the pass restarts over the current namespace.
-            None => self.restart_replacement_bucket_pass(current, retired).await?,
-            Some(_) if !retired.is_empty() => self.restart_replacement_bucket_pass(current, retired).await?,
-            // Buckets created since the snapshot join the pass so their identity
-            // is fenced like every other bucket, without discarding progress.
-            Some(identities) => {
-                let added = current
-                    .iter()
-                    .filter(|(bucket, _)| !identities.contains_key(*bucket))
-                    .map(|(bucket, incarnation)| (bucket.clone(), *incarnation))
-                    .collect::<Vec<_>>();
-                if !added.is_empty() {
-                    let mut state = self.state.write().await;
-                    if state.replacement_bucket_pass_order.is_empty() {
-                        state.replacement_bucket_pass_order = identities.keys().cloned().collect();
-                    }
-                    for (bucket, incarnation) in added {
+        if previous.replacement_bucket_incarnations.as_ref() != Some(&current) {
+            let mut state = self.state.write().await;
+            if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
+                for (bucket, expected) in identities {
+                    if current.get(bucket) != Some(expected) {
                         state
-                            .replacement_bucket_incarnations
-                            .get_or_insert_with(BTreeMap::new)
-                            .insert(bucket.clone(), incarnation);
-                        state.replacement_bucket_pass_order.push(bucket.clone());
-                        state.pending_buckets.push(bucket);
+                            .replacement_retired_bucket_incarnations
+                            .insert(bucket.clone(), *expected);
                     }
-                    drop(state);
-                    self.save_state_strict().await?;
                 }
             }
+            state.replacement_bucket_incarnations = Some(current.clone());
+            if state.replacement_generation.is_some() {
+                // Protocol-1 readers reject this before interpreting a new pass cursor.
+                state.replacement_execution_protocol = 2;
+            }
+            state.replacement_bucket_pass_reset_pending = true;
+            state.reset_for_retry();
+            state.pending_buckets = current.keys().cloned().collect();
+            state.total_objects = 0;
+            state.total_bytes = 0;
+            state.baseline_generation = None;
+            state.baseline_known = false;
+            drop(state);
+            self.save_state_strict().await?;
         }
         if self.get_state().await.replacement_bucket_pass_reset_pending {
             let checkpoint = if CheckpointManager::has_checkpoint(&self.disk, &previous.task_id).await {
@@ -116,46 +106,13 @@ impl ResumeManager {
             self.state.write().await.replacement_bucket_pass_reset_pending = false;
             self.save_state_strict().await?;
         }
-        Ok(self.get_state().await.replacement_bucket_pass_buckets())
-    }
-
-    /// Bind a fresh pass to `current`, publishing the checkpoint rewind first.
-    async fn restart_replacement_bucket_pass(
-        &self,
-        current: BTreeMap<String, Uuid>,
-        retired: BTreeMap<String, Uuid>,
-    ) -> Result<()> {
-        let mut state = self.state.write().await;
-        state.replacement_retired_bucket_incarnations.extend(retired);
-        state.replacement_bucket_pass_order = current.keys().cloned().collect();
-        state.pending_buckets = current.keys().cloned().collect();
-        state.replacement_bucket_incarnations = Some(current);
-        if state.replacement_generation.is_some() {
-            // Protocol-1 readers reject this before interpreting a new pass cursor.
-            state.replacement_execution_protocol = 2;
-        }
-        state.replacement_bucket_pass_reset_pending = true;
-        state.reset_for_retry();
-        state.total_objects = 0;
-        state.total_bytes = 0;
-        state.baseline_generation = None;
-        state.baseline_known = false;
-        drop(state);
-        self.save_state_strict().await
+        Ok(current.into_keys().collect())
     }
 
     pub(crate) async fn verify_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<()> {
         let state = self.get_state().await;
         let current = namespace_snapshot(storage, &state).await?;
-        let Some(identities) = state.replacement_bucket_incarnations.as_ref() else {
-            return Err(Error::ReplacementNamespaceChanged);
-        };
-        // Buckets created during the pass have nothing the pass could have
-        // missed; only a recorded identity that is gone or replaced fails it.
-        if identities
-            .iter()
-            .any(|(bucket, expected)| current.get(bucket) != Some(expected))
-        {
+        if state.replacement_bucket_incarnations.as_ref() != Some(&current) {
             return Err(Error::ReplacementNamespaceChanged);
         }
         Ok(())
@@ -451,125 +408,4 @@ mod tests {
             .await
             .expect("new pass matches current namespace");
     }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn replacement_bucket_pass_keeps_progress_when_a_bucket_is_created_mid_pass() {
-        let (_directories, store) = Box::pin(isolated_store_over_temp_disks()).await;
-        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        let recorded = "pass-recorded";
-        store
-            .make_bucket(recorded, &MakeBucketOptions::default())
-            .await
-            .expect("create the recorded bucket");
-        let storage = ECStoreHealStorage::new(store.clone());
-        let (_anchor_dir, disk) = super::super::tests::schema_test_disk().await;
-        let task_id = ResumeUtils::generate_task_id();
-        let manager = ResumeManager::new_replacement_intent(
-            disk.clone(),
-            task_id.clone(),
-            "pool_0_set_0".to_string(),
-            vec![recorded.to_owned()],
-            vec!["replacement-a".to_string()],
-            vec![ReplacementTargetIdentity {
-                endpoint: "replacement-a".to_string(),
-                canonical_path: "/mnt/replacement-a".to_string(),
-                physical_device_ids: vec!["device-a".to_string()],
-                filesystem_identity: "1:2:3".to_string(),
-            }],
-        )
-        .await
-        .expect("persist generation");
-        assert_eq!(
-            manager
-                .prepare_replacement_bucket_pass(&storage)
-                .await
-                .expect("admit the recorded pass"),
-            [recorded]
-        );
-        // Progress into the recorded bucket.
-        manager
-            .set_resume_cursor(Some("mid-pass-cursor".to_string()))
-            .await
-            .expect("persist mid-pass cursor");
-        let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
-            .await
-            .expect("admitted checkpoint");
-        checkpoint.complete_page(0, 7).await.expect("persist mid-pass position");
-
-        // The new name sorts before the recorded one, so a sorted rebuild would
-        // move the recorded bucket away from the checkpoint's index.
-        let created = "pass-created";
-        store
-            .make_bucket(created, &MakeBucketOptions::default())
-            .await
-            .expect("create a bucket during the pass");
-        let created_identity = store
-            .bucket_incarnation_id_from_disk(created)
-            .await
-            .expect("created identity");
-
-        manager
-            .verify_replacement_bucket_pass(&storage)
-            .await
-            .expect("a bucket created mid-pass does not fail verification");
-        assert_eq!(
-            manager
-                .prepare_replacement_bucket_pass(&storage)
-                .await
-                .expect("additions extend the pass"),
-            [recorded, created],
-            "additions are appended so the checkpoint's bucket index keeps its meaning"
-        );
-        let state = manager.get_state().await;
-        assert_eq!(
-            state
-                .replacement_bucket_incarnations
-                .as_ref()
-                .and_then(|identities| identities.get(created))
-                .copied(),
-            Some(created_identity)
-        );
-        assert!(state.pending_buckets.contains(&created.to_string()));
-        assert_eq!(state.resume_cursor.as_deref(), Some("mid-pass-cursor"));
-        assert!(!state.replacement_bucket_pass_reset_pending);
-        assert_eq!(state.replacement_execution_protocol, 2);
-        let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
-            .await
-            .expect("checkpoint after the addition")
-            .get_checkpoint()
-            .await;
-        assert_eq!(
-            (checkpoint.current_bucket_index, checkpoint.current_object_index),
-            (0, 7),
-            "the checkpoint must not rewind"
-        );
-
-        // A restart reads the persisted order instead of rebuilding it sorted.
-        let restarted = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
-            .await
-            .expect("restart the pass");
-        assert_eq!(
-            restarted
-                .prepare_replacement_bucket_pass(&storage)
-                .await
-                .expect("an unchanged namespace keeps the pass"),
-            [recorded, created]
-        );
-        restarted
-            .verify_replacement_bucket_pass(&storage)
-            .await
-            .expect("the extended pass matches the namespace");
-        assert_eq!(restarted.get_state().await.resume_cursor.as_deref(), Some("mid-pass-cursor"));
-
-        // Deleting the created bucket is still a namespace change for the pass.
-        store
-            .delete_bucket(created, &DeleteBucketOptions::default())
-            .await
-            .expect("delete the created bucket");
-        assert!(matches!(
-            restarted.verify_replacement_bucket_pass(&storage).await,
-            Err(Error::ReplacementNamespaceChanged)
-        ));
-    }
 }

crates/heal/src/heal/resume/handoff.rs

@@ -461,7 +461,6 @@ impl ResumeManager {
         if predecessor.replacement_execution_protocol == 2 {
             state.replacement_execution_protocol = 2;
             state.replacement_bucket_incarnations = predecessor.replacement_bucket_incarnations.clone();
-            state.replacement_bucket_pass_order = predecessor.replacement_bucket_pass_order.clone();
             state.replacement_retired_bucket_incarnations = predecessor.replacement_retired_bucket_incarnations.clone();
             state.replacement_bucket_pass_reset_pending = true;
         }

crates/heal/src/heal/storage_api.rs

@@ -22,9 +22,8 @@ pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError,
 pub(crate) use rustfs_ecstore::api::disk::{
     BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes,
     ConditionalFileUpdate as EcstoreConditionalFileUpdate, DeleteOptions as EcstoreDeleteOptions, DiskAPI as EcstoreDiskAPI,
-    DiskStore as EcstoreDiskStore, HEAL_REPLACEMENT_RECOVERY_DIR as ECSTORE_HEAL_REPLACEMENT_RECOVERY_DIR,
-    HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
-    ReplacementExecutionLease as EcstoreReplacementExecutionLease,
+    DiskStore as EcstoreDiskStore, HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH,
+    RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, ReplacementExecutionLease as EcstoreReplacementExecutionLease,
 };
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::disk::{
@@ -81,11 +80,11 @@ pub(crate) mod owner {
     pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
 
     pub(crate) use super::{
-        ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEAL_REPLACEMENT_RECOVERY_DIR,
-        ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME, ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate,
-        EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
-        EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore,
-        ecstore_is_reserved_or_invalid_bucket, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
+        ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
+        ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
+        EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
+        EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_is_reserved_or_invalid_bucket,
+        ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
     };
 
     pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};

crates/heal/src/heal/task/heal_bucket.rs

@@ -21,7 +21,6 @@ const MAX_DEFERRED_OBJECTS: usize = 256;
 const MAX_DEFERRED_BYTES: usize = 256 * 1024;
 const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
 const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
-const MAX_RATE_LIMITED_OBJECTS: usize = 3;
 
 struct DeferredObject {
     name: String,
@@ -30,7 +29,6 @@ struct DeferredObject {
     page: u64,
     first_failure: Option<tokio::time::Instant>,
     due: tokio::time::Instant,
-    rate_limited: bool,
 }
 
 impl DeferredObject {
@@ -42,7 +40,6 @@ impl DeferredObject {
             page,
             first_failure: None,
             due: tokio::time::Instant::now(),
-            rate_limited: false,
         }
     }
 
@@ -104,15 +101,8 @@ impl DeferredWindow {
         self.objects.iter().map(|item| item.due).min()
     }
 
-    fn can_admit_object(&self) -> bool {
-        // Probe past an isolated SlowDown, but do not start an entire page's
-        // retry age while several objects already report rate pressure.
-        self.objects.iter().filter(|item| item.rate_limited).count() < MAX_RATE_LIMITED_OBJECTS
-    }
-
     fn can_advance(&self, page: u64) -> bool {
-        self.can_admit_object()
-            && self.objects.len() < MAX_DEFERRED_OBJECTS
+        self.objects.len() < MAX_DEFERRED_OBJECTS
             && self.bytes < MAX_DEFERRED_BYTES
             && self
                 .objects
@@ -682,17 +672,14 @@ impl HealTask {
                             .is_some_and(|item| item.due <= tokio::time::Instant::now())
                         {
                             inline_retry.take()
-                        } else if inline_retry.is_none() && deferred.can_admit_object() {
+                        } else if inline_retry.is_none() {
                             pending.next().map(|item| DeferredObject::new(item, page_number))
                         } else {
                             None
                         }
                     });
                     let Some(mut item) = item else {
-                        let can_list = !listing_finished
-                            && pending.as_slice().is_empty()
-                            && inline_retry.is_none()
-                            && deferred.can_advance(page_number);
+                        let can_list = !listing_finished && inline_retry.is_none() && deferred.can_advance(page_number);
                         if can_list && listing_due <= tokio::time::Instant::now() {
                             let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
                                 self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
@@ -942,7 +929,6 @@ impl HealTask {
                                 "Heal bucket object retry scheduled"
                             );
                             item.defer(Self::bucket_object_retry_delay(&self.id, retry_attempt + 1));
-                            item.rate_limited |= err.is_slow_down();
                             if let Err(item) = deferred.push(item) {
                                 inline_retry = Some(item);
                             }

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -501,11 +501,7 @@ impl HealTask {
                     self.record_result_item(result).await;
                 }
                 Err(err) => {
-                    // Cancellation and timeout keep their classification; only
-                    // a storage failure may be reclassified as a namespace change.
-                    if let Some((_, manager, _)) = replacement_resume.as_ref()
-                        && !matches!(err, Error::TaskCancelled | Error::TaskTimeout)
-                    {
+                    if let Some((_, manager, _)) = replacement_resume.as_ref() {
                         manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
                     }
                     warn!(

crates/heal/src/heal/task/tests.rs

@@ -1661,7 +1661,6 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
 struct MockStorage {
     retry_test_pages: Option<Vec<Vec<HealListItem>>>,
     retry_test_delays: HashMap<String, Duration>,
-    retry_test_success_delays: HashMap<String, Duration>,
     retry_test_listing_delays: Mutex<VecDeque<Duration>>,
     retry_test_events: Mutex<Vec<String>>,
     listed: Mutex<bool>,
@@ -2661,9 +2660,6 @@ impl HealStorageAPI for MockStorage {
         if object == "object-dir/" {
             return Ok((HealResultItem::default(), Some(Error::Disk(DiskError::FileNotFound))));
         }
-        if let Some(delay) = self.retry_test_success_delays.get(object) {
-            tokio::time::sleep(*delay).await;
-        }
         self.healed_objects.lock().unwrap().push(object.to_string());
         if let Some(bucket_incarnation_id) = self.bucket_incarnation_after_object_heal.lock().unwrap().take() {
             *self.bucket_incarnation_id.lock().unwrap() = Some(bucket_incarnation_id);

crates/heal/src/heal/task/tests/deferred_retry.rs

@@ -220,90 +220,6 @@ async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unproc
     }
 }
 
-#[tokio::test(start_paused = true)]
-async fn rate_limited_page_does_not_age_out_behind_successful_repairs() {
-    let names: Vec<String> = (0..64).map(|index| format!("object-{index:04}")).collect();
-    let mut storage = MockStorage {
-        retry_test_pages: Some(vec![names.iter().map(|name| heal_item(name)).collect()]),
-        ..Default::default()
-    };
-    for name in &names {
-        storage.heal_object_outcomes.lock().expect("outcomes").insert(
-            name.clone(),
-            VecDeque::from([
-                MockHealObjectOutcome::RetryableSlowDown,
-                MockHealObjectOutcome::RetryableSlowDown,
-            ]),
-        );
-        storage
-            .retry_test_success_delays
-            .insert(name.clone(), Duration::from_millis(500));
-    }
-    let storage = Arc::new(storage);
-    let task = bucket_task(storage.clone());
-    let started = tokio::time::Instant::now();
-    task.execute()
-        .await
-        .expect("rate pressure must not start the whole page's retry age before serial repair capacity is available");
-    assert!(started.elapsed() > Duration::from_secs(30), "the page exceeds one object's retry age");
-    let outcome = task.get_outcome().await;
-    assert_eq!(
-        (outcome.counters.processed, outcome.counters.failed, outcome.counters.unknown),
-        (64, 0, 64)
-    );
-    assert_eq!(outcome.counters.attempt_failures, 128);
-    assert_eq!(task.get_progress().await.objects_healed, 64);
-    let calls = storage.heal_object_calls.lock().expect("calls");
-    for name in &names {
-        assert_eq!(calls.iter().filter(|called| *called == name).count(), 3);
-    }
-    assert_eq!(storage.healed_objects.lock().expect("healed").len(), 64);
-}
-
-#[tokio::test(start_paused = true)]
-async fn rate_pressure_abort_retains_unstarted_page_tail() {
-    for cancel in [true, false] {
-        let storage = Arc::new(pages_storage(&[&["a", "b", "c", "tail"], &["next-page"]]));
-        for name in ["a", "b", "c"] {
-            storage
-                .heal_object_outcomes
-                .lock()
-                .expect("outcomes")
-                .insert(name.to_string(), VecDeque::from([MockHealObjectOutcome::RetryableSlowDown]));
-        }
-        let mut task = bucket_task(storage.clone());
-        if !cancel {
-            task.options.timeout = Some(Duration::from_secs(1));
-        }
-        let execution = task.execute();
-        tokio::pin!(execution);
-        assert!(
-            tokio::time::timeout(Duration::from_millis(500), &mut execution)
-                .await
-                .is_err()
-        );
-        assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "c"]);
-        assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
-        if cancel {
-            task.cancel().await.expect("cancel");
-        }
-        assert!(matches!(
-            (execution.await, cancel),
-            (Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
-        ));
-        let outcome = task.get_outcome().await;
-        assert_eq!((outcome.counters.processed, outcome.counters.skipped, outcome.counters.failed), (3, 3, 0));
-        assert!(
-            !outcome
-                .objects
-                .iter()
-                .any(|item| item.identity.object == "tail" || item.identity.object == "next-page")
-        );
-        tokio::time::advance(Duration::from_secs(60)).await;
-        assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 3);
-    }
-}
-
 #[tokio::test(start_paused = true)]
 async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
     let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));

crates/io-core/Cargo.toml

@@ -42,7 +42,6 @@ rustfs-io-metrics = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]
-metrics = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs"] }
 
 [lib]

crates/io-core/src/pool.rs

@@ -83,10 +83,7 @@ struct PoolTier {
     /// Queue of available buffers for reuse
     available_buffers: Mutex<Vec<BytesMut>>,
     /// Metrics for tracking this tier
-    metrics: Arc<BytesPoolMetrics>,
-    /// Serialize acquisitions before taking buffers so a slow recorder does not
-    /// accumulate allocated buffers behind hit-rate publication.
-    acquire_metrics_lock: Mutex<()>,
+    metrics: Mutex<Option<Arc<BytesPoolMetrics>>>,
     /// Total acquisitions for this tier
     tier_total_acquires: AtomicU64,
     /// Total hits for this tier
@@ -188,10 +185,16 @@ impl BytesPool {
     /// ```
     pub fn with_config(config: BytesPoolConfig) -> Self {
         let metrics = Arc::new(BytesPoolMetrics::default());
-        let small_pool = Arc::new(PoolTier::new(config.small_size, config.small_max, "small", Arc::clone(&metrics)));
-        let medium_pool = Arc::new(PoolTier::new(config.medium_size, config.medium_max, "medium", Arc::clone(&metrics)));
-        let large_pool = Arc::new(PoolTier::new(config.large_size, config.large_max, "large", Arc::clone(&metrics)));
-        let xlarge_pool = Arc::new(PoolTier::new(config.xlarge_size, config.xlarge_max, "xlarge", Arc::clone(&metrics)));
+        let small_pool = Arc::new(PoolTier::new(config.small_size, config.small_max, "small"));
+        let medium_pool = Arc::new(PoolTier::new(config.medium_size, config.medium_max, "medium"));
+        let large_pool = Arc::new(PoolTier::new(config.large_size, config.large_max, "large"));
+        let xlarge_pool = Arc::new(PoolTier::new(config.xlarge_size, config.xlarge_max, "xlarge"));
+
+        // Set metrics reference in all tiers
+        small_pool.set_metrics(Arc::clone(&metrics));
+        medium_pool.set_metrics(Arc::clone(&metrics));
+        large_pool.set_metrics(Arc::clone(&metrics));
+        xlarge_pool.set_metrics(Arc::clone(&metrics));
 
         Self {
             small_pool,
@@ -222,7 +225,7 @@ impl BytesPool {
     /// ```
     pub async fn acquire_buffer(&self, size: usize) -> PooledBuffer {
         let tier = self.select_tier(size);
-        let mut buffer = tier.acquire_buffer(size).await;
+        let mut buffer = tier.acquire_buffer(size, &self.metrics).await;
         if buffer._permit.is_some() {
             buffer.tier = Some(Arc::clone(tier));
         }
@@ -249,7 +252,7 @@ impl BytesPool {
     /// ```
     pub fn try_acquire_buffer(&self, size: usize) -> Option<PooledBuffer> {
         let tier = self.select_tier(size);
-        let mut buffer = tier.try_acquire_buffer(size)?;
+        let mut buffer = tier.try_acquire_buffer(size, &self.metrics)?;
         // Set tier reference for return on drop
         buffer.tier = Some(Arc::clone(tier));
         Some(buffer)
@@ -287,38 +290,43 @@ impl BytesPool {
 }
 
 impl PoolTier {
-    fn new(buffer_size: usize, max_buffers: usize, name: &'static str, metrics: Arc<BytesPoolMetrics>) -> Self {
+    fn new(buffer_size: usize, max_buffers: usize, name: &'static str) -> Self {
         Self {
             buffer_size,
             max_buffers,
             semaphore: Arc::new(Semaphore::new(max_buffers)),
             name,
             available_buffers: Mutex::new(Vec::new()),
-            metrics,
-            acquire_metrics_lock: Mutex::new(()),
+            metrics: Mutex::new(None),
             tier_total_acquires: AtomicU64::new(0),
             tier_pool_hits: AtomicU64::new(0),
             tier_current_allocated_bytes: AtomicU64::new(0),
         }
     }
 
-    fn take_or_allocate_buffer(&self, size: usize) -> (BytesMut, bool) {
-        let pool_metrics = &self.metrics;
+    fn set_metrics(&self, metrics: Arc<BytesPoolMetrics>) {
+        *self.metrics.lock().unwrap_or_else(|e| e.into_inner()) = Some(metrics);
+    }
+
+    fn take_or_allocate_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> (BytesMut, bool) {
         let buffer_opt = {
             let mut available = self.available_buffers.lock().unwrap_or_else(|e| e.into_inner());
-            let buffer = available.pop();
-            if buffer.is_some() {
-                pool_metrics.available_buffers.fetch_sub(1, Ordering::Relaxed);
-            }
-            buffer
+            available.pop()
         };
         let was_reused = buffer_opt.is_some();
+        if was_reused {
+            // A reused buffer leaves the available pool. `return_buffer` does a
+            // matching `fetch_add(1)` on push, so without this the
+            // `available_buffers` gauge only ever grows and never reflects the real
+            // pool size (backlog#806).
+            pool_metrics.available_buffers.fetch_sub(1, Ordering::Relaxed);
+        }
 
         let buffer = if let Some(mut buf) = buffer_opt {
             let previous_capacity = buf.capacity();
             buf.clear();
             if previous_capacity < size {
-                buf.reserve(size);
+                buf.reserve(size - previous_capacity);
             }
             let current_capacity = buf.capacity();
             if current_capacity > previous_capacity {
@@ -345,10 +353,7 @@ impl PoolTier {
         (buffer, was_reused)
     }
 
-    fn record_acquire_metrics(&self, buffer_capacity: usize, was_reused: bool) {
-        let pool_metrics = &self.metrics;
-        pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
-        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
+    fn record_acquire_metrics(&self, pool_metrics: &BytesPoolMetrics, buffer_capacity: usize, was_reused: bool) {
         rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
 
         if was_reused {
@@ -369,7 +374,7 @@ impl PoolTier {
         rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 
-    async fn acquire_buffer(&self, size: usize) -> PooledBuffer {
+    async fn acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> PooledBuffer {
         // Acquire semaphore permit (owned for storage in PooledBuffer)
         let permit = match Arc::clone(&self.semaphore).acquire_owned().await {
             Ok(p) => p,
@@ -383,10 +388,17 @@ impl PoolTier {
             }
         };
 
-        let _guard = self.acquire_metrics_lock.lock().unwrap_or_else(|e| e.into_inner());
-        let (buffer, was_reused) = self.take_or_allocate_buffer(size);
+        // Use the pool's shared metrics for recording
+        let _metrics_lock = self.metrics.lock().unwrap_or_else(|e| e.into_inner());
+        let _metrics = _metrics_lock.as_ref().expect("operation should succeed");
+
+        // Record acquisition
+        pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
+
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-        self.record_acquire_metrics(buffer_capacity, was_reused);
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -395,14 +407,21 @@ impl PoolTier {
         }
     }
 
-    fn try_acquire_buffer(&self, size: usize) -> Option<PooledBuffer> {
+    fn try_acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> Option<PooledBuffer> {
         // Try to acquire permit without blocking
         let permit = Arc::clone(&self.semaphore).try_acquire_owned().ok()?;
 
-        let _guard = self.acquire_metrics_lock.lock().unwrap_or_else(|e| e.into_inner());
-        let (buffer, was_reused) = self.take_or_allocate_buffer(size);
+        // Use the pool's shared metrics for recording
+        let _metrics_lock = self.metrics.lock().unwrap_or_else(|e| e.into_inner());
+        let _metrics = _metrics_lock.as_ref().expect("operation should succeed");
+
+        // Record acquisition
+        pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
+
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-        self.record_acquire_metrics(buffer_capacity, was_reused);
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         Some(PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -411,7 +430,7 @@ impl PoolTier {
         })
     }
 
-    /// Return a buffer to the pool for reuse without waiting for the queue lock.
+    /// Return a buffer to the pool for reuse without ever blocking the caller.
     fn return_buffer(&self, buffer: BytesMut) {
         let mut buffer = Some(buffer);
 
@@ -419,7 +438,11 @@ impl PoolTier {
             && available.len() < self.max_buffers
         {
             available.push(buffer.take().expect("buffer should be present until returned"));
-            self.metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
+            if let Ok(metrics) = self.metrics.try_lock()
+                && let Some(metrics) = metrics.as_ref()
+            {
+                metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
+            }
         }
 
         if let Some(buffer) = buffer {
@@ -429,30 +452,21 @@ impl PoolTier {
                     Some(current.saturating_sub(released_bytes))
                 })
                 .ok();
-            self.metrics
-                .current_allocated_bytes
-                .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
-                    Some(current.saturating_sub(released_bytes))
-                })
-                .ok();
+            if let Ok(metrics) = self.metrics.try_lock()
+                && let Some(metrics) = metrics.as_ref()
+            {
+                metrics
+                    .current_allocated_bytes
+                    .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                        Some(current.saturating_sub(released_bytes))
+                    })
+                    .ok();
+            }
         }
         rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 }
 
-impl PooledBuffer {
-    /// A buffer outside the pool: it holds no permit and is never returned to
-    /// a tier. Callers use it when the pool is at capacity but the request
-    /// must not be refused.
-    pub fn unpooled(size: usize) -> Self {
-        Self {
-            buffer: ManuallyDrop::new(BytesMut::with_capacity(size)),
-            tier: None,
-            _permit: None,
-        }
-    }
-}
-
 impl Drop for PooledBuffer {
     // SAFETY: Drop has exclusive access to `self`; taking the `ManuallyDrop`
     // buffer moves it exactly once into the pool when a tier still owns it.
@@ -537,44 +551,6 @@ mod tests {
         assert!(buffer.capacity() >= 2048);
     }
 
-    #[tokio::test]
-    async fn reused_buffer_satisfies_larger_async_request() {
-        let pool = BytesPool::with_config(BytesPoolConfig {
-            small_max: 1,
-            ..Default::default()
-        });
-        let mut first = pool.acquire_buffer(32 * 1024).await;
-        first.extend_from_slice(b"previous contents");
-        assert!(first.capacity() < 48 * 1024, "fixture must require buffer growth");
-        drop(first);
-
-        let reused = pool.acquire_buffer(48 * 1024).await;
-        assert!(reused.is_empty());
-        assert!(reused.capacity() >= 48 * 1024);
-        assert_eq!(pool.metrics().pool_hits.load(Ordering::Relaxed), 1);
-        assert_eq!(
-            pool.metrics().current_allocated_bytes.load(Ordering::Relaxed),
-            u64::try_from(reused.capacity()).expect("buffer capacity fits in metrics")
-        );
-    }
-
-    #[test]
-    fn reused_buffer_satisfies_larger_try_request() {
-        let pool = BytesPool::with_config(BytesPoolConfig {
-            small_max: 1,
-            ..Default::default()
-        });
-        let mut first = pool.try_acquire_buffer(32 * 1024).expect("first buffer is available");
-        first.extend_from_slice(b"previous contents");
-        assert!(first.capacity() < 48 * 1024, "fixture must require buffer growth");
-        drop(first);
-
-        let reused = pool.try_acquire_buffer(48 * 1024).expect("returned buffer is available");
-        assert!(reused.is_empty());
-        assert!(reused.capacity() >= 48 * 1024);
-        assert_eq!(pool.metrics().pool_hits.load(Ordering::Relaxed), 1);
-    }
-
     #[tokio::test]
     async fn available_buffers_gauge_decrements_on_reuse() {
         // Regression (backlog#806): acquiring a pooled (reused) buffer must

crates/io-core/tests/bytes_pool_metrics_concurrency.rs

@@ -1,121 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::{Mutex, mpsc};
-use std::thread;
-use std::time::Duration;
-
-use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit};
-use rustfs_io_core::{BytesPool, BytesPoolConfig};
-
-const WAIT: Duration = Duration::from_secs(5);
-const GATE_WAIT: Duration = Duration::from_secs(20);
-
-struct AcquisitionGate {
-    entered: mpsc::Sender<()>,
-    release: Mutex<mpsc::Receiver<()>>,
-}
-
-impl Recorder for AcquisitionGate {
-    fn describe_counter(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
-
-    fn describe_gauge(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
-
-    fn describe_histogram(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
-
-    fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
-        if key.name() == "rustfs_bytes_pool_acquisitions_total" {
-            self.entered
-                .send(())
-                .expect("notify test that acquisition reached the recorder");
-            self.release
-                .lock()
-                .expect("lock acquisition gate")
-                .recv_timeout(GATE_WAIT)
-                .expect("release the acquisition recorder gate");
-        }
-        Counter::noop()
-    }
-
-    fn register_gauge(&self, _key: &Key, _metadata: &Metadata<'_>) -> Gauge {
-        Gauge::noop()
-    }
-
-    fn register_histogram(&self, _key: &Key, _metadata: &Metadata<'_>) -> Histogram {
-        Histogram::noop()
-    }
-}
-
-#[test]
-fn returned_buffer_is_counted_while_an_acquisition_recorder_is_busy() {
-    let pool = BytesPool::with_config(BytesPoolConfig {
-        small_max: 2,
-        ..Default::default()
-    });
-    let held = pool.try_acquire_buffer(8 * 1024).expect("acquire the buffer to return");
-    assert_eq!(pool.available_buffers(), 0);
-
-    // This integration test is a dedicated test process: the recorder is local
-    // to the worker, while the metrics enablement flag is process-wide.
-    rustfs_io_metrics::set_metrics_enabled(true);
-    let (entered_tx, entered_rx) = mpsc::channel();
-    let (release_tx, release_rx) = mpsc::channel();
-    let worker_pool = pool.clone();
-    let acquirer = thread::spawn(move || {
-        let gate = AcquisitionGate {
-            entered: entered_tx,
-            release: Mutex::new(release_rx),
-        };
-        metrics::with_local_recorder(&gate, || {
-            worker_pool
-                .try_acquire_buffer(8 * 1024)
-                .expect("acquire a second same-tier buffer")
-        })
-    });
-
-    let entered = entered_rx.recv_timeout(WAIT);
-    if entered.is_err() {
-        let _ = release_tx.send(());
-        let _ = acquirer.join();
-        panic!("acquisition did not reach the recorder gate: {entered:?}");
-    }
-
-    let (dropped_tx, dropped_rx) = mpsc::channel();
-    let returner = thread::spawn(move || {
-        drop(held);
-        let _ = dropped_tx.send(());
-    });
-    let returned_while_blocked = dropped_rx.recv_timeout(WAIT);
-
-    // Always release before joining so a blocking Drop cannot deadlock the test.
-    let _ = release_tx.send(());
-    let returned = returner.join();
-    let acquired = acquirer.join();
-    rustfs_io_metrics::set_metrics_enabled(false);
-
-    assert!(returned_while_blocked.is_ok(), "returning a buffer must not wait on the recorder");
-    returned.expect("return worker should finish");
-    let acquired = acquired.expect("acquisition worker should finish");
-    let after_return = pool.available_buffers();
-    let reused = pool.try_acquire_buffer(8 * 1024).expect("reuse the returned buffer");
-    let after_reuse = pool.available_buffers();
-    drop(reused);
-    drop(acquired);
-    let after_both_return = pool.available_buffers();
-    assert_eq!(
-        (after_return, after_reuse, after_both_return),
-        (1, 0, 2),
-        "returned buffers must be counted, and reuse must not underflow the available gauge"
-    );
-}

crates/lifecycle/src/core.rs

@@ -58,7 +58,6 @@ const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS: &str =
     "Days must be a positive integer and Date must not be specified inside Expiration with ExpiredObjectAllVersions";
 const ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS: &str = "Days must be a positive integer with DelMarkerExpiration";
 const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
-const ERR_LIFECYCLE_INVALID_RULE_ID_CHARS: &str = "Lifecycle rule ID must not contain control characters";
 const ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY: &str = "Rule ID must not be empty";
 const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
 const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering";
@@ -584,11 +583,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
                 if id.len() > 255 {
                     return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
                 }
-                // The ID is echoed in the x-amz-expiration header; a control
-                // character there would make every object GET, HEAD and PUT fail.
-                if id.chars().any(char::is_control) {
-                    return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_CHARS));
-                }
             }
             r.validate()?;
             if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref()
@@ -2052,34 +2046,6 @@ mod tests {
         assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG);
     }
 
-    #[tokio::test]
-    async fn validate_rejects_rule_id_with_control_characters() {
-        for id in ["rule\nid", "rule\rid", "rule\tid", "rule\u{1b}id"] {
-            let lc = BucketLifecycleConfiguration {
-                expiry_updated_at: None,
-                rules: vec![LifecycleRule {
-                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                    expiration: Some(LifecycleExpiration {
-                        days: Some(30),
-                        ..Default::default()
-                    }),
-                    abort_incomplete_multipart_upload: None,
-                    del_marker_expiration: None,
-                    filter: None,
-                    id: Some(id.to_owned()),
-                    noncurrent_version_expiration: None,
-                    noncurrent_version_transitions: None,
-                    prefix: None,
-                    transitions: None,
-                }],
-            };
-
-            let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
-
-            assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_CHARS, "{id:?}");
-        }
-    }
-
     #[tokio::test]
     async fn validate_rejects_duplicate_rule_ids() {
         let lc = BucketLifecycleConfiguration {

crates/lock/src/fast_lock/publication.rs

@@ -25,7 +25,7 @@ pub(super) struct PublicationQueue {
 }
 
 pub(super) struct PublicationTicket {
-    state: Arc<ObjectLockState>,
+    pub(super) state: Arc<ObjectLockState>,
     pub(super) queue: Arc<PublicationQueue>,
     token: Arc<()>,
     mode: LockMode,

crates/obs/Cargo.toml

@@ -166,7 +166,6 @@ libc = { workspace = true }
 
 
 [dev-dependencies]
-opentelemetry_sdk = { workspace = true, features = ["testing"] }
 tempfile = { workspace = true }
 temp-env = { workspace = true }
 log.workspace = true

crates/obs/src/telemetry/otel.rs

@@ -93,17 +93,6 @@ const LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
     0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
 ];
 
-const OBJECT_LOCK_DURATION_HISTOGRAM_METRICS: &[&str] = &[
-    "rustfs_object_lock_diag_acquire_duration_seconds",
-    "rustfs_object_lock_diag_hold_duration_seconds",
-];
-
-// Resolve millisecond acquisitions without losing long-held delete and recovery locks.
-const OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
-    0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002, 0.003, 0.004, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03, 0.05, 0.075,
-    0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0,
-];
-
 const DEFAULT_OTLP_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
 const OTEL_EXPORTER_OTLP_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TIMEOUT";
 const OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT";
@@ -487,7 +476,7 @@ fn build_meter_provider(
                         .with_interval(Duration::from_secs(meter_interval))
                         .build(),
                 )
-                .with_view(duration_histogram_view);
+                .with_view(get_object_duration_histogram_view);
             if use_stdout {
                 b.with_reader(create_periodic_reader(meter_interval))
             } else {
@@ -502,8 +491,8 @@ fn build_meter_provider(
     Ok(Some(provider))
 }
 
-fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
-    let boundaries = duration_histogram_buckets(instrument.name())?;
+fn get_object_duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
+    let boundaries = get_object_duration_histogram_buckets(instrument.name())?;
 
     Stream::builder()
         .with_aggregation(Aggregation::ExplicitBucketHistogram {
@@ -514,13 +503,11 @@ fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
         .ok()
 }
 
-fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
+fn get_object_duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
     if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
         Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
     } else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
         Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
-    } else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
-        Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
     } else {
         None
     }
@@ -1125,86 +1112,25 @@ mod tests {
     }
 
     #[test]
-    fn test_duration_histogram_metric_match_is_scoped() {
+    fn test_get_object_duration_histogram_metric_match_is_scoped() {
         for name in [
             "rustfs_io_get_object_stage_duration_seconds",
             "rustfs_io_get_object_request_duration_seconds",
         ] {
-            assert_eq!(duration_histogram_buckets(name), Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS));
+            assert_eq!(get_object_duration_histogram_buckets(name), Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS));
         }
         for name in LOCAL_BATCH_DURATION_HISTOGRAM_METRICS {
-            assert_eq!(duration_histogram_buckets(name), Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS));
-        }
-        for name in OBJECT_LOCK_DURATION_HISTOGRAM_METRICS {
-            assert_eq!(duration_histogram_buckets(name), Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS));
+            assert_eq!(get_object_duration_histogram_buckets(name), Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS));
         }
         for name in [
             "rustfs_io_put_object_request_duration_seconds",
             "rustfs_io_get_object_response_size_bytes",
             "rustfs_io_get_object_local_batch_shards",
-            "rustfs_object_lock_diag_slow_acquire_total",
-            "rustfs_object_lock_diag_acquire_duration_seconds_bucket",
         ] {
-            assert!(duration_histogram_buckets(name).is_none());
+            assert!(get_object_duration_histogram_buckets(name).is_none());
         }
     }
 
-    #[test]
-    fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
-        use metrics::Recorder as _;
-        use opentelemetry_sdk::metrics::{
-            InMemoryMetricExporter,
-            data::{AggregatedMetrics, MetricData},
-        };
-
-        let exporter = InMemoryMetricExporter::default();
-        let reader = PeriodicReader::builder(exporter.clone())
-            .with_interval(Duration::from_secs(60))
-            .build();
-        let (provider, recorder) = Recorder::builder("lock-histogram-test")
-            .with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
-            .build();
-        let metadata = metrics::Metadata::new(module_path!(), metrics::Level::INFO, None);
-
-        for (name, observations) in [
-            (OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[0], [0.002, 0.004, 0.012]),
-            (OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[1], [0.002, 12.9, 31.0]),
-        ] {
-            let histogram = recorder.register_histogram(&metrics::Key::from_name(name), &metadata);
-            for seconds in observations {
-                histogram.record(seconds);
-            }
-        }
-        provider.force_flush().expect("export lock duration observations");
-        let exported = exporter.get_finished_metrics().expect("read exported lock histograms");
-        let mut checked = 0;
-        for metric in exported
-            .iter()
-            .flat_map(|resource| resource.scope_metrics())
-            .flat_map(|scope| scope.metrics())
-        {
-            let expected = match metric.name() {
-                "rustfs_object_lock_diag_acquire_duration_seconds" => [(0.002, 1), (0.004, 1), (0.015, 1)],
-                "rustfs_object_lock_diag_hold_duration_seconds" => [(0.002, 1), (15.0, 1), (60.0, 1)],
-                _ => continue,
-            };
-            let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
-                panic!("lock duration must export as an f64 histogram");
-            };
-            let points: Vec<_> = histogram.data_points().collect();
-            assert_eq!(points.len(), 1, "one time series for each lock duration instrument");
-            let point = points[0];
-            assert_eq!(point.count(), 3, "all observations reach the exporter");
-            let counts: Vec<_> = point.bucket_counts().collect();
-            assert_eq!(counts.last(), Some(&0), "observations do not fall into the overflow bucket");
-            let occupied: Vec<_> = point.bounds().zip(counts).filter(|(_, count)| *count != 0).collect();
-            assert_eq!(occupied, expected, "seconds remain distinct at millisecond and long-hold scales");
-            checked += 1;
-        }
-        assert_eq!(checked, 2, "both acquisition and hold histograms were exported");
-        provider.shutdown().expect("stop lock histogram test provider");
-    }
-
     #[test]
     fn test_get_object_duration_histogram_buckets_are_sorted() {
         assert!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.windows(2).all(|pair| pair[0] < pair[1]));

crates/s3-client/src/api_get_object.rs

@@ -21,17 +21,17 @@
 
 use futures_util::ready;
 use http::HeaderMap;
-use std::io::{Error as IoError, ErrorKind as IoErrorKind, Read};
+use std::io::{Cursor, Error as IoError, ErrorKind as IoErrorKind, Read};
 use std::pin::Pin;
 use std::task::{Context, Poll};
+use tokio::io::BufReader;
 use tokio_util::io::StreamReader;
 
 use crate::{
     api_error_response::err_invalid_argument,
     api_get_options::GetObjectOptions,
     transition_api::{
-        ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, response_body_reader,
-        to_object_info_for_provider,
+        ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
     },
 };
 use futures_util::StreamExt;
@@ -86,11 +86,6 @@ impl TransitionClient {
         ))
     }
 
-    /// Open a remote object without collecting its response body.
-    ///
-    /// Request and response-header errors are returned here. Body transport,
-    /// idle-timeout and range-limit errors are returned when the reader is read.
-    /// The reader owns the response; dropping it releases an unfinished body.
     pub async fn get_object_inner(
         &self,
         bucket_name: &str,
@@ -125,8 +120,12 @@ impl TransitionClient {
 
         let h = resp.headers().clone();
 
-        let reader = response_body_reader(resp.into_body(), max_response_bytes, self.timeouts.response_body_idle_timeout);
-        Ok((object_stat, h, reader))
+        let body_vec = if let Some(limit) = max_response_bytes {
+            self.collect_response_body(resp.into_body(), limit).await?
+        } else {
+            self.collect_response_body_unbounded(resp.into_body()).await?
+        };
+        Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
     }
 }
 
@@ -270,13 +269,9 @@ mod bounded_response_tests {
         let mut opts = GetObjectOptions::default();
         opts.set_range(0, 6).expect("the probe range should be valid");
 
-        let (_, _, mut reader) = client
+        let err = client
             .get_object_inner("bucket", "probe", &opts)
             .await
-            .expect("the response headers should produce a reader");
-        let err = reader
-            .read_to_end(&mut Vec::new())
-            .await
             .expect_err("an eight-byte response must exceed the seven-byte range limit");
 
         assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
@@ -396,13 +391,9 @@ mod bounded_response_tests {
         let mut opts = GetObjectOptions::default();
         opts.set_range(0, 6).expect("the probe range should be valid");
 
-        let (_, _, mut reader) = client
+        let err = client
             .get_object_inner("bucket", "probe", &opts)
             .await
-            .expect("the response headers should produce a reader");
-        let err = reader
-            .read_to_end(&mut Vec::new())
-            .await
             .expect_err("body stalls after partial progress must be bounded");
 
         assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);

crates/s3-client/src/transition_api.rs

@@ -68,6 +68,7 @@ use s3s::dto::ReplicationStatus;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 use std::error::Error as StdError;
+use std::io::Cursor;
 use std::pin::Pin;
 use std::sync::atomic::{AtomicI32, Ordering};
 use std::task::{Context, Poll};
@@ -78,6 +79,7 @@ use std::{
 };
 use time::Duration;
 use time::OffsetDateTime;
+use tokio::io::BufReader;
 use tokio::io::{AsyncRead, AsyncReadExt};
 use tracing::{error, trace, warn};
 use url::{Url, form_urlencoded};
@@ -154,7 +156,7 @@ where
 
 async fn collect_response_body_inner<B>(
     body: B,
-    limit: usize,
+    limit: Option<usize>,
     idle_timeout: Option<StdDuration>,
 ) -> Result<Vec<u8>, std::io::Error>
 where
@@ -167,7 +169,7 @@ where
         let Some(new_len) = body_vec.len().checked_add(data.len()) else {
             return Err(response_body_exceeds_limit_error());
         };
-        if new_len > limit {
+        if limit.is_some_and(|limit| new_len > limit) {
             return Err(response_body_exceeds_limit_error());
         }
         body_vec.extend_from_slice(&data);
@@ -180,22 +182,7 @@ where
     B: Body<Data = Bytes>,
     B::Error: Into<Box<dyn StdError + Send + Sync>>,
 {
-    collect_response_body_inner(body, limit, None).await
-}
-
-/// Keep the remote body with its reader; idle and range errors surface while reading.
-pub(crate) fn response_body_reader(body: Incoming, limit: Option<usize>, idle_timeout: StdDuration) -> ReadCloser {
-    let stream = futures::stream::try_unfold((Box::pin(body), 0_usize), move |(mut body, received)| async move {
-        let Some(data) = next_response_body_data(body.as_mut(), Some(idle_timeout)).await? else {
-            return Ok::<_, std::io::Error>(None);
-        };
-        let received = received
-            .checked_add(data.len())
-            .filter(|size| limit.is_none_or(|limit| *size <= limit))
-            .ok_or_else(response_body_exceeds_limit_error)?;
-        Ok(Some((data, (body, received))))
-    });
-    ReadCloser::new(tokio_util::io::StreamReader::new(Box::pin(stream)))
+    collect_response_body_inner(body, Some(limit), None).await
 }
 
 const C_UNKNOWN: i32 = -1;
@@ -230,9 +217,9 @@ fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> s
     signer_error::signer_error_to_io_error(scope, error)
 }
 
-/// Streaming object body shared by the client and storage engine. The concrete
-/// reader (e.g. ecstore's `GetObjectReader` or a remote response body) stays
-/// behind this boxed `AsyncRead`.
+/// Streaming object body handed to the client by the storage engine. The
+/// client only ever reads it to completion, so the engine-side reader type
+/// (e.g. ecstore's `GetObjectReader`) stays behind this boxed `AsyncRead`.
 pub struct ObjectReader(Box<dyn AsyncRead + Send + Sync + Unpin>);
 
 impl ObjectReader {
@@ -268,8 +255,7 @@ pub enum ReaderImpl {
     ObjectBody(ObjectReader),
 }
 
-/// Remote object reader. Dropping it releases the response body without collecting it.
-pub type ReadCloser = ObjectReader;
+pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;
 
 #[derive(Debug)]
 pub struct TransitionClient {
@@ -828,7 +814,15 @@ impl TransitionClient {
         B: Body<Data = Bytes>,
         B::Error: Into<Box<dyn StdError + Send + Sync>>,
     {
-        collect_response_body_inner(body, limit, Some(self.timeouts.response_body_idle_timeout)).await
+        collect_response_body_inner(body, Some(limit), Some(self.timeouts.response_body_idle_timeout)).await
+    }
+
+    pub async fn collect_response_body_unbounded<B>(&self, body: B) -> Result<Vec<u8>, std::io::Error>
+    where
+        B: Body<Data = Bytes>,
+        B::Error: Into<Box<dyn StdError + Send + Sync>>,
+    {
+        collect_response_body_inner(body, None, Some(self.timeouts.response_body_idle_timeout)).await
     }
 
     async fn new_request(
@@ -1782,7 +1776,7 @@ mod tests {
 
         let err = tokio::time::timeout(
             StdDuration::from_millis(200),
-            collect_response_body_inner(body, 1, Some(StdDuration::from_millis(50))),
+            collect_response_body_inner(body, Some(1), Some(StdDuration::from_millis(50))),
         )
         .await
         .expect("the collector should enforce its own body idle timeout")

crates/s3-client/tests/get_object_streaming.rs

@@ -1,169 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use rustfs_s3_client::{
-    api_get_options::GetObjectOptions,
-    credentials::{Credentials, SignatureType, Static, Value},
-    transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts},
-};
-use std::time::Duration;
-use tokio::{
-    io::{AsyncReadExt, AsyncWriteExt},
-    net::TcpListener,
-};
-
-async fn run_case(ranged: bool) -> bool {
-    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fixture");
-    let endpoint = listener.local_addr().expect("local endpoint").to_string();
-    let (headers_sent, headers_ready) = tokio::sync::oneshot::channel();
-    let (release, released) = tokio::sync::oneshot::channel();
-    let server = tokio::spawn(async move {
-        let (mut socket, _) = listener.accept().await.expect("GET connection");
-        let mut request = Vec::new();
-        loop {
-            let mut buffer = [0_u8; 1024];
-            let n = socket.read(&mut buffer).await.expect("GET headers");
-            assert_ne!(n, 0);
-            request.extend_from_slice(&buffer[..n]);
-            if request.windows(4).any(|part| part == b"\r\n\r\n") {
-                break;
-            }
-        }
-        let request = String::from_utf8(request).expect("ASCII HTTP request");
-        assert!(request.starts_with("GET /bucket/object "), "{request}");
-        assert_eq!(request.to_ascii_lowercase().contains("\r\nrange: bytes=0-7\r\n"), ranged, "{request}");
-        let status = if ranged { "206 Partial Content" } else { "200 OK" };
-        let range_header = if ranged { "Content-Range: bytes 0-7/8\r\n" } else { "" };
-        let response = format!(
-            "HTTP/1.1 {status}\r\nContent-Length: 8\r\n{range_header}Content-Type: application/octet-stream\r\nETag: \"fixture\"\r\nConnection: close\r\n\r\nRust"
-        );
-        socket.write_all(response.as_bytes()).await.expect("first body bytes");
-        headers_sent.send(()).expect("signal response head");
-        released.await.expect("release remaining body");
-        socket.write_all(b"FS!!").await.expect("last body bytes");
-    });
-    let options = Options {
-        creds: Credentials::new(Static(Value {
-            access_key_id: "access-key".into(),
-            secret_access_key: "secret-key".into(),
-            signer_type: SignatureType::SignatureV4,
-            ..Default::default()
-        })),
-        region: "us-east-1".into(),
-        bucket_lookup: BucketLookupType::BucketLookupAuto,
-        max_retries: 1,
-        ..Default::default()
-    };
-    let client = TransitionClient::new_with_timeouts(&endpoint, options, "s3", TransitionClientTimeouts::default())
-        .await
-        .expect("production client");
-    let client = rustfs_s3_client::transition_api::TransitionCore(std::sync::Arc::new(client));
-    let mut opts = GetObjectOptions::default();
-    if ranged {
-        opts.set_range(0, 7).expect("closed production range");
-    }
-    let get = client.get_object("bucket", "object", &opts);
-    tokio::pin!(get);
-    let probe = async {
-        tokio::select! {
-            result = &mut get => Some(result),
-            _ = headers_ready => None,
-        }
-    }
-    .await;
-    let before_eof = match probe {
-        Some(result) => Ok(result),
-        None => tokio::time::timeout(Duration::from_secs(2), &mut get).await,
-    };
-    let returned_before_body_end = before_eof.is_ok();
-    let mut body = Vec::new();
-    let mut reader = match before_eof {
-        Ok(result) => {
-            let (_, _, mut reader) = result.expect("streaming GET");
-            let mut prefix = [0; 4];
-            tokio::time::timeout(Duration::from_secs(2), reader.read_exact(&mut prefix))
-                .await
-                .expect("first bytes must arrive before the remote body ends")
-                .expect("first body bytes");
-            body.extend_from_slice(&prefix);
-            release.send(()).expect("release fixture");
-            reader
-        }
-        Err(_) => {
-            release.send(()).expect("release fixture");
-            let (_, _, reader) = get.await.expect("buffered GET after full body");
-            reader
-        }
-    };
-    reader.read_to_end(&mut body).await.expect("response bytes");
-    assert_eq!(body, b"RustFS!!");
-    server.await.expect("fixture server");
-    returned_before_body_end
-}
-
-#[tokio::test]
-async fn get_object_returns_reader_before_remote_response_ends() {
-    let full = run_case(false).await;
-    let closed_range = run_case(true).await;
-    assert!(full && closed_range, "tier GET must stream: full={full}, closed_range={closed_range}");
-}
-
-#[tokio::test]
-async fn dropping_get_object_reader_closes_the_unfinished_response() {
-    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fixture");
-    let endpoint = listener.local_addr().expect("local endpoint").to_string();
-    let server = tokio::spawn(async move {
-        let (mut socket, _) = listener.accept().await.expect("GET connection");
-        let mut request = Vec::new();
-        while !request.windows(4).any(|part| part == b"\r\n\r\n") {
-            let mut buffer = [0; 1024];
-            let size = socket.read(&mut buffer).await.expect("GET headers");
-            assert_ne!(size, 0);
-            request.extend_from_slice(&buffer[..size]);
-        }
-        socket
-            .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nRust")
-            .await
-            .expect("partial response");
-        let closed = tokio::time::timeout(Duration::from_secs(5), socket.read(&mut [0; 1]))
-            .await
-            .expect("dropping the reader must release the remote connection");
-        match closed {
-            Ok(size) => assert_eq!(size, 0, "a canceled body must not send another request"),
-            Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::ConnectionReset),
-        }
-    });
-    let options = Options {
-        creds: Credentials::new(Static(Value {
-            access_key_id: "access-key".into(),
-            secret_access_key: "secret-key".into(),
-            signer_type: SignatureType::SignatureV4,
-            ..Default::default()
-        })),
-        region: "us-east-1".into(),
-        ..Default::default()
-    };
-    let client = TransitionClient::new_with_timeouts(&endpoint, options, "s3", TransitionClientTimeouts::default())
-        .await
-        .expect("production client");
-    let (_, _, reader) = tokio::time::timeout(
-        Duration::from_secs(2),
-        client.get_object_inner("bucket", "object", &GetObjectOptions::default()),
-    )
-    .await
-    .expect("headers must return before the response completes")
-    .expect("GET reader");
-    drop(reader);
-    server.await.expect("fixture server");
-}

crates/s3select-api/Cargo.toml

@@ -88,7 +88,7 @@ transform-stream.workspace = true
 url.workspace = true
 
 [dev-dependencies]
-rustfs-test-utils = { workspace = true }
+rustfs-test-utils = { workspace = true, features = ["put-object-commit-barrier"] }
 serial_test.workspace = true
 
 [lib]

crates/s3select-query/Cargo.toml

@@ -58,7 +58,7 @@ tracing = { workspace = true }
 
 [dev-dependencies]
 flate2.workspace = true
-rustfs-test-utils = { workspace = true }
+rustfs-test-utils = { workspace = true, features = ["put-object-commit-barrier"] }
 
 [lib]
 doctest = false

crates/s3select-query/README.md

@@ -34,9 +34,9 @@ For comprehensive documentation, examples, and usage guides, please visit the ma
 
 ## Resource limits
 
-JSON LINES schema inference reads the first 1,000 nonblank records, preserving Arrow's type inference and source field order. It buffers at most 4 MiB of decompressed input. Reaching that byte limit cuts the sample at the last complete record, exactly as a shorter object would; it is an error only when no complete record fits. CSV inference reads only the first non-comment record (the header when enabled), with the same 4 MiB prefix limit. Quoting, escaping, comments, header modes and record delimiters still apply.
+JSON LINES schema inference reads the first 1,000 nonblank records, preserving Arrow's type inference and source field order. It buffers at most 4 MiB of decompressed input. Reaching that byte limit before the inference sample or end of the object is an error; RustFS does not truncate a record or silently shorten the sample. CSV inference reads only the first non-comment record (the header when enabled), with the same 4 MiB prefix limit. Quoting, escaping, comments, header modes and record delimiters still apply.
 
-CSV schemas have at most 1,024 columns. JSON LINES schemas have at most 1,024 distinct fields across all nested objects in the inference sample. SQL expressions have at most 256 KiB and 1,024 projection expressions; wildcard expansion must also fit 1,024 output columns. Inputs exceeding these limits now return explicit errors, including inputs older versions attempted to process without these bounds. Parquet schema inference is unchanged apart from reading string and binary columns as view types; Parquet decoding and downstream execution use one-row batches on a single partition, with batch coalescing disabled to bound dictionary expansion during casts. JSON DOCUMENT input handling is unchanged.
+CSV schemas have at most 1,024 columns. JSON LINES schemas have at most 1,024 distinct fields across all nested objects in the inference sample. SQL expressions have at most 256 KiB and 1,024 projection expressions; wildcard expansion must also fit 1,024 output columns. Inputs exceeding these limits now return explicit errors, including inputs older versions attempted to process without these bounds. These changes do not alter Parquet schema inference or JSON DOCUMENT input handling.
 
 CSV decoding uses at most 256 rows per batch, reduced further for smaller configured query memory limits. At 1,024 columns, Arrow's two-offset-per-cell reservation is at most 4 MiB on 64-bit systems and at most one sixteenth of the configured query budget (with a minimum one-row batch). Schema prefix storage and temporary inference work are reserved against the existing query memory pool. The configured memory limit can reject a sample before the 4 MiB hard prefix limit.
 

crates/s3select-query/src/dispatcher/manager.rs

@@ -1898,60 +1898,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn bounded_json_lines_prefix_accepts_complete_sample_before_byte_limit() {
-        for rows in [1, 1000, 600] {
-            let mut input = test_input();
-            input.request.expression = "SELECT * FROM S3Object LIMIT 1".to_owned();
-            input.request.input_serialization.csv = None;
-            input.request.input_serialization.json = Some(JSONInput {
-                type_: Some(JSONType::from_static(JSONType::LINES)),
-            });
-            let input = Arc::new(input);
-            let optimizer = Arc::new(CascadeOptimizerBuilder::default().build());
-            let dispatcher = SimpleQueryDispatcherBuilder::default()
-                .with_input(Arc::clone(&input))
-                .with_default_table_provider(Arc::new(BaseTableProvider::default()))
-                .with_session_factory(Arc::new(SessionCtxFactory::new(true)))
-                .with_parser(Arc::new(DefaultParser::default()))
-                .with_query_execution_factory(Arc::new(SqlQueryExecutionFactory::new(optimizer, Arc::new(LocalScheduler {}))))
-                .with_func_manager(Arc::new(SimpleFunctionMetadataManager::default()))
-                .with_memory_limit_bytes(128 * 1024 * 1024)
-                .with_query_admission(Arc::new(Semaphore::new(1)))
-                .with_query_timeout(Duration::from_secs(30))
-                .build()
-                .expect("supported 128 MiB query budget");
-            let query = Query::new(
-                QueryContext {
-                    input: Arc::clone(&input),
-                },
-                input.request.expression.clone(),
-            );
-            let machine = dispatcher.build_query_state_machine(query).await.expect("query machine");
-            let url = ObjectStoreUrl::parse(format!("s3://{}", input.bucket)).expect("store URL");
-            let store = machine.session.inner().runtime_env().object_store(&url).expect("store");
-            let payload = "x".repeat(if rows == 1000 { 32 } else { 8192 });
-            let data = format!("{{\"payload\":\"{payload}\"}}\n").repeat(rows);
-            store
-                .put(&Path::from(input.key.as_str()), data.into_bytes().into())
-                .await
-                .expect("write valid JSON Lines object");
-            let plan = dispatcher
-                .build_logical_plan(Arc::clone(&machine))
-                .await
-                .unwrap_or_else(|error| panic!("{rows} valid records must plan from a complete bounded sample: {error}"))
-                .expect("SELECT plan");
-            let output = dispatcher.execute_logical_plan(plan, machine).await.expect("execute query");
-            let mut stream = output.into_record_batch_stream().expect("record batches");
-            let mut returned = 0;
-            while let Some(batch) = stream.next().await {
-                let batch = batch.expect("decode supported records");
-                returned += batch.num_rows();
-            }
-            assert_eq!(returned, 1, "LIMIT 1 yields one supported row for {rows} input records");
-        }
-    }
-
     #[tokio::test]
     async fn bounded_json_lines_prefix_rejects_overflow_without_truncating_records() {
         let mut input = test_input();
@@ -1960,9 +1906,9 @@ mod tests {
         input.request.input_serialization.json = Some(JSONInput {
             type_: Some(JSONType::from_static(JSONType::LINES)),
         });
-        // No complete record fits the inference prefix. Do not manufacture a
-        // partial record for the JSON parser at the byte limit.
-        let data = format!("{{\"payload\":\"{}\"}}\n", "x".repeat(4 * 1024 * 1024));
+        // Individually valid records remain below the existing 1 MiB record
+        // limit. The 1000-record sample, however, cannot fit its byte budget.
+        let data = format!("{{\"payload\":\"{}\"}}\n", "x".repeat(8192)).repeat(600);
         let (dispatcher, machine) = bounded_inference_machine(input, data.into_bytes()).await;
         let error = dispatcher.build_logical_plan(machine).await.err().expect("prefix limit");
         assert!(error.to_string().contains("4 MiB prefix"), "{error}");

crates/s3select-query/src/dispatcher/schema_inference.rs

@@ -98,7 +98,7 @@ impl PrefixBoundary {
                         reader.read_record(&input[consumed..], &mut output, &mut ends[..MAX_SELECT_COLUMNS + 1 - *fields]);
                     consumed += read;
                     *fields += count;
-                    if *fields > MAX_SELECT_COLUMNS {
+                    if *fields > MAX_SELECT_COLUMNS || result == ReadRecordResult::OutputEndsFull {
                         return Err(DataFusionError::ResourcesExhausted(
                             "S3 Select CSV schema exceeds 1024 columns".to_owned(),
                         ));
@@ -120,9 +120,8 @@ impl PrefixBoundary {
     }
 }
 
-/// Reads only complete inference records. Hitting the byte bound with no
-/// complete record is an error; with at least one complete record the sample
-/// is cut there, exactly as a short object or the record count would cut it.
+/// Reads only complete inference records. Hitting the byte bound is an error,
+/// never a shorter successful sample that can silently omit later fields.
 pub(super) async fn read_schema_prefix(
     mut stream: BoxStream<'static, std::result::Result<Bytes, ObjectStoreError>>,
     memory_pool: &Arc<dyn MemoryPool>,
@@ -136,12 +135,6 @@ pub(super) async fn read_schema_prefix(
         while offset < chunk.len() {
             let remaining = MAX_SCHEMA_PREFIX_BYTES - bytes.len();
             if remaining == 0 {
-                if let PrefixBoundary::JsonLines { records, line_start } = &boundary
-                    && *records > 0
-                {
-                    bytes.truncate(*line_start);
-                    return Ok(SchemaPrefix { bytes, reservation });
-                }
                 return Err(prefix_limit_error());
             }
             let end = chunk.len().min(offset + remaining);
@@ -219,65 +212,6 @@ mod tests {
         Arc::new(GreedyMemoryPool::new(64 * 1024 * 1024))
     }
 
-    #[tokio::test]
-    async fn bounded_json_prefix_keeps_only_complete_records_at_byte_limit() {
-        let first = b"{\"a\":1}\n";
-        let mut input = first.to_vec();
-        input.extend_from_slice(b"{\"a\":\"");
-        input.extend(std::iter::repeat_n(b'x', MAX_SCHEMA_PREFIX_BYTES));
-        input.extend_from_slice(b"\"}\n");
-        for width in [8192, MAX_SCHEMA_PREFIX_BYTES + 128] {
-            let chunks = input
-                .chunks(width)
-                .map(|chunk| Ok(Bytes::copy_from_slice(chunk)))
-                .collect::<Vec<_>>();
-            let prefix = read_schema_prefix(stream::iter(chunks).boxed(), &pool(), PrefixBoundary::json_lines())
-                .await
-                .expect("complete first record fits the inference sample");
-            let schema = super::super::json_lines_schema::infer_schema(&prefix.bytes, DEFAULT_SCHEMA_INFER_MAX_RECORD)
-                .expect("a complete JSON Lines object must not produce a truncated schema record");
-            assert_eq!(prefix.bytes, first, "the incomplete second record is outside the sample");
-            assert_eq!(schema.fields().len(), 1);
-        }
-    }
-
-    #[tokio::test]
-    async fn bounded_json_prefix_empty_and_complete_record_controls() {
-        for input in [b"".as_slice(), b"{\"a\":1}\n".as_slice(), b"{\"a\":1}".as_slice()] {
-            let prefix = read_schema_prefix(
-                stream::iter([Ok(Bytes::copy_from_slice(input))]).boxed(),
-                &pool(),
-                PrefixBoundary::json_lines(),
-            )
-            .await
-            .expect("short complete input is accepted");
-            let schema = super::super::json_lines_schema::infer_schema(&prefix.bytes, DEFAULT_SCHEMA_INFER_MAX_RECORD)
-                .expect("short JSON Lines schema");
-            assert_eq!(schema.fields().len(), usize::from(!input.is_empty()));
-        }
-    }
-
-    #[tokio::test]
-    async fn bounded_schema_prefix_rejects_an_incomplete_first_record() {
-        for json in [true, false] {
-            let bytes = if json {
-                format!("{{\"payload\":\"{}\"}}\n", "x".repeat(MAX_SCHEMA_PREFIX_BYTES))
-            } else {
-                format!("\"{}\"\n", "x".repeat(MAX_SCHEMA_PREFIX_BYTES))
-            };
-            let boundary = if json {
-                PrefixBoundary::json_lines()
-            } else {
-                PrefixBoundary::csv(&CsvFormat::default())
-            };
-            let error = read_schema_prefix(stream::iter([Ok(Bytes::from(bytes))]).boxed(), &pool(), boundary)
-                .await
-                .err()
-                .expect("a partial first record cannot provide an inference sample");
-            assert!(error.to_string().contains("4 MiB prefix"), "{error}");
-        }
-    }
-
     #[tokio::test]
     async fn bounded_schema_prefix_stops_polling_after_complete_sample() {
         let polls = Arc::new(AtomicUsize::new(0));

docs/architecture/s3-compatibility-matrix.md

@@ -117,23 +117,6 @@ Object keys are stored as file-system paths under each drive (`{drive}/{bucket}/
 | Object key with a `.` or `..` path segment, or an empty segment (`//`), such as `a//b/./c/../d` | `400 InvalidArgument` (`check_object_args` in `crates/ecstore/src/bucket/utils.rs`, mirroring MinIO `IsValidObjectPrefix`) | Accepted as an opaque key | A `..` segment would resolve to a parent directory and `.`/`//` segments would alias other keys on disk; encoding them would change the MinIO-compatible on-disk format. |
 | Directory marker (key ending in `/`, with or without a body) in a versioned bucket | Stored as the null version: `PutObject`, `CopyObject`, `HeadObject`, and `ListObjectVersions` report version id `null`, and a later PUT of the same key overwrites in place (`put_opts` in `rustfs/src/storage/options.rs`, mirroring MinIO `putOpts`: "for directory objects skip creating new versions") | A real version id per PUT, with a version history | The marker only exists to make an empty prefix listable; keeping a history for it would leave hidden versions behind every prefix delete. Replication still copies the marker as its null version (`test_bucket_replication_replicates_directory_marker_in_versioned_bucket` in `crates/e2e_test/src/replication_extension_test.rs`). |
 
-## Request Validation and Operational Bounds
-
-These behaviors are client-visible and differ from AWS S3 or MinIO. They bound hostile or malformed input; a client that depends on the permissive behavior must adapt.
-
-| Behavior | RustFS | AWS S3 / MinIO | Why |
-|---|---|---|---|
-| SigV2 configuration PUT (`?acl`, `?policy`, `?lifecycle`, `?versioning`, `?tagging`, `?retention`, `?legal-hold`, and the other bucket/object configuration subresources) without `Content-MD5` | `400 MissingContentMD5` (`control_body_digest_required` in `rustfs/src/server/http.rs`) unless the request declares `Content-Length: 0`, which has nothing to replay; a body without that header is refused even when it turns out empty | AWS requires `Content-MD5` for most of these APIs; MinIO verifies it only when present | SigV2 signs headers, not the body, so an unsigned configuration body could be replayed with other content. SigV4 keeps the optional-MD5 contract because its payload hash binds the body. |
-| `DeleteObjects` without `Content-MD5` | `400 MissingContentMD5`; under SigV4 an `x-amz-checksum-*` header substitutes for the digest, under SigV2 it does not | Same on AWS and MinIO (MinIO answers `MissingContentMD5`) | Parity; a supplied digest must match. |
-| Object key containing a C0 control character other than tab, LF, or CR (`PutObject`, `POST`, `CopyObject` destination, `CreateMultipartUpload`, `CompleteMultipartUpload`) | `400 InvalidArgument` | Accepted as an opaque key | Such keys are unprintable in listings and logs and cannot round-trip through XML 1.0 responses. Existing stored keys still load; inbound replication and on-demand migration of such a key from another server are refused the same way. |
-| Lifecycle rule `<ID>` containing a control character (including a tab or line break inside the element) | `400 InvalidArgument` on `PutBucketLifecycleConfiguration` | MinIO validates only the length | The ID is echoed in the `x-amz-expiration` response header; a line break there would inject a header. Stored configurations still load. |
-| Bucket tag key or value outside the AWS tag character class (`[\p{L}\p{N}\p{Z}+\-=._:/@]`) | `400 InvalidTag` on `PutBucketTagging` | AWS enforces the same class; MinIO is more permissive | AWS parity. Legacy stored tags still load. |
-| `CopyObject` with the default `x-amz-tagging-directive: COPY` from a source that carries tags | `403 AccessDenied` unless the caller also holds `s3:GetObjectTagging` (or `s3:GetObjectVersionTagging`) on the source; the destination's `s3:RequestObjectTag/*` conditions are evaluated against the copied tag set | AWS and MinIO copy the tags with `s3:GetObject` alone | A copy would otherwise disclose source tags to a principal that may not read them, and source tags could satisfy destination conditions the caller cannot assert (rustfs/backlog#3034). Use `REPLACE` with an explicit `x-amz-tagging` to copy without the tagging grant. |
-| `ListObjectVersions` / `ListObjectsV2` with the include-deleted extension | Requires `s3:ListBucketVersions` | Vendor extension | Delete markers and non-current versions are version-listing data. |
-| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits the request budget (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 4096 entries per drive, one entry per 64 plain children); `503 SlowDown` beyond that, deterministically, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions under the prefix and is bounded so a hostile prefix cannot pin the request. Prefixes with more than 262,144 direct children per drive, or roughly 1,400 to 4,096 UUID-named children, exceed the bound (rustfs/backlog#3038 residual). |
-| `PutObject` / `UploadPart` body delivering less than 64 KiB per 60 seconds of server wait | `400 RequestTimeout` (the minimum-progress floor, logged as `upload_body_progress_stalled`); independent of `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`, including when that timeout is 0 | No minimum upload rate | A trickle upload otherwise pins upload admission and its buffers indefinitely; roughly 1.1 KiB/s is below any practical client. |
-| First write after `SetBucketQuota`, or after the quota changes, on a bucket without an authoritative usage ledger | `503 SlowDown` (retryable) until a background scan of the bucket publishes the exact ledger; a failed scan retries after 30 seconds | AWS has no bucket quota; MinIO admits with unknown usage | Quota admission uses only the authoritative ledger (scanner reporting is informational, see `reporting_retirement` tests), and the request must not hold its bucket metadata guard across a whole-bucket listing. The window is one bucket listing long. |
-
 ## Update Rule
 
 When a feature starts passing, move its test entries from `unimplemented_tests.txt` to `implemented_tests.txt` and update the row here in the same PR. Do not change README wording beyond the supported coverage. Handler-level status (missing, stubbed, or diverging endpoints) is tracked in [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md).

docs/architecture/s3-tables-support-matrix.md

@@ -26,7 +26,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
 | `/_iceberg/v1` | Supported compatibility alias | MinIO AIStor-style alias; smoke profile defaults to signing name `s3tables`. |
 | S3 object data plane | Supported | Data, metadata, manifest, and delete files are ordinary S3 objects with table-aware policy checks on warehouse paths. |
 | S3 table-bucket listings | Supported | `ListObjects`, `ListObjectsV2`, `ListObjectVersions`, metadata-list extensions, and multipart-upload listings omit warehouse objects unless the caller has `admin:GetTableMetadata` for that table, and always omit internal `.rustfs-table` keys. Protected cursors are encrypted with a key derived from the cluster root credentials, so rotating those credentials invalidates cursors already issued. |
-| Object-backed warehouse-index recovery | Supported with bounded request fallback | Data-plane requests repair or verify a missing warehouse-prefix index by listing the catalog prefix. Only table entries count toward the 4,096-object limit, so maintenance reports and other per-table objects cannot exhaust it; the listing itself is bounded to 64 pages plus one page per table entry already listed, and after each page the cursor skips the rest of the maintenance subtree the page ended in, so a report flood under one table costs a bounded number of pages. Larger catalogs fail closed with a retryable service error and require operator backfill. An optional table-entry directory (see Maintenance Matrix) answers these lookups without the prefix listing. |
+| Object-backed warehouse-index recovery | Supported with bounded request fallback | Data-plane requests repair or verify a missing warehouse-prefix index against bounded pages totaling at most 4,096 catalog objects. Larger catalogs fail closed with a retryable service error and require `POST /iceberg/v1/{warehouse}/catalog/warehouse-index/backfill` with `admin:MigrateTableCatalog` instead of allowing one S3 request to trigger an unbounded scan. |
 | Table bucket enablement | Supported | A regular bucket is enabled for catalog use and addressed as the REST catalog warehouse. |
 | Catalog-vended table credentials | Automated when enabled | Disabled by default. LoadTable vends credentials only when `X-Iceberg-Access-Delegation` contains the exact `vended-credentials` token; the dedicated credentials endpoint uses the same issuer path. |
 | AWS S3 Tables endpoint shape | Profile generator | Generates the AWS catalog URI and warehouse ARN shape for migration docs. API parity not claimed. |
@@ -106,26 +106,12 @@ expiration and compaction plan/commit, remains object-backed only. A configured
 server has opt-in, allowlisted periodic queue admission; see the
 [scheduler runbook](../operations/s3-tables-maintenance-scheduler.md).
 
-Renamed tables retain their original metadata paths. Scheduling, planning,
-and cleanup on both backings fail closed with a conflict if another table
-shares a scanned metadata or warehouse directory, including a recreated source
-name after rename. The object backing rechecks this ownership inventory under
-the bucket publication fence before every delete pass; the inventory reads every
-table entry in the bucket, so maintenance cost grows with the table count. Such
-tables require disjoint cleanup roots before maintenance can run; enabling
+Renamed tables retain their original metadata paths. Durable-strong scheduling
+and cleanup fail closed with a conflict if another table shares a scanned
+metadata or warehouse directory, including a recreated source name after rename.
+Such tables require disjoint cleanup roots before maintenance can run; enabling
 deletion does not override this ownership check.
 
-Object-backed create, register, and commit publication share one preparation deadline, `RUSTFS_TABLE_CATALOG_PUBLICATION_PREPARATION_TIMEOUT_SECS` (default 30, never below the object lock-acquire timeout, capped at 86,400). Migration and bucket-lock queueing, reference HEAD validation, ownership inventories, metadata parsing, and snapshot preparation consume that same interval; repeated preparation does not renew it. A create writes its initial metadata file before registration without ending the deadline; when registration then fails before any catalog write may have begun, including on expiry, the handler removes that file, so the worst outcome of an expired create is a small orphan under a root no table owns, left by a crash or failed delete between the write and the removal. Expiry before a possible durable catalog write returns HTTP 503 and releases the preparation locks; a valid large operation that cannot finish within the interval keeps failing on retry until the interval is raised. Once a catalog write may have begun, this deadline no longer cancels that transaction. Individual PUT completion and persisted commit/rename recovery retain their existing cancellation guarantees. This is a preparation limit, not a hard bound on disk I/O or the durable tail. The durable-strong backing does not implement this deadline; it is being retired. A rename that the client abandons mid-flight completes or recovers on a detached task, so the bucket's `active_rename_id` is always cleared.
-
-Compaction commits hold inputs to a 512 MiB stored and decoded budget per rewrite group, validate the footer (including the row-group count the decoder would reserve for) and every page header before decoding a page (a recursive or over-deep Iceberg manifest Avro schema and a Parquet page that exceeds its budget are refused with HTTP 400), and run one compaction per node at a time; a concurrent request receives HTTP 503. The planner packs each rewrite group to the smaller of `max-rewrite-bytes-per-job` and that 512 MiB stored-input ceiling, so a larger configured value is accepted but yields several groups; a group whose stored inputs still exceed the ceiling at commit time (for example a plan produced by an older release) is refused with HTTP 400 naming the ceiling before any group's output is written (every group of the plan is checked first).
-
-Object-backed commit publication identifies every catalog object it validated by the storage layer's write-unique body revision and rechecks only that identity under the bucket publication fence. Objects without such a revision (bytes the publication itself wrote before the fence, existence-only observations, and objects whose storage metadata reports no body revision) are "legacy" bytes: they are re-read and compared by SHA-256 under the fence, within a total budget of twice the maximum admin request body size (2 MiB) per publication. A publication whose legacy bytes exceed that budget fails with HTTP 503 and the message `legacy catalog objects exceed the publication validation budget; rewrite the referenced objects and retry`; rewriting the referenced metadata, manifest-list and manifest objects on the current release gives them a body revision, after which the same commit can be retried.
-
-### Table-entry directory
-
-An optional per-bucket directory of table entries under `.rustfs.sys` answers data-plane index-miss lookups without listing the catalog prefix. It is off by default: `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX=true` and `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX_FLEET_CONFIRMED=true` on every node, then `POST /iceberg/v1/{warehouse}/catalog/directory` (`admin:MigrateTableCatalog`) advances a backfill of 128 catalog objects per call until the directory reports `Ready`; `GET` reports the phase and `DELETE` rolls the directory back. Entry writers add a directory candidate only while the directory exists, and the backfill covers earlier entries. The directory state occupies the backing-migration fence object: nodes without this feature answer catalog mutations on that bucket with a parse error until they are upgraded, which is what the fleet-confirmed flag attests, and a backing migration is refused with a conflict while the directory is active.
-
-
 Durable-strong rename preserves maintenance configuration, job identifiers,
 current/latest aliases, history, audit events, and quarantine state. A persisted
 rename intent protects sidecar migration across interruption: the source remains

docs/operations/drive-timeout-tuning.md

@@ -75,7 +75,6 @@ Explicit per-operation overrides always win over the profile, so you can select
 | `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS` | `5` | `60` | Timeout for metadata reads such as `read_metadata`. |
 | `RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS` | `5` | `60` | Timeout for `disk_info()` calls. |
 | `RUSTFS_OBJECT_DISK_READ_TIMEOUT` | `10` | `60` | Per-read stall budget while streaming an object body from disk. |
-| `RUSTFS_GET_DISK_READ_IDLE_TIMEOUT_SECS` | `120` | `30` floor | How long a GET body may make no downstream progress before its disk-read admission is released and the stream fails; a client that resumes within the window is unaffected. The effective value is never below 30 seconds or twice `RUSTFS_OBJECT_DISK_READ_TIMEOUT`. |
 | `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3` | `60` | Base cooperative budget for an object-capacity disk scan. |
 | `RUSTFS_CAPACITY_MAX_TIMEOUT` | `15` | `60` | Upper bound for the dynamically adjusted object-capacity scan budget. |
 | `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` | `30` | — | Deprecated global fallback for every per-operation knob without an explicit override. Prefer the per-operation knobs. |

docs/operations/on-demand-migration.md

@@ -283,7 +283,6 @@ Five provenance keys are written on every pulled object under both internal pref
 | Bandwidth limit | Source and network saturation | `bandwidth_limit_bytes_per_sec` (minimum 64 KiB/s) on the source client |
 | Retry budget | Transient source blips | Background pulls retry a retryable failure up to 3 times (1 s / 4 s / 16 s plus jitter). Inline pulls never retry: the bytes are already on their way to the client. The SDK retry policy is disabled (`RemoteS3RetryPolicy::Disabled`); HEAD 404 also requires one bucket probe. Replication targets keep their separately declared three SDK attempts |
 | Idle timeout | A source that answers and then goes quiet mid-body | `source_timeout.idle_ms` per body chunk on both paths. The budget measures the source read, upstream of the inline tee, so a slow client is never mistaken for an idle source; when it fires the client stream ends in an error and the write-back is discarded |
-| Inline client budget | A slow or paused client holding an inline pull slot and its buffers | On an inline-teed GET the client body ends once the client accepts no data for 30 s, or when the whole transfer would average below 64 KiB/s (at least 120 s in total); the write-back keeps draining to disk, so the next GET is served locally. Ordinary GETs are bounded by `RUSTFS_GET_DISK_READ_IDLE_TIMEOUT_SECS` instead. |
 | Anti-loop marker | Migration chains between RustFS/MinIO deployments | Every source request carries `x-rustfs-source-proxy-request` and `x-minio-source-proxy-request`; a request carrying it is always answered locally |
 | Outbound endpoint policy | SSRF | See [outbound-connection-policy.md](outbound-connection-policy.md) |
 

docs/operations/pool-metadata-recovery.md

@@ -42,11 +42,9 @@ When promotion is committed and those checks remain valid, a failed peer reload
 
 A process kill can leave a pool with two PREPARE heads and two COMMIT heads for the same V3 transaction. Repeating the ordinary read cannot repair that split: the latest-head selection still requires its original quorum. Startup reconciliation holds the same distributed `pool.bin` writer fence as runtime recovery.
 
-Conditional native recovery is limited to modern, single-part inline metadata. An ordinary readable COMMIT in another pool authorizes completing that transaction. Recovery requires one matching cluster identity and epoch on every disk and complete readable current and rollback evidence. Every current shard must match the selected transaction; older backup generations remain subject to full integrity validation. Reconstruction verifies bitrot and erasure integrity; two reconstructable shards do not establish commit authority. Missing, conflicting, unsupported, or unexplained higher-generation evidence remains an operator recovery case.
+Conditional native recovery is limited to modern, single-part inline metadata. It requires an ordinary readable COMMIT in another pool, one matching cluster identity and epoch on every disk, and complete readable current and rollback evidence. Every shard must match that COMMIT, its exact PREPARE, or a fully verified older-generation backup. Reconstruction verifies bitrot and erasure integrity; two reconstructable shards do not establish commit authority. Missing, conflicting, unsupported, or higher-generation evidence remains an operator recovery case.
 
-If no ordinary COMMIT selects the interrupted transaction, an ordinary readable V3 PREPARE can authorize restoring its embedded previous V3 COMMIT. Recovery does not promote the prepared state. It repairs unreadable pools first, retaining an ordinary PREPARE until every unreadable pool has been restored. The restored ordinary COMMIT then carries that authority if rewriting the last PREPARE is interrupted. A retained COMMIT backup from the interrupted transaction or competing ordinary PREPARE records prevents rollback. Retained PREPARE backups from an abandoned transaction do not prevent a later transaction from reusing that generation only when both have the same exact predecessor; those backups never grant commit authority.
-
-Recovery checks each disk ID, current SHA and complete backup inventory inside the native metadata and publication leases. Identity shards are validated on every member of their own hashed erasure set before Apply and after normal CAS/readback, under the same global writer fence. Co-located identity bytes are additionally pinned and bound into that disk's inventory SHA; co-location is not required. It publishes the proven COMMIT or acknowledged predecessor with a fresh physical file identity, retains rollback evidence, then performs normal conditional V3 writes and ordinary quorum readback. Partial repair or fence loss keeps writes blocked. A verified identity or transaction conflict retains `RecoveryRequired` in the original write state even if partial repair made the head readable; a later ordinary quorum cannot clear that known conflict. Transport failures, unsupported recovery layouts and missing evidence do not acquire a permanent validation latch merely because recovery could not verify them.
+Recovery checks each disk ID, current SHA and complete backup inventory inside the native metadata and publication leases. Identity shards are validated on every member of their own hashed erasure set before Apply and after normal CAS/readback, under the same global writer fence. Co-located identity bytes are additionally pinned and bound into that disk's inventory SHA; co-location is not required. It publishes the proven COMMIT with a fresh physical file identity, retains rollback evidence, then performs normal conditional V3 writes and ordinary quorum readback. Partial repair or fence loss keeps writes blocked. A verified identity or transaction conflict retains `RecoveryRequired` in the original write state even if partial repair made the head readable; a later ordinary quorum cannot clear that known conflict. Transport failures, unsupported recovery layouts and missing evidence do not acquire a permanent validation latch merely because recovery could not verify them.
 
 Raw local mutators reject protected source and destination paths before I/O, including accepted volume/path aliases that resolve into the `pool.bin` namespace. The shared `protected_path_kind` helper in `crates/ecstore/src/disk/local/pool_meta_native.rs` follows the native path resolver. Conditional metadata writes and Undo require the canonical object key so aliases cannot create another mutation-lease domain.
 

docs/operations/replacement-generation-recovery.md

@@ -192,12 +192,6 @@ A successor that has already started retains its recorded scan plan.
 Keep the new binary for an unfinished schema 7 recovery. An older binary rejects
 that schema and cannot safely continue its ownership protocol. Finish and verify
 recovery before any downgrade; never edit the schema number to force acceptance.
-The same holds for a drive replacement pass in flight: this release runs the
-identity-bound bucket pass as replacement execution protocol 2 (recording the
-pass order in `replacement_bucket_pass_order`), and a previous release refuses
-such an intent with `replacement recovery conflict: replacement execution
-protocol or predecessor is invalid`. Finish drive replacements before a
-downgrade.
 
 ## Acceptance after a VM restart
 

docs/operations/replication-outbound-transport.md

@@ -28,13 +28,8 @@
 | --- | --- | --- |
 | `RUSTFS_REPLICATION_STREAMING_CHECKSUMS` | unset (plain payloads) | `true` or `1` restores SDK trailer checksums (`RequestChecksumCalculation::WhenSupported`). Every streaming upload is then `aws-chunked` with an `x-amz-trailer`, except a single-part PUT that forwards the source's `x-amz-checksum-*` header, which is sent plain so the target does not receive a second algorithm; use only when every target decodes that framing. |
 | `RUSTFS_REPLICATION_REPLICA_ETAG_VERIFY` | enabled | `false` or `0` disables the post-PUT ETag comparison for targets whose 32-hex ETags are legitimately not the content MD5. |
-| `RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS` | `60` | SDK operation budget for replication requests, including retries; also the response deadline after an upload finishes sending its body. Values outside `1..=86400` fall back to the default. |
-| `RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS` | `900` | Independent operation budget for multipart completion, including a target that sends keepalive whitespace without completing. Values outside `1..=86400` fall back to the default. |
-| `RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC` | `1048576` | Rate used to calculate upload transmission headroom. Zero falls back to the default; positive values are accepted. This is a timeout calculation, not a bandwidth throttle. |
 
-These settings are read by the RustFS process that owns the replication target, at client build time; restart the server after changing them.
-
-PUT and UploadPart use an operation budget of the request timeout plus `ceil(body_bytes / minimum_transfer_rate) * max_attempts` seconds. The current retry policy allows three attempts; the separate response deadline still starts after each upload finishes sending. Multipart completion uses its own completion budget. These defaults apply to every replication target class, including custom S3-compatible endpoints; increase them when a supported target needs longer to acknowledge a request.
+Both knobs are read by the RustFS process that owns the replication target, at client build time; restart the server after changing them.
 
 ### Remote tier transport timeouts
 

docs/operations/reverse-proxy.md

@@ -2,7 +2,7 @@
 
 **Use this when:** a request succeeds against `http://<host>:9000` directly but fails, hangs, or resets through Caddy, Nginx, HAProxy, or Cloudflare.
 
-**Source of truth:** `crates/config/src/constants/tls.rs` (`DEFAULT_HTTP1_HEADER_READ_TIMEOUT`, `DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT`); the `upload_body_progress_stalled` log event.
+**Source of truth:** `crates/config/src/constants/tls.rs` (`DEFAULT_HTTP1_HEADER_READ_TIMEOUT`, `DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT`); the `put_object_body_read_stalled` log event.
 
 RustFS speaks plain S3 over HTTP/1.1 and HTTP/2. Most proxy problems are not RustFS storage bugs: if the same request works directly against `:9000`, the fault is in proxy/CDN request forwarding. Use the checklist below to find which forwarding behavior broke.
 
@@ -108,7 +108,7 @@ Cloudflare's proxy may buffer the entire request body before forwarding and can
 1. Bypass the proxy. Send the failing request to `http://<host>:9000` directly. Success confirms the fault is in the proxy/CDN path.
 2. Bypass the CDN, keep the proxy. Point the proxy straight at the origin (Cloudflare grey cloud / direct DNS). If it now works, the CDN was buffering or re-chunking the body.
 3. Check idle reuse. Intermittent failures that correlate with upload size are almost always the keep-alive mismatch. Lower the proxy keepalive (or disable it) and retry.
-4. Check for a stalled body. A client or intermediary can stop forwarding data without closing the connection. `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` defaults to 300 seconds. `PutObject` and `UploadPart` log `upload_body_progress_stalled` and return `RequestTimeout` (HTTP 400) when a body makes no progress for the timeout, or less than 64 KiB of progress in 60 seconds of server wait (the minimum-progress floor, which applies even when the timeout is set to 0); the log records `raw_bytes_received`, `expected_decoded_bytes`, `timeout_secs`, bucket, key, and request ID. The event identifies missing input progress, without attributing the cause to a particular proxy.
+4. Check for a stalled body. A client or intermediary can stop forwarding data without closing the connection. `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` defaults to 300 seconds. `PutObject` logs `put_object_body_read_stalled` when its body-read guard expires. `UploadPart` logs `upload_part_body_read_stalled` and returns `RequestTimeout` (HTTP 400); its log records `raw_bytes_received`, `expected_decoded_bytes`, `timeout_secs`, bucket, key, and request ID. The event identifies missing input progress, without attributing the cause to a particular proxy.
 5. Compare bytes. Confirm the proxy forwards exactly `Content-Length` body bytes with no compression or transformation.
 6. Confirm signed headers survive. `Host` and `x-amz-*` must reach RustFS unchanged; a `SignatureDoesNotMatch` (rather than a hang) points here.
 

docs/operations/s3-tables-maintenance-scheduler.md

@@ -30,7 +30,7 @@ For example:
 }
 ```
 
-`max-rewrite-bytes-per-job` is accepted up to any value at or above `target-file-size-bytes`, but each rewrite group is packed to at most the smaller of that value and the 512 MiB per-group stored-input ceiling that the commit path holds in memory; the 1 GiB example therefore produces groups of at most 512 MiB. The optional snapshot-expiration and compaction fields request plans, not commits. An empty JSON object requests ordinary metadata planning. A nonzero retention value overrides the effective maintenance configuration for this request; omission or zero uses the effective configured retention.
+The optional snapshot-expiration and compaction fields request plans, not commits. An empty JSON object requests ordinary metadata planning. A nonzero retention value overrides the effective maintenance configuration for this request; omission or zero uses the effective configured retention.
 
 In durable-strong mode, successful planning returns HTTP 200 with a completed `DRY_RUN` job and persists its named, current, and latest reports with audit events. Read-only means that table data, metadata pointers, and catalog generations are unchanged; the request still writes maintenance report sidecars and requires the table-scoped `admin:RunTableMaintenance` permission. Inspect the returned plan and any manual-review state before taking further action.
 

docs/operations/scanner-runtime-controls.md

@@ -633,8 +633,6 @@ Admin heal has three separate budgets. Increasing one does not extend the others
 
 Automatic replacement retries a transient object repair twice within its current page, waiting 250 ms and then 500 ms without holding page capacity. An exhausted retry remains unverified and still triggers the existing generation retry; target readback and checkpoint requirements are unchanged.
 
-Recursive admin traversal permits a small forward probe after an isolated `SlowDown`. Once three retained objects have reported rate pressure, it pauses admission of new objects and pages until a due retry finishes and frees that capacity. Existing retries keep their original age and attempt limits, and the unstarted page tail remains owned by the traversal. Lock contention still uses the normal bounded window, so an isolated busy object does not block healthy pages.
-
 For a large admin heal, select a finite configured execution budget appropriate to the expected work and contention, and inspect progress while it runs. Investigate stalled work and object failures before increasing this budget; a larger timeout must not hide lock or quorum problems. A longer HTTP or internode timeout does not keep a repair task alive past its execution budget. A successful start response returns the canonical token for accepted or merged work; it does not prove that repair has completed. Query that token without `forceStart`; terminal timeout/cancellation reports retain completed progress while the task report remains available.
 
 Capability preflight runs before constructing and admitting a start request. However, the public `cluster heal coordination unavailable` error can also follow a transport failure or an invalid coordinator response after a request was sent. An unknown or lost HTTP response is therefore not proof that no task was admitted.

docs/testing/distributed-e2e.md

@@ -112,8 +112,6 @@ The EC:2 metadata-delay case also holds the three scoped `ListDir` RPCs only aft
 
 The four-node rolling and direct upgrade cases seed a multipart upload on the verified previous release. Rolling checks compare part number, size and ETag through old and new nodes while multipart mutations are paused. With one old disk offline and the other three part directories temporarily absent, metadata remains readable: the previous coordinator returns an empty list while the current coordinator returns HTTP 503 `SlowDownRead`. After the directories and disk return, both coordinators must list the original ETag and all four original metadata, part and part-metadata files must remain byte-identical. Once all nodes are current, parity-zero probes prove each coordinator can write through every disk before the single completion request; both upgrade cases then check every reader. This read-only fault case does not permit multipart mutations during a mixed rollout.
 
-The rolling case binds its DELETE expectation to the previous binary's source revision. The pinned 1.0.4 source already supports the delete recovery protocol: a separate old-writer object must be deleted successfully through the current node and remain absent through every reader during the mixed phase, after full upgrade, and after rollback. The earlier CI source pins retain their HTTP 503 and seed-preservation assertions. The original seed, traffic volume, MPU assertions, and deadlines are unchanged.
-
 The nightly stale-multipart cleanup case also retains one and two EC:2 source copies on online disks. It waits for a path-specific background recheck diagnostic, verifies every surviving directory and file byte, and checks that the diagnostic is not attributed to a client UploadPart. This is evidence for conservative preservation and diagnosis, not automatic reclamation without a closure certificate. Node logs are retained when `RUSTFS_E2E_LOG_DIR` is set.
 
 The `distributed::multipart_fence_acceptance_test` cases separate remote storage RPC delays from lock RPC delays through a method-selective proxy. They require the selected RPC to reach its pause before checking quorum, recovery, or cancellation. The proxy forwards opaque bodies and gRPC trailers and is confined to the test process. It does not change server timeouts, locking, retry policy, or the commit path.

entrypoint.sh

@@ -322,207 +322,4 @@ else
   echo "Starting: $*"
 fi
 
-BOOTSTRAP_DIR="${RUSTFS_BOOTSTRAP_DIR:-/etc/rustfs/bootstrap}"
-BUCKETS_FILE="$BOOTSTRAP_DIR/buckets.json"
-
-bootstrap_credential() {
-  SOURCE="$1"
-  LABEL="$2"
-  case "$SOURCE" in
-    value:*) printf '%s' "${SOURCE#value:}" ;;
-    file:*)
-      FILE="${SOURCE#file:}"
-      IFS= read -r VALUE < "$FILE" || :
-      trim_credential "$VALUE"
-      ;;
-    *)
-      echo "ERROR: Bootstrap requires an explicit $LABEL or its _FILE variant." >&2
-      return 1
-      ;;
-  esac
-}
-
-bootstrap_bucket_config() {
-  BUCKET="$1"
-  CONFIG_DIR="$2"
-  CONFIG_FILE="$CONFIG_DIR/$BUCKET.json"
-  [ -f "$CONFIG_FILE" ] || return 0
-  if ! jq -e --arg bucket "$BUCKET" 'type == "array" and index($bucket) != null' "$BUCKETS_FILE" >/dev/null; then
-    echo "ERROR: Bootstrap configuration '$CONFIG_FILE' refers to a bucket absent from buckets.json." >&2
-    return 1
-  fi
-  return 0
-}
-
-cleanup_bootstrap() {
-  if [ -n "${RUSTFS_PID:-}" ]; then
-    kill "$RUSTFS_PID" 2>/dev/null || true
-    wait "$RUSTFS_PID" 2>/dev/null || true
-  fi
-  if [ -n "${RC_CONFIG_DIR:-}" ]; then
-    rm -rf "$RC_CONFIG_DIR" 2>/dev/null || true
-  fi
-}
-
-apply_bucket_bootstrap() {
-  BOOTSTRAP_ENDPOINT="${RUSTFS_BOOTSTRAP_ENDPOINT:-}"
-  case "$BOOTSTRAP_ENDPOINT" in
-    http://*|https://*) ;;
-    *) echo "ERROR: RUSTFS_BOOTSTRAP_ENDPOINT must be an http(s) S3 endpoint when buckets are configured." >&2; return 1 ;;
-  esac
-
-  BOOTSTRAP_ACCESS_KEY=$(bootstrap_credential "$ACCESS_SOURCE" RUSTFS_ACCESS_KEY) || return 1
-  BOOTSTRAP_SECRET_KEY=$(bootstrap_credential "$SECRET_SOURCE" RUSTFS_SECRET_KEY) || return 1
-  ENCODED_ACCESS_KEY=$(jq -rn --arg value "$BOOTSTRAP_ACCESS_KEY" '$value | @uri') || return 1
-  ENCODED_SECRET_KEY=$(jq -rn --arg value "$BOOTSTRAP_SECRET_KEY" '$value | @uri') || return 1
-  ENDPOINT_AUTHORITY="${BOOTSTRAP_ENDPOINT#*://}"
-  case "$ENDPOINT_AUTHORITY" in
-    */*) ENDPOINT_AUTHORITY="${ENDPOINT_AUTHORITY%%/*}" ;;
-  esac
-  case "$BOOTSTRAP_ENDPOINT" in
-    https://*) RC_HOST_bootstrap="https://${ENCODED_ACCESS_KEY}:${ENCODED_SECRET_KEY}@${ENDPOINT_AUTHORITY}" ;;
-    http://*) RC_HOST_bootstrap="http://${ENCODED_ACCESS_KEY}:${ENCODED_SECRET_KEY}@${ENDPOINT_AUTHORITY}" ;;
-  esac
-  export RC_HOST_bootstrap
-
-  RC_CONFIG_DIR=$(mktemp -d "${TMPDIR:-/tmp}/rustfs-bootstrap.XXXXXX") || return 1
-  export RC_CONFIG_DIR
-
-  BUCKETS=$(jq -r '.[]' "$BUCKETS_FILE")
-  for bucket in $BUCKETS; do
-    echo "Ensuring bootstrap bucket: $bucket"
-    /usr/bin/rc bucket create "bootstrap/$bucket" --ignore-existing || return 1
-    /usr/bin/rc bucket public-access-block set "bootstrap/$bucket" "$BOOTSTRAP_DIR/public-access-block.json" || return 1
-  done
-
-  for bucket in $BUCKETS; do
-    for config_kind in policies versioning lifecycle cors notifications; do
-      CONFIG_FILE="$BOOTSTRAP_DIR/$config_kind/$bucket.json"
-      [ -f "$CONFIG_FILE" ] || continue
-      bootstrap_bucket_config "$bucket" "$BOOTSTRAP_DIR/$config_kind" || return 1
-      case "$config_kind" in
-        policies)
-          /usr/bin/rc bucket anonymous set-json "$CONFIG_FILE" "bootstrap/$bucket" || return 1
-          ;;
-        versioning)
-          VERSIONING=$(jq -r 'if type == "boolean" then . else error("expected a JSON boolean") end' "$CONFIG_FILE") || return 1
-          if [ "$VERSIONING" = "true" ]; then
-            /usr/bin/rc bucket version enable "bootstrap/$bucket" || return 1
-          else
-            /usr/bin/rc bucket version suspend "bootstrap/$bucket" || return 1
-          fi
-          ;;
-        lifecycle)
-          /usr/bin/rc bucket lifecycle rule import "bootstrap/$bucket" "$CONFIG_FILE" || return 1
-          ;;
-        cors)
-          /usr/bin/rc bucket cors set "bootstrap/$bucket" "$CONFIG_FILE" || return 1
-          ;;
-        notifications)
-          while IFS="$(printf '\t')" read -r arn events; do
-            /usr/bin/rc bucket event add "bootstrap/$bucket" "$arn" --event "$events" || return 1
-          done <<EOF
-$(jq -r '.[] | [.arn, (.events | join(","))] | @tsv' "$CONFIG_FILE")
-EOF
-          ;;
-      esac
-    done
-  done
-  return 0
-}
-
-validate_bootstrap_assets() {
-  for config_kind in policies versioning lifecycle cors notifications; do
-    CONFIG_DIR="$BOOTSTRAP_DIR/$config_kind"
-    for CONFIG_FILE in "$CONFIG_DIR"/*.json; do
-      [ -f "$CONFIG_FILE" ] || continue
-      CONFIG_BUCKET=${CONFIG_FILE##*/}
-      CONFIG_BUCKET=${CONFIG_BUCKET%.json}
-      bootstrap_bucket_config "$CONFIG_BUCKET" "$CONFIG_DIR" || return 1
-      case "$config_kind" in
-        versioning)
-          jq -e 'type == "boolean"' "$CONFIG_FILE" >/dev/null || {
-            echo "ERROR: Versioning configuration '$CONFIG_FILE' must contain a JSON boolean." >&2
-            return 1
-          }
-          ;;
-        notifications)
-          jq -e 'type == "array" and all(.[]; keys == ["arn", "events"] and (.arn | type == "string" and length > 0) and (.events | type == "array" and length > 0 and all(.[]; type == "string" and length > 0))) and ((map(.arn) | unique | length) == length)' "$CONFIG_FILE" >/dev/null || {
-            echo "ERROR: Notification configuration '$CONFIG_FILE' must be an array of unique arn/events objects." >&2
-            return 1
-          }
-          ;;
-      esac
-    done
-  done
-}
-
-if [ "$1" = "/usr/bin/rustfs" ]; then
-  if [ ! -r "$BUCKETS_FILE" ] || ! jq -e '. as $buckets | (type == "array" and all(.[]; type == "string" and test("^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$") and (contains("..") | not))) and ((unique | length) == ($buckets | length))' "$BUCKETS_FILE" >/dev/null; then
-    echo "ERROR: $BUCKETS_FILE must be a JSON array of unique, valid S3 bucket names." >&2
-    exit 1
-  fi
-
-  validate_bootstrap_assets || exit 1
-  BOOTSTRAP_BUCKET_COUNT=$(jq 'length' "$BUCKETS_FILE")
-  if [ "$BOOTSTRAP_BUCKET_COUNT" -gt 0 ]; then
-    [ -r "$BOOTSTRAP_DIR/public-access-block.json" ] || {
-      echo "ERROR: Missing $BOOTSTRAP_DIR/public-access-block.json." >&2
-      exit 1
-    }
-    if ! jq -e 'type == "object" and keys == ["block_public_acls", "block_public_policy", "ignore_public_acls", "restrict_public_buckets"] and all(.[]; type == "boolean")' "$BOOTSTRAP_DIR/public-access-block.json" >/dev/null; then
-      echo "ERROR: Public Access Block configuration must define all four boolean flags and no other fields." >&2
-      exit 1
-    fi
-    BOOTSTRAP_ENDPOINT="${RUSTFS_BOOTSTRAP_ENDPOINT:-}"
-    case "$BOOTSTRAP_ENDPOINT" in
-      http://*|https://*) ;;
-      *) echo "ERROR: RUSTFS_BOOTSTRAP_ENDPOINT is required when buckets are configured." >&2; exit 1 ;;
-    esac
-    ENDPOINT_AUTHORITY="${BOOTSTRAP_ENDPOINT#*://}"
-    case "$ENDPOINT_AUTHORITY" in
-      */*|*\?*|*\#*|*@*) echo "ERROR: RUSTFS_BOOTSTRAP_ENDPOINT must contain only the scheme, host, and optional port." >&2; exit 1 ;;
-    esac
-    bootstrap_credential "$ACCESS_SOURCE" RUSTFS_ACCESS_KEY >/dev/null || exit 1
-    bootstrap_credential "$SECRET_SOURCE" RUSTFS_SECRET_KEY >/dev/null || exit 1
-    /usr/bin/rc bucket public-access-block --help >/dev/null 2>&1 || {
-      echo "ERROR: The bundled RustFS CLI does not support bucket Public Access Block; rebuild with an rc release that includes bootstrap support." >&2
-      exit 1
-    }
-    BOOTSTRAP_TIMEOUT="${RUSTFS_BOOTSTRAP_TIMEOUT:-120}"
-    case "$BOOTSTRAP_TIMEOUT" in
-      ''|*[!0-9]*|0) echo "ERROR: RUSTFS_BOOTSTRAP_TIMEOUT must be a positive integer number of seconds." >&2; exit 1 ;;
-    esac
-    [ "$BOOTSTRAP_TIMEOUT" -le 3600 ] || { echo "ERROR: RUSTFS_BOOTSTRAP_TIMEOUT must not exceed 3600 seconds." >&2; exit 1; }
-    MAX_BOOTSTRAP_ATTEMPTS=$(((BOOTSTRAP_TIMEOUT + 1) / 2))
-
-    "$@" &
-    RUSTFS_PID=$!
-    trap cleanup_bootstrap EXIT
-    trap 'exit 129' HUP
-    trap 'exit 130' INT
-    trap 'exit 143' TERM
-
-    BOOTSTRAP_HEALTH_URL="${BOOTSTRAP_ENDPOINT%/}/health/ready"
-    attempt=0
-    while ! curl -fsS "$BOOTSTRAP_HEALTH_URL" >/dev/null 2>&1; do
-      kill -0 "$RUSTFS_PID" 2>/dev/null || { echo "ERROR: RustFS exited before becoming ready for bootstrap." >&2; exit 1; }
-      attempt=$((attempt + 1))
-      [ "$attempt" -lt "$MAX_BOOTSTRAP_ATTEMPTS" ] || { echo "ERROR: RustFS did not become ready for bootstrap at RUSTFS_BOOTSTRAP_ENDPOINT within RUSTFS_BOOTSTRAP_TIMEOUT." >&2; exit 1; }
-      sleep 2
-    done
-
-    set +e
-    apply_bucket_bootstrap
-    BOOTSTRAP_STATUS=$?
-    set -e
-    if [ "$BOOTSTRAP_STATUS" -ne 0 ]; then
-      echo "ERROR: RustFS bootstrap failed." >&2
-      exit "$BOOTSTRAP_STATUS"
-    fi
-    wait "$RUSTFS_PID"
-    exit $?
-  fi
-fi
-
 exec "$@"

rustfs/src/admin/handlers/table_catalog/config.rs

@@ -293,46 +293,3 @@ impl Operation for SyncExternalCatalogBridgeHandler {
         build_json_response(StatusCode::OK, &response)
     }
 }
-
-pub struct GetTableDirectoryHandler {}
-
-#[async_trait::async_trait]
-impl Operation for GetTableDirectoryHandler {
-    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let warehouse = warehouse_from_params(&params)?;
-        let resource = TableCatalogResource::warehouse(&warehouse);
-        authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCatalogAction).await?;
-        ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
-        let store = table_catalog_object_store_from_extensions(&req.extensions)?;
-        let response = store.table_directory_status(&warehouse).await.map_err(catalog_store_error)?;
-        build_json_response(StatusCode::OK, &response)
-    }
-}
-
-pub struct AdvanceTableDirectoryHandler {}
-
-#[async_trait::async_trait]
-impl Operation for AdvanceTableDirectoryHandler {
-    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let warehouse = warehouse_from_params(&params)?;
-        authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
-        ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
-        let store = table_catalog_object_store_from_extensions(&req.extensions)?;
-        let response = store.advance_table_directory(&warehouse).await.map_err(catalog_store_error)?;
-        build_json_response(StatusCode::OK, &response)
-    }
-}
-
-pub struct DisableTableDirectoryHandler {}
-
-#[async_trait::async_trait]
-impl Operation for DisableTableDirectoryHandler {
-    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let warehouse = warehouse_from_params(&params)?;
-        authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
-        ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
-        let store = table_catalog_object_store_from_extensions(&req.extensions)?;
-        let response = store.disable_table_directory(&warehouse).await.map_err(catalog_store_error)?;
-        build_json_response(StatusCode::OK, &response)
-    }
-}

rustfs/src/admin/handlers/table_catalog/mod.rs

@@ -1364,14 +1364,8 @@ enum TableCommitObjectIdentity {
     ContentSha256(String),
 }
 
-// Legacy objects without a write-unique body identity retain SHA validation,
-// with a total byte budget independent of the manifest graph's larger limit.
-const TABLE_COMMIT_LEGACY_REVALIDATION_MAX_BYTES: usize = 2 * MAX_ADMIN_REQUEST_BODY_SIZE;
-
 #[derive(Clone, PartialEq, Eq)]
 struct TableCommitObservedObject {
-    body_revision: Option<Uuid>,
-    content_size: usize,
     identity: TableCommitObjectIdentity,
     max_size: Option<usize>,
 }
@@ -1383,7 +1377,6 @@ struct TableCommitPublicationState {
     table_fence: Option<(String, String, String)>,
     catalog_migration_read_permit: bool,
     observed_objects: BTreeMap<(String, String), TableCommitObservedObject>,
-    legacy_revalidation_bytes: usize,
     guards: Vec<crate::table_catalog::TableCatalogLockGuard>,
 }
 
@@ -1394,7 +1387,6 @@ struct TableCommitObjectBackend<B> {
     authorization_error: Arc<tokio::sync::Mutex<Option<S3Error>>>,
     publication: Arc<parking_lot::Mutex<TableCommitPublicationState>>,
     publication_fence_fleet_confirmed: bool,
-    preparation: Arc<crate::table_catalog::TablePublicationPreparation>,
 }
 
 impl<B> TableCommitObjectBackend<B>
@@ -1408,7 +1400,6 @@ where
             authorization_error: Arc::new(tokio::sync::Mutex::new(None)),
             publication: Arc::new(parking_lot::Mutex::new(TableCommitPublicationState::default())),
             publication_fence_fleet_confirmed,
-            preparation: Arc::new(crate::table_catalog::TablePublicationPreparation::default()),
         }
     }
 
@@ -1454,62 +1445,6 @@ where
         )
     }
 
-    async fn prepare_put_observation(
-        &self,
-        bucket: &str,
-        object: &str,
-        data: &[u8],
-    ) -> crate::table_catalog::TableCatalogStoreResult<TableCommitObservedObject> {
-        self.prepare_put_observation_with(bucket, object, data, true).await
-    }
-
-    /// `begins_mutation` is false for a write that precedes the catalog
-    /// transaction (see `put_object_before_publication`): it is still fenced
-    /// and authorized, but it leaves the preparation deadline armed.
-    async fn prepare_put_observation_with(
-        &self,
-        bucket: &str,
-        object: &str,
-        data: &[u8],
-        begins_mutation: bool,
-    ) -> crate::table_catalog::TableCatalogStoreResult<TableCommitObservedObject> {
-        let fence = {
-            let publication = self.publication.lock();
-            publication.bucket_fence.as_ref().map(|_| {
-                crate::table_catalog::TableCatalogObjectMutationFence::from_signals(
-                    publication
-                        .guards
-                        .iter()
-                        .map(crate::table_catalog::TableCatalogLockGuard::lock_lost_signal),
-                )
-            })
-        };
-        let authorize = self.authorize(bucket, object, S3Action::PutObjectAction);
-        if fence.is_some() {
-            self.preparation.read(self, authorize).await?;
-        } else {
-            authorize.await?;
-        }
-        self.ensure_observation_allowed(bucket, object)?;
-        let observation = TableCommitObservedObject {
-            identity: TableCommitObjectIdentity::ContentSha256(hex_sha256(data, str::to_string)),
-            body_revision: None,
-            content_size: data.len(),
-            max_size: Some(data.len()),
-        };
-        if let Some(fence) = fence {
-            // An unfenced candidate write precedes the publication transaction. Only a PUT
-            // under the bucket fence commits this preparation to finishing its durable tail.
-            fence.ensure_held()?;
-            if begins_mutation {
-                self.preparation.begin_mutation()?;
-            } else {
-                self.preparation.check_deadline()?;
-            }
-        }
-        Ok(observation)
-    }
-
     async fn authorize(&self, bucket: &str, object: &str, action: S3Action) -> crate::table_catalog::TableCatalogStoreResult<()> {
         let result = match &self.authorization {
             TableCommitObjectAuthorization::Request(req) => {
@@ -1567,62 +1502,31 @@ where
         bucket: &str,
         object: &str,
         observation: TableCommitObservedObject,
-        enforce_legacy_budget: bool,
     ) -> crate::table_catalog::TableCatalogStoreResult<()> {
         let key = (bucket.to_string(), object.to_string());
         let mut publication = self.publication.lock();
         if publication.phase == TableCommitPublicationPhase::Complete {
             return Ok(());
         }
-        let previous_legacy_bytes = publication.observed_objects.get(&key).map_or(0, |observed| {
-            if observed.body_revision.is_none() {
-                observed.content_size
-            } else {
-                0
-            }
-        });
-        let legacy_bytes = publication.legacy_revalidation_bytes - previous_legacy_bytes;
-        let legacy_bytes = legacy_bytes.checked_add(if observation.body_revision.is_none() {
-            observation.content_size
-        } else {
-            0
-        });
-        let Some(legacy_bytes) =
-            legacy_bytes.filter(|bytes| !enforce_legacy_budget || *bytes <= TABLE_COMMIT_LEGACY_REVALIDATION_MAX_BYTES)
-        else {
-            return Err(crate::table_catalog::TableCatalogStoreError::Unavailable(
-                "legacy catalog objects exceed the publication validation budget; rewrite the referenced objects and retry"
-                    .to_string(),
-            ));
-        };
         let Some(existing) = publication.observed_objects.get_mut(&key) else {
             if publication.phase != TableCommitPublicationPhase::Discovering {
                 return Err(crate::table_catalog::TableCatalogStoreError::Internal(format!(
                     "table commit accessed an unvalidated object after publication preparation: {bucket}/{object}"
                 )));
             }
-            publication.legacy_revalidation_bytes = legacy_bytes;
             publication.observed_objects.insert(key, observation);
             return Ok(());
         };
-        if existing.identity != observation.identity
-            || existing
-                .body_revision
-                .zip(observation.body_revision)
-                .is_some_and(|(before, after)| before != after)
-        {
+        if existing.identity != observation.identity {
             return Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!(
                 "table commit object changed during validation: {bucket}/{object}"
             )));
         }
-        existing.body_revision = observation.body_revision;
-        existing.content_size = observation.content_size;
         existing.max_size = match (existing.max_size, observation.max_size) {
             (Some(expected), Some(actual)) => Some(expected.min(actual)),
             (Some(expected), None) => Some(expected),
             (None, actual) => actual,
         };
-        publication.legacy_revalidation_bytes = legacy_bytes;
         Ok(())
     }
 
@@ -1634,12 +1538,7 @@ where
             None => TableCommitObjectIdentity::Missing,
             Some(object) => TableCommitObjectIdentity::ContentSha256(hex_sha256(&object.data, str::to_string)),
         };
-        TableCommitObservedObject {
-            identity,
-            max_size,
-            body_revision: object.and_then(|object| object.body_revision).filter(|id| !id.is_nil()),
-            content_size: object.map_or(0, |object| object.data.len()),
-        }
+        TableCommitObservedObject { identity, max_size }
     }
 
     fn observed_metadata(
@@ -1661,18 +1560,10 @@ where
         Ok(TableCommitObservedObject {
             identity,
             max_size: None,
-            body_revision: None,
-            content_size: 0,
         })
     }
 
     async fn begin_bucket_publication(&self, table_bucket: &str) -> crate::table_catalog::TableCatalogStoreResult<()> {
-        self.preparation
-            .read(self, self.begin_bucket_publication_inner(table_bucket))
-            .await
-    }
-
-    async fn begin_bucket_publication_inner(&self, table_bucket: &str) -> crate::table_catalog::TableCatalogStoreResult<()> {
         {
             let mut publication = self.publication.lock();
             if publication.phase != TableCommitPublicationPhase::Discovering {
@@ -1719,17 +1610,6 @@ where
         table_bucket: &str,
         namespace: &str,
         table: &str,
-    ) -> crate::table_catalog::TableCatalogStoreResult<()> {
-        self.preparation
-            .read(self, self.prepare_publication_inner(table_bucket, namespace, table))
-            .await
-    }
-
-    async fn prepare_publication_inner(
-        &self,
-        table_bucket: &str,
-        namespace: &str,
-        table: &str,
     ) -> crate::table_catalog::TableCatalogStoreResult<()> {
         let namespace = crate::table_catalog::Namespace::parse(namespace)
             .map_err(|err| crate::table_catalog::TableCatalogStoreError::Invalid(format!("invalid namespace: {err}")))?;
@@ -1739,12 +1619,6 @@ where
         let publication_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &table);
         let expected = {
             let mut publication = self.publication.lock();
-            if publication.legacy_revalidation_bytes > TABLE_COMMIT_LEGACY_REVALIDATION_MAX_BYTES {
-                return Err(crate::table_catalog::TableCatalogStoreError::Unavailable(
-                    "legacy catalog objects exceed the publication validation budget; rewrite the referenced objects and retry"
-                        .to_string(),
-                ));
-            }
             match publication.phase {
                 TableCommitPublicationPhase::Prepared if publication.table_fence.as_ref() == Some(&table_fence) => {
                     return Ok(());
@@ -1794,19 +1668,10 @@ where
                 if !matches!(&expected_object.identity, TableCommitObjectIdentity::ContentSha256(_)) {
                     continue;
                 }
-                if let Some(revision) = expected_object.body_revision {
-                    let actual = self.backend.object_metadata_unlocked(bucket, object).await?;
-                    if actual.and_then(|metadata| metadata.body_revision) != Some(revision) {
-                        return Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!(
-                            "table commit object changed before catalog publication: {bucket}/{object}"
-                        )));
-                    }
-                    continue;
-                }
-                let max_size = expected_object
-                    .max_size
-                    .map_or(expected_object.content_size, |limit| limit.min(expected_object.content_size));
-                let actual_object = self.backend.read_object_unlocked_limited(bucket, object, max_size).await?;
+                let actual_object = match expected_object.max_size {
+                    Some(max_size) => self.backend.read_object_unlocked_limited(bucket, object, max_size).await?,
+                    None => self.backend.read_object_unlocked(bucket, object).await?,
+                };
                 let actual = Self::observed_content(actual_object.as_ref(), expected_object.max_size);
                 if actual.identity != expected_object.identity {
                     return Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!(
@@ -1897,7 +1762,7 @@ where
         self.authorize(bucket, object, S3Action::GetObjectAction).await?;
         self.ensure_observation_allowed(bucket, object)?;
         let result = self.backend.read_object(bucket, object).await?;
-        self.record_observation(bucket, object, Self::observed_content(result.as_ref(), None), true)?;
+        self.record_observation(bucket, object, Self::observed_content(result.as_ref(), None))?;
         Ok(result)
     }
 
@@ -1910,7 +1775,7 @@ where
         self.authorize(bucket, object, S3Action::GetObjectAction).await?;
         self.ensure_observation_allowed(bucket, object)?;
         let result = self.backend.read_object_limited(bucket, object, max_size).await?;
-        self.record_observation(bucket, object, Self::observed_content(result.as_ref(), Some(max_size)), true)?;
+        self.record_observation(bucket, object, Self::observed_content(result.as_ref(), Some(max_size)))?;
         Ok(result)
     }
 
@@ -1937,7 +1802,7 @@ where
         let metadata = self.backend.object_metadata(bucket, object).await?;
         let observation = Self::observed_metadata(bucket, object, metadata)?;
         let exists = observation.identity != TableCommitObjectIdentity::Missing;
-        self.record_observation(bucket, object, observation, true)?;
+        self.record_observation(bucket, object, observation)?;
         Ok(exists)
     }
 
@@ -1952,24 +1817,14 @@ where
         data: Vec<u8>,
         precondition: crate::table_catalog::TableCatalogPutPrecondition,
     ) -> crate::table_catalog::TableCatalogStoreResult<()> {
-        let observation = self.prepare_put_observation(bucket, object, &data).await?;
+        self.authorize(bucket, object, S3Action::PutObjectAction).await?;
+        self.ensure_observation_allowed(bucket, object)?;
+        let observation = TableCommitObservedObject {
+            identity: TableCommitObjectIdentity::ContentSha256(hex_sha256(&data, str::to_string)),
+            max_size: None,
+        };
         self.backend.put_object(bucket, object, data, precondition).await?;
-        // The existing generated-metadata readback binds the written bytes to
-        // their revision. Until then, keep the original SHA expectation and
-        // enforce the fallback budget before publication preparation.
-        self.record_observation(bucket, object, observation, false)
-    }
-
-    async fn put_object_before_publication(
-        &self,
-        bucket: &str,
-        object: &str,
-        data: Vec<u8>,
-        precondition: crate::table_catalog::TableCatalogPutPrecondition,
-    ) -> crate::table_catalog::TableCatalogStoreResult<()> {
-        let observation = self.prepare_put_observation_with(bucket, object, &data, false).await?;
-        self.backend.put_object(bucket, object, data, precondition).await?;
-        self.record_observation(bucket, object, observation, false)
+        self.record_observation(bucket, object, observation)
     }
 
     async fn put_object_unlocked(
@@ -1980,11 +1835,16 @@ where
         precondition: crate::table_catalog::TableCatalogPutPrecondition,
         write_guards: Vec<crate::admin::storage_api::object::WriteCommitGuard>,
     ) -> crate::table_catalog::TableCatalogStoreResult<()> {
-        let observation = self.prepare_put_observation(bucket, object, &data).await?;
+        self.authorize(bucket, object, S3Action::PutObjectAction).await?;
+        self.ensure_observation_allowed(bucket, object)?;
+        let observation = TableCommitObservedObject {
+            identity: TableCommitObjectIdentity::ContentSha256(hex_sha256(&data, str::to_string)),
+            max_size: None,
+        };
         self.backend
             .put_object_unlocked(bucket, object, data, precondition, write_guards)
             .await?;
-        self.record_observation(bucket, object, observation, false)
+        self.record_observation(bucket, object, observation)
     }
 
     async fn delete_object(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> {
@@ -2030,10 +1890,6 @@ where
         self.backend.acquire_write_lock(bucket, object).await
     }
 
-    fn table_publication_preparation_budget(&self) -> Arc<crate::table_catalog::TablePublicationPreparation> {
-        self.preparation.clone()
-    }
-
     async fn begin_table_bucket_commit_publication(
         &self,
         table_bucket: &str,
@@ -2666,45 +2522,6 @@ where
         .map_err(catalog_store_error)
 }
 
-/// A create's initial metadata file sits under a root no table owns until its
-/// entry is published, and its name embeds this request's fresh table id, so
-/// nothing else can reference it. When registration fails before any catalog
-/// write may have begun, remove it; the publication fence is already released,
-/// and a failure here (or a crash before this point) leaves a small orphan.
-async fn remove_unpublished_initial_metadata(
-    metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend,
-    bucket: &str,
-    metadata_location: &str,
-) {
-    if crate::table_catalog::TableCommitPublication::preparation_budget(metadata_backend).mutation_started() {
-        return;
-    }
-    let _ = metadata_backend.delete_object(bucket, metadata_location).await;
-}
-
-/// Recheck the bucket registered before the publication lock. A missing entry
-/// here can indicate retirement while queued; do not recreate it under the fence.
-async fn require_table_bucket_for_publication<S>(
-    store: &S,
-    bucket: &str,
-    publication: &(dyn crate::table_catalog::TableCommitPublication + Sync),
-) -> S3Result<()>
-where
-    S: crate::table_catalog::TableCatalogStore + ?Sized,
-{
-    let entry = publication
-        .preparation_budget()
-        .read(publication, store.get_table_bucket(bucket))
-        .await
-        .map_err(catalog_store_error)?;
-    if entry.is_some_and(|entry| entry.state == crate::table_catalog::TableCatalogEntryState::Active) {
-        return Ok(());
-    }
-    Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
-        "table bucket {bucket}"
-    ))))
-}
-
 async fn table_bucket_response<S>(store: &S, bucket: &str, enabled: bool) -> S3Result<TableBucketResponse>
 where
     S: crate::table_catalog::TableCatalogStore + ?Sized,
@@ -6197,11 +6014,11 @@ where
         )));
     }
     let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend);
-    require_table_bucket_for_publication(store, bucket, metadata_backend).await?;
+    ensure_table_bucket_entry(store, bucket, true).await?;
     let metadata_data = serde_json::to_vec(&metadata)
         .map_err(|err| s3_error!(InternalError, "failed to serialize initial table metadata: {}", err))?;
     metadata_backend
-        .put_object_before_publication(
+        .put_object(
             bucket,
             &entry.metadata_location,
             metadata_data,
@@ -6209,10 +6026,10 @@ where
         )
         .await
         .map_err(catalog_store_already_exists_error)?;
-    if let Err(err) = store.register_table_with_publication(entry.clone(), metadata_backend).await {
-        remove_unpublished_initial_metadata(metadata_backend, bucket, &entry.metadata_location).await;
-        return Err(catalog_store_already_exists_error(err));
-    }
+    store
+        .register_table_with_publication(entry.clone(), metadata_backend)
+        .await
+        .map_err(catalog_store_already_exists_error)?;
     Ok(create_table_response_from_entry(entry, metadata, false))
 }
 
@@ -6238,7 +6055,7 @@ where
         )));
     }
     let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend);
-    require_table_bucket_for_publication(store, bucket, metadata_backend).await?;
+    ensure_table_bucket_entry(store, bucket, true).await?;
     let metadata_data = serde_json::to_vec(&metadata)
         .map_err(|err| s3_error!(InternalError, "failed to serialize initial view metadata: {}", err))?;
     metadata_backend
@@ -6661,18 +6478,15 @@ async fn publish_table_commit<S>(
 where
     S: crate::table_catalog::TableCatalogStore + ?Sized,
 {
-    // Real request backends acquire the migration permit here, before the store chooses its
-    // inline/child route. Nested publication reuses that same permit and preparation budget.
-    let _bucket_publication_completion =
-        if table_bucket_fence_required || metadata_backend.acquires_catalog_migration_read_permit() {
-            metadata_backend
-                .begin_table_bucket_commit_publication(&request.table_bucket)
-                .await
-                .map_err(catalog_store_error)?;
-            Some(crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend))
-        } else {
-            None
-        };
+    let _bucket_publication_completion = if table_bucket_fence_required {
+        metadata_backend
+            .begin_table_bucket_commit_publication(&request.table_bucket)
+            .await
+            .map_err(catalog_store_error)?;
+        Some(crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend))
+    } else {
+        None
+    };
     store
         .commit_table_with_publication(request, metadata_backend)
         .await
@@ -7029,7 +6843,6 @@ where
         .map_err(catalog_store_error)?;
     validate_table_metadata_snapshot_graph(metadata_backend, bucket, &entry, None, &metadata).await?;
 
-    ensure_table_bucket_entry(store, bucket, true).await?;
     crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket)
         .await
         .map_err(catalog_store_error)?;
@@ -7039,12 +6852,12 @@ where
         )));
     }
     let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend);
-    require_table_bucket_for_publication(store, bucket, metadata_backend).await?;
+    ensure_table_bucket_entry(store, bucket, true).await?;
     let metadata_data = serde_json::to_vec(&metadata).map_err(|err| {
         S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize initial table metadata: {err}"))
     })?;
     metadata_backend
-        .put_object_before_publication(
+        .put_object(
             bucket,
             &entry.metadata_location,
             metadata_data,
@@ -7052,10 +6865,10 @@ where
         )
         .await
         .map_err(catalog_store_already_exists_error)?;
-    if let Err(err) = store.register_table_with_publication(entry.clone(), metadata_backend).await {
-        remove_unpublished_initial_metadata(metadata_backend, bucket, &entry.metadata_location).await;
-        return Err(catalog_store_already_exists_error(err));
-    }
+    store
+        .register_table_with_publication(entry.clone(), metadata_backend)
+        .await
+        .map_err(catalog_store_already_exists_error)?;
 
     Ok(RestCommitTableResponse {
         metadata_location: table_metadata_location_for_client(bucket, &entry.metadata_location),

rustfs/src/admin/handlers/table_catalog/routes.rs

@@ -39,21 +39,6 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router<AdminOperation>, prefix
         format!("{prefix}/buckets/{{warehouse}}").as_str(),
         AdminOperation(&GET_TABLE_BUCKET_HANDLER),
     )?;
-    r.insert(
-        Method::GET,
-        format!("{prefix}/{{warehouse}}/catalog/directory").as_str(),
-        AdminOperation(&GetTableDirectoryHandler {}),
-    )?;
-    r.insert(
-        Method::POST,
-        format!("{prefix}/{{warehouse}}/catalog/directory").as_str(),
-        AdminOperation(&AdvanceTableDirectoryHandler {}),
-    )?;
-    r.insert(
-        Method::DELETE,
-        format!("{prefix}/{{warehouse}}/catalog/directory").as_str(),
-        AdminOperation(&DisableTableDirectoryHandler {}),
-    )?;
     r.insert(
         Method::GET,
         format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),

rustfs/src/admin/handlers/table_catalog/table.rs

@@ -56,25 +56,6 @@ impl Operation for RestRenameTableHandler {
         ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
 
         let store = table_catalog_store_from_extensions(&req.extensions)?;
-        rename_table_response(store, warehouse, source_namespace, source_table, destination_namespace, destination_table).await?;
-        Ok(empty_response(StatusCode::NO_CONTENT))
-    }
-}
-
-// Authorization has completed before ownership moves to the mutation task. A
-// disconnected request must not strand the persisted bucket-wide rename fence.
-pub(super) async fn rename_table_response<S>(
-    store: S,
-    warehouse: String,
-    source_namespace: crate::table_catalog::Namespace,
-    source_table: crate::table_catalog::IdentifierSegment,
-    destination_namespace: crate::table_catalog::Namespace,
-    destination_table: crate::table_catalog::IdentifierSegment,
-) -> S3Result<()>
-where
-    S: crate::table_catalog::TableCatalogStore + Send + 'static,
-{
-    crate::admin::handlers::supervise_admin_mutation("table rename", async move {
         store
             .rename_table(
                 &warehouse,
@@ -84,9 +65,9 @@ where
                 destination_table.as_str(),
             )
             .await
-            .map_err(catalog_store_error)
-    })
-    .await
+            .map_err(catalog_store_error)?;
+        Ok(empty_response(StatusCode::NO_CONTENT))
+    }
 }
 
 pub struct RestCreateTableHandler {}

rustfs/src/admin/route_policy.rs

@@ -1132,24 +1132,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         GET_TABLE_BUCKET,
         RouteRiskLevel::Sensitive,
     ),
-    admin(
-        HttpMethod::Get,
-        "/iceberg/v1/{warehouse}/catalog/directory",
-        GET_TABLE_CATALOG,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/iceberg/v1/{warehouse}/catalog/directory",
-        MIGRATE_TABLE_CATALOG,
-        RouteRiskLevel::High,
-    ),
-    admin(
-        HttpMethod::Delete,
-        "/iceberg/v1/{warehouse}/catalog/directory",
-        MIGRATE_TABLE_CATALOG,
-        RouteRiskLevel::High,
-    ),
     admin(
         HttpMethod::Get,
         "/iceberg/v1/{warehouse}/catalog/migration",
@@ -1458,24 +1440,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         GET_TABLE_BUCKET,
         RouteRiskLevel::Sensitive,
     ),
-    admin(
-        HttpMethod::Get,
-        "/_iceberg/v1/{warehouse}/catalog/directory",
-        GET_TABLE_CATALOG,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/_iceberg/v1/{warehouse}/catalog/directory",
-        MIGRATE_TABLE_CATALOG,
-        RouteRiskLevel::High,
-    ),
-    admin(
-        HttpMethod::Delete,
-        "/_iceberg/v1/{warehouse}/catalog/directory",
-        MIGRATE_TABLE_CATALOG,
-        RouteRiskLevel::High,
-    ),
     admin(
         HttpMethod::Get,
         "/_iceberg/v1/{warehouse}/catalog/migration",
@@ -2161,7 +2125,7 @@ mod tests {
         let table_specs = ADMIN_ROUTE_POLICY_SPECS
             .iter()
             .filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1"));
-        assert_eq!(table_specs.count(), 112);
+        assert_eq!(table_specs.count(), 106);
         assert_action(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
         assert_action(HttpMethod::Delete, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
         assert_action(HttpMethod::Delete, "/_iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
@@ -2340,12 +2304,6 @@ mod tests {
             "/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/quarantine",
             RUN_TABLE_MAINTENANCE,
         );
-        assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/catalog/directory", GET_TABLE_CATALOG);
-        assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
-        assert_action(HttpMethod::Delete, "/iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
-        assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/directory", GET_TABLE_CATALOG);
-        assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
-        assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
         assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
         assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
         assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);