mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-05 10:26:56 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3362421240 | |||
| 57df9181d0 | |||
| 371f5e4ffc | |||
| 8a5a2c9c93 | |||
| babfeaed00 | |||
| 736bf777b4 | |||
| 7babc7d621 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
sha256-darwin=a3e166a140f3150218b4a857fedb8749940fad0c8b053d0e3c1b957321bf5beb
|
||||
sha256-linux=dce4a8eb634a27b4589bcf6d2ebf248ede97aa59701acc8696391d2812a3177d
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
|
||||
|
||||
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
|
||||
| File | Description | Usage |
|
||||
| :--- | :--- | :--- |
|
||||
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
|
||||
|
||||
### Docker Compose Configurations
|
||||
|
||||
|
||||
Generated
+1
-3
@@ -10090,6 +10090,7 @@ version = "1.0.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
"metrics",
|
||||
"rustfs-io-metrics",
|
||||
"thiserror 2.0.21",
|
||||
"tokio",
|
||||
@@ -10276,7 +10277,6 @@ dependencies = [
|
||||
"hotpath",
|
||||
"parking_lot 0.13.0",
|
||||
"rand 0.10.3",
|
||||
"rustfs-common",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
@@ -10789,10 +10789,8 @@ version = "1.0.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"csv-core",
|
||||
"datafusion",
|
||||
"derive_builder",
|
||||
"flate2",
|
||||
"futures",
|
||||
"hotpath",
|
||||
"parking_lot 0.13.0",
|
||||
|
||||
@@ -191,7 +191,6 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl jq unzip
|
||||
WORKDIR /build
|
||||
@@ -69,6 +70,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
|
||||
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
|
||||
ca-certificates \
|
||||
coreutils \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN addgroup -g 10001 -S rustfs && \
|
||||
adduser -u 10001 -G rustfs -S rustfs -D && \
|
||||
@@ -124,6 +132,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.97.1-trixie
|
||||
FROM rust:1.99.1-trixie
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
@@ -21,6 +24,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -36,6 +40,7 @@ COPY . .
|
||||
RUN ./scripts/static.sh
|
||||
RUN cargo run --bin gproto
|
||||
RUN cargo build --release --locked --bin rustfs
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
RUN set -eux; \
|
||||
groupadd -g 10001 rustfs; \
|
||||
@@ -50,8 +55,12 @@ RUN set -eux; \
|
||||
chown -R rustfs:rustfs /data /logs /app; \
|
||||
chmod 0750 /data /logs
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
|
||||
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
|
||||
EXPOSE 9000 9001
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
@@ -76,6 +77,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
|
||||
|
||||
FROM ubuntu:26.04
|
||||
|
||||
ARG RELEASE=latest
|
||||
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN groupadd -g 10001 rustfs && \
|
||||
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
|
||||
@@ -131,6 +139,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
+19
-7
@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.97.1-trixie AS builder
|
||||
FROM rust:1.99.1-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES
|
||||
ARG RC_RELEASE
|
||||
|
||||
# Debug: print platforms
|
||||
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
|
||||
@@ -52,6 +54,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
# -----------------------------
|
||||
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
# We rely on cargo run to build and run
|
||||
EXPOSE 9000 9001
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["cargo", "run", "--bin", "rustfs", "--"]
|
||||
|
||||
# -----------------------------
|
||||
@@ -227,6 +235,7 @@ RUN set -eux; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
coreutils; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -248,8 +257,11 @@ RUN set -eux; \
|
||||
|
||||
# Copy the freshly built binary and the entrypoint
|
||||
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Default environment (override in docker run/compose as needed)
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
@@ -263,5 +275,5 @@ EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Keep root here; entrypoint will drop privileges using chroot --userspec
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# RustFS image bootstrap
|
||||
|
||||
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
|
||||
|
||||
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
|
||||
|
||||
Optional per-bucket files use the bucket name as the filename without an extension:
|
||||
|
||||
| Directory | File format | Applied operation |
|
||||
| --- | --- | --- |
|
||||
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
|
||||
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
|
||||
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
|
||||
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
|
||||
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
|
||||
|
||||
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
|
||||
|
||||
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
|
||||
|
||||
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"block_public_acls": true,
|
||||
"ignore_public_acls": true,
|
||||
"block_public_policy": true,
|
||||
"restrict_public_buckets": true
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -502,10 +502,163 @@ mod tests {
|
||||
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
|
||||
"<Code>AccessDenied</Code>"
|
||||
} else {
|
||||
"<Code>BucketAlreadyExists</Code>"
|
||||
"<Code>BucketAlreadyOwnedByYou</Code>"
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployment owns a single bucket namespace, so an IAM user that recreates
|
||||
/// an existing bucket is the owner, not another account. In the default region
|
||||
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
|
||||
/// semantics), including buckets created by root and by another IAM user.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
|
||||
let root = env.create_s3_client();
|
||||
|
||||
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-iam",
|
||||
"duplicate-iam-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
));
|
||||
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root creates the first bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
|
||||
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the IAM user creates its own bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the same IAM user recreating its bucket gets 200 OK");
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root recreating an IAM-created bucket gets 200 OK");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outside the default region the same ownership answer becomes
|
||||
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
|
||||
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
|
||||
let region = "eu-west-1";
|
||||
let root = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-eu-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(
|
||||
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readonly",
|
||||
"duplicate-eu-readonly",
|
||||
)
|
||||
.await?;
|
||||
let readonly = Client::from_conf(
|
||||
build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-eu-readonly",
|
||||
"duplicate-eu-readonly-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
)
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
|
||||
// Both the creator and the other authenticated identity own the namespace.
|
||||
for creator in [&root, &iam] {
|
||||
let error = creator
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a duplicate create outside us-east-1 must report ownership");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("BucketAlreadyOwnedByYou"),
|
||||
"{bucket}"
|
||||
);
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
|
||||
}
|
||||
}
|
||||
|
||||
let denied = readonly
|
||||
.create_bucket()
|
||||
.bucket("duplicate-eu-root")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -283,6 +283,9 @@ mod ssec_copy_test;
|
||||
#[cfg(test)]
|
||||
mod multipart_storage_class_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod multipart_upload_id_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_class_capability_test;
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Multipart upload ID regression tests.
|
||||
//!
|
||||
//! Every operation that addresses an existing multipart upload must report an
|
||||
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
|
||||
//! the ID is malformed or is a well-formed ID of an aborted upload.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "multipart-upload-id";
|
||||
const KEY: &str = "target.bin";
|
||||
const COPY_SOURCE_KEY: &str = "copy-source.bin";
|
||||
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
|
||||
// the object layer rejects this ID as malformed instead of looking it up.
|
||||
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
|
||||
const PART_NUMBER: i32 = 1;
|
||||
const PART_BODY: &[u8] = b"multipart upload id regression part";
|
||||
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
|
||||
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
|
||||
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
|
||||
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
|
||||
|
||||
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
|
||||
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
|
||||
where
|
||||
E: ProvideErrorMetadata + std::fmt::Debug,
|
||||
{
|
||||
let Err(error) = result else {
|
||||
return Some(format!("{operation}: succeeded"));
|
||||
};
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
|
||||
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
|
||||
}
|
||||
|
||||
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(Vec::new()).await?;
|
||||
env.create_test_bucket(BUCKET).await?;
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(COPY_SOURCE_KEY)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
|
||||
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
|
||||
let upload_part = client
|
||||
.upload_part()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await;
|
||||
let upload_part_copy = client
|
||||
.upload_part_copy()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
|
||||
.send()
|
||||
.await;
|
||||
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
|
||||
let complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
let abort = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let mismatches: Vec<String> = [
|
||||
no_such_upload_mismatch("UploadPart", upload_part),
|
||||
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
|
||||
no_such_upload_mismatch("ListParts", list_parts),
|
||||
no_such_upload_mismatch("CompleteMultipartUpload", complete),
|
||||
no_such_upload_mismatch("AbortMultipartUpload", abort),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(
|
||||
mismatches.is_empty(),
|
||||
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
|
||||
mismatches.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
|
||||
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
|
||||
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
|
||||
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
|
||||
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
|
||||
let status = head
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|error| error.raw_response())
|
||||
.map(|response| response.status().as_u16());
|
||||
assert_eq!(
|
||||
status,
|
||||
Some(OBJECT_NOT_FOUND_STATUS),
|
||||
"rejected operations must not publish {KEY}: {head:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
}
|
||||
@@ -94,12 +94,3 @@ pub(crate) mod scanner_peer_bucket {
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod bucket_tag_fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
|
||||
}
|
||||
|
||||
@@ -678,44 +678,6 @@ async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop and reap the directory's owning server before opening the fixture store.
|
||||
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
use crate::storage_api::bucket_tag_fixture::{
|
||||
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
|
||||
};
|
||||
|
||||
let server = env
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or("missing server process before offline fixture write")?;
|
||||
server.kill()?;
|
||||
server.wait()?;
|
||||
env.process = None;
|
||||
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
|
||||
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
|
||||
let context = std::sync::Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
|
||||
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
|
||||
let _cancel = context
|
||||
.background_cancel_token()
|
||||
.expect("store binds its shutdown token")
|
||||
.drop_guard();
|
||||
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let policy = metadata.policy_config_json.clone();
|
||||
assert!(!incarnation.is_nil());
|
||||
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
|
||||
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
|
||||
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
|
||||
metadata.tagging_config_xml = xml.to_vec();
|
||||
metadata.save_with_store(store.clone()).await?;
|
||||
let persisted = load_bucket_metadata(store, bucket).await?;
|
||||
assert_eq!(persisted.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(persisted.policy_config_json, policy);
|
||||
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -726,13 +688,12 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
init_logging();
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let opa_url = opa.url.clone();
|
||||
let server_env = [
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
];
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
])
|
||||
.await?;
|
||||
let secret = uuid::Uuid::new_v4().to_string();
|
||||
create_user(&env, "opabuckettags", &secret).await?;
|
||||
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
|
||||
@@ -924,9 +885,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
// Duplicate keys are refused on write; the ambiguous record below is seeded
|
||||
// offline the way an upgraded deployment may still hold it on disk.
|
||||
let invalid_write = admin
|
||||
admin
|
||||
.put_bucket_tagging()
|
||||
.bucket(destination)
|
||||
.tagging(
|
||||
@@ -936,24 +895,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("new duplicate tags must be rejected before publication");
|
||||
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
set_department(&admin, source, "finance").await?;
|
||||
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
|
||||
let read = client
|
||||
.get_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(destination)
|
||||
|
||||
@@ -192,7 +192,6 @@ pub mod bucket {
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
}
|
||||
|
||||
pub mod config_parse_mode {
|
||||
@@ -332,8 +331,8 @@ pub mod bucket {
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -452,15 +451,9 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
|
||||
pub use crate::disk::multipart_index::{
|
||||
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
|
||||
MultipartIndexRpcRequest,
|
||||
};
|
||||
pub use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
|
||||
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
|
||||
@@ -473,9 +466,7 @@ pub mod disk {
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
|
||||
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
|
||||
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
@@ -626,7 +617,7 @@ pub mod rebalance {
|
||||
pub mod rio {
|
||||
pub use crate::io_support::rio::{
|
||||
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
|
||||
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
boxed_reader, compression_metadata_value, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -663,9 +654,7 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::{
|
||||
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
|
||||
};
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
|
||||
@@ -16,8 +16,7 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::get_bucket_targets_config;
|
||||
use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
|
||||
UploadResponseDeadline, build_remote_s3_client,
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
@@ -120,7 +119,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
|
||||
// Replication has no retry budget of its own on the request path,
|
||||
// so it keeps the SDK's standard three attempts; stating it here
|
||||
// pins the behaviour to this line instead of an SDK default.
|
||||
@@ -1193,7 +1191,6 @@ impl BucketTargetSys {
|
||||
health_check_duration: target.health_check_duration,
|
||||
replicate_sync: target.replication_sync,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::from_env(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1925,83 +1922,6 @@ impl std::fmt::Display for S3ClientError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication request budgets, snapshotted when a target client is built.
|
||||
/// Streamed GET bodies are not subject to the upload-response budget.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TargetRequestTimeouts {
|
||||
request: Duration,
|
||||
complete: Duration,
|
||||
minimum_transfer_rate: u64,
|
||||
}
|
||||
|
||||
impl Default for TargetRequestTimeouts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request: Duration::from_secs(60),
|
||||
complete: Duration::from_secs(15 * 60),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetRequestTimeouts {
|
||||
fn from_env() -> Self {
|
||||
let defaults = Self::default();
|
||||
let seconds = |name: &str, fallback: Duration| {
|
||||
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
|
||||
Duration::from_secs(if (1..=86400).contains(&value) {
|
||||
value
|
||||
} else {
|
||||
fallback.as_secs()
|
||||
})
|
||||
};
|
||||
let minimum_transfer_rate =
|
||||
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
|
||||
Self {
|
||||
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
|
||||
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
|
||||
minimum_transfer_rate: if minimum_transfer_rate == 0 {
|
||||
defaults.minimum_transfer_rate
|
||||
} else {
|
||||
minimum_transfer_rate
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
|
||||
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
|
||||
// The SDK can make three attempts. Leave transmission headroom for
|
||||
// all three; the separate response deadline still bounds a trickling
|
||||
// target after each upload finishes.
|
||||
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
|
||||
RemoteS3RetryPolicy::Disabled => 1,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
|
||||
};
|
||||
let seconds = size
|
||||
.div_ceil(self.minimum_transfer_rate)
|
||||
.checked_mul(attempts)
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
let timeout = self
|
||||
.request
|
||||
.checked_add(Duration::from_secs(seconds))
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
if Instant::now().checked_add(timeout).is_none() {
|
||||
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
|
||||
}
|
||||
Ok(timeout)
|
||||
}
|
||||
|
||||
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
|
||||
Ok(aws_sdk_s3::config::Builder::new()
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.upload_timeout(size)?)
|
||||
.build(),
|
||||
)
|
||||
.interceptor(UploadResponseDeadline(self.request)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TargetClient {
|
||||
pub endpoint: String,
|
||||
@@ -2015,7 +1935,6 @@ pub struct TargetClient {
|
||||
pub health_check_duration: Duration,
|
||||
pub replicate_sync: bool,
|
||||
pub client: Arc<S3Client>,
|
||||
pub timeouts: TargetRequestTimeouts,
|
||||
}
|
||||
|
||||
impl TargetClient {
|
||||
@@ -2441,11 +2360,12 @@ impl TargetClient {
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
let mut config = self.timeouts.upload_override(size)?;
|
||||
if forwards_source_checksum {
|
||||
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
}
|
||||
operation = operation.config_override(config);
|
||||
match operation.send().await {
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
@@ -2581,7 +2501,6 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(self.timeouts.upload_override(size)?)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2619,13 +2538,6 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(
|
||||
aws_sdk_s3::config::Builder::new().timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.timeouts.complete)
|
||||
.build(),
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -3054,241 +2966,11 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TimedResponseConnector {
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: bytes::Bytes,
|
||||
frames: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ResponseDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TimedResponseConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let settings = self.clone();
|
||||
HttpConnectorFuture::new(async move {
|
||||
http_body_util::BodyExt::collect(request.into_body())
|
||||
.await
|
||||
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
|
||||
tokio::time::sleep(settings.upload_delay).await;
|
||||
let guard = ResponseDrop(Arc::clone(&settings.dropped));
|
||||
let stream = futures::stream::unfold(
|
||||
(settings.prefix_bytes, Some(settings.response), true, guard),
|
||||
move |(remaining, mut body, first, guard)| {
|
||||
let frames = Arc::clone(&settings.frames);
|
||||
async move {
|
||||
if remaining == 0 && body.is_none() {
|
||||
return None;
|
||||
}
|
||||
if !first {
|
||||
tokio::time::sleep(settings.response_delay).await;
|
||||
}
|
||||
let bytes = if remaining > 0 {
|
||||
bytes::Bytes::from_static(b" ")
|
||||
} else {
|
||||
body.take()?
|
||||
};
|
||||
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Some((
|
||||
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
|
||||
(remaining.saturating_sub(1), body, false, guard),
|
||||
))
|
||||
}
|
||||
},
|
||||
)
|
||||
.fuse();
|
||||
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
|
||||
let mut response =
|
||||
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
|
||||
response.headers_mut().insert("etag", "\"test-etag\"");
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn timed_response_target(
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: &'static [u8],
|
||||
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(TimedResponseConnector {
|
||||
upload_delay,
|
||||
response_delay,
|
||||
prefix_bytes,
|
||||
response: bytes::Bytes::from_static(response),
|
||||
frames: Arc::clone(&frames),
|
||||
dropped: Arc::clone(&dropped),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut target, _) = header_recording_target_client(Vec::new());
|
||||
target.timeouts = TargetRequestTimeouts {
|
||||
request: Duration::from_secs(2),
|
||||
complete: Duration::from_secs(10),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
};
|
||||
target.client = Arc::new(S3Client::from_conf(
|
||||
target
|
||||
.client
|
||||
.config()
|
||||
.to_builder()
|
||||
.http_client(http_client)
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(target.timeouts.request)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
));
|
||||
(target, frames, dropped)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
|
||||
let policy = TargetRequestTimeouts::default();
|
||||
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
|
||||
assert_eq!(
|
||||
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
|
||||
Duration::from_secs(15420)
|
||||
);
|
||||
assert!(policy.upload_timeout(-1).is_err());
|
||||
let slow = TargetRequestTimeouts {
|
||||
minimum_transfer_rate: 1,
|
||||
..policy
|
||||
};
|
||||
assert!(slow.upload_timeout(i64::MAX).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
|
||||
for operation in ["put", "forwarded-checksum", "part"] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
let started = tokio::time::Instant::now();
|
||||
let error = if operation == "part" {
|
||||
target
|
||||
.put_object_part(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
1,
|
||||
size,
|
||||
ByteStream::from(body),
|
||||
&PutObjectPartOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
} else {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
if operation == "forwarded-checksum" {
|
||||
opts.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_owned(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
|
||||
);
|
||||
}
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_secs(6),
|
||||
"upload gets its full independent window: {error}"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
|
||||
"response must end before the larger total deadline"
|
||||
);
|
||||
assert!(
|
||||
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
|
||||
"trickle delivered progress before failure"
|
||||
);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
|
||||
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("slow upload, prompt response");
|
||||
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
|
||||
let response = target
|
||||
.get_object("bucket", "key", None, None, None, HeaderMap::new())
|
||||
.await
|
||||
.expect("GET headers");
|
||||
let started = tokio::time::Instant::now();
|
||||
let body = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("streamed GET remains outside upload deadline")
|
||||
.into_bytes();
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert_eq!(&body[..], b" payload");
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
|
||||
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
|
||||
for prefix in [5, 100] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = target
|
||||
.complete_multipart_upload(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
if prefix == 5 {
|
||||
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(11));
|
||||
}
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_control_operation_deadline_covers_response_trickle() {
|
||||
let (target, frames, dropped) =
|
||||
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
|
||||
let started = tokio::time::Instant::now();
|
||||
assert!(target.get_bucket_versioning("bucket").await.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(3));
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
@@ -3597,7 +3279,6 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_uris,
|
||||
)
|
||||
@@ -3910,7 +3591,6 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -5487,14 +5487,6 @@ pub async fn put_restore_opts(
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = oi.expires {
|
||||
meta.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(std::io::Error::other)?,
|
||||
);
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
@@ -5514,6 +5506,7 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
@@ -7151,25 +7144,6 @@ mod tests {
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
let unread = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("open an unread transitioned response");
|
||||
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
|
||||
drop(unread);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"canceling an unread response must release the tier lease"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -256,38 +256,6 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
|
||||
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
|
||||
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
|
||||
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
|
||||
/// Shared tag-entry validation for object requests and persisted bucket tags.
|
||||
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
|
||||
VALID_TAG_TEXT
|
||||
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
}
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub const MAX_BUCKET_TAGS: usize = 50;
|
||||
|
||||
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
|
||||
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
|
||||
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
|
||||
@@ -1022,20 +990,6 @@ impl BucketMetadata {
|
||||
self.encryption_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
if !data.is_empty() {
|
||||
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
|
||||
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
|
||||
})?;
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"Cannot have more than 50 tags per bucket".to_owned(),
|
||||
));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set)
|
||||
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
|
||||
}
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
@@ -1830,31 +1784,6 @@ mod test {
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
|
||||
let mut metadata = BucketMetadata::new("validated-tags");
|
||||
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
|
||||
.expect("empty value is valid");
|
||||
let updated = metadata.tagging_config_updated_at;
|
||||
for invalid in [
|
||||
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
|
||||
] {
|
||||
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
|
||||
assert_eq!(metadata.tagging_config_xml, valid);
|
||||
assert_eq!(metadata.tagging_config_updated_at, updated);
|
||||
}
|
||||
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("deletion repairs legacy corruption");
|
||||
assert!(metadata.tagging_config_xml.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
|
||||
@@ -3428,7 +3428,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
@@ -4556,13 +4556,8 @@ mod tests {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
|
||||
// Model legacy persisted corruption without passing the write-time validator.
|
||||
bm.tagging_config_xml = raw.to_vec();
|
||||
} else {
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
}
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
@@ -5743,8 +5738,7 @@ mod tests {
|
||||
// update only needs one interleaving to show up.
|
||||
const ROUNDS: usize = 8;
|
||||
for round in 0..ROUNDS {
|
||||
let tagging =
|
||||
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
|
||||
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
|
||||
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
|
||||
|
||||
let start = Arc::new(tokio::sync::Barrier::new(2));
|
||||
@@ -5817,7 +5811,7 @@ mod tests {
|
||||
|
||||
let blocked = tokio::spawn({
|
||||
let node_b = node_b.clone();
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
|
||||
});
|
||||
|
||||
// Long enough for the write to have finished had it not waited: the
|
||||
|
||||
@@ -119,21 +119,6 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
// Exercise the real storage entry point before testing the new
|
||||
// reporting semantics: the same ledger must admit and refuse
|
||||
// these writes even when a scanner floor claims a million bytes.
|
||||
if protocol == 1 {
|
||||
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
),
|
||||
"a missing authoritative ledger must not admit a write from scanner reporting"
|
||||
);
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative usage through the production worker");
|
||||
}
|
||||
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
|
||||
assert!(
|
||||
matches!(
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::disk::{DiskAPI, error::DiskError};
|
||||
use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
|
||||
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
@@ -36,8 +36,6 @@ use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
|
||||
const RECONCILE_RETRY_SECONDS: i64 = 30;
|
||||
const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
|
||||
const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
|
||||
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
|
||||
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
|
||||
@@ -98,10 +96,6 @@ struct QuotaLedger {
|
||||
version: u8,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
/// Changes on every durable write, including zero-growth reservations.
|
||||
/// Missing on an older writer's output, which invalidates a concurrent scan.
|
||||
#[serde(default)]
|
||||
writer_revision: Uuid,
|
||||
accounted_usage: u64,
|
||||
reservations: BTreeMap<Uuid, PersistedReservation>,
|
||||
#[serde(default)]
|
||||
@@ -116,7 +110,6 @@ impl QuotaLedger {
|
||||
version: QUOTA_LEDGER_FORMAT_VERSION,
|
||||
bucket_incarnation,
|
||||
quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
|
||||
writer_revision: Uuid::nil(),
|
||||
accounted_usage,
|
||||
reservations: BTreeMap::new(),
|
||||
reconcile_required: false,
|
||||
@@ -492,23 +485,22 @@ impl QuotaContext {
|
||||
let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
|
||||
let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
|
||||
let mut ledger =
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
|
||||
.await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
|
||||
let Some(mut ledger) = ledger.take() else {
|
||||
// Bootstrap has no authoritative usage yet. Release every
|
||||
// request-owned fence by returning; the worker owns no request.
|
||||
job.schedule();
|
||||
return Err(StorageError::SlowDown);
|
||||
};
|
||||
let mut ledger = load_current_ledger_locked(
|
||||
Arc::clone(&store),
|
||||
&ledger_data.bucket,
|
||||
&ledger_data.ledger_object,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let growth = reservation.growth();
|
||||
let current_usage = ledger.admitted_usage()?;
|
||||
let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if ledger.reconcile_required
|
||||
|| (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
|
||||
{
|
||||
job.schedule();
|
||||
let mut current_usage = ledger.admitted_usage()?;
|
||||
let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
|
||||
reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
|
||||
current_usage = ledger.admitted_usage()?;
|
||||
expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
}
|
||||
if growth > 0 && expected_usage > quota_limit {
|
||||
return Err(StorageError::QuotaExceeded {
|
||||
@@ -526,7 +518,7 @@ impl QuotaContext {
|
||||
});
|
||||
}
|
||||
ledger.reserve(operation_id, reservation)?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
|
||||
|
||||
Ok(QuotaReservation {
|
||||
ledger: Some(ReservationLedgerData::Single(ledger_data)),
|
||||
@@ -1090,7 +1082,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
|
||||
fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
|
||||
let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
|
||||
ledger.mark_commit_started(data.operation_id, &data.reservation)?;
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
|
||||
@@ -1121,7 +1113,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
} else {
|
||||
ledger.abort(operation_id, &reservation)?;
|
||||
}
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
|
||||
@@ -1129,15 +1121,22 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
|
||||
async fn load_current_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
ledger_object: &str,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
) -> Result<Option<QuotaLedger>> {
|
||||
match load_ledger_locked(store, ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
|
||||
Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
|
||||
) -> Result<QuotaLedger> {
|
||||
match load_ledger_locked(Arc::clone(&store), ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
|
||||
Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
Ok(_) => Err(StorageError::PartMissingOrCorrupt),
|
||||
Err(StorageError::ConfigNotFound) => Ok(None),
|
||||
Err(StorageError::ConfigNotFound) => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -1246,11 +1245,15 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
|
||||
for (object, target) in orphan_commit_targets {
|
||||
fence_namespace_mutations(&store, bucket, &object, target).await?;
|
||||
}
|
||||
let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
if ledger.reservations.is_empty() && ledger.reconcile_required {
|
||||
reconcile_exact(&store, bucket, &mut ledger).await?;
|
||||
removed = true;
|
||||
}
|
||||
if !removed && !cursor_changed {
|
||||
return Ok(());
|
||||
}
|
||||
save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
|
||||
save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
|
||||
}
|
||||
|
||||
fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
|
||||
@@ -1270,217 +1273,13 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ReconcileControl {
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
next_attempt_at: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ReconcileJob {
|
||||
store: Arc<ECStore>,
|
||||
bucket: String,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
}
|
||||
|
||||
type ReconcileKey = (Uuid, String, Uuid, i128);
|
||||
|
||||
fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
|
||||
static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
|
||||
ACTIVE.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
struct ActiveReconciliation(ReconcileKey);
|
||||
|
||||
impl Drop for ActiveReconciliation {
|
||||
fn drop(&mut self) {
|
||||
active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
impl ReconcileJob {
|
||||
fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
|
||||
Self {
|
||||
store,
|
||||
bucket,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule(self) {
|
||||
let key = (
|
||||
self.store.id,
|
||||
self.bucket.clone(),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
if !active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(key.clone())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let owner = ActiveReconciliation(key);
|
||||
// The owner carries no object, metadata, lifecycle or reservation guard.
|
||||
// Cancellation of the rejected request cannot cancel its reconciliation.
|
||||
tokio::spawn(async move {
|
||||
let _owner = owner;
|
||||
if let Err(error) = self.run().await {
|
||||
warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
|
||||
error = %error, "Quota reconciliation deferred");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn control_object(&self) -> String {
|
||||
format!(
|
||||
"{}.reconcile/{}/{}",
|
||||
ledger_object(&self.bucket),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos()
|
||||
)
|
||||
}
|
||||
|
||||
async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
|
||||
}
|
||||
|
||||
async fn run(&self) -> Result<()> {
|
||||
let control_object = self.control_object();
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
|
||||
return Ok(());
|
||||
};
|
||||
let guard = Arc::new(guard);
|
||||
let now = now_unix();
|
||||
match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
|
||||
Ok(bytes) => {
|
||||
let control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
if control.bucket_incarnation != self.bucket_incarnation
|
||||
|| control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
|
||||
{
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
if control.next_attempt_at > now {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
|
||||
let mut control = ReconcileControl {
|
||||
bucket_incarnation: self.bucket_incarnation,
|
||||
quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
|
||||
next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
|
||||
};
|
||||
// Persist the throttle before work, including bootstrap and failed scans.
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
let result = self.scan_and_publish(&guard).await;
|
||||
control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
result
|
||||
}
|
||||
|
||||
/// Take a short authoritative configuration snapshot without recursively
|
||||
/// acquiring a reader behind a queued metadata/lifecycle writer.
|
||||
async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
|
||||
let guard = snapshot
|
||||
.metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
|
||||
.ok_or(StorageError::PreconditionFailed)?;
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(self.bucket_incarnation),
|
||||
object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
};
|
||||
let (quota, incarnation, revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
|
||||
.await?;
|
||||
if incarnation != self.bucket_incarnation
|
||||
|| revision != self.quota_revision
|
||||
|| !quota.as_ref().is_some_and(|quota| {
|
||||
quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
})
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
|
||||
let ledger_object = ledger_object(&self.bucket);
|
||||
let expected_revision = {
|
||||
// Lock order: reconcile owner, lifecycle, metadata, ledger.
|
||||
// Request paths never wait for the reconcile owner.
|
||||
let _configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(mut ledger) => {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// Seed a non-nil token even for a legacy ledger. An old
|
||||
// node drops this field on write and cannot pass the CAS.
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
|
||||
Some(ledger.writer_revision)
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
|
||||
// Only the independent reconcile owner is held during this whole-bucket
|
||||
// walk. Concurrent reserve/settle/reap writes invalidate its result.
|
||||
let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
|
||||
let configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(ledger) => {
|
||||
expected_revision == Some(ledger.writer_revision)
|
||||
&& !ledger.writer_revision.is_nil()
|
||||
&& ledger.reservations.is_empty()
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
if owner.is_lock_lost()
|
||||
|| !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
|
||||
{
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
|
||||
async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
|
||||
ledger.reconcile_required = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
|
||||
@@ -2136,7 +1935,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
|
||||
async fn save_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
ledger_object: &str,
|
||||
ledger: &mut QuotaLedger,
|
||||
ledger: &QuotaLedger,
|
||||
ledger_guard: &Arc<NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
if ledger_guard.is_lock_lost() {
|
||||
@@ -2160,71 +1959,9 @@ async fn save_ledger_locked(
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
ledger.writer_revision = Uuid::new_v4();
|
||||
save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
|
||||
}
|
||||
|
||||
/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
|
||||
/// The production reaper, ownership fence, scan and compare-and-swap still run.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
|
||||
let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let active = active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.contains(&key);
|
||||
if !active {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| StorageError::SlowDown)?;
|
||||
reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
|
||||
let object = job.control_object();
|
||||
{
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match read_config_no_lock(Arc::clone(&store), &object).await {
|
||||
Ok(bytes) => {
|
||||
let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
|
||||
job.save_control(&object, &control, &guard).await?;
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
job.run().await
|
||||
}
|
||||
|
||||
/// Bootstrap a test bucket through the real background reconciliation path.
|
||||
/// Fixtures call this after enabling V1 quota, before issuing writes whose
|
||||
/// contract is independent of the initial retryable admission response.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
if !quota
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
|
||||
.run()
|
||||
.await?;
|
||||
match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
|
||||
Some(_) => Ok(()),
|
||||
None => Err(StorageError::SlowDown),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn fail_next_quota_ledger_save_for_test() {
|
||||
@@ -2266,434 +2003,6 @@ mod tests {
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use serial_test::serial;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum ReconcilePauseStage {
|
||||
BeforeScan,
|
||||
BeforePublish,
|
||||
}
|
||||
|
||||
struct ReconcilePause {
|
||||
bucket: String,
|
||||
stage: ReconcilePauseStage,
|
||||
armed: std::sync::atomic::AtomicBool,
|
||||
scans: std::sync::atomic::AtomicUsize,
|
||||
fail_scan: std::sync::atomic::AtomicBool,
|
||||
reached: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
|
||||
|
||||
struct ReconcilePauseGuard(Arc<ReconcilePause>);
|
||||
|
||||
impl ReconcilePauseGuard {
|
||||
fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
|
||||
let pause = Arc::new(ReconcilePause {
|
||||
bucket: bucket.to_owned(),
|
||||
stage,
|
||||
armed: true.into(),
|
||||
scans: 0.into(),
|
||||
fail_scan: false.into(),
|
||||
reached: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
|
||||
Self(pause)
|
||||
}
|
||||
|
||||
async fn reached(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
|
||||
.await
|
||||
.expect("reconciliation reached barrier");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReconcilePauseGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.release.notify_one();
|
||||
let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
|
||||
if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
|
||||
use std::sync::atomic::Ordering;
|
||||
let hook = RECONCILE_PAUSE
|
||||
.lock()
|
||||
.expect("reconcile hooks")
|
||||
.as_ref()
|
||||
.filter(|hook| hook.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(hook) = hook {
|
||||
if stage == ReconcilePauseStage::BeforeScan {
|
||||
hook.scans.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
|
||||
return Err(StorageError::other("injected quota usage failure"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
|
||||
use crate::object_api::PutObjReader;
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
|
||||
let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let mut seed = PutObjReader::from_vec(vec![1; 10]);
|
||||
store
|
||||
.put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("existing usage");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
|
||||
)
|
||||
.await
|
||||
.expect("enable quota");
|
||||
let (_, incarnation, revision) = store
|
||||
.bucket_quota_and_incarnation_from_disk(&bucket)
|
||||
.await
|
||||
.expect("quota identity");
|
||||
(directories, ReconcileJob::new(store, bucket, incarnation, revision))
|
||||
}
|
||||
|
||||
async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
|
||||
begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
|
||||
.await?
|
||||
.reserve(old, new)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("ledger")
|
||||
}
|
||||
|
||||
async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
|
||||
let key = (
|
||||
job.store.id,
|
||||
job.bucket.clone(),
|
||||
job.bucket_incarnation,
|
||||
job.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while active_reconciliations().lock().expect("active workers").contains(&key) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned reconciliation finished");
|
||||
}
|
||||
|
||||
async fn expire_reconciliation_backoff(job: &ReconcileJob) {
|
||||
drain_reconciliation_scheduler(job).await;
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
|
||||
let mut control: ReconcileControl =
|
||||
serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
|
||||
.expect("control JSON");
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("control fence");
|
||||
job.save_control(&object, &control, &guard).await.expect("advance test clock");
|
||||
}
|
||||
|
||||
async fn wait_for_reconcile_owner(job: &ReconcileJob) {
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
|
||||
.await
|
||||
.expect("worker finished")
|
||||
.expect("owner guard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
|
||||
"unknown usage cannot admit a reservation"
|
||||
);
|
||||
pause.reached().await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"bootstrap must not publish zero usage"
|
||||
);
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
|
||||
// Bypass the local scheduler, exercising the cross-node ownership boundary.
|
||||
tokio::time::timeout(Duration::from_secs(2), job.run())
|
||||
.await
|
||||
.expect("contender must not wait for a scan")
|
||||
.expect("another owner is active");
|
||||
let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
|
||||
.await
|
||||
.expect("scan does not retain lifecycle readers")
|
||||
.expect("lifecycle writer");
|
||||
let metadata = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
|
||||
)
|
||||
.await
|
||||
.expect("scan does not retain metadata readers")
|
||||
.expect("metadata writer");
|
||||
drop(metadata);
|
||||
drop(lifecycle);
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
let ledger = read_test_ledger(&job).await;
|
||||
assert_eq!(ledger.accounted_usage, 10);
|
||||
assert!(ledger.reservations.is_empty());
|
||||
assert!(!ledger.writer_revision.is_nil());
|
||||
for _ in 0..3 {
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
|
||||
job.run().await.expect("persisted cooldown");
|
||||
}
|
||||
drain_reconciliation_scheduler(&job).await;
|
||||
assert_eq!(
|
||||
pause.0.scans.load(Ordering::SeqCst),
|
||||
1,
|
||||
"denied requests cannot repeatedly scan a full bucket"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
job.run().await.expect("bootstrap");
|
||||
for mutation in ["abort", "commit", "reap", "legacy"] {
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
let initial = read_test_ledger(&job).await.writer_revision;
|
||||
if mutation == "legacy" {
|
||||
let object = ledger_object(&job.bucket);
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
|
||||
let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
|
||||
document.as_object_mut().expect("ledger map").remove("writer_revision");
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("ledger fence");
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(
|
||||
Arc::clone(&job.store),
|
||||
&object,
|
||||
serde_json::to_vec(&document).expect("old node JSON"),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("old node drops new field");
|
||||
} else {
|
||||
let mut reservation = quota_reservation(&job, 10, 10)
|
||||
.await
|
||||
.expect("zero-growth admission while scanning");
|
||||
let reserved = read_test_ledger(&job).await.writer_revision;
|
||||
assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
|
||||
reservation.mark_commit_started().await.expect("commit marker");
|
||||
assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
|
||||
match mutation {
|
||||
"abort" => reservation.abort().await,
|
||||
"commit" => reservation.commit().await,
|
||||
"reap" => {
|
||||
reservation.defer_after_fence();
|
||||
reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("reap orphan");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let written = read_test_ledger(&job).await;
|
||||
assert!(written.reservations.is_empty());
|
||||
assert_ne!(written.writer_revision, initial);
|
||||
pause.0.release.notify_one();
|
||||
worker.await.expect("worker join").expect("stale scan discarded");
|
||||
let current = read_test_ledger(&job).await;
|
||||
assert_eq!(
|
||||
current.writer_revision, written.writer_revision,
|
||||
"{mutation}: stale usage must not replace a concurrent writer"
|
||||
);
|
||||
assert_eq!(current.accounted_usage, written.accounted_usage);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
pause.0.fail_scan.store(true, Ordering::SeqCst);
|
||||
pause.0.release.notify_one();
|
||||
assert!(job.run().await.is_err());
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
job.run().await.expect("failure cooldown");
|
||||
assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
|
||||
pause.0.fail_scan.store(false, Ordering::SeqCst);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("recover after failure");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("delete stored usage");
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
|
||||
"stale usage remains conservative during cooldown"
|
||||
);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("refresh deleted usage");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
|
||||
quota_reservation(&job, 0, 1)
|
||||
.await
|
||||
.expect("delete must not leave bucket permanently full")
|
||||
.abort()
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let caller = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel caller").is_cancelled());
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
|
||||
)
|
||||
.await
|
||||
.expect("quota changes while scanning");
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"old config cannot publish bootstrap usage"
|
||||
);
|
||||
drop(pause);
|
||||
let (_, incarnation, revision) = job
|
||||
.store
|
||||
.bucket_quota_and_incarnation_from_disk(&job.bucket)
|
||||
.await
|
||||
.expect("current configuration");
|
||||
let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
let caller = tokio::spawn({
|
||||
let job = current.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel new caller").is_cancelled());
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(¤t).await;
|
||||
assert!(
|
||||
read_test_ledger(¤t).await.matches(incarnation, revision),
|
||||
"owned scan publishes after its waiter is canceled"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
|
||||
use crate::storage_api_contracts::bucket::DeleteBucketOptions;
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("empty old bucket");
|
||||
job.store
|
||||
.delete_bucket(&job.bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("retire bucket while scan is paused");
|
||||
job.store
|
||||
.make_bucket(&job.bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("replacement bucket");
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
|
||||
)
|
||||
.await
|
||||
.expect("replacement quota");
|
||||
pause.0.release.notify_one();
|
||||
assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
drop(pause);
|
||||
initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
|
||||
.await
|
||||
.expect("replacement can bootstrap independently");
|
||||
let replacement = read_test_ledger(&job).await;
|
||||
assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
|
||||
assert_eq!(replacement.accounted_usage, 0);
|
||||
}
|
||||
|
||||
fn ledger(accounted_usage: u64) -> QuotaLedger {
|
||||
QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
|
||||
}
|
||||
@@ -2781,7 +2090,8 @@ mod tests {
|
||||
.expect("persist crash-left reservation fixture");
|
||||
|
||||
assert!(matches!(
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await,
|
||||
Err(StorageError::PartMissingOrCorrupt)
|
||||
));
|
||||
assert!(matches!(
|
||||
@@ -2799,10 +2109,10 @@ mod tests {
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
let replacement =
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await
|
||||
.expect("new quota admission must not inherit a fresh stale reservation");
|
||||
assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
|
||||
assert!(replacement.reservations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,9 +40,7 @@ use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::Intercept;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::{
|
||||
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
|
||||
@@ -157,8 +155,6 @@ pub struct RemoteS3EndpointSpec {
|
||||
pub ca_cert_pem: Option<String>,
|
||||
pub connect_timeout: Option<Duration>,
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Entire SDK call, including non-streaming response bodies and retries.
|
||||
pub operation_timeout: Option<Duration>,
|
||||
/// How many wire requests one logical call may cost. Every consumer
|
||||
/// declares it; see [`RemoteS3RetryPolicy`].
|
||||
pub retry: RemoteS3RetryPolicy,
|
||||
@@ -256,78 +252,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct UploadResponseDeadline(pub Duration);
|
||||
|
||||
impl Intercept for UploadResponseDeadline {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsUploadResponseDeadline"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let response = context.response_mut();
|
||||
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
|
||||
inner: Some(inner),
|
||||
deadline: Box::pin(tokio::time::sleep(self.0)),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct UploadResponseBody {
|
||||
inner: Option<SdkBody>,
|
||||
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
|
||||
}
|
||||
|
||||
impl http_body::Body for UploadResponseBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = BoxError;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
use std::future::Future as _;
|
||||
use std::task::Poll;
|
||||
if self.inner.is_none() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
// The absolute response budget never renews on data, empty frames,
|
||||
// or trailers. A retry gets its own bounded response window.
|
||||
if self.deadline.as_mut().poll(cx).is_ready() {
|
||||
self.inner.take();
|
||||
return Poll::Ready(Some(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"replication response body deadline exceeded",
|
||||
)
|
||||
.into())));
|
||||
}
|
||||
let Some(inner) = self.inner.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
let result = std::pin::Pin::new(inner).poll_frame(cx);
|
||||
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
|
||||
self.inner.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
|
||||
}
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
|
||||
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
|
||||
/// signature stays valid.
|
||||
@@ -385,7 +309,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
|
||||
let mut timeouts = TimeoutConfig::builder();
|
||||
if let Some(connect_timeout) = spec.connect_timeout {
|
||||
timeouts = timeouts.connect_timeout(connect_timeout);
|
||||
@@ -393,9 +317,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
if let Some(read_timeout) = spec.read_timeout {
|
||||
timeouts = timeouts.read_timeout(read_timeout);
|
||||
}
|
||||
if let Some(operation_timeout) = spec.operation_timeout {
|
||||
timeouts = timeouts.operation_timeout(operation_timeout);
|
||||
}
|
||||
config_builder = config_builder.timeout_config(timeouts.build());
|
||||
}
|
||||
|
||||
@@ -753,7 +674,6 @@ mod tests {
|
||||
ca_cert_pem: None,
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: None,
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
user_agent_suffix: "",
|
||||
}
|
||||
@@ -1107,16 +1027,11 @@ mod tests {
|
||||
spec.user_agent_suffix = "RustFS-Test/0.0";
|
||||
spec.connect_timeout = Some(Duration::from_secs(5));
|
||||
spec.read_timeout = Some(Duration::from_secs(5));
|
||||
spec.operation_timeout = Some(Duration::from_secs(12));
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("spec should build")
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
assert_eq!(
|
||||
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
|
||||
Some(Duration::from_secs(12))
|
||||
);
|
||||
S3Client::from_conf(config)
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
|
||||
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
|
||||
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
|
||||
@@ -1581,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A source generation already queued or being uploaded is not driven a
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
@@ -2404,47 +2404,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name, version and source generation. `queue_replica_task` admits a generation only once
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct ReplicationInFlightKey {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<uuid::Uuid>,
|
||||
generation: ReplicationGenerationSnapshot,
|
||||
}
|
||||
|
||||
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
|
||||
fn from(info: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
bucket: info.bucket.clone(),
|
||||
object: info.name.clone(),
|
||||
version_id: info.version_id,
|
||||
generation: info.replication_generation.clone(),
|
||||
}
|
||||
}
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert(ReplicationInFlightKey::from(ri))
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&ReplicationInFlightKey::from(ri));
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2457,21 +2438,26 @@ impl ReplicationInFlight {
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicationInFlightKey,
|
||||
key: ReplicateObjectInfo,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicationInFlightKey::from(ri),
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.lock().remove(&self.key);
|
||||
self.in_flight.finish(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4068,48 +4054,6 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let initial = ReplicateObjectInfo {
|
||||
bucket: "pending-tags-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let metadata = ReplicateObjectInfo {
|
||||
op_type: ReplicationType::Metadata,
|
||||
user_tags: String::new(),
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..initial.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
|
||||
assert_eq!(pool.in_flight.len(), 1);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
for expected in [&initial, &metadata] {
|
||||
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
|
||||
panic!("expected object replication");
|
||||
};
|
||||
assert_eq!(queued.replication_generation, expected.replication_generation);
|
||||
}
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
|
||||
@@ -6609,7 +6609,6 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
timeouts: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -259,10 +259,6 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
|
||||
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
|
||||
check_put_object_args(bucket, object)?;
|
||||
|
||||
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
@@ -274,14 +270,6 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// New public keys must remain representable in an unencoded XML listing.
|
||||
/// Existing keys still use the read/delete validators so they can be removed.
|
||||
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
|
||||
object
|
||||
.chars()
|
||||
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
|
||||
}
|
||||
|
||||
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
use tracing::{Instrument, debug, error, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
|
||||
@@ -224,8 +224,6 @@ pub struct ListPathRawOptions {
|
||||
pub recursive: bool,
|
||||
pub incl_deleted: bool,
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
pub hidden_entry_budget: usize,
|
||||
pub pending_prefix: Option<String>,
|
||||
pub filter_prefix: Option<String>,
|
||||
pub forward_to: Option<String>,
|
||||
pub min_disks: usize,
|
||||
@@ -262,8 +260,6 @@ impl Clone for ListPathRawOptions {
|
||||
recursive: self.recursive,
|
||||
incl_deleted: self.incl_deleted,
|
||||
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: self.hidden_entry_budget,
|
||||
pending_prefix: self.pending_prefix.clone(),
|
||||
filter_prefix: self.filter_prefix.clone(),
|
||||
forward_to: self.forward_to.clone(),
|
||||
min_disks: self.min_disks,
|
||||
@@ -293,8 +289,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
recursive: opts.recursive,
|
||||
incl_deleted: opts.incl_deleted,
|
||||
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: opts.hidden_entry_budget,
|
||||
pending_prefix: opts.pending_prefix.clone(),
|
||||
report_notfound: opts.report_not_found,
|
||||
filter_prefix: opts.filter_prefix.clone(),
|
||||
forward_to: opts.forward_to.clone(),
|
||||
@@ -307,15 +301,13 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
}
|
||||
|
||||
/// How a merged raw listing stream ended.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ListPathRawOutcome {
|
||||
/// At least one producer stopped at its per-disk limit instead of at the
|
||||
/// end of its namespace, so the closed stream is not proof of exhaustion.
|
||||
/// Carried in-band by the metacache stream trailer, so remote producers
|
||||
/// report it as well.
|
||||
pub limit_reached: bool,
|
||||
/// Inclusive sorted boundary proved by every reader before a bounded stop.
|
||||
pub scan_cut: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
@@ -371,10 +363,15 @@ async fn list_path_raw_inner(
|
||||
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
|
||||
|
||||
let cancel_rx = CancellationToken::new();
|
||||
let walk_options = Arc::new(walk_dir_options(&opts));
|
||||
|
||||
for (disk_idx, disk) in opts.disks.iter().enumerate() {
|
||||
let opdisk = disk.clone();
|
||||
let opts_clone = opts.clone();
|
||||
let walk_options = walk_options.clone();
|
||||
#[cfg(test)]
|
||||
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
|
||||
#[cfg(test)]
|
||||
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
|
||||
let fallback_claim_tracker = fallback_claim_tracker.clone();
|
||||
let fds_clone = fds.clone();
|
||||
#[cfg(test)]
|
||||
@@ -384,17 +381,17 @@ async fn list_path_raw_inner(
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64 * 1024);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
|
||||
if let Some(observer) = &test_disk_io_context_observed {
|
||||
observer.store(
|
||||
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
|
||||
let test_primary_error = if let Some(behavior) = test_reader_behavior {
|
||||
match behavior {
|
||||
TestReaderBehavior::Eof => return Ok(()),
|
||||
TestReaderBehavior::Entries(entries) => {
|
||||
@@ -451,8 +448,6 @@ async fn list_path_raw_inner(
|
||||
None
|
||||
};
|
||||
|
||||
let wakl_opts = walk_dir_options(&opts_clone);
|
||||
|
||||
let mut need_fallback = false;
|
||||
let mut last_err = None;
|
||||
#[cfg(test)]
|
||||
@@ -467,7 +462,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
if !need_fallback && let Some(disk) = opdisk {
|
||||
let primary_walk_started = std::time::Instant::now();
|
||||
match disk.walk_dir(wakl_opts, &mut wr).await {
|
||||
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_res) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_primary",
|
||||
@@ -491,8 +486,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -503,8 +498,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -606,8 +601,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing_for_path",
|
||||
error = ?err,
|
||||
@@ -618,8 +613,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing",
|
||||
error = ?err,
|
||||
@@ -634,7 +629,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
let fallback_walk_started = std::time::Instant::now();
|
||||
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
|
||||
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_r) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_fallback",
|
||||
@@ -660,8 +655,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -672,8 +667,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -695,12 +690,13 @@ async fn list_path_raw_inner(
|
||||
|
||||
// warn!("list_path_raw: while need_fallback done");
|
||||
Ok(())
|
||||
}));
|
||||
});
|
||||
jobs.spawn(producer.in_current_span());
|
||||
}
|
||||
|
||||
let revjob_rx = rx.clone();
|
||||
let mut revjobs = JoinSet::new();
|
||||
revjobs.spawn(async move {
|
||||
let merge = async move {
|
||||
// Consumer-side peek timeout, and a caveat worth understanding
|
||||
// (rustfs/backlog#1217).
|
||||
//
|
||||
@@ -730,8 +726,6 @@ async fn list_path_raw_inner(
|
||||
// Sticky across rounds: once any reader's trailer says its producer
|
||||
// stopped at the per-disk limit, the merged stream may hide more keys.
|
||||
let mut limit_reached = false;
|
||||
let mut scan_cut = None;
|
||||
let mut last_scanned_key: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -743,7 +737,7 @@ async fn list_path_raw_inner(
|
||||
// );
|
||||
|
||||
if revjob_rx.is_cancelled() {
|
||||
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
|
||||
return Ok(ListPathRawOutcome { limit_reached });
|
||||
}
|
||||
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
@@ -803,12 +797,7 @@ async fn list_path_raw_inner(
|
||||
continue;
|
||||
}
|
||||
// eof
|
||||
if r.limit_reached() == Some(true) {
|
||||
limit_reached = true;
|
||||
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
|
||||
DiskError::other("bounded listing reader ended without a scan boundary")
|
||||
})?);
|
||||
}
|
||||
limit_reached |= r.limit_reached() == Some(true);
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek eof, disk: {}", i);
|
||||
continue;
|
||||
@@ -882,7 +871,7 @@ async fn list_path_raw_inner(
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -941,7 +930,7 @@ async fn list_path_raw_inner(
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -991,14 +980,6 @@ async fn list_path_raw_inner(
|
||||
return Err(classify_listing_quorum_failure(&failures));
|
||||
}
|
||||
|
||||
// The next head from a bounded reader is unknown. The previous
|
||||
// globally consumed group is a safe cut; consuming another reader's
|
||||
// later head would turn this absence of evidence into a skipped key.
|
||||
// Check quorum failures above before accepting the bounded result.
|
||||
if limit_reached {
|
||||
break;
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() {
|
||||
if has_err > 0
|
||||
@@ -1006,7 +987,7 @@ async fn list_path_raw_inner(
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -1017,7 +998,6 @@ async fn list_path_raw_inner(
|
||||
break;
|
||||
}
|
||||
|
||||
last_scanned_key = Some(current.name.clone());
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
if !r.skip_current() {
|
||||
@@ -1029,7 +1009,7 @@ async fn list_path_raw_inner(
|
||||
// warn!("list_path_raw: agreed_fn start, current: {:?}", ¤t.name);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
result = agreed_fn(current) => result?
|
||||
}
|
||||
// warn!("list_path_raw: agreed_fn done");
|
||||
@@ -1053,13 +1033,14 @@ async fn list_path_raw_inner(
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ListPathRawOutcome { limit_reached, scan_cut })
|
||||
});
|
||||
Ok(ListPathRawOutcome { limit_reached })
|
||||
};
|
||||
revjobs.spawn(merge.in_current_span());
|
||||
|
||||
let merge_started = std::time::Instant::now();
|
||||
let Some(revjob) = revjobs.join_next().await else {
|
||||
@@ -1223,24 +1204,6 @@ mod tests {
|
||||
assert!(!options.skip_total_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||
let request = ListPathRawOptions {
|
||||
bucket: "versioned".to_owned(),
|
||||
hidden_entry_budget: 100,
|
||||
pending_prefix: Some("pending/".to_owned()),
|
||||
forward_to: Some("pending/0099".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
// Primary and fallback readers both construct their walk from the
|
||||
// per-job clone, rather than directly from the request options.
|
||||
let job = request.clone();
|
||||
let walk = walk_dir_options(&job);
|
||||
assert_eq!(walk.hidden_entry_budget, 100);
|
||||
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
|
||||
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_empty_disks_returns_read_quorum() {
|
||||
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
|
||||
@@ -1291,6 +1254,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let baseline = Arc::strong_count(&disk);
|
||||
|
||||
for primary_count in [1, 4, 16] {
|
||||
for fallback_count in [0, 2] {
|
||||
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let behaviors = entered
|
||||
.iter()
|
||||
.zip(&stopped)
|
||||
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
|
||||
entered: entered.clone(),
|
||||
stopped: stopped.clone(),
|
||||
})
|
||||
.collect();
|
||||
let cancel = CancellationToken::new();
|
||||
let listing = tokio::spawn(list_path_raw(
|
||||
cancel.clone(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk.clone()); primary_count],
|
||||
fallback_disks: vec![Some(disk.clone()); fallback_count],
|
||||
min_disks: primary_count,
|
||||
test_reader_behaviors: behaviors,
|
||||
peek_timeout: Some(Duration::from_secs(60)),
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
for signal in &entered {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("every producer must enter the real fanout before counting references");
|
||||
}
|
||||
|
||||
// The merge owns the configured primary/fallback slots, each producer
|
||||
// owns its primary, and the shared fallback queue owns each fallback.
|
||||
let expected = baseline + 2 * (primary_count + fallback_count);
|
||||
let retained = Arc::strong_count(&disk);
|
||||
cancel.cancel();
|
||||
timeout(Duration::from_secs(2), listing)
|
||||
.await
|
||||
.expect("cancellation must stop the listing")
|
||||
.expect("listing task must join")
|
||||
.expect("external cancellation must finish cleanly");
|
||||
for signal in &stopped {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("cancellation must drop every producer");
|
||||
}
|
||||
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
|
||||
assert_eq!(
|
||||
retained, expected,
|
||||
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn list_path_raw_producer_and_merge_preserve_request_span() {
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt};
|
||||
|
||||
struct ProducerSpan(Arc<AtomicBool>);
|
||||
#[derive(Default)]
|
||||
struct StateField(bool);
|
||||
impl tracing::field::Visit for StateField {
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
|
||||
}
|
||||
}
|
||||
impl<S> Layer<S> for ProducerSpan
|
||||
where
|
||||
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
|
||||
{
|
||||
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
|
||||
let mut state = StateField::default();
|
||||
event.record(&mut state);
|
||||
if state.0
|
||||
&& context
|
||||
.lookup_current()
|
||||
.is_some_and(|span| span.name() == "flink_list_request")
|
||||
{
|
||||
self.0.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
|
||||
let producer_observed = Arc::new(AtomicBool::new(false));
|
||||
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let error = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk)],
|
||||
bucket: "missing-bucket".to_owned(),
|
||||
min_disks: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request.clone())
|
||||
.await
|
||||
.expect_err("missing bucket must fail the real disk producer");
|
||||
assert_eq!(error, DiskError::VolumeNotFound);
|
||||
assert!(
|
||||
producer_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"disk error must retain the request span"
|
||||
);
|
||||
|
||||
let merge_observed = Arc::new(AtomicBool::new(false));
|
||||
let merge_sink = merge_observed.clone();
|
||||
let request_id = request.id();
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
|
||||
agreed: Some(Box::new(move |_| {
|
||||
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
|
||||
Box::pin(async { Ok(()) })
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request)
|
||||
.await
|
||||
.expect("merge should deliver the test entry");
|
||||
assert!(
|
||||
merge_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"merge callback must retain the request span"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
|
||||
let root = tempfile::tempdir().expect("local disk root should be created");
|
||||
@@ -1358,7 +1478,6 @@ mod tests {
|
||||
outcome.limit_reached,
|
||||
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
|
||||
);
|
||||
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1588,13 +1588,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)?;
|
||||
// A body-bound request also carries a usable v2 signature. Consume both
|
||||
// nonces so stripping the replay-scope headers cannot authorize it again.
|
||||
if content_sha256 != UNSIGNED_PAYLOAD {
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
|
||||
}
|
||||
Ok(())
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -2585,43 +2579,6 @@ mod tests {
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_consumes_the_body_bound_v2_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "node-a:9000";
|
||||
let path = "/node_service.NodeService/RecoverDeleteTransaction";
|
||||
let digest = "a".repeat(64);
|
||||
for v3_first in [true, false] {
|
||||
let mut headers =
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
|
||||
.expect("body-bound v2 headers should build");
|
||||
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, &digest, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let mut downgraded = headers.clone();
|
||||
for header in [
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
] {
|
||||
downgraded.remove(header);
|
||||
}
|
||||
let (first, replay) = if v3_first {
|
||||
(&headers, &downgraded)
|
||||
} else {
|
||||
(&downgraded, &headers)
|
||||
};
|
||||
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
|
||||
.expect("the first delivery must be accepted");
|
||||
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
|
||||
.expect_err("a mutation cannot replay through the other signature generation");
|
||||
assert_eq!(error.to_string(), "RPC request replay detected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -2014,21 +2014,9 @@ impl PeerRestClient {
|
||||
}
|
||||
result
|
||||
})
|
||||
.await;
|
||||
self.settle_bucket_metadata_reload(bucket, result).await
|
||||
}
|
||||
|
||||
async fn settle_bucket_metadata_reload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
|
||||
) -> Result<()> {
|
||||
match result {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
// A local operation deadline does not establish that the peer is
|
||||
// unreachable. In particular, it must not gate IAM revocations.
|
||||
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
|
||||
}
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
|
||||
self.finalize_result(result).await
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
@@ -4765,31 +4753,6 @@ mod tests {
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
|
||||
let client = test_peer_client();
|
||||
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("slow-policy", elapsed)
|
||||
.await
|
||||
.expect_err("local budget expires");
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
assert!(
|
||||
!client.offline.load(Ordering::Acquire),
|
||||
"local work must not suppress subsequent IAM RPCs"
|
||||
);
|
||||
client
|
||||
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
|
||||
.await
|
||||
.expect("peer reply");
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
|
||||
.await
|
||||
.expect_err("real transport failure");
|
||||
assert!(error.to_string().contains("connection refused"));
|
||||
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
|
||||
let client = test_peer_client();
|
||||
|
||||
@@ -1082,10 +1082,7 @@ impl RemoteDisk {
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
}
|
||||
let settle_timeout = if matches!(
|
||||
&operation,
|
||||
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
|
||||
) {
|
||||
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
|
||||
Some(crate::disk::get_delete_recovery_timeout()?)
|
||||
} else {
|
||||
None
|
||||
@@ -2719,54 +2716,6 @@ impl RemoteDisk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
|
||||
let expected_disk = self
|
||||
.id
|
||||
.lock()
|
||||
.await
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query = MultipartIndexRpcRequest::new(bucket, marker);
|
||||
let endpoint = self.endpoint.to_string();
|
||||
let canonical = query.canonical_request(&endpoint)?;
|
||||
let opts = serde_json::to_string(&query)?;
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DiskInfoRequest {
|
||||
disk: endpoint.clone(),
|
||||
opts: opts.clone(),
|
||||
});
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = client.disk_info(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// An old server ignores the additive query and returns ordinary DiskInfo.
|
||||
// It cannot satisfy the dedicated reply schema or the fresh challenge.
|
||||
let reply: MultipartIndexRpcReply =
|
||||
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
|
||||
reply.validate(&query, expected_disk)?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&query.canonical_response(&endpoint, &reply.page)?,
|
||||
&reply.proof,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
Ok(reply.page)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
@@ -4350,35 +4299,6 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
// Older peers reject this variant. Do not retry through the unbounded protocol.
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::DiscoverBounded {
|
||||
object: object.to_owned(),
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
Err(Error::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4414,58 +4334,6 @@ impl DiskAPI for RemoteDisk {
|
||||
_ => Err(Error::other("unexpected delete discovery reply")),
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
Err(Error::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::InspectBounded {
|
||||
object: object.to_owned(),
|
||||
transaction,
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
Err(Error::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4986,113 +4854,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("test peer must bind");
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk"),
|
||||
);
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let prefix = directory.path().join("bucket/prefix");
|
||||
std::fs::create_dir(&prefix).expect("prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
let fixture = Arc::new(TestBoundedDiscovery {
|
||||
disk: Arc::clone(&disk),
|
||||
response_gate: TestFormatReadGate::default(),
|
||||
});
|
||||
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
|
||||
.await
|
||||
.expect("disk completed bounded scan");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("caller cancelled").is_cancelled());
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
fixture.response_gate.release.cancel();
|
||||
peer.stop().await;
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
assert!(prefix.join("child-63").is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
remote
|
||||
.set_disk_id(Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("expected disk identity");
|
||||
assert!(matches!(
|
||||
remote.list_multipart_index_roots("bucket", None).await,
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
));
|
||||
assert_eq!(
|
||||
peer.peer.disk_info_calls.load(Ordering::Acquire),
|
||||
1,
|
||||
"exercise the actual legacy JSON handler"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert_eq!(
|
||||
remote
|
||||
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
|
||||
.await
|
||||
.expect_err("old peer cannot prove a bounded inventory"),
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
remote
|
||||
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
|
||||
.await
|
||||
.expect_err("old peer cannot inspect with a budget"),
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.settle_delete_transaction_bounded(
|
||||
"bucket",
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "prefix".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: [0; 32],
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: 65536,
|
||||
},
|
||||
None
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot settle with a budget"),
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
);
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods"),
|
||||
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5985,16 +5746,9 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestBoundedDiscovery {
|
||||
disk: Arc<crate::disk::local::LocalDisk>,
|
||||
response_gate: TestFormatReadGate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TestFormatReadGate {
|
||||
entered: tokio::sync::Notify,
|
||||
@@ -6011,7 +5765,6 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
bounded_discovery: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -6111,56 +5864,6 @@ mod tests {
|
||||
Ok(grpc.unary(method, request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/RecoverDeleteTransaction"
|
||||
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
|
||||
{
|
||||
#[derive(Clone)]
|
||||
struct BoundedSvc(Arc<TestBoundedDiscovery>);
|
||||
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
|
||||
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
|
||||
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
|
||||
let fixture = Arc::clone(&self.0);
|
||||
Box::pin(async move {
|
||||
let request = request.into_inner();
|
||||
let operation: DeleteRecoveryRpcOperation =
|
||||
rustfs_protos::decode_delete_recovery_payload(&request.operation)
|
||||
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
|
||||
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
|
||||
return Err(Status::invalid_argument("unexpected fallback"));
|
||||
};
|
||||
let page = fixture
|
||||
.disk
|
||||
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
|
||||
.await
|
||||
.map_err(|_| Status::internal("bounded discovery failed"))?;
|
||||
// Lose the response only after the real disk operation has completed.
|
||||
fixture.response_gate.entered.notify_one();
|
||||
fixture.response_gate.release.cancelled().await;
|
||||
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
|
||||
.map_err(|_| Status::internal("encode bounded reply"))?;
|
||||
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
|
||||
success: true,
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
result: result.into(),
|
||||
error: None,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
let fixture = self
|
||||
.peer
|
||||
.bounded_discovery
|
||||
.lock()
|
||||
.expect("bounded fixture")
|
||||
.clone()
|
||||
.expect("configured fixture");
|
||||
Box::pin(async move {
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec);
|
||||
Ok(grpc.unary(BoundedSvc(fixture), request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/ReadAll" => {
|
||||
#[derive(Clone)]
|
||||
struct ReadAllSvc(AuthenticatedReadPeer);
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::cluster::rpc::client::{
|
||||
use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
@@ -32,16 +33,17 @@ use rustfs_protos::{
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{Instant, timeout};
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::{Request, Response};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
|
||||
|
||||
@@ -49,18 +51,38 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
/// Work to run if an RPC that already timed out for its caller completes later.
|
||||
/// Work to run if an RPC completes after its caller stops waiting.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
|
||||
|
||||
struct RpcCleanup<T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
/// Own admission before connecting as well as while an RPC is in flight.
|
||||
struct LockRequestGuard<'a> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
}
|
||||
|
||||
impl<T> RpcCleanup<T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
|
||||
impl LockRequestGuard<'_> {
|
||||
fn into_admission(mut self) -> Option<LockRequestAdmission> {
|
||||
self.admission.take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LockRequestGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.client.release_lock_request(self.op, self.admission.take());
|
||||
}
|
||||
}
|
||||
|
||||
struct RpcCleanup<'a, T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestGuard<'a>>,
|
||||
}
|
||||
|
||||
impl<'a, T> RpcCleanup<'a, T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
|
||||
Self {
|
||||
late,
|
||||
late_failure,
|
||||
@@ -69,6 +91,25 @@ impl<T> RpcCleanup<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
|
||||
struct InFlightRpc<'a, T: Send + 'static> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
resource_summary: &'a str,
|
||||
future: Option<RpcFuture<T>>,
|
||||
cleanup: Option<RpcCleanup<'a, T>>,
|
||||
}
|
||||
|
||||
impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(future) = self.future.take()
|
||||
&& let Some(cleanup) = self.cleanup.take()
|
||||
{
|
||||
self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The liveness window is this many RPC deadlines: a peer that completed a
|
||||
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
|
||||
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
|
||||
@@ -137,6 +178,9 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
|
||||
let mut peers = lock_peer_channel_health()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(health) = peers.get_mut(addr) {
|
||||
return update(health);
|
||||
}
|
||||
update(peers.entry(addr.to_string()).or_default())
|
||||
}
|
||||
|
||||
@@ -158,6 +202,15 @@ fn reset_lock_peer_health_for_test(addr: &str) {
|
||||
.remove(addr);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
#[cfg(test)]
|
||||
fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
|
||||
static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
|
||||
PAUSES.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum EvictionTrigger {
|
||||
/// The caller's deadline expired while the stream was still open.
|
||||
@@ -506,6 +559,14 @@ impl RemoteClient {
|
||||
admission
|
||||
}
|
||||
|
||||
fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
|
||||
self.admit_lock_request(op).map(|admission| LockRequestGuard {
|
||||
client: self,
|
||||
op,
|
||||
admission: Some(admission),
|
||||
})
|
||||
}
|
||||
|
||||
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
@@ -587,6 +648,17 @@ impl RemoteClient {
|
||||
trigger: EvictionTrigger,
|
||||
deadline: Duration,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
let pause = eviction_pauses_for_test()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.addr);
|
||||
#[cfg(test)]
|
||||
if let Some((entered, resume)) = pause {
|
||||
let _ = entered.send(());
|
||||
let _ = resume.await;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let cooldown = Self::eviction_cooldown();
|
||||
let liveness_window = Self::liveness_window(deadline);
|
||||
@@ -617,22 +689,26 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
|
||||
/// Keep a timed-out RPC running instead of cancelling its stream.
|
||||
/// Keep an abandoned RPC running instead of cancelling its stream.
|
||||
///
|
||||
/// Dropping the future sends `RST_STREAM`; under load those resets pile up
|
||||
/// in the server's pending-accept queue until it answers `GOAWAY
|
||||
/// too_many_resets` and kills every stream on the connection. A detached
|
||||
/// stream ends on its own within the internode RPC timeout, the number per
|
||||
/// peer is bounded, and a lock granted after its caller gave up is released.
|
||||
fn detach_timed_out_rpc<T: Send + 'static>(
|
||||
fn detach_rpc<T: Send + 'static>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
resource_summary: &str,
|
||||
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
future: RpcFuture<T>,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
) {
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let admission = admission.and_then(LockRequestGuard::into_admission);
|
||||
let limit = Self::detached_rpc_limit();
|
||||
let admitted = with_lock_peer_health(&self.addr, |health| {
|
||||
if health.detached_rpcs >= limit {
|
||||
@@ -643,25 +719,29 @@ impl RemoteClient {
|
||||
}
|
||||
});
|
||||
if !admitted {
|
||||
handle.abort();
|
||||
drop(future);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "detached_budget_exhausted",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
limit,
|
||||
"Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
|
||||
"Abandoned remote lock RPC cancelled"
|
||||
);
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = handle.await;
|
||||
let outcome = future.await;
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
@@ -688,19 +768,31 @@ impl RemoteClient {
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_failed",
|
||||
addr = %addr,
|
||||
op,
|
||||
tonic_code = ?status.code(),
|
||||
tonic_message = status.message(),
|
||||
"Detached remote lock RPC failed after its caller timed out"
|
||||
"Abandoned remote lock RPC failed"
|
||||
);
|
||||
}
|
||||
Err(join_error) => {
|
||||
Err(_) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
|
||||
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_panicked",
|
||||
addr = %addr,
|
||||
op,
|
||||
"Abandoned remote lock RPC panicked"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -788,16 +880,28 @@ impl RemoteClient {
|
||||
self.release_late_release();
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
|
||||
if outcome == "released" {
|
||||
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_released",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
"Released remote locks granted after their caller stopped waiting"
|
||||
);
|
||||
} else {
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_unreleased",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
"Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
@@ -811,25 +915,47 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
deadline: Duration,
|
||||
future: Fut,
|
||||
cleanup: RpcCleanup<T>,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
) -> std::result::Result<T, LockError>
|
||||
where
|
||||
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let RpcCleanup {
|
||||
late,
|
||||
let mut rpc = InFlightRpc {
|
||||
client: self,
|
||||
op,
|
||||
resource_summary,
|
||||
future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
|
||||
cleanup: Some(cleanup),
|
||||
};
|
||||
let outcome = match rpc.future.as_mut() {
|
||||
Some(future) => tokio::select! {
|
||||
// A response becoming ready on the deadline tick must not turn an
|
||||
// expired caller budget into a transport error and channel eviction.
|
||||
biased;
|
||||
_ = tokio::time::sleep(deadline) => None,
|
||||
outcome = future => Some(outcome),
|
||||
},
|
||||
None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
|
||||
};
|
||||
if outcome.is_some() {
|
||||
rpc.future.take();
|
||||
}
|
||||
let Some(RpcCleanup {
|
||||
late: _,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let mut handle = tokio::spawn(future);
|
||||
match timeout(deadline, &mut handle).await {
|
||||
Ok(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, admission);
|
||||
self.release_lock_request(op, admission);
|
||||
}) = rpc.cleanup.as_mut()
|
||||
else {
|
||||
return Err(LockError::internal("remote lock RPC has no cleanup owner"));
|
||||
};
|
||||
let token = admission.as_ref().and_then(|guard| guard.admission);
|
||||
let result = match outcome {
|
||||
Some(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, token);
|
||||
Ok(response)
|
||||
}
|
||||
Ok(Ok(Err(err))) => {
|
||||
Some(Ok(Err(err))) => {
|
||||
let reason = err.to_string();
|
||||
if err.code() == tonic::Code::DeadlineExceeded {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
@@ -862,12 +988,11 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC deadline"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
// Only evict (and re-dial) the cached channel when the failure is a genuine
|
||||
@@ -928,30 +1053,32 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
if transport_failure {
|
||||
// Ambiguous-grant cleanup owns its work before channel eviction can suspend.
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
|
||||
.await;
|
||||
}
|
||||
self.release_lock_request(op, admission);
|
||||
if transport_failure && let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
|
||||
}
|
||||
Ok(Err(join_error)) => {
|
||||
Some(Err(_)) => {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "rpc_panicked",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
error = %join_error,
|
||||
"Remote lock RPC task ended abnormally"
|
||||
"Remote lock RPC panicked"
|
||||
);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
|
||||
Err(LockError::internal(format!("{op} RPC future panicked")))
|
||||
}
|
||||
Err(_) => {
|
||||
None => {
|
||||
let reason = format!("RPC timed out after {deadline:?}");
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
@@ -983,12 +1110,17 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC timeout"
|
||||
);
|
||||
}
|
||||
// Continue draining the abandoned stream while channel eviction waits.
|
||||
drop(rpc);
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
|
||||
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
};
|
||||
if rpc.future.is_none() {
|
||||
rpc.cleanup.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
|
||||
@@ -1079,7 +1211,7 @@ impl RemoteClient {
|
||||
#[async_trait]
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
let admission = match self.admit_lock_request("lock") {
|
||||
let admission = match self.admit_lock_request_guard("lock") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_response(request)),
|
||||
};
|
||||
@@ -1090,7 +1222,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock", rpc_timeout);
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1098,14 +1229,12 @@ impl LockClient for RemoteClient {
|
||||
let args = match serde_json::to_string(&bounded_request) {
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(GenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_hook(request.lock_id.clone());
|
||||
@@ -1147,7 +1276,7 @@ impl LockClient for RemoteClient {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let admission = match self.admit_lock_request("lock_batch") {
|
||||
let admission = match self.admit_lock_request_guard("lock_batch") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
|
||||
};
|
||||
@@ -1161,7 +1290,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock_batch", rpc_timeout);
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1175,14 +1303,12 @@ impl LockClient for RemoteClient {
|
||||
{
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(BatchGenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
@@ -1710,6 +1836,351 @@ mod tests {
|
||||
assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_connection_releases_request_admission() {
|
||||
ensure_test_rpc_secret();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
|
||||
let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
|
||||
for batch in [false, true] {
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
|
||||
});
|
||||
let mut acquisition = Box::pin(async {
|
||||
if batch {
|
||||
client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
|
||||
} else {
|
||||
client.acquire_lock(&request).await.map(|_| ())
|
||||
}
|
||||
});
|
||||
std::future::poll_fn(|cx| {
|
||||
assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
|
||||
std::task::Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
let pending = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
|
||||
assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
|
||||
|
||||
drop(acquisition);
|
||||
let cancelled = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
|
||||
assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
|
||||
let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
}
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-grant";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
Ok(grant_rx.await.expect("test decides when peer grants lock"))
|
||||
},
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"cancelled RPC uses bounded detached ownership"
|
||||
);
|
||||
grant_tx.send(true).expect("cancellation must preserve the running RPC");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up")
|
||||
.expect("cleanup result")
|
||||
);
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-failure";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("ambiguous cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock_batch",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
failure_rx.await.expect("test decides when transport fails");
|
||||
Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
failure_tx.send(()).expect("cancellation must preserve the running RPC");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("ambiguous grant must be cleaned up")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
|
||||
for (label, status) in [
|
||||
("transport", tonic::Status::cancelled("ambiguous transport failure")),
|
||||
("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
|
||||
] {
|
||||
let addr = format!("http://cancelled-completed-{label}");
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.clone(), (evicting_tx, resume_rx));
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx
|
||||
.send(())
|
||||
.expect("completed-failure cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move { Err::<(), _>(status) },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("completed error must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("cleanup must transfer before channel eviction suspends")
|
||||
.expect("cleanup signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(&addr).detached_rpcs,
|
||||
0,
|
||||
"completed RPC cannot consume a detached slot"
|
||||
);
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
|
||||
let addr = "http://same-tick-rpc-deadline";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let budget = Duration::from_millis(50);
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("same-tick cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
let result = client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
budget,
|
||||
async move {
|
||||
tokio::time::sleep(budget).await;
|
||||
Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"expired caller budget must retain timeout classification"
|
||||
);
|
||||
let health = lock_peer_health_for_test(addr);
|
||||
assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
|
||||
assert!(
|
||||
health.last_eviction.is_none(),
|
||||
"a recently serving peer keeps its channel after a caller timeout"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("same-tick ambiguous completion must trigger cleanup")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"same-tick completion returns detached admission"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
|
||||
let addr = "http://timeout-late-grant-during-eviction";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.to_string(), (evicting_tx, resume_rx));
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_millis(25),
|
||||
async move { Ok(grant_rx.await.expect("test controls the late grant")) },
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("timed-out RPC must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"eviction waits with a detached RPC owner"
|
||||
);
|
||||
grant_tx
|
||||
.send(true)
|
||||
.expect("timed-out stream remains available for a late grant");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up before eviction resumes")
|
||||
.expect("late cleanup result")
|
||||
);
|
||||
assert!(!caller.is_finished(), "channel eviction is still paused");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"late completion returns detached admission"
|
||||
);
|
||||
resume_tx.send(()).expect("resume channel eviction");
|
||||
let result = caller.await.expect("timed-out caller completes without panicking");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"caller preserves its RPC timeout result"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_success_polls_rpc_on_callers_task() {
|
||||
tokio::spawn(async {
|
||||
let client = RemoteClient::new("http://direct-poll-success".to_string());
|
||||
let caller_id = tokio::task::id();
|
||||
let rpc_id = client
|
||||
.execute_rpc(
|
||||
"ping",
|
||||
"health-lock-online",
|
||||
Duration::from_secs(1),
|
||||
async { Ok(tokio::task::id()) },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await
|
||||
.expect("successful RPC must return its result");
|
||||
assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
|
||||
})
|
||||
.await
|
||||
.expect("caller task must complete");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let client = RemoteClient::new("http://direct-poll-panic".to_string());
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("panic cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc::<(), _>(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(1),
|
||||
async { panic!("test peer RPC panic") },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let result = caller.await.expect("RPC panic must not escape to its caller");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Internal { .. })),
|
||||
"RPC panic remains an internal lock failure"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("panic must trigger ambiguous-grant cleanup")
|
||||
.expect("cleanup signal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {
|
||||
|
||||
@@ -317,9 +317,8 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn stop_endpoint_monitor_for_test(&self) {
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
|
||||
@@ -55,20 +55,6 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
|
||||
/// Completion may mean legacy fallback after a failed scan; callers must assert
|
||||
/// the expected index result separately. Remote disks have no local worker.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
|
||||
match disk.as_ref() {
|
||||
crate::disk::Disk::Local(local) => {
|
||||
local.disk.wait_for_multipart_index_bootstrap().await;
|
||||
Ok(())
|
||||
}
|
||||
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disk health status constants
|
||||
const DISK_HEALTH_OK: u32 = 0;
|
||||
const DISK_HEALTH_FAULTY: u32 = 1;
|
||||
@@ -2457,7 +2443,6 @@ impl LocalDiskWrapper {
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2465,7 +2450,7 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
|
||||
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2481,7 +2466,6 @@ impl LocalDiskWrapper {
|
||||
transaction: uuid::Uuid,
|
||||
journal: &crate::disk::DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2489,14 +2473,7 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2751,18 +2728,6 @@ impl LocalDiskWrapper {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.list_multipart_index_roots(bucket, marker).await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
// The local implementation shares this binary's capability. Availability
|
||||
// and disk identity remain mutation checks and write-quorum inputs.
|
||||
@@ -2816,62 +2781,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
self.track_disk_health_with_op(
|
||||
"discover_delete_transactions",
|
||||
|| async {
|
||||
self.disk
|
||||
.discover_delete_transactions_bounded(volume, object, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
self.track_disk_health_with_op(
|
||||
"inspect_delete_transaction",
|
||||
|| async {
|
||||
self.disk
|
||||
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
self.disk
|
||||
.settle_delete_transaction_bounded(volume, request, namespace_owner)
|
||||
.await
|
||||
},
|
||||
crate::disk::get_delete_recovery_timeout()?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -3840,11 +3749,7 @@ mod tests {
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(
|
||||
!logs.contents().contains("disk_operation_slow"),
|
||||
"normal disk latency should not warn: {}",
|
||||
logs.contents()
|
||||
);
|
||||
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
|
||||
time::advance(Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
let warning = logs.contents();
|
||||
|
||||
@@ -172,8 +172,6 @@ pub enum DiskError {
|
||||
|
||||
#[error("guarded batch deletion is not supported by this disk")]
|
||||
GuardedDeleteUnsupported,
|
||||
#[error("foreground delete recovery work limit reached")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
|
||||
#[error("erasure write quorum")]
|
||||
ErasureWriteQuorum,
|
||||
@@ -772,7 +770,6 @@ impl Clone for DiskError {
|
||||
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
@@ -832,7 +829,6 @@ impl DiskError {
|
||||
DiskError::RemoteClientUnavailable(_) => 0x2B,
|
||||
DiskError::ConditionalDeleteConflict => 0x2C,
|
||||
DiskError::GuardedDeleteUnsupported => 0x2D,
|
||||
DiskError::DeleteRecoveryWorkLimit => 0x2E,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -883,7 +879,6 @@ impl DiskError {
|
||||
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
|
||||
0x2C => Some(DiskError::ConditionalDeleteConflict),
|
||||
0x2D => Some(DiskError::GuardedDeleteUnsupported),
|
||||
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1667,7 +1662,6 @@ mod tests {
|
||||
for (error, code) in [
|
||||
(DiskError::ConditionalDeleteConflict, 0x2C),
|
||||
(DiskError::GuardedDeleteUnsupported, 0x2D),
|
||||
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
|
||||
] {
|
||||
assert_eq!(error.to_u32(), code);
|
||||
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
|
||||
|
||||
@@ -25,7 +25,6 @@ pub(super) mod delete_recovery;
|
||||
#[cfg(all(unix, test))]
|
||||
#[path = "fd_cache_tests.rs"]
|
||||
mod fd_cache_tests;
|
||||
mod multipart_index;
|
||||
pub(in crate::disk) mod multipart_source_abort;
|
||||
mod multipart_source_recovery;
|
||||
pub(in crate::disk) mod multipart_source_state;
|
||||
@@ -186,12 +185,6 @@ const DELETE_RESIDUE_PROBE_LIMIT: i32 = 8;
|
||||
const DELETE_RESIDUE_PROBE_READ_BUDGET: usize = 32;
|
||||
|
||||
/// One directory read owed by the delete-residue probe.
|
||||
enum PrefixListingProbe {
|
||||
Visible,
|
||||
Empty,
|
||||
Cut(Vec<MetaCacheEntry>),
|
||||
}
|
||||
|
||||
enum ResidueProbeStep {
|
||||
/// Read a bounded batch of `dir` and probe what it holds.
|
||||
Sample(String),
|
||||
@@ -651,7 +644,6 @@ async fn write_delete_rollback_files(
|
||||
let object_dir_buf = object_dir.to_path_buf();
|
||||
let volume_str = volume.to_owned();
|
||||
let inspect_object = inspect_object.map(str::to_owned);
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation_with_sync(first_lease, sync, move || {
|
||||
for (path, _) in &removed_entries {
|
||||
reject_local_disk_symlink_components(&io_root, path).map_err(std::io::Error::from)?;
|
||||
@@ -710,21 +702,10 @@ async fn write_delete_rollback_files(
|
||||
}
|
||||
}
|
||||
drop(staging_dir);
|
||||
if entries.iter().any(|(name, _, _, _)| *name == DELETE_QUORUM_COMMITTED_FILE)
|
||||
&& let Some(scope) = recovery_scope.as_ref()
|
||||
{
|
||||
scope.receipt_published();
|
||||
}
|
||||
let inspected = inspect_object
|
||||
.map(|object| {
|
||||
delete_recovery::inspection_with_markers_bounded(
|
||||
&object_dir_buf,
|
||||
&volume_str,
|
||||
&object,
|
||||
rollback_dir,
|
||||
recovery_scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)
|
||||
.map(|(state, _)| state)
|
||||
delete_recovery::inspection_with_markers(&object_dir_buf, &volume_str, &object, rollback_dir)
|
||||
.map(|(state, _)| state)
|
||||
})
|
||||
.transpose();
|
||||
drop((removed_entries, entries, inspect_lease));
|
||||
@@ -796,10 +777,7 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
let mut remove_new_metadata = false;
|
||||
match fs::read_dir(&rollback_path).await {
|
||||
Ok(mut entries) => {
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
let name = entry.file_name();
|
||||
if name == DELETE_MARKER_ROLLBACK_FILE {
|
||||
remove_new_metadata = true;
|
||||
@@ -833,10 +811,7 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
if recovery_journal.is_some() {
|
||||
let mut entries = fs::read_dir(&src).await.map_err(to_file_error)?;
|
||||
let mut files = Vec::new();
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
let name = entry.file_name();
|
||||
let name = name.to_str().ok_or(DiskError::FileCorrupt)?;
|
||||
if !entry.file_type().await.map_err(to_file_error)?.is_file()
|
||||
@@ -934,16 +909,10 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
match restored {
|
||||
Ok(()) => {}
|
||||
Err(DiskError::FileNotFound) if remove_new_metadata => {
|
||||
let preserves_child_prefix = recovery_journal
|
||||
.as_ref()
|
||||
.is_some_and(delete_recovery::missing_marker_preserves_child_prefix)
|
||||
&& fs::symlink_metadata(xl_path).await.is_ok_and(|metadata| metadata.is_dir());
|
||||
if !preserves_child_prefix {
|
||||
match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
Err(DiskError::FileNotFound) if had_staged_paths => return Err(DiskError::FileNotFound),
|
||||
@@ -993,10 +962,7 @@ async fn remove_version_delete_markers(
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
};
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
if !entry.file_type().await.map_err(to_file_error)?.is_dir()
|
||||
|| !entry.file_name().to_str().is_some_and(|name| Uuid::parse_str(name).is_ok())
|
||||
{
|
||||
@@ -7510,7 +7476,6 @@ pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
multipart_proof_authority: OnceLock<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_index: Arc<multipart_index::IndexOwner>,
|
||||
/// I/O root pinned to the mount instance that was opened while the disk
|
||||
/// was initialized. On Linux this is `/proc/self/fd/<dirfd>/.`; resolving
|
||||
/// paths beneath it keeps repair I/O on that mount even if the configured
|
||||
@@ -8053,10 +8018,7 @@ impl LocalDisk {
|
||||
|
||||
// TODO(backlog): add O_DIRECT I/O support for performance-critical paths
|
||||
// TODO(backlog): populate DiskInfo in constructor
|
||||
let multipart_index = multipart_index::IndexOwner::acquire(&publication_root, &root, &io_root)?;
|
||||
let blocking_queue = multipart_index.attach_blocking_queue(&root, &io_root);
|
||||
let mut disk = Self {
|
||||
multipart_index,
|
||||
root: root.clone(),
|
||||
publication_root,
|
||||
multipart_proof_authority: OnceLock::new(),
|
||||
@@ -8092,7 +8054,7 @@ impl LocalDisk {
|
||||
)
|
||||
.await,
|
||||
file_sync_permits: os::disk_file_sync_limiter(&root),
|
||||
blocking_queue,
|
||||
blocking_queue: os::register_disk_blocking_queue(&root, &io_root),
|
||||
snapshot_leases: Arc::new(ParkingLotMutex::new(SnapshotLeaseRegistry::default())),
|
||||
#[cfg(unix)]
|
||||
pool_meta_durable_head: ParkingLotMutex::new(None),
|
||||
@@ -8135,8 +8097,6 @@ impl LocalDisk {
|
||||
Self::check_startup_writable(&disk.publication_root, &disk.root.join(directory)).await?;
|
||||
}
|
||||
|
||||
disk.multipart_index.start();
|
||||
|
||||
let (exit_tx, exit_rx) = tokio::sync::broadcast::channel(1);
|
||||
disk.exit_signal = Some(exit_tx);
|
||||
|
||||
@@ -9949,10 +9909,7 @@ impl LocalDisk {
|
||||
let mut transaction_cleanup = false;
|
||||
match fs::read_dir(&transaction_path).await {
|
||||
Ok(mut entries) => {
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
let name = entry.file_name();
|
||||
if !entry.file_type().await.map_err(to_file_error)?.is_file()
|
||||
|| (name != STORAGE_FORMAT_FILE_BACKUP
|
||||
@@ -10168,7 +10125,6 @@ impl LocalDisk {
|
||||
let data_path = file_path.clone();
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let keep_rollback_witnesses = opt.recoverable_delete;
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
if let Some(check) = &multipart_check {
|
||||
check.callback()()?;
|
||||
@@ -10186,9 +10142,7 @@ impl LocalDisk {
|
||||
let mut parts = Vec::new();
|
||||
let mut markers = Vec::new();
|
||||
let mut has_committed_marker = false;
|
||||
for entry in
|
||||
delete_recovery::recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref()))
|
||||
{
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name = name.to_str().ok_or_else(corrupt)?;
|
||||
@@ -10546,6 +10500,7 @@ impl LocalDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn read_listing_metadata(&self, volume: &str, object_name: &str) -> Result<ListingMetadataRead> {
|
||||
let mut batch = self
|
||||
.spawn_listing_metadata_batch(volume, vec![object_name.to_owned()], None)
|
||||
@@ -10692,14 +10647,6 @@ impl LocalDisk {
|
||||
let rollback_dir = opts.old_data_dir;
|
||||
let incarnation = opts.delete_recovery_bucket_incarnation;
|
||||
let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
// An independent child key owns this directory. Reject before creating
|
||||
// recovery evidence that could otherwise make a harmless prefix unhealable.
|
||||
match fs::symlink_metadata(&xl_path).await {
|
||||
Ok(metadata) if metadata.is_dir() => return Err(DiskError::IsNotRegular),
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
let mut recovery_target = None;
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
if opts.recoverable_delete {
|
||||
@@ -11404,9 +11351,6 @@ impl LocalDisk {
|
||||
admission: Option<multipart_staging::MultipartSourceAdmission>,
|
||||
) -> Result<()> {
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
}
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
@@ -11827,9 +11771,9 @@ impl LocalDisk {
|
||||
continue;
|
||||
}
|
||||
|
||||
let metadata = with_walk_stall_timeout(stall, self.read_listing_metadata(bucket, ¤t))
|
||||
.await?
|
||||
.bytes;
|
||||
let metadata =
|
||||
with_walk_stall_timeout(stall, self.read_metadata(bucket, format!("{}/{}", current, entry).as_str()))
|
||||
.await?;
|
||||
|
||||
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
|
||||
let name = entry.trim_end_matches(SLASH_SEPARATOR);
|
||||
@@ -11873,27 +11817,19 @@ impl LocalDisk {
|
||||
});
|
||||
}
|
||||
|
||||
type PendingDirectory = (String, bool, Option<HashSet<String>>, bool, bool);
|
||||
let mut dir_stack: Vec<PendingDirectory> = Vec::with_capacity(5);
|
||||
let mut dir_stack: Vec<(String, bool, Option<HashSet<String>>, bool)> = Vec::with_capacity(5);
|
||||
// Explicit directory markers and real directories can resolve to the same logical path.
|
||||
let schedule_dir = |dir_stack: &mut Vec<PendingDirectory>,
|
||||
let schedule_dir = |dir_stack: &mut Vec<(String, bool, Option<HashSet<String>>, bool)>,
|
||||
dir_name: String,
|
||||
skip_object: bool,
|
||||
dir_to_skip: Option<HashSet<String>>,
|
||||
scan_required: bool,
|
||||
verify_prefix: bool| {
|
||||
if let Some((
|
||||
last_dir_name,
|
||||
existing_skip_object,
|
||||
existing_dir_to_skip,
|
||||
existing_scan_required,
|
||||
existing_verify_prefix,
|
||||
)) = dir_stack.last_mut()
|
||||
scan_required: bool| {
|
||||
if let Some((last_dir_name, existing_skip_object, existing_dir_to_skip, existing_scan_required)) =
|
||||
dir_stack.last_mut()
|
||||
&& *last_dir_name == dir_name
|
||||
{
|
||||
*existing_skip_object |= skip_object;
|
||||
*existing_scan_required |= scan_required;
|
||||
*existing_verify_prefix |= verify_prefix;
|
||||
if let Some(existing_dir_to_skip) = existing_dir_to_skip {
|
||||
if let Some(new_dir_to_skip) = &dir_to_skip {
|
||||
existing_dir_to_skip.extend(new_dir_to_skip.iter().cloned());
|
||||
@@ -11902,12 +11838,9 @@ impl LocalDisk {
|
||||
*existing_dir_to_skip = dir_to_skip;
|
||||
}
|
||||
} else {
|
||||
dir_stack.push((dir_name, skip_object, dir_to_skip, scan_required, verify_prefix));
|
||||
dir_stack.push((dir_name, skip_object, dir_to_skip, scan_required));
|
||||
}
|
||||
};
|
||||
let bounded_probe =
|
||||
!opts.recursive && !opts.incl_deleted && !opts.skip_hidden_prefix_check && opts.hidden_entry_budget > 0;
|
||||
let mut remaining_hidden = opts.hidden_entry_budget;
|
||||
prefix = "".to_owned();
|
||||
let mut prefetched_batch: Option<ListingMetadataBatch> = None;
|
||||
let mut adaptive_batch_cap = WALK_DIR_BATCH_SIZE;
|
||||
@@ -11923,7 +11856,7 @@ impl LocalDisk {
|
||||
|
||||
let name = path_join_buf(&[current.as_str(), entry.as_str()]);
|
||||
|
||||
while let Some((last_name, _, _, _, _)) = dir_stack.last()
|
||||
while let Some((last_name, _, _, _)) = dir_stack.last()
|
||||
&& *last_name < name
|
||||
{
|
||||
// A prior iteration of this same loop may have just recursed
|
||||
@@ -11937,20 +11870,7 @@ impl LocalDisk {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let (pop, skip_object, dir_to_skip, scan_required, verify_prefix) =
|
||||
dir_stack.pop().expect("operation should succeed");
|
||||
if verify_prefix {
|
||||
match self.probe_listing_prefix(opts, &pop, &mut remaining_hidden).await? {
|
||||
PrefixListingProbe::Empty => continue,
|
||||
PrefixListingProbe::Visible => {}
|
||||
PrefixListingProbe::Cut(entries) => {
|
||||
for entry in entries {
|
||||
write_metacache_obj(out, &entry).await?;
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (pop, skip_object, dir_to_skip, scan_required) = dir_stack.pop().expect("operation should succeed");
|
||||
let more_ready_in_batch = !(opts.recursive && scan_required)
|
||||
&& prefetched_batch.as_ref().is_some_and(|b| b.remaining > 0 && !b.rx.is_empty());
|
||||
if more_ready_in_batch {
|
||||
@@ -12058,7 +11978,7 @@ impl LocalDisk {
|
||||
};
|
||||
let batch_names = if batch_limit > 1
|
||||
&& dir_objes.is_empty()
|
||||
&& dir_stack.iter().all(|(_, _, _, scan_required, _)| !*scan_required)
|
||||
&& dir_stack.iter().all(|(_, _, _, scan_required)| !*scan_required)
|
||||
{
|
||||
entries[entry_idx..]
|
||||
.iter()
|
||||
@@ -12081,7 +12001,23 @@ impl LocalDisk {
|
||||
}
|
||||
.await
|
||||
} else {
|
||||
with_walk_stall_timeout(stall, self.read_listing_metadata(&opts.bucket, &meta.name)).await
|
||||
let fname = format!("{}/{}", meta.name, STORAGE_FORMAT_FILE);
|
||||
with_walk_stall_timeout(stall, self.read_metadata(&opts.bucket, fname.as_str()))
|
||||
.await
|
||||
.map(|metadata| {
|
||||
let bytes = Vec::from(metadata);
|
||||
let (file_meta, data_dirs, counts_toward_limit) = parse_listing_metadata_bytes(&bytes);
|
||||
#[cfg(not(test))]
|
||||
let _ = file_meta;
|
||||
ListingMetadataRead {
|
||||
bytes,
|
||||
#[cfg(test)]
|
||||
file_meta,
|
||||
data_dirs,
|
||||
has_namespace_child_candidate: true,
|
||||
counts_toward_limit,
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
match metadata_read {
|
||||
@@ -12127,20 +12063,12 @@ impl LocalDisk {
|
||||
true,
|
||||
if dir_to_skip.is_empty() { None } else { Some(dir_to_skip) },
|
||||
has_namespace_child_candidate,
|
||||
false,
|
||||
);
|
||||
} else if !is_dir_obj
|
||||
&& has_namespace_child_candidate
|
||||
&& (bounded_probe
|
||||
|| self
|
||||
.object_dir_has_listable_child(
|
||||
&opts.bucket,
|
||||
&meta.name,
|
||||
&dir_to_skip,
|
||||
opts.incl_deleted,
|
||||
stall,
|
||||
)
|
||||
.await?)
|
||||
&& self
|
||||
.object_dir_has_listable_child(&opts.bucket, &meta.name, &dir_to_skip, opts.incl_deleted, stall)
|
||||
.await?
|
||||
{
|
||||
// A plain object `a` shares its backing directory with any
|
||||
// children `a/...`, and non-recursive walks never descend into
|
||||
@@ -12150,7 +12078,7 @@ impl LocalDisk {
|
||||
// separate real directory entry handled above.
|
||||
let mut dir_name = meta.name.clone();
|
||||
dir_name.push_str(SLASH_SEPARATOR);
|
||||
schedule_dir(&mut dir_stack, dir_name, true, None, true, bounded_probe);
|
||||
schedule_dir(&mut dir_stack, dir_name, true, None, true);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -12166,7 +12094,7 @@ impl LocalDisk {
|
||||
meta.name.push_str(SLASH_SEPARATOR);
|
||||
// Conservative listings verify physical prefixes. Never-versioned
|
||||
// buckets first rule out the data dirs a deleted version leaves behind.
|
||||
let listable = if opts.recursive || opts.incl_deleted || bounded_probe {
|
||||
let listable = if opts.recursive || opts.incl_deleted {
|
||||
true
|
||||
} else if opts.skip_hidden_prefix_check {
|
||||
!self.directory_is_delete_residue(&opts.bucket, &meta.name, stall).await?
|
||||
@@ -12175,7 +12103,7 @@ impl LocalDisk {
|
||||
.await?
|
||||
};
|
||||
if listable {
|
||||
schedule_dir(&mut dir_stack, meta.name, false, None, true, bounded_probe);
|
||||
schedule_dir(&mut dir_stack, meta.name, false, None, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12197,23 +12125,11 @@ impl LocalDisk {
|
||||
};
|
||||
}
|
||||
|
||||
while let Some((dir, skip_object, dir_to_skip, scan_required, verify_prefix)) = dir_stack.pop() {
|
||||
while let Some((dir, skip_object, dir_to_skip, scan_required)) = dir_stack.pop() {
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
if verify_prefix {
|
||||
match self.probe_listing_prefix(opts, &dir, &mut remaining_hidden).await? {
|
||||
PrefixListingProbe::Empty => continue,
|
||||
PrefixListingProbe::Visible => {}
|
||||
PrefixListingProbe::Cut(entries) => {
|
||||
for entry in entries {
|
||||
write_metacache_obj(out, &entry).await?;
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
write_metacache_obj(
|
||||
out,
|
||||
&MetaCacheEntry {
|
||||
@@ -12431,86 +12347,6 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe a prefix in the same byte order as the listing stream. Keep the
|
||||
/// ordinary one-prefix result when a live object is found; only publish
|
||||
/// hidden entries when their budget forces a resumable stream cut.
|
||||
async fn probe_listing_prefix(
|
||||
&self,
|
||||
opts: &WalkDirOptions,
|
||||
prefix: &str,
|
||||
remaining: &mut usize,
|
||||
) -> Result<PrefixListingProbe> {
|
||||
if opts.forward_to.as_ref().is_some_and(|marker| prefix <= marker.as_str())
|
||||
&& opts.pending_prefix.as_deref() != Some(prefix)
|
||||
{
|
||||
return Ok(PrefixListingProbe::Empty);
|
||||
}
|
||||
let root = prefix.trim_end_matches('/').to_owned();
|
||||
let mut pending = std::collections::BTreeMap::from([(root.clone(), root.clone())]);
|
||||
let mut hidden = Vec::new();
|
||||
let mut bytes = 0usize;
|
||||
let stall = opts.stall_timeout_duration();
|
||||
while let Some((name, path)) = pending.pop_first() {
|
||||
#[cfg(test)]
|
||||
let _previous_probe_count =
|
||||
DIRECTORY_LISTING_ENTRY_PROBE_COUNT.try_with(|count| count.fetch_add(1, Ordering::Relaxed));
|
||||
let mut data_dirs = HashSet::new();
|
||||
match with_walk_stall_timeout(stall, self.read_listing_metadata(&opts.bucket, &path)).await {
|
||||
Ok(read) => {
|
||||
data_dirs.extend(read.data_dirs.into_iter().map(|id| id.to_string()));
|
||||
// The root object's visibility does not prove that its
|
||||
// child prefix exists. Ancestor metadata still identifies
|
||||
// internal data directories that must never be traversed.
|
||||
if path != root && opts.forward_to.as_ref().is_none_or(|marker| &name > marker) {
|
||||
if read.counts_toward_limit {
|
||||
return Ok(PrefixListingProbe::Visible);
|
||||
}
|
||||
bytes = bytes.saturating_add(read.bytes.len());
|
||||
hidden.push(MetaCacheEntry {
|
||||
name: name.clone(),
|
||||
metadata: read.bytes,
|
||||
..Default::default()
|
||||
});
|
||||
*remaining = remaining.saturating_sub(1);
|
||||
if *remaining == 0 || bytes >= LISTING_META_BATCH_MAX_BYTES {
|
||||
return Ok(PrefixListingProbe::Cut(hidden));
|
||||
}
|
||||
}
|
||||
if !read.has_namespace_child_candidate {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(Error::FileNotFound | Error::IsNotRegular) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let children = match with_walk_stall_timeout(stall, self.list_dir("", &opts.bucket, &path, -1)).await {
|
||||
Ok(children) => children,
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound) => continue,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
for child in children {
|
||||
let Some(child) = child.strip_suffix('/') else { continue };
|
||||
if child.is_empty() || data_dirs.contains(child) {
|
||||
continue;
|
||||
}
|
||||
let path = path_join_buf(&[&path, child]);
|
||||
let name = decode_dir_object(&path);
|
||||
if name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if opts
|
||||
.forward_to
|
||||
.as_ref()
|
||||
.is_some_and(|marker| &name <= marker && !marker.starts_with(&format!("{name}/")))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
pending.insert(name, path);
|
||||
}
|
||||
}
|
||||
Ok(PrefixListingProbe::Empty)
|
||||
}
|
||||
|
||||
/// Whether anything under `dir_name` would appear in a listing. With
|
||||
/// `incl_deleted`, any `xl.meta` counts (versioned listings surface
|
||||
/// delete-marker-only objects too); otherwise the metadata must hold a
|
||||
@@ -12553,9 +12389,10 @@ impl LocalDisk {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
match with_walk_stall_timeout(stall, self.read_listing_metadata(bucket, ¤t)).await {
|
||||
let metadata_path = path_join_buf(&[current.as_str(), STORAGE_FORMAT_FILE]);
|
||||
match with_walk_stall_timeout(stall, self.read_metadata(bucket, metadata_path.as_str())).await {
|
||||
Ok(metadata) => {
|
||||
let file_meta = match FileMeta::load(&metadata.bytes) {
|
||||
let file_meta = match FileMeta::load(&metadata) {
|
||||
Ok(file_meta) => file_meta,
|
||||
Err(_) => return Ok(true),
|
||||
};
|
||||
@@ -13139,7 +12976,6 @@ impl LocalDisk {
|
||||
let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}");
|
||||
let scan_object_path = object_path.clone();
|
||||
let scan_reserved_name = reserved_name.clone();
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
let (matching_dirs, recovery_owned) = self
|
||||
.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| {
|
||||
@@ -13149,10 +12985,7 @@ impl LocalDisk {
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let mut matching = Vec::new();
|
||||
for (idx, entry) in
|
||||
delete_recovery::recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref()))
|
||||
.enumerate()
|
||||
{
|
||||
for (idx, entry) in entries.enumerate() {
|
||||
if idx & 63 == 0 && is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
@@ -13454,7 +13287,6 @@ impl LocalDisk {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
let matching_data_dirs = self
|
||||
.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| {
|
||||
@@ -13474,10 +13306,7 @@ impl LocalDisk {
|
||||
};
|
||||
let marker_name = format!("{prefix}{rollback_dir}");
|
||||
let mut matched = Vec::new();
|
||||
for (idx, entry) in
|
||||
delete_recovery::recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref()))
|
||||
.enumerate()
|
||||
{
|
||||
for (idx, entry) in entries.enumerate() {
|
||||
if idx & 63 == 0 && is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
@@ -13742,14 +13571,6 @@ fn measure_local_batch_stage<T>(
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
self.multipart_index_page(bucket, marker).await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -13795,14 +13616,6 @@ impl DiskAPI for LocalDisk {
|
||||
async fn has_delete_transaction_evidence(&self, volume: &str, object: &str) -> Result<bool> {
|
||||
self.delete_recovery_has_evidence(volume, object).await
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<super::BoundedDeleteRecoveryDiscovery> {
|
||||
self.delete_recovery_discover_bounded(volume, object, max_entries).await
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -13812,24 +13625,6 @@ impl DiskAPI for LocalDisk {
|
||||
) -> Result<super::DeleteRecoveryPage> {
|
||||
self.delete_recovery_discover(volume, object, cursor, limit).await
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: super::BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<super::BoundedDeleteRecoverySettlementReply> {
|
||||
self.delete_recovery_settle_bounded(volume, request, namespace_owner).await
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<super::BoundedDeleteRecoveryInspection> {
|
||||
self.delete_recovery_inspect_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -15295,16 +15090,20 @@ impl DiskAPI for LocalDisk {
|
||||
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
||||
if let Ok(data) = with_walk_stall_timeout(
|
||||
stall,
|
||||
self.read_listing_metadata(
|
||||
self.read_metadata(
|
||||
&opts.bucket,
|
||||
&format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX),
|
||||
path_join_buf(&[
|
||||
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(),
|
||||
STORAGE_FORMAT_FILE,
|
||||
])
|
||||
.as_str(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let meta = MetaCacheEntry {
|
||||
name: opts.base_dir.clone(),
|
||||
metadata: data.bytes,
|
||||
metadata: data.to_vec(),
|
||||
..Default::default()
|
||||
};
|
||||
write_metacache_obj(&mut out, &meta).await?;
|
||||
@@ -15322,9 +15121,15 @@ impl DiskAPI for LocalDisk {
|
||||
} && meta.is_file()
|
||||
{
|
||||
skip_current_dir_object = true;
|
||||
if let Ok(meta_bytes) =
|
||||
with_walk_stall_deadline(stall, self.read_listing_metadata(&opts.bucket, &opts.base_dir)).await?
|
||||
&& let Ok(file_meta) = FileMeta::load(&meta_bytes.bytes)
|
||||
if let Ok(meta_bytes) = with_walk_stall_deadline(
|
||||
stall,
|
||||
self.read_metadata(
|
||||
opts.bucket.as_str(),
|
||||
path_join_buf(&[opts.base_dir.as_str(), STORAGE_FORMAT_FILE]).as_str(),
|
||||
),
|
||||
)
|
||||
.await?
|
||||
&& let Ok(file_meta) = FileMeta::load(&meta_bytes)
|
||||
&& let Ok(data_dirs) = file_meta.get_data_dirs()
|
||||
{
|
||||
for data_dir in data_dirs.iter().flatten() {
|
||||
@@ -15681,10 +15486,7 @@ impl DiskAPI for LocalDisk {
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
|
||||
let xl_rel_path = format!("{path}/{STORAGE_FORMAT_FILE}");
|
||||
// The common publisher indexes the merged owner metadata before
|
||||
// a multipart head becomes durable, including legacy source heads.
|
||||
if volume != RUSTFS_META_MULTIPART_BUCKET
|
||||
&& self.multipart_source_admission(volume, &xl_rel_path, true, false)?.is_none()
|
||||
if self.multipart_source_admission(volume, &xl_rel_path, true, false)?.is_none()
|
||||
&& !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, &xl_rel_path)
|
||||
&& !has_delete_version_test_failpoints()
|
||||
&& !has_rename_data_test_failpoints(&xl_rel_path)
|
||||
@@ -26861,162 +26663,6 @@ mod test {
|
||||
assert!(FileMeta::load(&corrupt_read.bytes).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_prefix_probe_resumes_hidden_subtrees_and_keeps_live_prefix_fast_path() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
let directory = tempfile::tempdir().expect("disk");
|
||||
let bucket = "bounded-prefix-probe";
|
||||
let modified = OffsetDateTime::from_unix_timestamp(1_705_312_500).expect("time");
|
||||
for index in 0..600 {
|
||||
let name = format!("prefix/{index:04}");
|
||||
let mut info = if index < 250 {
|
||||
FileInfo {
|
||||
name: name.clone(),
|
||||
deleted: true,
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
FileInfo::new(&name, 1, 1)
|
||||
};
|
||||
info.version_id = Some(Uuid::new_v4());
|
||||
info.mod_time = Some(modified);
|
||||
let mut metadata = FileMeta::default();
|
||||
metadata.add_version(info).expect("version");
|
||||
assert_eq!(file_meta_counts_toward_limit(&metadata), index >= 250);
|
||||
let path = directory.path().join(bucket).join(&name);
|
||||
fs::create_dir_all(&path).await.expect("object directory");
|
||||
fs::write(path.join(STORAGE_FORMAT_FILE), metadata.marshal_msg().expect("metadata"))
|
||||
.await
|
||||
.expect("object metadata");
|
||||
}
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("path")).expect("endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("disk");
|
||||
let mut marker = None;
|
||||
for page in 0..3 {
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let (reader, mut writer) = tokio::io::duplex(1024 * 1024);
|
||||
DIRECTORY_LISTING_ENTRY_PROBE_COUNT
|
||||
.scope(
|
||||
count.clone(),
|
||||
disk.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
recursive: false,
|
||||
hidden_entry_budget: 100,
|
||||
forward_to: marker.clone(),
|
||||
pending_prefix: marker.as_ref().map(|_| "prefix/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("bounded walk");
|
||||
drop(writer);
|
||||
let mut reader = MetacacheReader::new(reader);
|
||||
let mut entries = reader.read_all().await.expect("stream");
|
||||
assert!(count.load(Ordering::Relaxed) <= 102, "one page must not read the entire hidden subtree");
|
||||
if page < 2 {
|
||||
assert_eq!(entries.len(), 100);
|
||||
assert_eq!(reader.limit_reached(), Some(true));
|
||||
assert!(entries.iter_mut().all(|entry| entry.is_latest_delete_marker()));
|
||||
marker = entries.last().map(|entry| entry.name.clone());
|
||||
assert_eq!(marker, Some(format!("prefix/{:04}", (page + 1) * 100 - 1)));
|
||||
} else {
|
||||
assert_eq!(entries.iter().map(|entry| entry.name.as_str()).collect::<Vec<_>>(), ["prefix/"]);
|
||||
assert_eq!(reader.limit_reached(), Some(false));
|
||||
assert!(
|
||||
count.load(Ordering::Relaxed) < 60,
|
||||
"a live witness must keep the whole remaining subtree out of the probe"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_strips_inline_bodies_from_base_objects_and_directory_markers() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
let directory = tempfile::tempdir().expect("disk directory");
|
||||
let bucket = "inline-listing";
|
||||
let root = directory.path().join(bucket);
|
||||
let mut expected = std::collections::HashMap::new();
|
||||
for name in ["parent", "parent/child", "folder/", "folder/child"] {
|
||||
let object_dir = root.join(encode_dir_object(name));
|
||||
fs::create_dir_all(&object_dir).await.expect("object directory");
|
||||
let mut info = FileInfo::new(name, 1, 1);
|
||||
info.size = 128 * 1024;
|
||||
info.data_dir = Some(Uuid::new_v4());
|
||||
info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
info.data = Some(Bytes::from(vec![42; 128 * 1024]));
|
||||
info.set_inline_data();
|
||||
let mut meta = FileMeta::default();
|
||||
meta.add_version(info).expect("inline metadata");
|
||||
let full = meta.marshal_msg().expect("serialize metadata");
|
||||
let envelope = read_xl_meta_no_data_sync(&mut std::io::Cursor::new(&full), full.len()).expect("metadata envelope");
|
||||
assert!(full.len() > envelope.len() + 120 * 1024);
|
||||
expected.insert(name.to_string(), envelope);
|
||||
fs::write(object_dir.join(STORAGE_FORMAT_FILE), full)
|
||||
.await
|
||||
.expect("inline xl.meta");
|
||||
}
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().unwrap()).unwrap();
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
|
||||
for (base, recursive) in [
|
||||
("", true),
|
||||
("", false),
|
||||
("parent", true),
|
||||
("parent/", true),
|
||||
("folder/", true),
|
||||
] {
|
||||
let (reader, mut writer) = tokio::io::duplex(1024 * 1024);
|
||||
disk.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: bucket.into(),
|
||||
base_dir: base.into(),
|
||||
recursive,
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
)
|
||||
.await
|
||||
.expect("walk");
|
||||
drop(writer);
|
||||
let entries = MetacacheReader::new(reader).read_all().await.expect("listing stream");
|
||||
let objects: Vec<_> = entries.iter().filter(|entry| !entry.metadata.is_empty()).collect();
|
||||
let expected_names: &[&str] = match (base, recursive) {
|
||||
("", true) => &["folder/", "folder/child", "parent", "parent/child"],
|
||||
("", false) => &["folder/", "parent"],
|
||||
("parent", true) => &["parent/", "parent/child"],
|
||||
("parent/", true) => &["parent/child"],
|
||||
("folder/", true) => &["folder/", "folder/child"],
|
||||
_ => unreachable!(),
|
||||
};
|
||||
assert_eq!(
|
||||
objects.iter().map(|entry| entry.name.as_str()).collect::<Vec<_>>(),
|
||||
expected_names,
|
||||
"base {base}, recursive {recursive}"
|
||||
);
|
||||
for entry in objects {
|
||||
// A walk rooted at a plain object's backing directory emits its
|
||||
// own metadata with a trailing slash, unlike a root-level walk.
|
||||
let physical_name = if base == "parent" && entry.name == "parent/" {
|
||||
"parent"
|
||||
} else {
|
||||
entry.name.as_str()
|
||||
};
|
||||
assert_eq!(
|
||||
&entry.metadata,
|
||||
expected.get(physical_name).expect("known object"),
|
||||
"inline data leaked into listing of {} from base {base}, recursive {recursive}",
|
||||
entry.name
|
||||
);
|
||||
}
|
||||
}
|
||||
// Raw metadata is still required by non-listing recovery consumers.
|
||||
let raw = disk.read_metadata(bucket, "parent/xl.meta").await.expect("raw metadata");
|
||||
assert!(raw.len() > 128 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_listing_metadata_bytes_covers_large_envelope_versions_and_non_recursive_walk_coexistence() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
|
||||
@@ -486,9 +486,6 @@ impl LocalDisk {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_info(dst_path, &fi).await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
|
||||
@@ -5,28 +5,6 @@ use crate::disk::{
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(super) struct RecoveryOperationScope {
|
||||
pub(super) budget: Arc<crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
may_publish_receipt: bool,
|
||||
receipt_published: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static RECOVERY_OPERATION: Arc<RecoveryOperationScope>;
|
||||
}
|
||||
|
||||
pub(super) fn recovery_operation_scope() -> Option<Arc<RecoveryOperationScope>> {
|
||||
RECOVERY_OPERATION.try_with(Arc::clone).ok()
|
||||
}
|
||||
|
||||
impl RecoveryOperationScope {
|
||||
pub(super) fn receipt_published(&self) {
|
||||
if self.may_publish_receipt {
|
||||
self.receipt_published.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) const PREPARED: &str = "delete-prepared";
|
||||
pub(super) const TARGET: &str = "delete-target";
|
||||
pub(super) const FINISHED: &str = "delete-commit-finished";
|
||||
@@ -43,27 +21,22 @@ struct DiscoverySession {
|
||||
|
||||
impl DiscoverySession {
|
||||
/// Consume one bounded step, returning ownership only when traversal remains.
|
||||
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
|
||||
.map(|(transactions, next, _)| (transactions, next))
|
||||
}
|
||||
|
||||
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
|
||||
fn advance(mut self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
if self.frames.is_empty() {
|
||||
match std::fs::read_dir(&self.root) {
|
||||
Ok(entries) => self.frames.push((String::new(), entries)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
return Ok((Vec::new(), None, 0));
|
||||
return Ok((Vec::new(), None));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut seen_candidates = HashSet::new();
|
||||
let mut visited = 0u32;
|
||||
let mut visited = 0usize;
|
||||
while !self.frames.is_empty()
|
||||
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
|
||||
&& visited < max_entries
|
||||
&& visited < 4096
|
||||
{
|
||||
let (parent, entries) = self
|
||||
.frames
|
||||
@@ -183,7 +156,7 @@ impl DiscoverySession {
|
||||
.map(|(object, transaction)| DeleteTransactionId { object, transaction })
|
||||
.collect();
|
||||
let next = (!self.frames.is_empty()).then_some(self);
|
||||
Ok((transactions, next, visited))
|
||||
Ok((transactions, next))
|
||||
}
|
||||
|
||||
async fn has_evidence(mut self, disk: &LocalDisk, path: &Path) -> Result<bool> {
|
||||
@@ -607,41 +580,12 @@ fn explicit_delete_referenced_data_dirs(
|
||||
Ok(Some(referenced))
|
||||
}
|
||||
|
||||
pub(super) async fn next_recovery_entry(entries: &mut tokio::fs::ReadDir) -> std::io::Result<Option<tokio::fs::DirEntry>> {
|
||||
if let Some(scope) = recovery_operation_scope() {
|
||||
scope.budget.reserve(1).map_err(std::io::Error::from)?;
|
||||
}
|
||||
entries.next_entry().await
|
||||
}
|
||||
|
||||
pub(super) fn recovery_entries(
|
||||
mut entries: std::fs::ReadDir,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> impl Iterator<Item = std::io::Result<std::fs::DirEntry>> + '_ {
|
||||
let mut finished = false;
|
||||
std::iter::from_fn(move || {
|
||||
if finished {
|
||||
return None;
|
||||
}
|
||||
if let Some(budget) = budget
|
||||
&& let Err(error) = budget.reserve(1)
|
||||
{
|
||||
finished = true;
|
||||
return Some(Err(error.into()));
|
||||
}
|
||||
let entry = entries.next();
|
||||
finished = entry.is_none();
|
||||
entry
|
||||
})
|
||||
}
|
||||
|
||||
fn explicit_delete_plan_bounded(
|
||||
fn explicit_delete_plan(
|
||||
root: &Path,
|
||||
transaction: Uuid,
|
||||
original: Option<&FileMeta>,
|
||||
journal: Option<&DeleteRecoveryJournal>,
|
||||
current_missing: bool,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Result<Option<ExplicitDeletePlan>> {
|
||||
let Some(referenced) = explicit_delete_referenced_data_dirs(original, journal, current_missing)? else {
|
||||
return Ok(None);
|
||||
@@ -658,7 +602,7 @@ fn explicit_delete_plan_bounded(
|
||||
let mut candidates = Vec::new();
|
||||
match std::fs::read_dir(&rollback) {
|
||||
Ok(entries) => {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let kind = entry.file_type().map_err(to_file_error)?;
|
||||
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
@@ -690,7 +634,7 @@ fn explicit_delete_plan_bounded(
|
||||
}
|
||||
match std::fs::read_dir(root) {
|
||||
Ok(entries) => {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil()) else {
|
||||
@@ -714,7 +658,7 @@ fn explicit_delete_plan_bounded(
|
||||
candidates.dedup();
|
||||
for dir in candidates {
|
||||
let mut has_parts = false;
|
||||
for entry in recovery_entries(std::fs::read_dir(&dir).map_err(to_file_error)?, budget) {
|
||||
for entry in std::fs::read_dir(&dir).map_err(to_file_error)? {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
if !entry.file_type().map_err(to_file_error)?.is_file() {
|
||||
return Ok(None);
|
||||
@@ -753,27 +697,15 @@ fn canonical_uuid(name: &str) -> Option<Uuid> {
|
||||
Uuid::parse_str(name).ok().filter(|id| name == id.hyphenated().to_string())
|
||||
}
|
||||
|
||||
pub(super) fn missing_marker_preserves_child_prefix(journal: &DeleteRecoveryJournal) -> bool {
|
||||
journal.original_digest.is_none()
|
||||
&& journal.original_versions.is_empty()
|
||||
&& journal.data_dirs.is_empty()
|
||||
&& journal.target_digest.is_some()
|
||||
&& !journal.requests.is_empty()
|
||||
&& journal
|
||||
.requests
|
||||
.iter()
|
||||
.all(|request| request.deleted && request.version_id.is_some_and(|id| !id.is_nil()))
|
||||
}
|
||||
|
||||
pub(super) fn inspection_with_markers_bounded(
|
||||
pub(super) fn inspection_with_markers(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Result<(DeleteRecoveryInspection, bool)> {
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?;
|
||||
let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?;
|
||||
let prepared = regular_bytes(&rollback.join(PREPARED))?;
|
||||
let target = regular_metadata_bytes(&rollback.join(TARGET))?;
|
||||
let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
@@ -807,20 +739,6 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// A failed creation of a marker over a child named `xl.meta` did not
|
||||
// replace that directory. Older releases nevertheless left a journal.
|
||||
// Only this bound, originally absent generation may treat it as absence.
|
||||
let current_path = root.join(STORAGE_FORMAT_FILE);
|
||||
let current = if !committed
|
||||
&& before.is_none()
|
||||
&& (journal.as_ref().is_some_and(missing_marker_preserves_child_prefix)
|
||||
|| (journal.is_none() && target.is_none() && receipt_bytes.is_none()))
|
||||
&& std::fs::symlink_metadata(¤t_path).is_ok_and(|metadata| metadata.is_dir())
|
||||
{
|
||||
None
|
||||
} else {
|
||||
regular_metadata_bytes(¤t_path)?
|
||||
};
|
||||
let backup_digest = before.as_deref().map(digest);
|
||||
let current_digest = current.as_deref().map(digest);
|
||||
let prepared_digest = prepared.as_deref().map(digest);
|
||||
@@ -863,7 +781,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
};
|
||||
if let Some(entries) = entries {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let kind = entry.file_type().map_err(to_file_error)?;
|
||||
if !kind.is_file() && !kind.is_dir() {
|
||||
@@ -880,7 +798,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len()));
|
||||
if kind.is_dir() && id.is_some_and(|id| !id.is_nil()) {
|
||||
has_other_uuid_dir |= id != Some(transaction);
|
||||
for child in recovery_entries(std::fs::read_dir(entry.path()).map_err(to_file_error)?, budget) {
|
||||
for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? {
|
||||
let child = child.map_err(to_file_error)?;
|
||||
let kind = child.file_type().map_err(to_file_error)?;
|
||||
if !kind.is_dir() && !kind.is_file() {
|
||||
@@ -926,7 +844,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
}
|
||||
if name == transaction.to_string() && kind.is_dir() {
|
||||
has_staged_dir = true;
|
||||
for file in recovery_entries(std::fs::read_dir(child.path()).map_err(to_file_error)?, budget) {
|
||||
for file in std::fs::read_dir(child.path()).map_err(to_file_error)? {
|
||||
let file = file.map_err(to_file_error)?;
|
||||
let kind = file.file_type().map_err(to_file_error)?;
|
||||
let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
@@ -1050,8 +968,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
.is_some_and(|referenced| !referenced.contains(&transaction))
|
||||
}
|
||||
DeleteRecoveryState::LegacyAmbiguous | DeleteRecoveryState::NewPrepared => {
|
||||
explicit_delete_plan_bounded(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none(), budget)?
|
||||
.is_some()
|
||||
explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some()
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
@@ -1122,7 +1039,6 @@ impl LocalDisk {
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
|
||||
let recovery_scope = recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
Ok((|| {
|
||||
let entries = match std::fs::read_dir(&root) {
|
||||
@@ -1132,7 +1048,7 @@ impl LocalDisk {
|
||||
};
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut visited = 0usize;
|
||||
for entry in recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())) {
|
||||
for entry in entries {
|
||||
visited += 1;
|
||||
if visited > 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
@@ -1285,42 +1201,6 @@ impl LocalDisk {
|
||||
.map_err(|error| to_file_error(error).into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn public_delete_discovery_sessions_for_test(&self) -> usize {
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.filter(|session| session.disk_root == self.io_root())
|
||||
.count()
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_discover_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<super::super::BoundedDeleteRecoveryDiscovery> {
|
||||
if !(1..=super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES).contains(&max_entries) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let path = self.delete_discovery_path(volume, Some(object))?;
|
||||
let session = self.new_delete_discovery_session(volume, Some(object), &path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
|
||||
self.validate_delete_discovery_owner(&session)?;
|
||||
// The blocking owner drops every directory handle even if the RPC waiter disappears.
|
||||
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
|
||||
Ok(super::super::BoundedDeleteRecoveryDiscovery {
|
||||
transactions: if next.is_none() { transactions } else { Vec::new() },
|
||||
visited,
|
||||
complete: next.is_none(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|error| to_file_error(error).into())
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_discover(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1368,39 +1248,6 @@ impl LocalDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_inspect_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<super::super::BoundedDeleteRecoveryInspection> {
|
||||
if !safe_path(object, false)
|
||||
|| transaction.is_nil()
|
||||
|| !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&max_entries)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let root = self.get_object_path(volume, object)?;
|
||||
let owner = os::acquire_metadata_mutation_lease(&root, None).await;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, Some(owner)).await;
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let budget = crate::object_api::DeleteRecoveryWorkBudget::new(max_entries);
|
||||
Ok(
|
||||
inspection_with_markers_bounded(&root, &volume, &object, transaction, Some(&budget)).map(|(inspection, _)| {
|
||||
super::super::BoundedDeleteRecoveryInspection {
|
||||
inspection,
|
||||
visited: max_entries - budget.remaining(),
|
||||
}
|
||||
}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_inspect(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1440,19 +1287,12 @@ impl LocalDisk {
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
|
||||
let scope = recovery_operation_scope();
|
||||
// Journal inspections ride the durable-mutation lane (the fused delete
|
||||
// prepare already does): a bulk delete publishes one transaction per
|
||||
// key, and the one-worker foreground namespace lane would serialize
|
||||
// those per-drive publishes behind every other mutation.
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
Ok(inspection_with_markers_bounded(
|
||||
&root,
|
||||
&volume,
|
||||
&object,
|
||||
transaction,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
))
|
||||
Ok(inspection_with_markers(&root, &volume, &object, transaction))
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
@@ -1653,49 +1493,6 @@ impl LocalDisk {
|
||||
self.activate_delete_preparation(&journal, original, target, owner).await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_settle_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: super::super::BoundedDeleteRecoverySettlement,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<super::super::BoundedDeleteRecoverySettlementReply> {
|
||||
if !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&request.max_entries) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let scope = Arc::new(RecoveryOperationScope {
|
||||
budget: Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(request.max_entries)),
|
||||
may_publish_receipt: matches!(
|
||||
&request.action,
|
||||
DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. }
|
||||
),
|
||||
receipt_published: std::sync::atomic::AtomicBool::new(false),
|
||||
});
|
||||
let result = RECOVERY_OPERATION
|
||||
.scope(
|
||||
Arc::clone(&scope),
|
||||
self.delete_recovery_settle(
|
||||
volume,
|
||||
&request.object,
|
||||
request.transaction,
|
||||
request.expected_fingerprint,
|
||||
request.action,
|
||||
owner,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
let outcome = match result {
|
||||
Ok(inspection) => super::super::DeleteRecoverySettlementOutcome::Inspected(Box::new(inspection)),
|
||||
Err(DiskError::DeleteRecoveryWorkLimit) if scope.receipt_published.load(std::sync::atomic::Ordering::Acquire) => {
|
||||
super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
Ok(super::super::BoundedDeleteRecoverySettlementReply {
|
||||
outcome,
|
||||
visited: request.max_entries - scope.budget.remaining(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_recovery_settle(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1894,7 +1691,6 @@ impl LocalDisk {
|
||||
let plan_root = root.clone();
|
||||
let missing = observed.current_digest.is_none();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
|
||||
let scope = recovery_operation_scope();
|
||||
// Same durable-mutation lane as the journal inspection above: this
|
||||
// plan closure runs inside every settle on the bulk-delete publish
|
||||
// path.
|
||||
@@ -1905,15 +1701,7 @@ impl LocalDisk {
|
||||
.as_deref()
|
||||
.map(FileMeta::load)
|
||||
.transpose()?;
|
||||
explicit_delete_plan_bounded(
|
||||
&plan_root,
|
||||
transaction,
|
||||
before.as_ref(),
|
||||
None,
|
||||
missing,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)?
|
||||
.ok_or(DiskError::FileCorrupt)
|
||||
explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
@@ -2178,7 +1966,6 @@ impl LocalDisk {
|
||||
let (rollback_buf, root_buf, volume_str, object_str) =
|
||||
(rollback.clone(), root.clone(), volume.to_owned(), object.to_owned());
|
||||
let publication_root = self.publication_root.clone();
|
||||
let scope = recovery_operation_scope();
|
||||
return os::run_blocking_namespace_operation_with_sync(receipt_lease, sync, move || {
|
||||
let _publication_root = publication_root;
|
||||
for (path, _) in &file_leases {
|
||||
@@ -2211,14 +1998,8 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
let inspected = inspection_with_markers_bounded(
|
||||
&root_buf,
|
||||
&volume_str,
|
||||
&object_str,
|
||||
transaction,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)
|
||||
.map(|(state, _)| state);
|
||||
let inspected =
|
||||
inspection_with_markers(&root_buf, &volume_str, &object_str, transaction).map(|(state, _)| state);
|
||||
drop((file_leases, prune_dirs));
|
||||
Ok(inspected)
|
||||
})
|
||||
@@ -2375,7 +2156,6 @@ impl LocalDisk {
|
||||
let volume_str = volume.to_owned();
|
||||
let object_str = object.to_owned();
|
||||
|
||||
let recovery_scope = recovery_operation_scope();
|
||||
let res = os::run_blocking_namespace_operation_with_sync(
|
||||
lease,
|
||||
sync,
|
||||
@@ -2422,9 +2202,7 @@ impl LocalDisk {
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in
|
||||
recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
|
||||
{
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
@@ -2637,25 +2415,12 @@ impl LocalDisk {
|
||||
Ok(Some(()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local_with_budget(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
@@ -2734,7 +2499,7 @@ impl LocalDisk {
|
||||
if sole_transaction {
|
||||
match std::fs::read_dir(&root) {
|
||||
Ok(entries) => {
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let Ok(name) = entry.file_name().into_string() else {
|
||||
sole_transaction = false;
|
||||
@@ -2782,7 +2547,6 @@ impl LocalDisk {
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2790,26 +2554,6 @@ impl LocalDisk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
self.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
|
||||
match res {
|
||||
@@ -2876,7 +2620,7 @@ impl LocalDisk {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else {
|
||||
@@ -4380,100 +4124,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_marker_collision_rejects_new_journals_and_recovers_legacy_proof() {
|
||||
let (temp, disk, _info, original) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
let staged_child = temp.path().join("child-fixture");
|
||||
fs::rename(&root, &staged_child).await.expect("move intact object");
|
||||
fs::create_dir(&root).await.expect("create parent prefix");
|
||||
fs::rename(staged_child, root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("seed an independent object named object/xl.meta");
|
||||
let transaction = Uuid::new_v4();
|
||||
let marker = FileInfo {
|
||||
volume: "recovery".to_owned(),
|
||||
name: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
disk.delete_version_recoverable(
|
||||
"recovery",
|
||||
"object",
|
||||
marker.clone(),
|
||||
true,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a child xl.meta directory cannot be replaced by a marker");
|
||||
assert!(!root.join(transaction.to_string()).exists(), "rejection must precede journal publication");
|
||||
|
||||
// Reproduce the durable evidence written by releases that prepared a
|
||||
// missing-object marker before attempting to replace the directory.
|
||||
disk.prepare_delete_recovery(
|
||||
"recovery",
|
||||
"object",
|
||||
None,
|
||||
&[marker],
|
||||
None,
|
||||
DeletePreparationContext {
|
||||
transaction,
|
||||
bucket_incarnation: None,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed legacy prepared marker");
|
||||
let journal_path = root.join(transaction.to_string()).join(PREPARED);
|
||||
let journal_bytes = fs::read(&journal_path).await.expect("prepared journal");
|
||||
let mut incompatible: DeleteRecoveryJournal = serde_json::from_slice(&journal_bytes).expect("journal");
|
||||
incompatible.original_digest = Some([7; 32]);
|
||||
fs::write(&journal_path, serde_json::to_vec(&incompatible).expect("serialize journal"))
|
||||
.await
|
||||
.expect("write incompatible evidence");
|
||||
assert!(
|
||||
disk.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.is_err(),
|
||||
"a journal that claims existing metadata cannot ignore a directory collision"
|
||||
);
|
||||
fs::write(&journal_path, journal_bytes)
|
||||
.await
|
||||
.expect("restore exact legacy proof");
|
||||
let observed = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("inspect legacy proof");
|
||||
assert_eq!(observed.state, DeleteRecoveryState::NewPrepared);
|
||||
let recovered = disk
|
||||
.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback)
|
||||
.await
|
||||
.expect("rollback legacy marker without unlinking the child directory");
|
||||
assert_eq!(recovered.state, DeleteRecoveryState::Absent);
|
||||
assert!(!root.join(transaction.to_string()).exists());
|
||||
assert_eq!(
|
||||
disk.read_all("recovery", "object/xl.meta/xl.meta")
|
||||
.await
|
||||
.expect("child metadata")
|
||||
.as_ref(),
|
||||
original
|
||||
);
|
||||
assert!(
|
||||
disk.read_version("", "recovery", "object/xl.meta", "", &Default::default())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
discover_all_keys(&disk, Some("object")).await.is_empty(),
|
||||
"no proof may keep replacement heal pending"
|
||||
);
|
||||
}
|
||||
|
||||
// Data movement copies a delete marker onto a target that holds no
|
||||
// metadata yet. Rolling back that sub-quorum write removes the marker it
|
||||
// published, and must not report the backup it never wrote as missing:
|
||||
@@ -6117,151 +5767,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_inspection_charges_root_and_uuid_children() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let full = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
|
||||
.await
|
||||
.expect("full bounded inspection");
|
||||
assert!(full.visited > 1, "both root and transaction directory entries are charged");
|
||||
let error = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, full.visited - 1)
|
||||
.await
|
||||
.expect_err("nested entry exceeds allowance");
|
||||
assert_eq!(error, DiskError::DeleteRecoveryWorkLimit);
|
||||
let unchanged = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("unchanged proof");
|
||||
assert_eq!(full.inspection.fingerprint, unchanged.fingerprint);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_receipt_publication_distinguishes_deferred_inspection_from_write_failure() {
|
||||
for write_fails in [false, true] {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let full = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
|
||||
.await
|
||||
.expect("measure preparation work");
|
||||
let receipt = disk
|
||||
.io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}"))
|
||||
.expect("receipt path");
|
||||
if write_fails {
|
||||
let destination = receipt.clone();
|
||||
set_owned_file_before_open(&receipt, move || std::fs::create_dir(&destination).expect("reject receipt rename"));
|
||||
}
|
||||
let result = disk
|
||||
.settle_delete_transaction_bounded(
|
||||
"recovery",
|
||||
super::super::super::BoundedDeleteRecoverySettlement {
|
||||
object: "object".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: full.inspection.fingerprint,
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: full.visited,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if write_fails {
|
||||
assert!(result.is_err(), "a failed receipt write never counts as publication");
|
||||
assert!(receipt.is_dir());
|
||||
} else {
|
||||
let result = result.expect("persisted receipt is acknowledged independently of post-write inspection");
|
||||
assert!(matches!(
|
||||
result.outcome,
|
||||
super::super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
|
||||
));
|
||||
assert_eq!(result.visited, full.visited);
|
||||
let state = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("durable receipt");
|
||||
assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
|
||||
fs::create_dir_all(&path).await.expect("physical prefix");
|
||||
for index in 0..64 {
|
||||
fs::create_dir(path.join(format!("child-{index}")))
|
||||
.await
|
||||
.expect("legitimate child");
|
||||
}
|
||||
for _ in 0..140 {
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 8)
|
||||
.await
|
||||
.expect("bounded probe");
|
||||
assert_eq!(page.visited, 8);
|
||||
assert!(!page.complete, "an empty partial page cannot establish absence");
|
||||
assert!(page.transactions.is_empty());
|
||||
}
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 128)
|
||||
.await
|
||||
.expect("complete probe");
|
||||
assert!(page.complete);
|
||||
assert_eq!(page.visited, 64);
|
||||
assert!(page.transactions.is_empty());
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let before = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("before probe");
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "object", 4096)
|
||||
.await
|
||||
.expect("complete evidence");
|
||||
assert!(page.complete);
|
||||
assert!(page.transactions.iter().any(|id| id.transaction == transaction));
|
||||
let after = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("after probe");
|
||||
assert_eq!(before.fingerprint, after.fingerprint);
|
||||
assert!(
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.all(|session| session.disk_root != disk.io_root())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_cancelled_bounded_discovery_drops_private_session() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("object")).expect("probe scope");
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
|
||||
let mut probe = Box::pin(disk.discover_delete_transactions_bounded("recovery", "object", 4096));
|
||||
assert!(futures::poll!(probe.as_mut()).is_pending());
|
||||
drop(probe);
|
||||
assert!(
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.all(|session| session.disk_root != disk.io_root())
|
||||
);
|
||||
drop(lease);
|
||||
assert!(
|
||||
disk.discover_delete_transactions_bounded("recovery", "object", 4096)
|
||||
.await
|
||||
.expect("subsequent probe")
|
||||
.complete
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_evidence_probes_do_not_register_public_cursors() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,7 +117,6 @@ pub(super) struct LoadedSource {
|
||||
}
|
||||
|
||||
pub(super) struct SourceStore {
|
||||
multipart_index: Arc<super::multipart_index::IndexOwner>,
|
||||
pub disk_id: Uuid,
|
||||
pub physical_slot: usize,
|
||||
pub disk_ids: Vec<Uuid>,
|
||||
@@ -187,7 +186,6 @@ impl LocalDisk {
|
||||
let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
|
||||
Ok(SourceStore {
|
||||
multipart_index: self.multipart_index.clone(),
|
||||
disk_id,
|
||||
physical_slot,
|
||||
disk_ids,
|
||||
@@ -757,9 +755,6 @@ impl SourceStore {
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
.record_head(&format!("{}/xl.meta", self.path), bytes)
|
||||
.await?;
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
|
||||
@@ -27,7 +27,6 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub mod multipart_index;
|
||||
pub mod multipart_source;
|
||||
pub mod os;
|
||||
|
||||
@@ -415,11 +414,10 @@ impl Disk {
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(None),
|
||||
@@ -433,19 +431,11 @@ impl Disk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(false),
|
||||
}
|
||||
@@ -522,17 +512,6 @@ impl Disk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for Disk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
|
||||
@@ -576,17 +555,6 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -599,35 +567,6 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1574,15 +1513,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
fn host_name(&self) -> String;
|
||||
fn endpoint(&self) -> Endpoint;
|
||||
async fn close(&self) -> Result<()>;
|
||||
/// Return candidates only while this disk maintains complete boot-local coverage.
|
||||
/// Unsupported or unready disks must never return an authoritative empty page.
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
|
||||
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
|
||||
|
||||
@@ -1760,33 +1690,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
/// Bounded foreground inventory owns all traversal state until return or cancellation.
|
||||
/// Unsupported peers must fail closed; callers must never fall back to paginated discovery.
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_transaction: Uuid,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_request: BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
/// A read-only capability check; unsupported implementations must fail closed.
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Err(DiskError::GuardedDeleteUnsupported)
|
||||
@@ -2172,13 +2075,6 @@ pub struct WalkDirOptions {
|
||||
#[serde(default)]
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
|
||||
/// Bound hidden metadata read while proving a non-recursive prefix visible.
|
||||
#[serde(default)]
|
||||
pub hidden_entry_budget: usize,
|
||||
/// A prefix whose visibility was not established before the preceding cut.
|
||||
#[serde(default)]
|
||||
pub pending_prefix: Option<String>,
|
||||
|
||||
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
|
||||
pub report_notfound: bool,
|
||||
|
||||
@@ -2511,62 +2407,9 @@ pub enum DeleteRecoveryAction {
|
||||
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
|
||||
}
|
||||
|
||||
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryDiscovery {
|
||||
pub transactions: Vec<DeleteTransactionId>,
|
||||
pub visited: u32,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlement {
|
||||
pub object: String,
|
||||
pub transaction: Uuid,
|
||||
pub expected_fingerprint: DeleteRecoveryDigest,
|
||||
pub action: DeleteRecoveryAction,
|
||||
pub max_entries: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoverySettlementOutcome {
|
||||
Inspected(Box<DeleteRecoveryInspection>),
|
||||
ReceiptPublishedInspectionDeferred,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlementReply {
|
||||
pub outcome: DeleteRecoverySettlementOutcome,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryInspection {
|
||||
pub inspection: DeleteRecoveryInspection,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoveryRpcOperation {
|
||||
SettleBounded(BoundedDeleteRecoverySettlement),
|
||||
InspectBounded {
|
||||
object: String,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
},
|
||||
DiscoverBounded {
|
||||
object: String,
|
||||
max_entries: u32,
|
||||
},
|
||||
InspectOrphanDataDir {
|
||||
path: String,
|
||||
},
|
||||
@@ -2604,11 +2447,7 @@ impl DeleteRecoveryRpcOperation {
|
||||
/// Full metadata application and adoption share the existing mutation envelope.
|
||||
pub fn max_wire_payload_bytes(&self) -> usize {
|
||||
match self {
|
||||
Self::SettleBounded(BoundedDeleteRecoverySettlement {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
..
|
||||
})
|
||||
| Self::ApplyVersion { .. }
|
||||
Self::ApplyVersion { .. }
|
||||
| Self::ApplyVersions { .. }
|
||||
| Self::Settle {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
@@ -2624,11 +2463,8 @@ impl DeleteRecoveryRpcOperation {
|
||||
pub enum DeleteRecoveryRpcReply {
|
||||
OrphanDataDir { plain: bool },
|
||||
Discovered(DeleteRecoveryPage),
|
||||
DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
|
||||
Inspected(DeleteRecoveryInspection),
|
||||
InspectedBounded(BoundedDeleteRecoveryInspection),
|
||||
Settled(DeleteRecoveryInspection),
|
||||
SettledBounded(BoundedDeleteRecoverySettlementReply),
|
||||
AppliedVersion,
|
||||
AppliedVersions(Vec<Option<u32>>),
|
||||
Evidence { present: bool },
|
||||
@@ -3106,8 +2942,6 @@ mod tests {
|
||||
recursive: true,
|
||||
incl_deleted: false,
|
||||
skip_hidden_prefix_check: false,
|
||||
hidden_entry_budget: 0,
|
||||
pending_prefix: None,
|
||||
report_notfound: false,
|
||||
filter_prefix: Some("prefix_".to_string()),
|
||||
forward_to: Some("object/path".to_string()),
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A candidate index never replaces multipart metadata or quorum validation.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
|
||||
pub const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
|
||||
|
||||
/// A bounded page from one disk's continuously maintained, boot-local index.
|
||||
/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
|
||||
/// pages from different epochs or physical disk identities.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexPage {
|
||||
pub bucket: String,
|
||||
pub disk_id: Uuid,
|
||||
pub epoch: Uuid,
|
||||
pub roots: Vec<String>,
|
||||
pub next_marker: Option<String>,
|
||||
}
|
||||
|
||||
pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
|
||||
pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
|
||||
|
||||
/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
|
||||
/// Old peers may ignore this field, but their ordinary response cannot decode
|
||||
/// as the required signed, challenge-bound MultipartIndexRpcReply.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcRequest {
|
||||
pub disk_id: String,
|
||||
pub metrics: bool,
|
||||
pub noop: bool,
|
||||
pub multipart_index: MultipartIndexQuery,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexQuery {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub bucket: String,
|
||||
pub marker: Option<String>,
|
||||
pub page_cap: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcReply {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub page: MultipartIndexPage,
|
||||
pub proof: Vec<u8>,
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcRequest {
|
||||
pub fn new(bucket: &str, marker: Option<&str>) -> Self {
|
||||
Self {
|
||||
disk_id: String::new(),
|
||||
metrics: false,
|
||||
noop: true,
|
||||
multipart_index: MultipartIndexQuery {
|
||||
operation: "list_multipart_index_roots".into(),
|
||||
version: 1,
|
||||
challenge: Uuid::new_v4(),
|
||||
bucket: bucket.to_owned(),
|
||||
marker: marker.map(str::to_owned),
|
||||
page_cap: MULTIPART_INDEX_PAGE_SIZE,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> crate::disk::error::Result<()> {
|
||||
let query = &self.multipart_index;
|
||||
if !self.disk_id.is_empty()
|
||||
|| self.metrics
|
||||
|| !self.noop
|
||||
|| query.operation != "list_multipart_index_roots"
|
||||
|| query.version != 1
|
||||
|| query.challenge.is_nil()
|
||||
|| query.bucket.is_empty()
|
||||
|| query.bucket.len() > 255
|
||||
|| query.bucket.contains('/')
|
||||
|| matches!(query.bucket.as_str(), "." | "..")
|
||||
|| query.page_cap != MULTIPART_INDEX_PAGE_SIZE
|
||||
|| query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::MethodNotAllowed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
|
||||
}
|
||||
|
||||
pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_root(root: &str) -> bool {
|
||||
root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
impl MultipartIndexPage {
|
||||
pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
if expected_disk.is_nil()
|
||||
|| self.disk_id != expected_disk
|
||||
|| self.epoch.is_nil()
|
||||
|| self.bucket != bucket
|
||||
|| self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
|
||||
|| self
|
||||
.roots
|
||||
.iter()
|
||||
.any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
|
||||
|| self.roots.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
|| self
|
||||
.next_marker
|
||||
.as_ref()
|
||||
.is_some_and(|marker| self.roots.last() != Some(marker) || self.roots.len() != MULTIPART_INDEX_PAGE_SIZE)
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcReply {
|
||||
/// This is structural validation only; the caller must also verify `proof`.
|
||||
pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
let query = &request.multipart_index;
|
||||
request.validate()?;
|
||||
if self.operation != query.operation
|
||||
|| self.version != query.version
|
||||
|| self.challenge != query.challenge
|
||||
|| self.proof.len() != 32
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
|
||||
}
|
||||
}
|
||||
|
||||
/// A disk can attest coverage only after every page belongs to the same physical
|
||||
/// identity and index generation. An interrupted/changed stream contributes no
|
||||
/// partial candidates to the erasure set's coverage count.
|
||||
pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
|
||||
where
|
||||
F: FnMut(Option<String>) -> Fut,
|
||||
Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
|
||||
{
|
||||
let mut epoch = None;
|
||||
let mut marker = None;
|
||||
let mut roots = Vec::new();
|
||||
loop {
|
||||
let page = fetch(marker.clone()).await?;
|
||||
page.validate(bucket, marker.as_deref(), identity)?;
|
||||
if epoch.is_some_and(|epoch| epoch != page.epoch) {
|
||||
return Err(crate::disk::error::DiskError::DiskNotFound);
|
||||
}
|
||||
epoch = Some(page.epoch);
|
||||
roots.extend(page.roots);
|
||||
marker = page.next_marker;
|
||||
if marker.is_none() {
|
||||
return Ok(roots);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
|
||||
let request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let id = Uuid::new_v4();
|
||||
let mut reply = MultipartIndexRpcReply {
|
||||
operation: request.multipart_index.operation.clone(),
|
||||
version: 1,
|
||||
challenge: request.multipart_index.challenge,
|
||||
page: MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec!["a".repeat(64)],
|
||||
next_marker: None,
|
||||
},
|
||||
proof: vec![0; 32],
|
||||
};
|
||||
reply.validate(&request, id).expect("valid structural envelope");
|
||||
assert!(
|
||||
reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
|
||||
"old challenge cannot replay"
|
||||
);
|
||||
assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
|
||||
reply.page.next_marker = Some("a".repeat(64));
|
||||
assert!(reply.validate(&request, id).is_err(), "short page cannot claim a continuation");
|
||||
reply.page.next_marker = None;
|
||||
reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
|
||||
assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_index_response_proof_binds_the_complete_query_and_page() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let mut request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let mut page = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: Uuid::new_v4(),
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let canonical = request.canonical_response("disk", &page).expect("canonical response");
|
||||
let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
|
||||
let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&request.canonical_response("disk", page).expect("canonical"),
|
||||
&proof,
|
||||
)
|
||||
};
|
||||
verify(&request, &page).expect("exact response");
|
||||
request.multipart_index.marker = Some("0".repeat(64));
|
||||
assert!(verify(&request, &page).is_err(), "cursor is signed");
|
||||
request.multipart_index.marker = None;
|
||||
page.epoch = Uuid::new_v4();
|
||||
assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
|
||||
let id = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
for replace_disk in [false, true] {
|
||||
let first = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch,
|
||||
roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
|
||||
next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
|
||||
};
|
||||
let second = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: if replace_disk { Uuid::new_v4() } else { id },
|
||||
epoch: if replace_disk { epoch } else { Uuid::new_v4() },
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let mut replies = std::collections::VecDeque::from([first, second]);
|
||||
let result = collect_pages("bucket", id, |marker| {
|
||||
let page = replies.pop_front().expect("bounded fixture pages");
|
||||
if marker.is_some() {
|
||||
assert!(page.roots.is_empty());
|
||||
}
|
||||
std::future::ready(Ok(page))
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "changed physical source cannot contribute partial coverage");
|
||||
assert!(replies.is_empty(), "exercise the continuation response");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2731,11 +2731,6 @@ pub(crate) struct NamespaceMutationLease {
|
||||
}
|
||||
|
||||
impl NamespaceMutationLease {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn blocking_queue_for_test(&self) -> Arc<DiskBlockingQueue> {
|
||||
self.blocking_queue.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn attach_external_guard(&self, guard: Arc<dyn Send + Sync>) {
|
||||
*self.external_guard.lock() = Some(guard);
|
||||
}
|
||||
@@ -5387,29 +5382,6 @@ impl PublicationRoot {
|
||||
})
|
||||
}
|
||||
|
||||
/// Distinguish reconnects from replacement mounts before reusing disk-local state.
|
||||
pub(crate) fn physical_identity(&self) -> io::Result<(u64, [u8; 16])> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(target_os = "linux")]
|
||||
let metadata = self.directory.metadata()?;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let metadata = std::fs::metadata(&*self.path)?;
|
||||
let mut inode = [0; 16];
|
||||
inode[..8].copy_from_slice(&metadata.ino().to_le_bytes());
|
||||
Ok((metadata.dev(), inode))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
windows_file_identity(&self.directory.handle)
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
Err(io::Error::new(io::ErrorKind::Unsupported, "physical root identity unavailable"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
@@ -269,11 +269,6 @@ pub enum StorageError {
|
||||
/// retained for operator inspection.
|
||||
#[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
|
||||
DeleteRecoveryLegacyAmbiguous,
|
||||
/// Delete recovery exhausted the per-request work budget before it could
|
||||
/// classify every residue. Retrying later makes progress, so callers treat
|
||||
/// it like `SlowDown`; it stays distinct so the disk error round-trips.
|
||||
#[error("delete recovery work limit reached; please retry")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
#[error("Decommission not started")]
|
||||
DecommissionNotStarted,
|
||||
#[error("Decommission already running")]
|
||||
@@ -485,7 +480,6 @@ impl From<DiskError> for StorageError {
|
||||
DiskError::InvalidPath => StorageError::InvalidPath,
|
||||
DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
|
||||
}
|
||||
}
|
||||
@@ -513,7 +507,6 @@ impl StorageError {
|
||||
StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
StorageError::StorageFull => DiskError::DiskFull,
|
||||
StorageError::SlowDown => DiskError::TooManyOpenFiles,
|
||||
StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
||||
StorageError::InconsistentDisk => DiskError::InconsistentDisk,
|
||||
StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
|
||||
@@ -717,7 +710,6 @@ impl Clone for StorageError {
|
||||
StorageError::StorageFull => StorageError::StorageFull,
|
||||
StorageError::SlowDown => StorageError::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
|
||||
StorageError::InvalidUploadIDKeyCombination(a, b) => {
|
||||
StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
|
||||
@@ -860,7 +852,6 @@ impl StorageError {
|
||||
StorageError::StorageFull => StorageErrorCode::StorageFull,
|
||||
StorageError::SlowDown => StorageErrorCode::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
|
||||
StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
|
||||
StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
|
||||
@@ -967,7 +958,6 @@ impl StorageError {
|
||||
StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
|
||||
StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
|
||||
StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
|
||||
StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
|
||||
StorageErrorCode::PrefixAccessDenied => {
|
||||
Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
|
||||
}
|
||||
|
||||
@@ -79,14 +79,6 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
|
||||
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
|
||||
}
|
||||
|
||||
/// Validate that replicated compression metadata can be read by this build.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for an unsupported compression scheme.
|
||||
pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
|
||||
compression_scheme_to_read_plan(scheme).map(|_| ())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadEncryptionBackend {
|
||||
Legacy,
|
||||
@@ -452,12 +444,10 @@ impl WritePlan {
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
let etag_nonce = match encryption.mode {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
|
||||
WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
|
||||
// Each part write needs an independent fingerprint, including a
|
||||
// retry at the same part number. This salt affects only the ETag
|
||||
// MAC; encryption framing and its data nonce stay unchanged.
|
||||
WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
|
||||
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
|
||||
base_nonce
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
|
||||
.with_inner_etag_delegation(delegate_inner_etag);
|
||||
@@ -568,54 +558,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
|
||||
let plaintext = b"same confidential part";
|
||||
let size = i64::try_from(plaintext.len()).expect("test length");
|
||||
for object_key_mode in [false, true] {
|
||||
let mut tags = std::collections::HashSet::new();
|
||||
for part_number in [1, 2, 1] {
|
||||
let encryption = if object_key_mode {
|
||||
WriteEncryption::multipart_object_key([7; 32], part_number)
|
||||
} else {
|
||||
WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
|
||||
};
|
||||
let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
|
||||
let mut reader = WritePlan::new()
|
||||
.with_encryption(encryption)
|
||||
.apply(source, size)
|
||||
.expect("part plan");
|
||||
let mut ciphertext = Vec::new();
|
||||
reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
|
||||
let tag = reader.try_resolve_etag().expect("part fingerprint");
|
||||
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
|
||||
assert!(
|
||||
tags.insert(tag),
|
||||
"different part numbers and retries must not identify matching plaintext"
|
||||
);
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let mut decrypt = if object_key_mode {
|
||||
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
|
||||
} else {
|
||||
DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
|
||||
};
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let mut decrypt = DecryptReader::new_multipart(
|
||||
Cursor::new(ciphertext),
|
||||
[7; 32],
|
||||
if object_key_mode { [0; 12] } else { [9; 12] },
|
||||
vec![part_number as usize],
|
||||
);
|
||||
let mut decoded = Vec::new();
|
||||
decrypt
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect("existing data nonce still decrypts");
|
||||
assert_eq!(decoded, plaintext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
|
||||
@@ -949,65 +949,8 @@ pub enum ShardIntegrityWriteMode {
|
||||
Protected,
|
||||
}
|
||||
|
||||
/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
|
||||
/// Credits cover directory entries (including a minimum charge for an empty probe) and
|
||||
/// transaction inspections. Failed/cancelled probes keep their worst-case reservation.
|
||||
#[derive(Debug)]
|
||||
pub struct DeleteRecoveryWorkBudget {
|
||||
remaining: std::sync::atomic::AtomicU32,
|
||||
}
|
||||
|
||||
/// Entries one disk may reserve up front per recovery step (discovery plus
|
||||
/// inspection). Every disk of every set a request fans out to reserves this
|
||||
/// much before any refund arrives, so a budget must cover that many disks.
|
||||
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
|
||||
|
||||
impl Default for DeleteRecoveryWorkBudget {
|
||||
/// Covers a full 16-drive set with headroom; `for_disks` scales it when a
|
||||
/// request spans more drives than that.
|
||||
fn default() -> Self {
|
||||
Self::new(16 * 1024 * 1024)
|
||||
}
|
||||
}
|
||||
|
||||
impl DeleteRecoveryWorkBudget {
|
||||
/// A per-request budget sized for `disks` concurrent up-front reservations.
|
||||
pub(crate) fn for_disks(disks: usize) -> Self {
|
||||
let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
|
||||
let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
|
||||
let default = u64::from(Self::default().remaining());
|
||||
Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
|
||||
}
|
||||
|
||||
pub(crate) fn new(entries: u32) -> Self {
|
||||
Self {
|
||||
remaining: std::sync::atomic::AtomicU32::new(entries),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
|
||||
use std::sync::atomic::Ordering;
|
||||
self.remaining
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
|
||||
(remaining > 0).then(|| remaining.saturating_sub(maximum))
|
||||
})
|
||||
.map(|remaining| remaining.min(maximum))
|
||||
.map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
|
||||
pub(crate) fn remaining(&self) -> u32 {
|
||||
self.remaining.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn refund(&self, unused: u32) {
|
||||
self.remaining.fetch_add(unused, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
#[doc(hidden)]
|
||||
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
pub max_parity: bool,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
@@ -1159,12 +1102,6 @@ pub struct ObjectOptions {
|
||||
pub expiration: ExpirationOptions,
|
||||
pub lifecycle_audit_event: LcAuditEvent,
|
||||
|
||||
/// In-process request gate evaluated on authoritative completion metadata.
|
||||
/// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
|
||||
/// implementations may own customer encryption keys.
|
||||
#[doc(hidden)]
|
||||
pub multipart_completion_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
|
||||
pub eval_metadata: Option<HashMap<String, String>>,
|
||||
/// Internal compare-and-set condition for replication workers publishing
|
||||
/// terminal status after remote I/O. Storage validates it while holding
|
||||
@@ -1178,10 +1115,6 @@ pub struct ObjectOptions {
|
||||
/// Authoritative bucket Object Lock snapshot installed inside `ECStore`
|
||||
/// before a destructive commit reaches the set layer.
|
||||
pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
|
||||
/// ECStore single PUTs acquire their destructive-write configuration guards
|
||||
/// only after staging the client body. External lock owners supply a snapshot.
|
||||
#[doc(hidden)]
|
||||
pub defer_put_object_lock_snapshot: bool,
|
||||
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
@@ -1213,11 +1146,6 @@ pub enum ReplicationStatusWritebackMode {
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
|
||||
self.delete_recovery_work_budget
|
||||
.get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
|
||||
}
|
||||
|
||||
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
|
||||
match self.shard_integrity_write_mode {
|
||||
Some(ShardIntegrityWriteMode::Protected) => true,
|
||||
|
||||
@@ -786,7 +786,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
bytes.len()
|
||||
};
|
||||
|
||||
Ok(ReadCloser::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
|
||||
@@ -37,6 +37,7 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::{
|
||||
select,
|
||||
sync::{Notify, RwLock},
|
||||
@@ -8182,7 +8183,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
@@ -8247,7 +8248,7 @@ mod tests {
|
||||
_opts: WarmBackendGetOpts,
|
||||
) -> std::result::Result<ReadCloser, std::io::Error> {
|
||||
if self.healthy {
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
} else {
|
||||
Err(std::io::Error::other("mock get failed"))
|
||||
}
|
||||
@@ -9894,7 +9895,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
|
||||
@@ -195,14 +195,7 @@ impl SetDisks {
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
// With no decodable version, the per-version quorum loop cannot
|
||||
// reject corrupt or unavailable replicas. Only complete, readable
|
||||
// evidence may establish that this key has no authoritative version.
|
||||
return if complete {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
};
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(MetaCacheEntry {
|
||||
name,
|
||||
|
||||
@@ -357,7 +357,8 @@ use crate::io_support::rio::HashReader;
|
||||
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
||||
pub const MAX_PARTS_COUNT: usize = 10000;
|
||||
pub(crate) use crate::disk::multipart_index::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
|
||||
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
|
||||
|
||||
|
||||
@@ -141,7 +141,6 @@ pub(crate) struct DeleteRecoverySummary {
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
pub incarnation: Option<Uuid>,
|
||||
pub work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
pub dry_run: bool,
|
||||
pub explicit_delete: bool,
|
||||
/// Only the locked heal path may repair a live marker-creation source.
|
||||
@@ -157,7 +156,6 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
pub(crate) fn new(incarnation: Option<Uuid>, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self {
|
||||
Self {
|
||||
incarnation,
|
||||
work_budget: None,
|
||||
dry_run: false,
|
||||
explicit_delete: false,
|
||||
repair_marker_source: false,
|
||||
@@ -232,54 +230,19 @@ async fn inspect(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Vec<Result<DeleteRecoveryInspection>> {
|
||||
join_all(disks.iter().map(|disk| async move {
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let state = if let Some(budget) = budget {
|
||||
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let reply = disk
|
||||
.inspect_delete_transaction_bounded(bucket, object, transaction, reserved)
|
||||
.await?;
|
||||
if reply.visited > reserved {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.refund(reserved - reply.visited.max(1));
|
||||
reply.inspection
|
||||
} else {
|
||||
disk.inspect_delete_transaction(bucket, object, transaction).await?
|
||||
};
|
||||
let state = disk
|
||||
.as_ref()
|
||||
.ok_or(DiskError::DiskNotFound)?
|
||||
.inspect_delete_transaction(bucket, object, transaction)
|
||||
.await?;
|
||||
validate(&state, object, transaction)?;
|
||||
Ok(state)
|
||||
}))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_bounded(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
budget: &crate::object_api::DeleteRecoveryWorkBudget,
|
||||
) -> Result<Vec<DeleteTransactionId>> {
|
||||
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
|
||||
let page = disk.discover_delete_transactions_bounded(bucket, object, reserved).await?;
|
||||
if page.visited > reserved
|
||||
|| page.transactions.len() > 256
|
||||
|| page
|
||||
.transactions
|
||||
.iter()
|
||||
.any(|id| id.object != object || id.transaction.is_nil())
|
||||
|| (!page.complete && !page.transactions.is_empty())
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.refund(reserved - page.visited.max(1));
|
||||
if !page.complete {
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
Ok(page.transactions)
|
||||
}
|
||||
|
||||
async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result<Vec<DeleteTransactionId>> {
|
||||
let mut cursor = None;
|
||||
let mut ids = BTreeSet::new();
|
||||
@@ -310,92 +273,30 @@ async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Resul
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RecoveryWorkScope<'a> {
|
||||
namespace_owner: Option<&'a Arc<dyn Send + Sync>>,
|
||||
budget: Option<&'a crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
}
|
||||
|
||||
async fn settle_outcome(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
) -> Result<crate::disk::DeleteRecoverySettlementOutcome> {
|
||||
use crate::disk::DeleteRecoverySettlementOutcome;
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let outcome = if let Some(budget) = scope.budget {
|
||||
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
|
||||
let reply = disk
|
||||
.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: state.id.object.clone(),
|
||||
transaction: state.id.transaction,
|
||||
expected_fingerprint: state.fingerprint,
|
||||
action,
|
||||
max_entries: reserved,
|
||||
},
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?;
|
||||
if reply.visited > reserved
|
||||
|| (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.refund(reserved - reply.visited.max(1));
|
||||
reply.outcome
|
||||
} else {
|
||||
DeleteRecoverySettlementOutcome::Inspected(Box::new(
|
||||
disk.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?,
|
||||
))
|
||||
};
|
||||
if let DeleteRecoverySettlementOutcome::Inspected(next) = &outcome {
|
||||
validate(next, &state.id.object, state.id.transaction)?;
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn settle(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
match settle_outcome(disk, bucket, state, action, can_commit, scope).await? {
|
||||
crate::disk::DeleteRecoverySettlementOutcome::Inspected(inspection) => Ok(*inspection),
|
||||
crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_receipt_quorum(outcomes: &[Result<crate::disk::DeleteRecoverySettlementOutcome>], quorum: usize) -> bool {
|
||||
outcomes.iter().filter(|outcome| published_receipt(outcome)).count() >= quorum
|
||||
}
|
||||
|
||||
fn published_receipt(outcome: &Result<crate::disk::DeleteRecoverySettlementOutcome>) -> bool {
|
||||
match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => receipt(state.state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => true,
|
||||
Err(_) => false,
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let result = disk
|
||||
.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
namespace_owner.cloned(),
|
||||
)
|
||||
.await;
|
||||
let next = result?;
|
||||
validate(&next, &state.id.object, state.id.transaction)?;
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
/// Each phase fans out across independent drives, and awaits all started work
|
||||
@@ -406,16 +307,23 @@ async fn settle_group(
|
||||
copies: &mut [DeleteRecoveryInspection],
|
||||
action: impl Fn(&DeleteRecoveryInspection) -> Option<DeleteRecoveryAction> + Sync,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
removed_versions: &mut HashSet<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| {
|
||||
let action = action(copy);
|
||||
async move {
|
||||
match action {
|
||||
Some(action) => settle(disk.as_ref().ok_or(DiskError::DiskNotFound)?, bucket, copy, action, can_commit, scope)
|
||||
.await
|
||||
.map(Some),
|
||||
Some(action) => settle(
|
||||
disk.as_ref().ok_or(DiskError::DiskNotFound)?,
|
||||
bucket,
|
||||
copy,
|
||||
action,
|
||||
can_commit,
|
||||
namespace_owner,
|
||||
)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
@@ -531,31 +439,16 @@ impl SetDisks {
|
||||
true
|
||||
}
|
||||
};
|
||||
let mut fast_receipt_quorum = false;
|
||||
if can_try_local_fast_path && can_commit() {
|
||||
let fast_published = join_all(disks.iter().map(|disk| async {
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
disk.try_publish_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
context.namespace_owner.clone(),
|
||||
context.work_budget.clone(),
|
||||
)
|
||||
.await
|
||||
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
fast_receipt_quorum = fast_published.iter().filter(|result| matches!(result, Ok(Some(_)))).count() >= write_quorum;
|
||||
if fast_receipt_quorum
|
||||
&& fast_published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let all_fast_published = fast_published.len() == self.set_drive_count
|
||||
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
|
||||
if all_fast_published && can_commit() {
|
||||
@@ -563,7 +456,6 @@ impl SetDisks {
|
||||
test_hooks::record_cleanup_attempt(bucket, object);
|
||||
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
|
||||
let ns_owner = context.namespace_owner.clone();
|
||||
let work_budget = context.work_budget.clone();
|
||||
async move {
|
||||
if !can_commit() {
|
||||
return Ok(false);
|
||||
@@ -574,24 +466,11 @@ impl SetDisks {
|
||||
let Ok(Some((journal, _, _))) = pub_res else {
|
||||
return Ok(false);
|
||||
};
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
ns_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
|
||||
.await
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
if finalized
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if finalized.iter().all(|res| matches!(res, Ok(true))) {
|
||||
if let Some(ticket) = context.recovery_ticket.as_ref() {
|
||||
self.ctx.delete_recovery_hints().ack(ticket);
|
||||
@@ -600,17 +479,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
let states = inspect(disks, bucket, object, transaction, context.work_budget.as_deref()).await;
|
||||
if states
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return if fast_receipt_quorum {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
};
|
||||
}
|
||||
let states = inspect(disks, bucket, object, transaction).await;
|
||||
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
|
||||
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
|
||||
&& states.iter().all(|state| {
|
||||
@@ -630,50 +499,25 @@ impl SetDisks {
|
||||
if state.state == DeleteRecoveryState::LegacyAmbiguous {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
settle_outcome(
|
||||
settle(
|
||||
disk,
|
||||
bucket,
|
||||
state,
|
||||
DeleteRecoveryAction::PublishCommit,
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: context.namespace_owner.as_ref(),
|
||||
budget: context.work_budget.as_deref(),
|
||||
},
|
||||
context.namespace_owner.as_ref(),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if !has_receipt_quorum(&published, write_quorum) {
|
||||
return Err(
|
||||
if published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
} else {
|
||||
DiskError::ErasureWriteQuorum
|
||||
},
|
||||
);
|
||||
if published
|
||||
.iter()
|
||||
.filter(|state| matches!(state, Ok(state) if receipt(state.state)))
|
||||
.count()
|
||||
< write_quorum
|
||||
{
|
||||
return Err(DiskError::ErasureWriteQuorum);
|
||||
}
|
||||
if published.iter().any(|outcome| {
|
||||
matches!(
|
||||
outcome,
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
)
|
||||
}) {
|
||||
return Ok(());
|
||||
}
|
||||
let published = published
|
||||
.into_iter()
|
||||
.map(|outcome| match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => Ok(*state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
// Physical cleanup is retryable after the certified decision. An
|
||||
// unavailable participant retains the decision for its eventual return.
|
||||
let ticket = context.recovery_ticket.clone();
|
||||
@@ -826,7 +670,6 @@ impl SetDisks {
|
||||
}
|
||||
let DeleteRecoveryContext {
|
||||
incarnation,
|
||||
work_budget,
|
||||
dry_run,
|
||||
explicit_delete,
|
||||
repair_marker_source,
|
||||
@@ -838,13 +681,7 @@ impl SetDisks {
|
||||
} = context;
|
||||
let disks = self.disk_inventory().await;
|
||||
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| async {
|
||||
match work_budget.as_deref() {
|
||||
Some(budget) => discover_bounded(disk, bucket, object, budget).await,
|
||||
None => discover(disk, bucket, Some(object)).await,
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await;
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut inspected = 0;
|
||||
let mut offline = !complete_inventory;
|
||||
@@ -875,7 +712,6 @@ impl SetDisks {
|
||||
..Default::default()
|
||||
};
|
||||
let mut all_states = BTreeMap::new();
|
||||
let mut registered_evidence_hint = false;
|
||||
for transaction in transactions {
|
||||
let mut complete = Vec::with_capacity(disks.len());
|
||||
let inspections = if let Some((published_tx, published_states)) = published_hint
|
||||
@@ -887,14 +723,8 @@ impl SetDisks {
|
||||
{
|
||||
published_states.to_vec()
|
||||
} else {
|
||||
inspect(&disks, bucket, object, transaction, work_budget.as_deref()).await
|
||||
inspect(&disks, bucket, object, transaction).await
|
||||
};
|
||||
if inspections
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
for state in inspections {
|
||||
match state {
|
||||
Ok(state) => {
|
||||
@@ -906,33 +736,6 @@ impl SetDisks {
|
||||
summary.blocked += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
// Candidate UUID directories are not evidence. Only a bound, inspected
|
||||
// journal can create a foreground hint; empty/partial scans rely on the
|
||||
// existing raw anti-entropy obligation instead of queuing arbitrary prefixes.
|
||||
if work_budget.is_some()
|
||||
&& !registered_evidence_hint
|
||||
&& journal.protocol_version == DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
&& journal.volume == bucket
|
||||
&& journal.object == object
|
||||
&& journal.transaction == transaction
|
||||
&& matches!(
|
||||
state.state,
|
||||
DeleteRecoveryState::NewPrepared
|
||||
| DeleteRecoveryState::QuorumCommitted
|
||||
| DeleteRecoveryState::CommitFinished
|
||||
)
|
||||
{
|
||||
self.ctx.delete_recovery_hints().register(
|
||||
crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: self.pool_index,
|
||||
set: self.set_index,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
incarnation,
|
||||
},
|
||||
);
|
||||
registered_evidence_hint = true;
|
||||
}
|
||||
}
|
||||
complete.push(state);
|
||||
}
|
||||
@@ -964,7 +767,7 @@ impl SetDisks {
|
||||
};
|
||||
if let Some((transaction, source)) = prerequisite {
|
||||
Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit, namespace_owner.as_ref())).await?;
|
||||
let refreshed = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
let refreshed = inspect(&disks, bucket, object, transaction)
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1050,7 +853,7 @@ impl SetDisks {
|
||||
if mutated {
|
||||
// A sibling may have restored metadata or reclaimed shared
|
||||
// evidence. Refresh fingerprints before acting on this group.
|
||||
*copies = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
*copies = inspect(&disks, bucket, object, transaction)
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1073,10 +876,7 @@ impl SetDisks {
|
||||
(copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy)
|
||||
},
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1187,10 +987,7 @@ impl SetDisks {
|
||||
)
|
||||
},
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1204,10 +1001,7 @@ impl SetDisks {
|
||||
.then_some(DeleteRecoveryAction::FinishCommit)
|
||||
},
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1254,10 +1048,7 @@ impl SetDisks {
|
||||
copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback),
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1285,7 +1076,7 @@ impl SetDisks {
|
||||
settled.push(copies);
|
||||
} else {
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction, work_budget.as_deref()).await;
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
@@ -1295,7 +1086,7 @@ impl SetDisks {
|
||||
}
|
||||
for (index, mut copies) in settled.into_iter().enumerate() {
|
||||
if index > 0 {
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction, work_budget.as_deref())
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction)
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1315,10 +1106,7 @@ impl SetDisks {
|
||||
&mut copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit),
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1570,457 +1358,6 @@ mod tests {
|
||||
body
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_mixed_inspected_and_deferred_receipts_count_only_real_quorum() {
|
||||
let bucket = "recover-bounded-mixed-quorum";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![73; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let mut outcomes = Vec::new();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let state = disk
|
||||
.inspect_delete_transaction_bounded(bucket, "object", transaction, 65536)
|
||||
.await
|
||||
.expect("preparation inspection");
|
||||
let max_entries = match index {
|
||||
0 => 65536,
|
||||
1 | 2 => state.visited,
|
||||
_ => 1,
|
||||
};
|
||||
outcomes.push(
|
||||
disk.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "object".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: state.inspection.fingerprint,
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map(|reply| reply.outcome),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
matches!(&outcomes[0], Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) if receipt(state.state))
|
||||
);
|
||||
assert!(matches!(
|
||||
&outcomes[1],
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
));
|
||||
assert!(outcomes[3].is_err());
|
||||
assert!(has_receipt_quorum(&outcomes, 3));
|
||||
assert!(!has_receipt_quorum(&outcomes, 4));
|
||||
outcomes[2] = Err(DiskError::FileAccessDenied);
|
||||
assert!(
|
||||
!has_receipt_quorum(&outcomes, 3),
|
||||
"a write failure cannot substitute for a durable receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_single_and_batch_share_bounded_work_across_versions_and_retries() {
|
||||
use crate::object_api::DeleteRecoveryWorkBudget;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
let bucket = "recover-bounded-request";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
for dir in &dirs {
|
||||
let prefix = dir.path().join(bucket).join("prefix");
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
}
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"prefix",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(DeleteRecoveryWorkBudget::new(8))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("incomplete discovery is not absence");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "prefix".to_owned(),
|
||||
incarnation,
|
||||
}),
|
||||
"an unproven prefix must not move its scan into background recovery"
|
||||
);
|
||||
}
|
||||
for distinct_versions in [false, true] {
|
||||
let budget = Arc::new(DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|_| ObjectToDelete {
|
||||
object_name: "prefix".to_owned(),
|
||||
version_id: distinct_versions.then(Uuid::new_v4),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"repeated actions must not each obtain a new allowance: {distinct_versions}"
|
||||
);
|
||||
assert!(
|
||||
dirs.iter()
|
||||
.all(|dir| dir.path().join(bucket).join("prefix/child-63").is_dir())
|
||||
);
|
||||
}
|
||||
let error = store
|
||||
.delete_object(bucket, "prefix", ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("ordinary missing key");
|
||||
assert!(
|
||||
crate::error::is_err_object_not_found(&error),
|
||||
"small complete inventory retains no-op semantics: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
let dir = tempfile::tempdir().expect("temporary disk");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(set_index);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
dirs.push(dir);
|
||||
}
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("bounded-recovery-{pool_index}"),
|
||||
platform: "test".to_owned(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("local disks");
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("address"), endpoints, cancel.clone(), ctx)
|
||||
.await
|
||||
.expect("two pools with two sets");
|
||||
cancel.cancel();
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "recover-bounded-fanout";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let candidates = (0..100).map(|index| format!("prefix-{index}")).collect::<Vec<_>>();
|
||||
let mut keys = BTreeSet::new();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
keys.insert(
|
||||
candidates
|
||||
.iter()
|
||||
.find(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set))
|
||||
.expect("fixture key for set")
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let keys = keys.into_iter().collect::<Vec<_>>();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
assert!(
|
||||
keys.iter().any(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set)),
|
||||
"fixture must cover every set"
|
||||
);
|
||||
}
|
||||
}
|
||||
for dir in &dirs {
|
||||
for key in &keys {
|
||||
let prefix = dir.path().join(bucket).join(key);
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for child in 0..8 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{child}"))).expect("child");
|
||||
}
|
||||
}
|
||||
}
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|index| crate::storage_api_contracts::object::ObjectToDelete {
|
||||
object_name: keys[index % keys.len()].clone(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"all pools and sets must share the finite allowance"
|
||||
);
|
||||
assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
|
||||
assert!(
|
||||
dirs.iter()
|
||||
.all(|dir| dir.path().join(bucket).join(&keys[0]).join("child-7").is_dir())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fast_commit_keeps_success_when_cleanup_budget_is_empty() {
|
||||
let bucket = "recover-bounded-fast-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![74; 2 * 1024 * 1024], false).await;
|
||||
// Inline cleanup needs no directory enumeration and may legitimately
|
||||
// finish with zero scan credits. Require real part files here.
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, "object", "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read non-inline source");
|
||||
assert!(!info.inline_data(), "the fixture must require physical part cleanup");
|
||||
let data_dir = info.data_dir.expect("physical data directory");
|
||||
let part = std::path::PathBuf::from(disk.endpoint().to_string())
|
||||
.join(bucket)
|
||||
.join("object")
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
assert!(part.is_file(), "the cleanup scan must encounter a real part");
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(0))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a cached prepared receipt commits independently of cleanup budget");
|
||||
assert!(
|
||||
store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation")
|
||||
),
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
!disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("retained recovery proof")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background finishes committed cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_unverified_uuid_candidate_does_not_enqueue_background_work() {
|
||||
let bucket = "recover-bounded-candidate";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
let transaction = Uuid::new_v4();
|
||||
for directory in &dirs {
|
||||
let candidate = directory.path().join(bucket).join("candidate").join(transaction.to_string());
|
||||
std::fs::create_dir_all(&candidate).expect("candidate path");
|
||||
std::fs::write(candidate.join("delete-prepared"), b"not a bound journal").expect("unverified candidate");
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.delete_object(bucket, "candidate", ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "candidate".to_owned(),
|
||||
incarnation,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_budget_refusal_preserves_evidence_for_background_replay() {
|
||||
let bucket = "recover-bounded-evidence";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
let payload = vec![71; 4096];
|
||||
put(&store, bucket, &payload, false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let before = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("before refusal");
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(1))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("no incomplete recovery admission");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
let after = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("after refusal");
|
||||
assert_eq!(before.fingerprint, after.fingerprint);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background-compatible recovery");
|
||||
assert_eq!(body(&store, bucket, None).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_committed_delete_defers_exhausted_cleanup_without_rollback() {
|
||||
let bucket = "recover-bounded-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![72; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let key = crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: None,
|
||||
};
|
||||
let ticket = set.ctx.delete_recovery_hints().register(key.clone());
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::default());
|
||||
let exhausted_after_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
// This hook disables local fast cleanup and runs only after receipt
|
||||
// publication and the shared recovery path's key-wide GC inspection.
|
||||
// Exhausting preparation instead would correctly reject the delete.
|
||||
test_hooks::before_gc_finalize(bucket, "object", {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Arc::new(move || {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Box::pin(async move {
|
||||
assert!(budget.reserve(u32::MAX).expect("credits remain until committed cleanup") > 0);
|
||||
exhausted_after_commit.store(true, std::sync::atomic::Ordering::Release);
|
||||
})
|
||||
})
|
||||
});
|
||||
set.publish_delete_commit(
|
||||
bucket,
|
||||
"object",
|
||||
transaction,
|
||||
&set.disk_inventory().await,
|
||||
set.set_drive_count / 2 + 1,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: Some(Arc::clone(&budget)),
|
||||
recovery_ticket: ticket,
|
||||
..DeleteRecoveryContext::new(None, &|| true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the durable decision still succeeds");
|
||||
assert!(exhausted_after_commit.load(std::sync::atomic::Ordering::Acquire));
|
||||
assert_eq!(budget.remaining(), 0);
|
||||
assert!(set.ctx.delete_recovery_hints().is_pending(&key));
|
||||
let state = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("retained decision");
|
||||
assert!(receipt(state.state));
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("eventual cleanup");
|
||||
assert!(
|
||||
disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("cleaned proofs")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_set_rolls_back_undecided_inline_bytes() {
|
||||
let bucket = "recover-undecided";
|
||||
|
||||
@@ -99,7 +99,6 @@ use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
|
||||
#[cfg(test)]
|
||||
use rustfs_utils::http::SUFFIX_COMPRESSION;
|
||||
use rustfs_utils::http::{SUFFIX_INLINE_DATA, SUFFIX_MAX_TOTAL_OBJECT_SIZE, get_consistent_str};
|
||||
use rustfs_utils::path::{decode_dir_object, encode_dir_object};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
@@ -973,48 +972,12 @@ fn multipart_upload_leaf_is_safe(leaf: &str) -> bool {
|
||||
!leaf.is_empty() && leaf != "." && leaf != ".." && !leaf.contains(['/', '\\', '\0'])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static MULTIPART_DISCOVERY_DIRECTORIES: std::sync::LazyLock<std::sync::Mutex<HashMap<String, Vec<String>>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(test)]
|
||||
fn observe_multipart_discovery_directory(bucket: &str, directory: &str) {
|
||||
if let Some(directories) = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("multipart discovery observer lock")
|
||||
.get_mut(bucket)
|
||||
{
|
||||
directories.push(directory.to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
async fn multipart_index_roots_on_disk(disk: DiskStore, bucket: &str) -> disk::error::Result<(Uuid, Vec<String>)> {
|
||||
if !disk.is_online().await {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
let identity = disk
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query_bucket = bucket.to_owned();
|
||||
let roots = crate::disk::multipart_index::collect_pages(bucket, identity, move |marker| {
|
||||
let disk = disk.clone();
|
||||
let bucket = query_bucket.clone();
|
||||
async move { disk.list_multipart_index_roots(&bucket, marker.as_deref()).await }
|
||||
})
|
||||
.await?;
|
||||
Ok((identity, roots))
|
||||
}
|
||||
|
||||
async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_prefix: &str) -> disk::error::Result<Vec<String>> {
|
||||
if !disk.is_online().await {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
#[cfg(test)]
|
||||
observe_multipart_discovery_directory(bucket, root_prefix);
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1029,8 +992,6 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.collect());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
observe_multipart_discovery_directory(bucket, root_prefix);
|
||||
let sha_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1053,8 +1014,6 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.map(|sha_dir| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
#[cfg(test)]
|
||||
observe_multipart_discovery_directory(bucket, &sha_dir);
|
||||
match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1).await {
|
||||
Ok(entries) => Ok((sha_dir, entries)),
|
||||
Err(DiskError::FileNotFound) => Ok((sha_dir, Vec::new())),
|
||||
@@ -1079,12 +1038,6 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
Ok(upload_paths)
|
||||
}
|
||||
|
||||
struct MultipartUploadListScope<'a> {
|
||||
bucket: &'a str,
|
||||
prefix: &'a str,
|
||||
exact_key: bool,
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
fn validated_multipart_upload_path(bucket: &str, object: &str, upload_id: &str, data_movement: bool) -> Result<String> {
|
||||
let path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
@@ -1181,25 +1134,13 @@ impl SetDisks {
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
self.discover_multipart_upload_paths_on_disks(disks, orig_bucket, error_path, root_prefix, require_all_disks)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_multipart_upload_paths_on_disks(
|
||||
&self,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
if disks.len() != self.set_drive_count || self.set_drive_count == 0 {
|
||||
if disks.is_empty() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
let discovery_quorum = if self.default_parity_count == 0 {
|
||||
self.set_drive_count
|
||||
disks.len()
|
||||
} else {
|
||||
(self.set_drive_count / 2).max(1)
|
||||
(disks.len() / 2).max(1)
|
||||
};
|
||||
let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::<Vec<_>>();
|
||||
let mut candidate_counts = HashMap::<String, usize>::new();
|
||||
@@ -1878,127 +1819,8 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
self.list_multipart_uploads_in_scope(
|
||||
MultipartUploadListScope {
|
||||
bucket,
|
||||
prefix,
|
||||
exact_key: false,
|
||||
},
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
delimiter,
|
||||
max_uploads,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Probe only the legacy staging directory belonging to this encoded key.
|
||||
pub(crate) async fn has_multipart_upload_for_key(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let key = decode_dir_object(object);
|
||||
let result = self
|
||||
.list_multipart_uploads_in_scope(
|
||||
MultipartUploadListScope {
|
||||
bucket,
|
||||
prefix: &key,
|
||||
exact_key: true,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
1,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
Ok(!result.uploads.is_empty())
|
||||
}
|
||||
|
||||
/// Index coverage only locates candidates. Every candidate still traverses
|
||||
/// the original directory and metadata quorum checks on this disk snapshot.
|
||||
async fn multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Option<(Vec<Option<DiskStore>>, std::collections::BTreeSet<String>, usize)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
// Coverage describes configured physical slots, not the size of a
|
||||
// transient inventory. Missing slots must remain explicit None entries.
|
||||
if disks.len() != self.set_drive_count || self.set_drive_count == 0 {
|
||||
return None;
|
||||
}
|
||||
let quorum = if self.default_parity_count == 0 {
|
||||
self.set_drive_count
|
||||
} else {
|
||||
(self.set_drive_count / 2).max(1)
|
||||
};
|
||||
let required = self.set_drive_count.checked_sub(quorum)?.checked_add(1)?.max(quorum);
|
||||
let mut tasks = JoinSet::new();
|
||||
for disk in disks.iter().flatten().cloned() {
|
||||
let bucket = bucket.to_owned();
|
||||
tasks.spawn(async move { multipart_index_roots_on_disk(disk, &bucket).await });
|
||||
}
|
||||
let mut covered_ids = HashSet::new();
|
||||
let mut roots = std::collections::BTreeSet::new();
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
if let Ok(Ok((disk_id, candidates))) = result {
|
||||
// Duplicate physical identities cannot count as independent slots.
|
||||
if !covered_ids.insert(disk_id) {
|
||||
return None;
|
||||
}
|
||||
roots.extend(candidates);
|
||||
}
|
||||
}
|
||||
// Every returnable metadata read quorum has at least `quorum` slots.
|
||||
// N-D+1 independently indexed slots intersect every such quorum, even
|
||||
// when an old coordinator wrote the upload without knowing this index.
|
||||
(covered_ids.len() >= required).then_some((disks, roots, quorum))
|
||||
}
|
||||
|
||||
async fn list_multipart_uploads_in_scope(
|
||||
&self,
|
||||
scope: MultipartUploadListScope<'_>,
|
||||
key_marker: Option<String>,
|
||||
upload_id_marker: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let MultipartUploadListScope {
|
||||
bucket,
|
||||
prefix,
|
||||
exact_key,
|
||||
} = scope;
|
||||
let root = if exact_key {
|
||||
Self::get_multipart_sha_dir(bucket, &encode_dir_object(prefix))
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let covered = if exact_key {
|
||||
None
|
||||
} else {
|
||||
self.multipart_index_roots(bucket).await
|
||||
};
|
||||
let (disks, candidate_paths, discovery_quorum, _) = if let Some((disks, mut roots, quorum)) = covered {
|
||||
// Legacy no-incarnation callers can list an exact-prefix orphan after
|
||||
// completion moved xl.meta. Its SHA is known without foreign scanning.
|
||||
if expected_incarnation_id.is_none() && !prefix.is_empty() {
|
||||
roots.insert(Self::get_multipart_sha_dir(bucket, &encode_dir_object(prefix)));
|
||||
}
|
||||
let mut candidates = Vec::new();
|
||||
for root in roots {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths_on_disks(disks.clone(), bucket, prefix, &root, false)
|
||||
.await?;
|
||||
candidates.extend(paths);
|
||||
}
|
||||
(disks, candidates, quorum, false)
|
||||
} else {
|
||||
self.discover_multipart_upload_paths(bucket, prefix, &root, false).await?
|
||||
};
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -2022,11 +1844,7 @@ impl SetDisks {
|
||||
.iter()
|
||||
.filter(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::VolumeNotFound)))
|
||||
.count();
|
||||
// At half-set parity, both successful heads and missing or corrupt
|
||||
// replies can reach discovery quorum. Resolve the surviving heads
|
||||
// through the authoritative layout and identity checks in that tie.
|
||||
let below_discovery_quorum = errs.iter().filter(|err| err.is_none()).count() < discovery_quorum;
|
||||
if below_discovery_quorum && missing_metadata >= discovery_quorum {
|
||||
if missing_metadata >= discovery_quorum {
|
||||
if expected_incarnation_id.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -2035,7 +1853,7 @@ impl SetDisks {
|
||||
// crash in that window intentionally leaves a reclaimable
|
||||
// upload directory whose object name can still be proven for
|
||||
// an exact-key listing by matching the namespace hash.
|
||||
if !prefix.is_empty() && sha_dir == Self::get_multipart_sha_dir(bucket, &encode_dir_object(prefix)) {
|
||||
if !prefix.is_empty() && sha_dir == Self::get_multipart_sha_dir(bucket, prefix) {
|
||||
let initiated = raw_upload_id
|
||||
.rsplit_once('x')
|
||||
.and_then(|(_, timestamp)| timestamp.parse::<i128>().ok())
|
||||
@@ -2064,7 +1882,7 @@ impl SetDisks {
|
||||
.iter()
|
||||
.filter(|err| err.as_ref().is_some_and(is_corrupt_upload_metadata_error))
|
||||
.count();
|
||||
if below_discovery_quorum && corrupt_metadata > 0 && missing_metadata + corrupt_metadata >= discovery_quorum {
|
||||
if corrupt_metadata > 0 && missing_metadata + corrupt_metadata >= discovery_quorum {
|
||||
debug!(
|
||||
bucket,
|
||||
upload_path = %upload_path,
|
||||
@@ -2129,8 +1947,7 @@ impl SetDisks {
|
||||
// retry instead of silently losing entries.
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let object = decode_dir_object(&object);
|
||||
if (exact_key && object != prefix) || !object.starts_with(prefix) {
|
||||
if !object.starts_with(prefix) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
@@ -2234,9 +2051,14 @@ impl SetDisks {
|
||||
.zip(errors)
|
||||
.map(|(fi, error)| error.is_none().then_some(fi))
|
||||
.collect();
|
||||
let (completed, mut matching_minority) = self
|
||||
.completed_multipart_write_quorum_candidate((bucket, object), upload_id, uploaded_parts, source, candidates, opts)
|
||||
.await?;
|
||||
let (completed, mut matching_minority) = self.completed_multipart_write_quorum_candidate(
|
||||
(bucket, object),
|
||||
upload_id,
|
||||
uploaded_parts,
|
||||
source,
|
||||
candidates,
|
||||
opts,
|
||||
)?;
|
||||
if let Some(completed) = completed {
|
||||
return self
|
||||
.acknowledge_completed_multipart(&disks, bucket, object, upload_id, completed, opts)
|
||||
@@ -2286,9 +2108,14 @@ impl SetDisks {
|
||||
}
|
||||
candidates.push(candidate);
|
||||
}
|
||||
let (completed, historical_minority) = self
|
||||
.completed_multipart_write_quorum_candidate((bucket, object), upload_id, uploaded_parts, source, candidates, opts)
|
||||
.await?;
|
||||
let (completed, historical_minority) = self.completed_multipart_write_quorum_candidate(
|
||||
(bucket, object),
|
||||
upload_id,
|
||||
uploaded_parts,
|
||||
source,
|
||||
candidates,
|
||||
opts,
|
||||
)?;
|
||||
matching_minority |= historical_minority;
|
||||
if let Some(completed) = completed {
|
||||
return self
|
||||
@@ -2343,7 +2170,7 @@ impl SetDisks {
|
||||
/// Count exact completion identities once per physical slot. Each call
|
||||
/// evaluates one complete fanout; observations from different reads never
|
||||
/// combine to manufacture a write quorum.
|
||||
async fn completed_multipart_write_quorum_candidate(
|
||||
fn completed_multipart_write_quorum_candidate(
|
||||
&self,
|
||||
target: (&str, &str),
|
||||
upload_id: &str,
|
||||
@@ -2388,16 +2215,6 @@ impl SetDisks {
|
||||
matching_minority |= count < required
|
||||
&& rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_MULTIPART_UPLOAD_ID)
|
||||
== Some(upload_id);
|
||||
// Select a write-quorum candidate belonging to this upload before
|
||||
// authorizing it; never compare requested parts or confirm/reclaim a
|
||||
// completed object using a different upload's authorization evidence.
|
||||
if count >= required
|
||||
&& rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_MULTIPART_UPLOAD_ID)
|
||||
== Some(upload_id)
|
||||
&& let Some(authorization) = &opts.multipart_completion_authorization
|
||||
{
|
||||
authorization.authorize(&fi.metadata).await?;
|
||||
}
|
||||
if count >= required
|
||||
&& completed_multipart_upload_matches(
|
||||
&fi.metadata,
|
||||
@@ -2992,7 +2809,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let mut ret = ListPartsInfo {
|
||||
bucket: bucket.to_owned(),
|
||||
object: decode_dir_object(object),
|
||||
object: object.to_owned(),
|
||||
upload_id: upload_id.to_owned(),
|
||||
storage_class,
|
||||
max_parts,
|
||||
@@ -3371,7 +3188,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
Ok(MultipartInfo {
|
||||
bucket: bucket.to_owned(),
|
||||
object: decode_dir_object(object),
|
||||
object: object.to_owned(),
|
||||
upload_id: upload_id.to_owned(),
|
||||
user_defined: {
|
||||
strip_internal_multipart_metadata(&mut fi.metadata);
|
||||
@@ -3585,12 +3402,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||
.await?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
// Recovery can restore a source after the request's metadata preflight
|
||||
// missed it. Reauthorize that exact source before checksum comparison or
|
||||
// publication while retaining the original object/upload locks.
|
||||
if let Some(authorization) = &opts.multipart_completion_authorization {
|
||||
authorization.authorize(&fi.metadata).await?;
|
||||
}
|
||||
if !initial_multipart_source_observations(&fi, &files_metas)
|
||||
&& let Some(completed) = self
|
||||
.replay_completed_multipart_upload(bucket, object, upload_id, &uploaded_parts, Some(&fi), opts)
|
||||
@@ -5442,45 +5253,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_upload_paths_reject_aliases_before_disk_or_lock_use() {
|
||||
let bucket = "multipart-path-owner";
|
||||
let key = "allowed-key";
|
||||
let leaf = format!("{}x1000000000", Uuid::new_v4());
|
||||
let encode = |deployment: &str, suffix: &str| {
|
||||
base64_simd::URL_SAFE_NO_PAD.encode_to_string(format!("{deployment}.{suffix}").as_bytes())
|
||||
};
|
||||
for data_movement in [false, true] {
|
||||
let canonical = SetDisks::validated_multipart_upload_path(bucket, key, &encode("deployment", &leaf), data_movement)
|
||||
.expect("issued upload leaf is accepted");
|
||||
assert_eq!(
|
||||
canonical,
|
||||
SetDisks::validated_multipart_upload_path(bucket, key, &encode("another-deployment", &leaf), data_movement)
|
||||
.expect("deployment aliases resolve to the identical disk and lock path")
|
||||
);
|
||||
for suffix in [
|
||||
format!("{leaf}/"),
|
||||
format!("{leaf}//"),
|
||||
format!("./{leaf}"),
|
||||
format!("../{}/{leaf}", SetDisks::get_multipart_sha_dir(bucket, "denied-key")),
|
||||
format!("{leaf}\\"),
|
||||
format!("{leaf}\0"),
|
||||
] {
|
||||
assert!(
|
||||
matches!(
|
||||
SetDisks::validated_multipart_upload_path(bucket, key, &encode("deployment", &suffix), data_movement),
|
||||
Err(Error::InvalidUploadID(..))
|
||||
),
|
||||
"unsafe upload suffix must not reach path normalization: {suffix:?}"
|
||||
);
|
||||
}
|
||||
assert!(matches!(
|
||||
SetDisks::validated_multipart_upload_path(bucket, key, "not-base64!", data_movement),
|
||||
Err(Error::InvalidUploadID(..))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_multipart_identity_requires_canonical_upload_id() {
|
||||
let created = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
|
||||
@@ -9026,82 +8798,6 @@ mod tests {
|
||||
put_object_part_fences_upload_lock_loss_before_rename_case(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_part_retry_persists_new_etag_and_rejects_the_previous_receipt() {
|
||||
use crate::io_support::rio::{WriteEncryption, WritePlan};
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "encrypted-etag-retry";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect("create encrypted upload");
|
||||
let mut receipts = Vec::new();
|
||||
for part_number in [1, 2, 1, 3] {
|
||||
let bytes = b"same confidential part";
|
||||
let size = i64::try_from(bytes.len()).expect("test length");
|
||||
let source = HashReader::from_stream(Cursor::new(bytes.to_vec()), size, size, None, None, false).expect("source");
|
||||
let encrypted = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::multipart([7; 32], [9; 12], part_number))
|
||||
.apply(source, size)
|
||||
.expect("encrypt part");
|
||||
let mut reader = PutObjReader::new(encrypted);
|
||||
let mut part_opts = opts.clone();
|
||||
if part_number == 3 {
|
||||
part_opts.preserve_etag = Some("0123456789abcdef0123456789abcdef".to_string());
|
||||
}
|
||||
let part = set
|
||||
.put_object_part(bucket, "object", &upload.upload_id, part_number, &mut reader, &part_opts)
|
||||
.await
|
||||
.expect("persist encrypted part");
|
||||
if part_number == 3 {
|
||||
assert_eq!(part.etag, part_opts.preserve_etag);
|
||||
} else {
|
||||
receipts.push(part.etag);
|
||||
}
|
||||
}
|
||||
assert_ne!(receipts[0], receipts[1], "spare parts must not identify another part's plaintext");
|
||||
assert_ne!(receipts[0], receipts[2], "same-number retries get a new receipt");
|
||||
for _ in 0..2 {
|
||||
let listed = set
|
||||
.list_object_parts(bucket, "object", &upload.upload_id, None, 10, &opts)
|
||||
.await
|
||||
.expect("list parts");
|
||||
assert_eq!(listed.parts[0].etag, receipts[2], "stored ETags do not change on reads");
|
||||
}
|
||||
let stale = CompletePart {
|
||||
part_num: 1,
|
||||
etag: receipts[0].clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let error = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload.upload_id, vec![stale], &opts)
|
||||
.await
|
||||
.expect_err("the old part receipt must not complete a rewritten part");
|
||||
assert!(matches!(error, Error::InvalidPart(..)), "{error:?}");
|
||||
let current = CompletePart {
|
||||
part_num: 1,
|
||||
etag: receipts[2].clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let committed = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload.upload_id, vec![current.clone()], &opts)
|
||||
.await
|
||||
.expect("complete with the stored receipt");
|
||||
let replay = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload.upload_id, vec![current], &opts)
|
||||
.await
|
||||
.expect("committed completion replay is stable");
|
||||
assert_eq!(replay.etag, committed.etag);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_encrypted_multipart_marks_quorum_validated_layout() {
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
|
||||
@@ -11481,237 +11177,6 @@ mod tests {
|
||||
.unwrap_or(decoded)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_shortened_inventory_cannot_lower_configured_quorum() {
|
||||
let (_roots, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::disk::disk_store::wait_for_multipart_index_bootstrap(disk),
|
||||
)
|
||||
.await
|
||||
.expect("bootstrap completes")
|
||||
.expect("local bootstrap worker");
|
||||
}
|
||||
let bucket = "multipart-index-short-inventory";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let full = set.disks.read().await.clone();
|
||||
assert_eq!(set.set_drive_count, 4);
|
||||
for available in 0..4 {
|
||||
*set.disks.write().await = full[..available].to_vec();
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_none(),
|
||||
"{available} inventory slots cannot redefine configured N"
|
||||
);
|
||||
assert!(matches!(
|
||||
set.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await,
|
||||
Err(Error::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
*set.disks.write().await = full.clone();
|
||||
set.disks.write().await[3] = None;
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_some(),
|
||||
"three covered slots still satisfy configured 4-slot coverage"
|
||||
);
|
||||
set.disks.write().await[2] = None;
|
||||
assert!(set.multipart_index_roots(bucket).await.is_none());
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("two explicit available slots still satisfy legacy D=2");
|
||||
assert!(listed.uploads.is_empty());
|
||||
*set.disks.write().await = full;
|
||||
assert!(set.multipart_index_roots(bucket).await.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_covering_set_finds_old_write_on_minimum_metadata_quorum() {
|
||||
let (_roots, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::disk::disk_store::wait_for_multipart_index_bootstrap(disk),
|
||||
)
|
||||
.await
|
||||
.expect("bootstrap completes")
|
||||
.expect("local bootstrap worker");
|
||||
}
|
||||
let bucket = "multipart-index-minimum-quorum";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let key = "old-writer";
|
||||
let upload = format!("{}x1", Uuid::new_v4());
|
||||
let path = format!("{}/{upload}", SetDisks::get_multipart_sha_dir(bucket, key));
|
||||
let mut metadata = FileInfo::new(&path, 2, 2);
|
||||
metadata.data_dir = Some(Uuid::new_v4());
|
||||
metadata.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.fresh = true;
|
||||
metadata.metadata.insert(RUSTFS_MULTIPART_BUCKET_KEY.into(), bucket.into());
|
||||
metadata.metadata.insert(RUSTFS_MULTIPART_OBJECT_KEY.into(), key.into());
|
||||
// Exactly one of the two surviving source copies intersects the three
|
||||
// covered disks. No coordinator-side index update is involved.
|
||||
for (index, disk) in disks.iter().take(2).enumerate() {
|
||||
let mut info = metadata.clone();
|
||||
info.erasure.index = metadata.erasure.distribution[index];
|
||||
disk.write_metadata(bucket, RUSTFS_META_MULTIPART_BUCKET, &path, info)
|
||||
.await
|
||||
.expect("legacy metadata publication");
|
||||
}
|
||||
// Raw head replacement requires an existing source. Initialize a separate
|
||||
// upload before damaging it so the two readable target copies stay intact.
|
||||
let damaged_key = "damaged-source";
|
||||
let damaged_path = format!("{}/{}x1", SetDisks::get_multipart_sha_dir(bucket, damaged_key), Uuid::new_v4());
|
||||
let mut damaged = metadata.clone();
|
||||
damaged.name.clone_from(&damaged_path);
|
||||
damaged.data_dir = Some(Uuid::new_v4());
|
||||
damaged.erasure.index = damaged.erasure.distribution[0];
|
||||
damaged
|
||||
.metadata
|
||||
.insert(RUSTFS_MULTIPART_OBJECT_KEY.into(), damaged_key.into());
|
||||
disks[0]
|
||||
.write_metadata(bucket, RUSTFS_META_MULTIPART_BUCKET, &damaged_path, damaged)
|
||||
.await
|
||||
.expect("initialize the source that will lose index coverage");
|
||||
disks[0]
|
||||
.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{damaged_path}/xl.meta"),
|
||||
Bytes::from_static(b"old damaged head"),
|
||||
)
|
||||
.await
|
||||
.expect("one uncovered disk");
|
||||
assert!(matches!(
|
||||
multipart_index_roots_on_disk(disks[0].clone(), bucket).await,
|
||||
Err(disk::error::DiskError::MethodNotAllowed)
|
||||
));
|
||||
let target_root = SetDisks::get_multipart_sha_dir(bucket, key);
|
||||
for (index, disk) in disks.iter().enumerate().skip(1) {
|
||||
let (_, roots) = multipart_index_roots_on_disk(disk.clone(), bucket)
|
||||
.await
|
||||
.expect("the other three physical disks attest coverage");
|
||||
assert_eq!(roots.contains(&target_root), index == 1, "only covered disk 1 contains the target source");
|
||||
}
|
||||
let (_, roots, quorum) = set
|
||||
.multipart_index_roots(bucket)
|
||||
.await
|
||||
.expect("three covered disks intersect D=2");
|
||||
assert_eq!(quorum, 2);
|
||||
assert_eq!(roots, std::collections::BTreeSet::from([target_root]));
|
||||
let (head, _) = set
|
||||
.check_upload_id_exists(bucket, key, &runtime_sources::deployment_upload_id(&upload), false)
|
||||
.await
|
||||
.expect("the two surviving legacy source heads satisfy an ordinary metadata read");
|
||||
assert_eq!(head.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).map(String::as_str), Some(key));
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("minimum readable source quorum remains discoverable");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(listed.uploads[0].object, key);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_covering_set_avoids_foreign_scans_and_falls_back_without_coverage() {
|
||||
let (_roots, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::disk::disk_store::wait_for_multipart_index_bootstrap(disk),
|
||||
)
|
||||
.await
|
||||
.expect("bootstrap completes")
|
||||
.expect("local bootstrap worker");
|
||||
}
|
||||
let bucket = "multipart-index-covered-bucket";
|
||||
let foreign = "multipart-index-foreign-bucket";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("target bucket");
|
||||
disk.make_volume(foreign).await.expect("foreign bucket");
|
||||
}
|
||||
let mine = set
|
||||
.new_multipart_upload(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("target upload");
|
||||
let other = set
|
||||
.new_multipart_upload(foreign, "other", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("foreign upload");
|
||||
let foreign_path = SetDisks::get_upload_id_dir(foreign, "other", &other.upload_id);
|
||||
let target_root = SetDisks::get_multipart_sha_dir(bucket, "object");
|
||||
// Model one disk that cannot attest coverage. The real raw legacy write
|
||||
// revokes only that physical disk's index, without changing the target.
|
||||
disks[0]
|
||||
.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{foreign_path}/xl.meta"),
|
||||
Bytes::from_static(b"legacy damaged head"),
|
||||
)
|
||||
.await
|
||||
.expect("unindexable legacy publication");
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_some(),
|
||||
"3 of 4 intersects every 2-copy metadata quorum"
|
||||
);
|
||||
MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.insert(bucket.into(), Vec::new());
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("covered listing");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&mine.upload_id));
|
||||
let directories = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.remove(bucket)
|
||||
.expect("observed discovery");
|
||||
assert!(!directories.is_empty());
|
||||
assert!(
|
||||
directories.iter().all(|directory| directory == &target_root),
|
||||
"covered requests cannot enumerate the staging root or foreign uploads"
|
||||
);
|
||||
|
||||
disks[1]
|
||||
.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{foreign_path}/xl.meta"),
|
||||
Bytes::from_static(b"another legacy damaged head"),
|
||||
)
|
||||
.await
|
||||
.expect("second unavailable index");
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_none(),
|
||||
"2 of 4 does not cover the disjoint 2-copy quorum"
|
||||
);
|
||||
MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.insert(bucket.into(), Vec::new());
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("complete legacy fallback");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&mine.upload_id));
|
||||
let directories = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.remove(bucket)
|
||||
.expect("observed fallback");
|
||||
assert!(
|
||||
directories.iter().any(String::is_empty),
|
||||
"fallback uses the complete old discovery algorithm"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): the multipart staging namespace is flat —
|
||||
/// `sha256(bucket/object)` directories from every bucket share one volume —
|
||||
/// so a bucket-scoped listing reads metadata belonging to other buckets'
|
||||
@@ -11880,66 +11345,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_multipart_pool_probe_is_scoped_to_bucket_and_encoded_key() {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-exact-probe";
|
||||
let other_bucket = "multipart-other-probe";
|
||||
for disk in &disks {
|
||||
for bucket in [bucket, other_bucket] {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
}
|
||||
for (upload_bucket, key) in [(bucket, "object-long"), (bucket, "object/"), (other_bucket, "object")] {
|
||||
set.new_multipart_upload(upload_bucket, &encode_dir_object(key), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create unrelated upload");
|
||||
}
|
||||
// An unrelated broken SHA directory makes a full staging-root walk fail.
|
||||
// It must not be touched by an exact-key pool probe.
|
||||
let unrelated = SetDisks::get_multipart_sha_dir(other_bucket, "broken");
|
||||
for dir in &dirs {
|
||||
tokio::fs::write(dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&unrelated), b"not a directory")
|
||||
.await
|
||||
.expect("seed unrelated discovery failure");
|
||||
}
|
||||
MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer lock")
|
||||
.insert(bucket.to_owned(), Vec::new());
|
||||
assert!(
|
||||
!set.has_multipart_upload_for_key(bucket, "object", None)
|
||||
.await
|
||||
.expect("unrelated keys cannot match")
|
||||
);
|
||||
set.new_multipart_upload(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create exact legacy-layout upload");
|
||||
assert!(
|
||||
set.has_multipart_upload_for_key(bucket, "object", None)
|
||||
.await
|
||||
.expect("exact upload remains discoverable")
|
||||
);
|
||||
let probes = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer lock")
|
||||
.remove(bucket)
|
||||
.expect("recorded probes");
|
||||
assert_eq!(probes.len(), 2 * disks.len(), "one directory per disk and probe");
|
||||
let expected = SetDisks::get_multipart_sha_dir(bucket, "object");
|
||||
assert!(probes.iter().all(|directory| directory == &expected));
|
||||
assert!(
|
||||
set.has_multipart_upload_for_key(bucket, &encode_dir_object("object/"), None)
|
||||
.await
|
||||
.expect("directory key keeps its encoded staging layout")
|
||||
);
|
||||
set.disks.write().await.fill(None);
|
||||
assert!(
|
||||
set.has_multipart_upload_for_key(bucket, "object", None).await.is_err(),
|
||||
"a failed discovery quorum must not become a negative pool match"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -12129,7 +11534,7 @@ mod tests {
|
||||
/// surviving disks still identify it, so it stays listed.
|
||||
#[tokio::test]
|
||||
async fn list_multipart_uploads_survives_sub_quorum_corruption() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-partial-corrupt-bucket";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
@@ -12140,29 +11545,32 @@ mod tests {
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
|
||||
// Exercise both a minority of corrupt replies and a tie between corruption
|
||||
// and the remaining readable metadata. Each list must resolve the real ID.
|
||||
let path = SetDisks::get_upload_id_dir(bucket, "partial/object.bin", &upload.upload_id);
|
||||
for disk in disk_stores.iter().take(2) {
|
||||
disk.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{path}/xl.meta"),
|
||||
Bytes::from_static(b"not an xl.meta"),
|
||||
)
|
||||
.await
|
||||
.expect("replace one existing head and invalidate its cached metadata");
|
||||
let error = disk
|
||||
.read_version(bucket, RUSTFS_META_MULTIPART_BUCKET, &path, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("the damaged head must no longer return cached metadata");
|
||||
assert!(is_corrupt_upload_metadata_error(&error), "unexpected damage classification: {error:?}");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "partial/", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("a readable upload survives corrupt replies");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&upload.upload_id));
|
||||
// Corrupt the upload's metadata on exactly one disk (discovery quorum
|
||||
// on this 4-disk set is 2): the other replicas keep it identifiable.
|
||||
let sha = SetDisks::get_multipart_sha_dir(bucket, "partial/object.bin");
|
||||
let mut corrupted = 0usize;
|
||||
for temp_dir in &temp_dirs {
|
||||
let sha_dir = temp_dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&sha);
|
||||
for meta in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
if meta.starts_with(sha_dir.to_string_lossy().as_ref()) {
|
||||
tokio::fs::write(&meta, b"not an xl.meta")
|
||||
.await
|
||||
.expect("corrupting xl.meta should succeed");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
if corrupted > 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(corrupted, 1, "exactly one disk's metadata should be corrupted");
|
||||
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "partial/", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("sub-quorum corruption must not fail the listing");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
/// Pins the degrade-vs-propagate classification (issue #5716 review): only
|
||||
@@ -12594,108 +12002,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
struct TestCompletionAuthorization {
|
||||
allowed: std::sync::atomic::AtomicBool,
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
expected_marker: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartCompletionAuthorization for TestCompletionAuthorization {
|
||||
type Error = Error;
|
||||
|
||||
async fn authorize(&self, metadata: &HashMap<String, String>) -> Result<()> {
|
||||
if let Some(expected) = self.expected_marker {
|
||||
assert_eq!(metadata.get("authorization-source").map(String::as_str), Some(expected));
|
||||
}
|
||||
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if self.allowed.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::PrefixAccessDenied("bucket".to_string(), "object".to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_authorizes_selected_source_and_historical_replay_before_part_comparison() {
|
||||
use std::sync::atomic::Ordering;
|
||||
for versioned in [false, true] {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "completion-authorization";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
versioned,
|
||||
user_defined: HashMap::from([("authorization-source".to_string(), "original".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload, parts) = stage_upload_with_create_opts(&set, bucket, object, &[0x61; 4096], &create_opts).await;
|
||||
let gate = Arc::new(TestCompletionAuthorization {
|
||||
allowed: std::sync::atomic::AtomicBool::new(false),
|
||||
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
expected_marker: Some("original"),
|
||||
});
|
||||
let opts = ObjectOptions {
|
||||
multipart_completion_authorization: Some(gate.clone()),
|
||||
..create_opts.clone()
|
||||
};
|
||||
let denied = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts.clone(), &opts)
|
||||
.await;
|
||||
assert!(matches!(denied, Err(Error::PrefixAccessDenied(..))));
|
||||
assert!(set.get_multipart_info(bucket, object, &upload, &create_opts).await.is_ok());
|
||||
gate.allowed.store(true, Ordering::SeqCst);
|
||||
let committed = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts.clone(), &opts)
|
||||
.await
|
||||
.expect("authorized commit");
|
||||
if versioned {
|
||||
let other_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([("authorization-source".to_string(), "other".to_string())]),
|
||||
..create_opts.clone()
|
||||
};
|
||||
let (other_upload, other_parts) =
|
||||
stage_upload_with_create_opts(&set, bucket, object, &[0x62; 4096], &other_opts).await;
|
||||
set.clone()
|
||||
.complete_multipart_upload(bucket, object, &other_upload, other_parts, &other_opts)
|
||||
.await
|
||||
.expect("new current version");
|
||||
}
|
||||
gate.allowed.store(false, Ordering::SeqCst);
|
||||
let mut guessed_parts = parts.clone();
|
||||
guessed_parts[0].etag = Some("guessed-part-etag".to_string());
|
||||
let denied = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, guessed_parts, &opts)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(denied, Err(Error::PrefixAccessDenied(..))),
|
||||
"authorization must precede InvalidPart disclosure"
|
||||
);
|
||||
gate.allowed.store(true, Ordering::SeqCst);
|
||||
let replay_opts = ObjectOptions {
|
||||
want_checksum: Some(rustfs_rio::Checksum {
|
||||
checksum_type: rustfs_rio::ChecksumType::SHA256,
|
||||
encoded: "a-guess-is-not-compared-on-replay".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..opts
|
||||
};
|
||||
let replay = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts, &replay_opts)
|
||||
.await
|
||||
.expect("authorized replay");
|
||||
assert!(replay.multipart_completion_replayed);
|
||||
assert_eq!(replay.version_id, committed.version_id);
|
||||
assert_eq!(replay.etag, committed.etag);
|
||||
assert!(gate.calls.load(Ordering::SeqCst) >= 4);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_replay_requires_write_quorum_after_uncertain_publication() {
|
||||
use crate::set_disk::core::io_primitives::{rename_fault_injection, rollback_fault_injection};
|
||||
@@ -14806,30 +14112,9 @@ mod tests {
|
||||
}
|
||||
assert_eq!(seals, 2, "two local seals do not establish a terminal quorum");
|
||||
assert_completion_files_preserved(&disks, RUSTFS_META_MULTIPART_BUCKET, &source_files).await;
|
||||
let gate = Arc::new(TestCompletionAuthorization {
|
||||
allowed: std::sync::atomic::AtomicBool::new(false),
|
||||
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
expected_marker: None,
|
||||
});
|
||||
let guarded_opts = ObjectOptions {
|
||||
multipart_completion_authorization: Some(gate.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let denied = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts.clone(), &guarded_opts)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(denied, Err(Error::PrefixAccessDenied(..))),
|
||||
"restored source must be authorized before publication"
|
||||
);
|
||||
gate.allowed.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let retried = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts, &guarded_opts)
|
||||
let retried = complete(&set, bucket, object, &upload, parts)
|
||||
.await
|
||||
.expect("authorized recovery retains all-N blocked restore and activation");
|
||||
assert!(gate.calls.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
.expect("H remaining present heads authorize all-N blocked restore and activation");
|
||||
assert!(
|
||||
!retried.multipart_completion_replayed,
|
||||
"a partial seal cannot acknowledge the old minority"
|
||||
|
||||
@@ -3914,7 +3914,6 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
explicit_delete: true,
|
||||
transaction_guard: transaction_guard.cloned(),
|
||||
namespace_owner: Some(Arc::new((
|
||||
@@ -4790,27 +4789,6 @@ impl SetDisks {
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Lock order: lifecycle, metadata transaction, then object namespace.
|
||||
// No body reads occur below this point. The snapshot must describe the
|
||||
// configuration at commit, not the pre-body advisory object lookup.
|
||||
let mut commit_opts;
|
||||
let opts = if opts.defer_put_object_lock_snapshot {
|
||||
commit_opts = opts.clone();
|
||||
let expected = source_bucket_incarnation_id
|
||||
.ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let snapshot = store.object_lock_config_snapshot_for_single_put(bucket).await?;
|
||||
if !snapshot.is_valid_for_destructive_put(store.id, bucket, expected) {
|
||||
return Err(Error::other("PUT bucket generation or configuration changed while staging"));
|
||||
}
|
||||
snapshot.add_lock_fences(&mut commit_opts);
|
||||
commit_opts.object_lock_config_snapshot = Some(snapshot);
|
||||
commit_opts.defer_put_object_lock_snapshot = false;
|
||||
&commit_opts
|
||||
} else {
|
||||
opts
|
||||
};
|
||||
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -5304,7 +5282,6 @@ impl SetDisks {
|
||||
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
|
||||
let commit_publication_guard = publication_commit_guard.take();
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let commit_object_lock_config_snapshot = opts.object_lock_config_snapshot.clone();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
|
||||
// A scanner publication scope owns the movement permit until the
|
||||
@@ -5367,7 +5344,6 @@ impl SetDisks {
|
||||
let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
|
||||
let mut _publication_guard = commit_publication_guard;
|
||||
let mut _bucket_lifecycle_guard = commit_bucket_lifecycle_guard;
|
||||
let mut _object_lock_config_snapshot = commit_object_lock_config_snapshot;
|
||||
let mut _decommission_capacity_guard = commit_decommission_capacity_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let rename_stage_start = Instant::now();
|
||||
@@ -5701,7 +5677,6 @@ impl SetDisks {
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let publication_guard = _publication_guard.take();
|
||||
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
|
||||
let object_lock_config_snapshot = _object_lock_config_snapshot.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
let decommission_capacity_guard = _decommission_capacity_guard.take();
|
||||
let cleanup_bucket = commit_bucket.clone();
|
||||
@@ -5724,7 +5699,6 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_config_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5746,7 +5720,6 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_config_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5760,7 +5733,6 @@ impl SetDisks {
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
drop(object_lock_config_snapshot);
|
||||
cleanup_runtime::run(cleanup_reservation, async move {
|
||||
let zero_target_cleanup = targets.is_empty();
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
@@ -9175,9 +9147,6 @@ impl SetDisks {
|
||||
&disks,
|
||||
write_quorum,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: commit_context
|
||||
.options
|
||||
.and_then(|options| options.delete_recovery_work_budget.clone()),
|
||||
recovery_ticket,
|
||||
transaction_guard: commit_context.transaction_guard.clone(),
|
||||
namespace_owner: namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
@@ -9741,26 +9710,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if let Some(etag) = &src_info.etag {
|
||||
replacement_metadata.insert("etag".to_owned(), etag.clone());
|
||||
}
|
||||
// ObjectInfo externalizes these fields; copy callers supply the effective
|
||||
// values, including empty tags/absent expiry for explicit replacement.
|
||||
if !src_info.user_tags.is_empty() {
|
||||
replacement_metadata
|
||||
.entry(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string())
|
||||
.or_insert_with(|| (*src_info.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = src_info.expires
|
||||
&& !replacement_metadata.contains_key("expires")
|
||||
{
|
||||
replacement_metadata.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(Error::other)?,
|
||||
);
|
||||
}
|
||||
fi.metadata = replacement_metadata.clone();
|
||||
|
||||
let mod_time = dst_opts.mod_time.unwrap_or_else(OffsetDateTime::now_utc);
|
||||
let mod_time = OffsetDateTime::now_utc();
|
||||
fi.mod_time = Some(mod_time);
|
||||
fi.version_id = version_id;
|
||||
fi.versioned = src_opts.versioned || src_opts.version_suspended;
|
||||
@@ -9883,7 +9835,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
objects: Vec<ObjectToDelete>,
|
||||
mut opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
opts.ensure_delete_recovery_work_budget();
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let delete_config_snapshot = opts
|
||||
@@ -10155,7 +10106,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
bucket,
|
||||
&dobj.object_name,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
dry_run: !opts.no_lock,
|
||||
transaction_guard: transaction_guards.get(lock_names[i].as_ref()).cloned(),
|
||||
namespace_owner: opts.delete_mutation_owner.clone(),
|
||||
@@ -10764,7 +10714,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
disks,
|
||||
self.set_drive_count / 2 + 1,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
recovery_ticket,
|
||||
namespace_owner,
|
||||
transaction_guard,
|
||||
@@ -10948,7 +10897,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
bucket,
|
||||
&object.object_name,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
explicit_delete,
|
||||
namespace_owner: Some(Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
@@ -11007,7 +10955,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self, opts))]
|
||||
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
opts.ensure_delete_recovery_work_budget();
|
||||
let _scope_outcome_guard = opts
|
||||
.scanner_publication_commit_scope
|
||||
.clone()
|
||||
@@ -11304,7 +11251,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
transaction_guard: transaction_guard.clone(),
|
||||
namespace_owner: Some(Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
@@ -11916,10 +11862,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// as a second object header set: forwarding SSE intent or wrapped DEKs
|
||||
// would request a second encryption pass and disclose local envelope
|
||||
// material to the remote provider.
|
||||
transition_meta.retain(|key, _| {
|
||||
!rustfs_utils::http::is_replication_stripped_encryption_key(key)
|
||||
&& !rustfs_utils::http::is_snowball_transport_key(key)
|
||||
});
|
||||
transition_meta.retain(|key, _| !rustfs_utils::http::is_replication_stripped_encryption_key(key));
|
||||
transition_meta.insert("name".to_string(), object.to_string());
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
@@ -16395,170 +16338,6 @@ mod metadata_mutation_generation_tests {
|
||||
assert_retired(&set_disks, &metadata_key).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_metadata_copy_preserves_tags_expiry_and_version_order() {
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use s3s::dto::RestoreRequest;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "restore-metadata-preservation";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
|
||||
let latest_time = old_time + time::Duration::days(1);
|
||||
let expires = latest_time + time::Duration::days(365);
|
||||
for deleted in [false, true] {
|
||||
let object = if deleted { "deleted" } else { "live" };
|
||||
let tags = "classification=secret&owner=alice";
|
||||
let old = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(b"old protected content".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
user_defined: HashMap::from([
|
||||
(AMZ_OBJECT_TAGGING.to_string(), tags.to_string()),
|
||||
("x-amz-tagging".to_string(), tags.to_string()),
|
||||
("expires".to_string(), expires.format(&Rfc3339).expect("expiry")),
|
||||
]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let latest = if deleted {
|
||||
let marker = FileInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
mod_time: Some(latest_time),
|
||||
deleted: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
for disk in &disks {
|
||||
disk.write_metadata(bucket, bucket, object, marker.clone())
|
||||
.await
|
||||
.expect("publish current delete marker");
|
||||
}
|
||||
marker.version_id
|
||||
} else {
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(b"latest content".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(latest_time),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write latest version")
|
||||
.version_id
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
version_id: old.version_id.map(|id| id.to_string()),
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
};
|
||||
let mut source = set.get_object_info(bucket, object, &opts).await.expect("read old metadata");
|
||||
assert_eq!(source.user_tags.as_str(), tags);
|
||||
assert_eq!(source.expires, Some(expires));
|
||||
assert!(!source.user_defined.contains_key(AMZ_OBJECT_TAGGING));
|
||||
assert!(source.user_defined.contains_key("x-amz-tagging"));
|
||||
source.metadata_only = true;
|
||||
Arc::make_mut(&mut source.user_defined)
|
||||
.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string());
|
||||
set.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("restore metadata update");
|
||||
let restored = set
|
||||
.get_object_info(bucket, object, &opts)
|
||||
.await
|
||||
.expect("read restored metadata");
|
||||
assert_eq!(restored.user_tags.as_str(), tags);
|
||||
assert_eq!(restored.expires, Some(expires));
|
||||
assert_eq!(restored.mod_time, Some(old_time));
|
||||
for disk in &disks {
|
||||
let bytes = disk
|
||||
.read_all(bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("persisted metadata");
|
||||
let meta = FileMeta::load(&bytes).expect("decode versions");
|
||||
let versions = meta.list_versions(bucket, object, false).expect("list versions");
|
||||
assert_eq!(versions.len(), 2);
|
||||
assert_eq!(versions[0].version_id, latest);
|
||||
assert_eq!(versions[0].mod_time, Some(latest_time));
|
||||
assert_eq!(versions[0].deleted, deleted);
|
||||
assert_eq!(versions[1].version_id, old.version_id);
|
||||
assert_eq!(versions[1].mod_time, Some(old_time));
|
||||
}
|
||||
let restore_opts = put_restore_opts(bucket, object, &RestoreRequest::default(), &restored)
|
||||
.await
|
||||
.expect("background restore options");
|
||||
assert_eq!(restore_opts.user_defined.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some(tags));
|
||||
assert_eq!(restore_opts.user_defined.get("expires"), Some(&expires.format(&Rfc3339).expect("expiry")));
|
||||
assert_eq!(restore_opts.mod_time, Some(old_time));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_copy_honors_explicit_tag_and_expiry_replacement() {
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "copy-metadata-replacement";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
|
||||
let expires = old_time + time::Duration::days(365);
|
||||
set.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut PutObjReader::from_vec(b"content".to_vec()),
|
||||
&ObjectOptions {
|
||||
mod_time: Some(old_time),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original");
|
||||
let opts = ObjectOptions::default();
|
||||
let mut source = set.get_object_info(bucket, "object", &opts).await.expect("original metadata");
|
||||
source.metadata_only = true;
|
||||
source.user_tags = Arc::new("old=value".to_string());
|
||||
source.expires = Some(expires - time::Duration::days(1));
|
||||
let metadata = Arc::make_mut(&mut source.user_defined);
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
|
||||
metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit metadata wins over externalized fields");
|
||||
let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
|
||||
assert_eq!(copied.user_tags.as_str(), "new=value");
|
||||
assert_eq!(copied.expires, Some(expires));
|
||||
assert!(copied.mod_time.is_some_and(|time| time > old_time));
|
||||
|
||||
copied.metadata_only = true;
|
||||
copied.user_tags = Arc::new(String::new());
|
||||
copied.expires = None;
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit empty replacement");
|
||||
let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
|
||||
assert!(cleared.user_tags.is_empty());
|
||||
assert!(cleared.expires.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -17011,19 +16790,6 @@ mod transition_commit_failure_tests {
|
||||
.collect();
|
||||
user_defined.insert("x-amz-meta-owner".to_string(), "finance".to_string());
|
||||
|
||||
let snowball_keys = [
|
||||
"snowball-auto-extract",
|
||||
"x-amz-meta-snowball-auto-extract",
|
||||
"x-amz-snowball-auto-extract",
|
||||
"snowball-prefix",
|
||||
"x-amz-meta-rustfs-snowball-prefix",
|
||||
"x-amz-meta-minio-snowball-ignore-dirs",
|
||||
"other-vendor-snowball-ignore-errors",
|
||||
];
|
||||
for key in snowball_keys {
|
||||
user_defined.insert(key.to_string(), "true".to_string());
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"stored bytes the tier keeps opaque ".repeat(64));
|
||||
set_disks
|
||||
.put_object(
|
||||
@@ -17073,10 +16839,6 @@ mod transition_commit_failure_tests {
|
||||
.await
|
||||
.expect("the tier must have received the object");
|
||||
|
||||
for key in snowball_keys {
|
||||
assert!(!remote_metadata.contains_key(key), "tier must receive opaque bytes, not {key}");
|
||||
assert_eq!(transitioned.user_defined.get(key).map(String::as_str), Some("true"));
|
||||
}
|
||||
for (key, _) in encryption_metadata {
|
||||
assert!(
|
||||
!remote_metadata.keys().any(|stored| stored.eq_ignore_ascii_case(key)),
|
||||
|
||||
@@ -52,8 +52,8 @@ pub(crate) mod list {
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
|
||||
MultipartUploadInspection, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::{
|
||||
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState},
|
||||
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState, table_bucket_catalog_metadata_prefix},
|
||||
utils::is_meta_bucketname,
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
@@ -145,41 +145,11 @@ async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> Vec<String> {
|
||||
use crate::bucket::metadata::*;
|
||||
|
||||
let mut prefixes = vec![table_bucket_catalog_metadata_prefix(bucket)];
|
||||
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
|
||||
if bucket != "ahm-replacement" {
|
||||
prefixes.push(root);
|
||||
return prefixes;
|
||||
}
|
||||
|
||||
// This valid user bucket shares its metadata directory with raw drive
|
||||
// replacement intents, seals and recovery proofs. Delete only bucket-owned
|
||||
// paths, including legacy standalone configuration, never the shared root.
|
||||
// S3 objects live in the separate user volume; configuration APIs write the
|
||||
// aggregate metadata file, so clients cannot populate arbitrary siblings.
|
||||
prefixes.extend(
|
||||
[
|
||||
BUCKET_METADATA_FILE,
|
||||
BUCKET_INCARNATION_FILE,
|
||||
BUCKET_POLICY_CONFIG,
|
||||
BUCKET_QUOTA_CONFIG_FILE,
|
||||
BUCKET_TARGETS_FILE,
|
||||
BUCKET_ACL_CONFIG,
|
||||
BUCKET_TABLE_CONFIG,
|
||||
BUCKET_DURABILITY_CONFIG,
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG,
|
||||
".replication",
|
||||
"integrity-jobs",
|
||||
crate::data_usage::DATA_USAGE_CACHE_NAME,
|
||||
]
|
||||
.into_iter()
|
||||
.chain(XML_BUCKET_CONFIG_FILES)
|
||||
.map(|file| format!("{root}/{file}")),
|
||||
);
|
||||
prefixes
|
||||
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
|
||||
[
|
||||
table_bucket_catalog_metadata_prefix(bucket),
|
||||
format!("{BUCKET_META_PREFIX}/{bucket}"),
|
||||
]
|
||||
}
|
||||
|
||||
fn bucket_deleted_marker_prefix(bucket: &str) -> String {
|
||||
@@ -190,48 +160,41 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
|
||||
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
|
||||
}
|
||||
|
||||
// Box the child before constructing either lease wrapper. Keeping F inline in
|
||||
// nested select futures multiplies poll-frame temporaries along DELETE -> quota
|
||||
// cleanup -> metadata GET. This boundary must not inline back into that chain.
|
||||
#[inline(never)]
|
||||
pub(crate) fn await_bucket_namespace_operation<'a, T: 'a, F>(
|
||||
guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &'a str,
|
||||
pub(crate) async fn await_bucket_namespace_operation<T, F>(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
operation: &'static str,
|
||||
future: F,
|
||||
) -> impl Future<Output = Result<T>> + 'a
|
||||
) -> Result<T>
|
||||
where
|
||||
F: Future<Output = Result<T>> + 'a,
|
||||
F: Future<Output = Result<T>>,
|
||||
{
|
||||
let future = Box::pin(future);
|
||||
async move {
|
||||
let Some(guard) = guard else {
|
||||
return future.await;
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost before {operation}: {bucket}"
|
||||
)));
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost during {operation}: {bucket}"
|
||||
))),
|
||||
result = future => result,
|
||||
}
|
||||
let Some(guard) = guard else {
|
||||
return future.await;
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost before {operation}: {bucket}"
|
||||
)));
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost during {operation}: {bucket}"
|
||||
))),
|
||||
result = future => result,
|
||||
}
|
||||
}
|
||||
|
||||
fn await_bucket_lifecycle_operation<'a, T: 'a, F>(
|
||||
lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
namespace_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &'a str,
|
||||
async fn await_bucket_lifecycle_operation<T, F>(
|
||||
lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
operation: &'static str,
|
||||
future: F,
|
||||
) -> impl Future<Output = Result<T>> + 'a
|
||||
) -> Result<T>
|
||||
where
|
||||
F: Future<Output = Result<T>> + 'a,
|
||||
F: Future<Output = Result<T>>,
|
||||
{
|
||||
await_bucket_namespace_operation(
|
||||
lifecycle_guard,
|
||||
@@ -239,6 +202,7 @@ where
|
||||
operation,
|
||||
await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
|
||||
@@ -514,18 +478,10 @@ impl ECStore {
|
||||
let mut tags = HashMap::with_capacity(tagging.tag_set.len());
|
||||
for tag in &tagging.tag_set {
|
||||
let (Some(key), Some(value)) = (&tag.key, &tag.value) else {
|
||||
return Err(crate::bucket::metadata::unreadable_config_error(
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
metadata.tagging_config_xml.len(),
|
||||
));
|
||||
return Err(Error::other("stored bucket tag is missing a key or value"));
|
||||
};
|
||||
if key.is_empty() || tags.insert(key.clone(), value.clone()).is_some() {
|
||||
return Err(crate::bucket::metadata::unreadable_config_error(
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
metadata.tagging_config_xml.len(),
|
||||
));
|
||||
return Err(Error::other("stored bucket tags contain empty or duplicate keys"));
|
||||
}
|
||||
}
|
||||
Ok(tags)
|
||||
@@ -1717,17 +1673,7 @@ impl ECStore {
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket retirement publication",
|
||||
crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
|
||||
)
|
||||
.await?;
|
||||
// The lifecycle fence excludes live reservation owners. A crash-left
|
||||
// V1 ledger must not block quota admission after same-name recreation.
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"retired bucket quota ledger cleanup",
|
||||
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
|
||||
crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1878,20 +1824,6 @@ mod tests {
|
||||
assert!(!xlmeta.diagnostic_truncated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_namespace_operation_keeps_nested_future_state_off_the_poll_stack() {
|
||||
let payload = std::hint::black_box([7u8; 128 * 1024]);
|
||||
let operation = super::await_bucket_lifecycle_operation(None, None, "bucket", "large metadata operation", async move {
|
||||
tokio::task::yield_now().await;
|
||||
Ok(std::hint::black_box(payload)[0])
|
||||
});
|
||||
assert!(
|
||||
std::mem::size_of_val(&operation) < 1024,
|
||||
"nested lease wrappers must not retain the concrete storage future inline"
|
||||
);
|
||||
assert_eq!(operation.await.expect("the boxed operation must still execute"), 7);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
|
||||
let ttl = Duration::from_millis(20);
|
||||
@@ -2308,13 +2240,7 @@ mod tests {
|
||||
let result = store.get_bucket_tags_for_policy(bucket).await;
|
||||
match expected {
|
||||
Some(tags) => assert_eq!(&result.expect("valid tag configuration"), tags),
|
||||
None => assert!(
|
||||
crate::bucket::metadata::unreadable_config_refusal(
|
||||
&result.expect_err("corrupt tags must not become an untagged bucket")
|
||||
)
|
||||
.is_some_and(|failure| failure.config_file == super::BUCKET_TAGGING_CONFIG),
|
||||
"corrupt tags must carry a typed tagging refusal: {xml:?}"
|
||||
),
|
||||
None => assert!(result.is_err(), "corrupt tags must not become an untagged bucket: {xml:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2516,187 +2442,6 @@ mod tests {
|
||||
assert!(prefixes.contains(&"buckets/analytics".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replacement_named_bucket_delete_preserves_recovery_and_recreates_clean_metadata() {
|
||||
use crate::bucket::metadata::{BUCKET_QUOTA_CONFIG_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
|
||||
let (_temp, store) = setup_bucket_quorum_test_env(&[4], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "ahm-replacement";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create legal bucket");
|
||||
let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
|
||||
for (config, bytes) in [
|
||||
(BUCKET_QUOTA_CONFIG_FILE, br#"{"quota":1024,"quota_type":"Hard"}"#.as_slice()),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
b"<Tagging><TagSet><Tag><Key>old</Key><Value>value</Value></Tag></TagSet></Tagging>".as_slice(),
|
||||
),
|
||||
(
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".as_slice(),
|
||||
),
|
||||
] {
|
||||
metadata_sys::update_in(&store.ctx, bucket, config, bytes.to_vec())
|
||||
.await
|
||||
.expect("persist old configuration");
|
||||
}
|
||||
let ledger_path = format!("config/quota-ledger/{bucket}.json");
|
||||
// V1 precharges outstanding growth: four live bytes plus the
|
||||
// crash-left reservation's 100 bytes must already be accounted.
|
||||
let ledger = serde_json::to_vec(&serde_json::json!({
|
||||
"version": 1, "bucket_incarnation": old_incarnation,
|
||||
"quota_revision_unix_nanos": 0, "accounted_usage": 104,
|
||||
"reservations": {Uuid::new_v4().to_string(): {
|
||||
"object": "crashed-put", "old_size": 0, "new_size": 100,
|
||||
"created_at": 0, "pool_index": 0, "set_index": 0, "commit_started": false
|
||||
}}, "reconcile_required": true
|
||||
}))
|
||||
.expect("encode crash-left ledger");
|
||||
|
||||
// The storage API requires the same legacy-quota admission supplied
|
||||
// by the S3 handler, even for this small fixture object.
|
||||
let mut live_options = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(live_options.set_quota_admission(0, 1024));
|
||||
// Failed deletion must leave both the live incarnation and its ledger alone.
|
||||
store
|
||||
.put_object(bucket, "live", &mut PutObjReader::from_vec(b"live".to_vec()), &live_options)
|
||||
.await
|
||||
.expect("write live object");
|
||||
crate::config::com::save_config(store.clone(), &ledger_path, ledger.clone())
|
||||
.await
|
||||
.expect("persist old ledger");
|
||||
assert!(matches!(
|
||||
store.delete_bucket(bucket, &DeleteBucketOptions::default()).await,
|
||||
Err(StorageError::BucketNotEmpty(_))
|
||||
));
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(store.clone(), &ledger_path)
|
||||
.await
|
||||
.expect("live ledger remains"),
|
||||
ledger
|
||||
);
|
||||
// Remove every version; the configured bucket is otherwise still empty.
|
||||
let versions = store
|
||||
.clone()
|
||||
.list_object_versions(bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list live versions");
|
||||
for object in versions.objects {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
&object.name,
|
||||
ObjectOptions {
|
||||
version_id: object.version_id.map(|id| id.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove live version");
|
||||
}
|
||||
|
||||
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
|
||||
let task = Uuid::new_v4();
|
||||
let recovery_names = [
|
||||
"ahm_replacement_intent.json",
|
||||
"ahm_replacement_intent_seal",
|
||||
"ahm_replacement_completion_proof.json",
|
||||
]
|
||||
.map(|suffix| format!("{task}_{suffix}"));
|
||||
let disk_paths = store.pools[0].disk_set[0]
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|disk| PathBuf::from(disk.endpoint().to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disk_paths {
|
||||
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
|
||||
for name in &recovery_names {
|
||||
tokio::fs::write(directory.join(name), b"durable heal record")
|
||||
.await
|
||||
.expect("seed raw recovery record");
|
||||
}
|
||||
// Legacy standalone settings and bucket-owned subtrees must not survive.
|
||||
for name in [
|
||||
"quota.json",
|
||||
"versioning.xml",
|
||||
"policy.json",
|
||||
"bucket-targets.json",
|
||||
"integrity-jobs/job.json",
|
||||
".replication/resync.bin",
|
||||
".usage-cache.bin",
|
||||
] {
|
||||
let path = directory.join(name);
|
||||
tokio::fs::create_dir_all(path.parent().expect("fixture parent"))
|
||||
.await
|
||||
.expect("legacy parent");
|
||||
tokio::fs::write(path, b"old bucket configuration")
|
||||
.await
|
||||
.expect("seed legacy configuration");
|
||||
}
|
||||
}
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete legal same-name bucket");
|
||||
assert!(matches!(
|
||||
crate::config::com::read_config(store.clone(), &ledger_path).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
for disk in &disk_paths {
|
||||
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
|
||||
for name in &recovery_names {
|
||||
assert_eq!(
|
||||
tokio::fs::read(directory.join(name)).await.expect("recovery record survives"),
|
||||
b"durable heal record"
|
||||
);
|
||||
}
|
||||
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket)
|
||||
.into_iter()
|
||||
.filter(|p| p.starts_with(&root))
|
||||
{
|
||||
assert!(
|
||||
!disk.join(RUSTFS_META_BUCKET).join(prefix).exists(),
|
||||
"bucket-owned metadata must be removed"
|
||||
);
|
||||
}
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate legal bucket");
|
||||
let fresh = metadata_sys::get_in(&store.ctx, bucket).await.expect("fresh metadata");
|
||||
assert_ne!(fresh.bucket_incarnation_id, old_incarnation);
|
||||
assert!(fresh.quota_config_json.is_empty());
|
||||
assert!(fresh.tagging_config_xml.is_empty());
|
||||
assert!(fresh.versioning_config_xml.is_empty());
|
||||
assert!(fresh.policy_config_json.is_empty());
|
||||
// The strict purge/recovery cleanup must share the same ownership boundary.
|
||||
store
|
||||
.cleanup_deleted_bucket_metadata_strict(bucket, true)
|
||||
.await
|
||||
.expect("strict metadata cleanup");
|
||||
for disk in &disk_paths {
|
||||
for name in &recovery_names {
|
||||
assert_eq!(
|
||||
tokio::fs::read(disk.join(RUSTFS_META_BUCKET).join(&root).join(name))
|
||||
.await
|
||||
.expect("strict cleanup preserves recovery"),
|
||||
b"durable heal record"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
|
||||
assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
|
||||
|
||||
@@ -368,7 +368,7 @@ impl ECStore {
|
||||
Some(expected) => expected,
|
||||
None => self.bucket_incarnation_id_from_disk(bucket).await?,
|
||||
};
|
||||
let object = object.map(encode_dir_object);
|
||||
let object = object.map(str::to_owned);
|
||||
let prefix = prefix.map(str::to_owned);
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
|
||||
let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
|
||||
@@ -430,7 +430,7 @@ impl ECStore {
|
||||
HealItemType::Bucket
|
||||
}
|
||||
.to_string(),
|
||||
object: object.map(|object| decode_dir_object(&object)).or(prefix).unwrap_or_default(),
|
||||
object: object.or(prefix).unwrap_or_default(),
|
||||
bucket,
|
||||
detail: format!(
|
||||
"delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}, failed={}",
|
||||
@@ -656,48 +656,12 @@ impl ECStore {
|
||||
Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
|
||||
}
|
||||
|
||||
/// Read the published bucket identity without fabricating or migrating metadata.
|
||||
/// Absence is distinct from quorum, corruption and transaction-fence failures.
|
||||
pub async fn replacement_bucket_incarnation_from_disk(self: &Arc<Self>, bucket: &str) -> Result<Option<uuid::Uuid>> {
|
||||
let transaction =
|
||||
crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let incarnation = crate::bucket::metadata::load_bucket_incarnation(self.clone(), bucket).await?;
|
||||
if transaction.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(incarnation)
|
||||
}
|
||||
|
||||
/// Restore the bucket configuration and incarnation before retiring a replacement marker.
|
||||
pub async fn heal_replacement_bucket_metadata(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
// Reject an invalid request before touching the drives so callers keep
|
||||
// the PreconditionFailed contract for bad buckets and options.
|
||||
if opts.pool.is_none()
|
||||
|| opts.set.is_none()
|
||||
|| targets.is_empty()
|
||||
|| opts.dry_run
|
||||
|| opts.no_lock
|
||||
|| check_valid_bucket_name_strict(bucket).is_err()
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let expected = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
self.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Keep a replacement pass bound to its admitted bucket through metadata commit.
|
||||
pub async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
|
||||
use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
|
||||
@@ -734,8 +698,9 @@ impl ECStore {
|
||||
|
||||
// Read the persisted identity without lazy migration: a replacement
|
||||
// must never turn lost configuration into newly fabricated defaults.
|
||||
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let targets = targets.to_vec();
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, opts| async move {
|
||||
self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
|
||||
// Match config writers: lifecycle, metadata transaction, object locks.
|
||||
// Keep the transaction stable through target readback, including when
|
||||
// the caller is cancelled and the storage owner finishes its write.
|
||||
@@ -1057,7 +1022,6 @@ impl ECStore {
|
||||
set_index: usize,
|
||||
targets: &[String],
|
||||
) -> Result<bool> {
|
||||
let object = encode_dir_object(object);
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -1070,7 +1034,7 @@ impl ECStore {
|
||||
)
|
||||
})?;
|
||||
|
||||
set.replacement_targets_have_version(bucket, &object, version_id, targets)
|
||||
set.replacement_targets_have_version(bucket, object, version_id, targets)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
@@ -2895,105 +2859,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_directory_object_readback_and_exact_recovery_use_encoded_key() {
|
||||
let (_root, store, shutdown) = heal_store_with_pool_count(1).await;
|
||||
let bucket = "directory-replacement-readback";
|
||||
let object = "folder/";
|
||||
let encoded = encode_dir_object(object);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(Vec::new()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write directory object through the public store entry");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let targets = vec![set.set_endpoints[0].to_string()];
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("readback")
|
||||
);
|
||||
let disk = set.disks.read().await[0].clone().expect("target disk");
|
||||
disk.delete(
|
||||
bucket,
|
||||
&encoded,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove target directory shard");
|
||||
assert!(
|
||||
!store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("missing target")
|
||||
);
|
||||
let opts = HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
let (result, error) = store
|
||||
.heal_object(bucket, object, "", &opts)
|
||||
.await
|
||||
.expect("heal directory object");
|
||||
assert!(error.is_none(), "heal failed: {error:?}");
|
||||
assert_eq!(result.object, object);
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("healed readback")
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, &encoded, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("encoded readback")
|
||||
);
|
||||
|
||||
// Exact-object recovery must inspect the same physical namespace and
|
||||
// reject corrupt proof rather than silently report no transactions.
|
||||
let metadata = disk
|
||||
.read_all(bucket, &format!("{encoded}/xl.meta"))
|
||||
.await
|
||||
.expect("healed metadata");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{encoded}/{}/delete-prepared", Uuid::new_v4()),
|
||||
b"invalid proof".to_vec().into(),
|
||||
)
|
||||
.await
|
||||
.expect("seed corrupt recovery proof");
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some(object), None, &HealOpts { dry_run: true, ..opts })
|
||||
.await
|
||||
.expect_err("directory recovery must discover and reject the corrupt proof");
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, &format!("{encoded}/xl.meta"))
|
||||
.await
|
||||
.expect("unchanged metadata"),
|
||||
metadata
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
|
||||
+278
-298
@@ -5155,11 +5155,6 @@ mod tests {
|
||||
let offline_disk = force_set_disk_range_offline_for_test(set, 0..1).await;
|
||||
let _without_proof = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
|
||||
if !sharded {
|
||||
crate::bucket::quota::reservation::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("bootstrap quota with one disk offline");
|
||||
}
|
||||
|
||||
let payload = vec![0x51; 4096];
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
@@ -21012,299 +21007,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_retained_null_version(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
protection: &str,
|
||||
latest_is_marker: bool,
|
||||
) {
|
||||
let metadata = if protection == "legal-hold" {
|
||||
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
|
||||
} else {
|
||||
HashMap::from([
|
||||
("x-amz-object-lock-mode".to_string(), protection.to_string()),
|
||||
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
|
||||
])
|
||||
};
|
||||
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut original,
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed retained null version");
|
||||
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut latest,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed newer unprotected UUID version");
|
||||
if latest_is_marker {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a UUID delete marker above the retained null version");
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retained null version must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read retained null version");
|
||||
assert_eq!(body, b"protected null version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_delete_checks_actual_target() {
|
||||
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
|
||||
let temp = tempfile::tempdir().expect("create null-version delete store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
let configs = [
|
||||
VersioningConfiguration::default(),
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
for (case, config) in configs.into_iter().enumerate() {
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
|
||||
"batch delete must judge the retained null version: {object}: {errors:?}"
|
||||
);
|
||||
let error = store
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("suspended single delete must also judge the retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
if protection == "GOVERNANCE" {
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
|
||||
bypass_governance: true,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
|
||||
let removed = store
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match removed {
|
||||
Ok(marker) => {
|
||||
assert!(marker.delete_marker, "only a null delete marker may replace the payload");
|
||||
assert_eq!(marker.size, 0);
|
||||
assert!(marker.data_dir.is_none());
|
||||
}
|
||||
Err(error) => assert!(
|
||||
matches!(
|
||||
error,
|
||||
StorageError::MethodNotAllowed
|
||||
| StorageError::ObjectNotFound(_, _)
|
||||
| StorageError::VersionNotFound(_, _, _)
|
||||
),
|
||||
"unexpected bypass result: {error:?}"
|
||||
),
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.get_object_reader(
|
||||
&bucket,
|
||||
&object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"authorized deletion must make the retained null payload unreadable"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
|
||||
let temp = tempfile::tempdir().expect("create null-version overwrite store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
|
||||
let error = store
|
||||
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("PUT must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
|
||||
let upload = store
|
||||
.clone()
|
||||
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
|
||||
let part = store
|
||||
.clone()
|
||||
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart replacement");
|
||||
let error = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("multipart completion must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
|
||||
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
&object,
|
||||
&mut next_version,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a new UUID version must remain allowed");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
|
||||
@@ -27650,4 +27352,282 @@ mod tests {
|
||||
.expect("recovery must preserve the local source object");
|
||||
assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn seed_retained_null_version(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
protection: &str,
|
||||
latest_is_marker: bool,
|
||||
) {
|
||||
let metadata = if protection == "legal-hold" {
|
||||
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
|
||||
} else {
|
||||
HashMap::from([
|
||||
("x-amz-object-lock-mode".to_string(), protection.to_string()),
|
||||
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
|
||||
])
|
||||
};
|
||||
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut original,
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed retained null version");
|
||||
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut latest,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed newer unprotected UUID version");
|
||||
if latest_is_marker {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a UUID delete marker above the retained null version");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retained null version must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read retained null version");
|
||||
assert_eq!(body, b"protected null version");
|
||||
}
|
||||
|
||||
/// A versionless delete removes the null version, so the Object Lock check
|
||||
/// must judge that version even when a UUID version or delete marker is
|
||||
/// the key's latest entry (rustfs/backlog#3017).
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_delete_checks_actual_target() {
|
||||
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
|
||||
let temp = tempfile::tempdir().expect("create null-version delete store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
let configs = [
|
||||
VersioningConfiguration::default(),
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
for (case, config) in configs.into_iter().enumerate() {
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
|
||||
"batch delete must judge the retained null version: {object}: {errors:?}"
|
||||
);
|
||||
let error = store
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("suspended single delete must also judge the retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
if protection == "GOVERNANCE" {
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
|
||||
bypass_governance: true,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
|
||||
assert!(
|
||||
store
|
||||
.get_object_reader(
|
||||
&bucket,
|
||||
&object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"authorized deletion must make the retained null payload unreadable"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A versionless PUT or multipart completion replaces the null version in
|
||||
/// place, so the commit-time WORM gate must judge that version even when a
|
||||
/// UUID version or delete marker is the key's latest entry
|
||||
/// (rustfs/backlog#3019).
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
|
||||
let temp = tempfile::tempdir().expect("create null-version overwrite store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
|
||||
let error = store
|
||||
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("PUT must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
|
||||
let upload = store
|
||||
.clone()
|
||||
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
|
||||
let part = store
|
||||
.clone()
|
||||
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart replacement");
|
||||
let error = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("multipart completion must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
|
||||
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
&object,
|
||||
&mut next_version,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a new UUID version must remain allowed");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -423,6 +423,8 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
!is_meta_bucketname(&oi.bucket)
|
||||
}
|
||||
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
|
||||
@@ -273,28 +273,6 @@ async fn pause_data_movement_multipart_before_selected_completion(bucket: &str)
|
||||
}
|
||||
}
|
||||
|
||||
async fn pool_has_multipart_upload_for_key(
|
||||
pool: &crate::core::sets::Sets,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let results = stream::iter(pool.disk_set.iter().cloned())
|
||||
.map(|set| async move {
|
||||
set.has_multipart_upload_for_key(bucket, object, expected_incarnation_id)
|
||||
.await
|
||||
})
|
||||
.buffer_unordered(MULTIPART_LIST_SET_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let mut found = false;
|
||||
for result in results {
|
||||
// Preserve errors from every set instead of hiding them behind a hit.
|
||||
found |= result?;
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
async fn list_pool_multipart_uploads_for_incarnation(
|
||||
pool: &crate::core::sets::Sets,
|
||||
bucket: &str,
|
||||
@@ -512,8 +490,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ListPartsInfo> {
|
||||
check_list_parts_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
|
||||
@@ -644,8 +620,6 @@ impl ECStore {
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_object_args(bucket, object, opts.data_movement)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
if self.single_pool() {
|
||||
@@ -701,7 +675,21 @@ impl ECStore {
|
||||
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
if pool_has_multipart_upload_for_key(pool, bucket, object, opts.expected_bucket_incarnation_id).await? {
|
||||
let res = list_pool_multipart_uploads_for_incarnation(
|
||||
pool,
|
||||
bucket,
|
||||
&MultipartUploadListRequest {
|
||||
prefix: object.to_string(),
|
||||
key_marker: None,
|
||||
upload_id_marker: None,
|
||||
delimiter: None,
|
||||
max_uploads: MAX_UPLOADS_LIST,
|
||||
expected_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !res.uploads.is_empty() {
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
|
||||
@@ -784,8 +772,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
@@ -820,8 +806,6 @@ impl ECStore {
|
||||
.part_number
|
||||
.ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||
}
|
||||
@@ -854,8 +838,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<MultipartInfo> {
|
||||
check_list_parts_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
if self.single_pool() {
|
||||
@@ -889,8 +871,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
@@ -1072,8 +1052,6 @@ impl ECStore {
|
||||
for upload_id in upload_ids {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
}
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart abort requires data_movement options"));
|
||||
}
|
||||
@@ -1368,8 +1346,6 @@ impl ECStore {
|
||||
let request_opts = opts;
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
}
|
||||
@@ -1491,362 +1467,6 @@ mod tests {
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_multipart_pool_selection_uses_only_an_exact_key_upload() {
|
||||
use crate::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let mut endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
dirs.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("exact-multipart-pool-{pool_index}"),
|
||||
platform: "test".to_owned(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("initialize local disks");
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), endpoints, cancel.clone(), ctx)
|
||||
.await
|
||||
.expect("initialize two pools");
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "exact-pool-selection";
|
||||
let other_bucket = "other-pool-selection";
|
||||
for bucket in [bucket, other_bucket] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
for (upload_bucket, key) in [(bucket, "report-long"), (bucket, "report/"), (other_bucket, "report")] {
|
||||
let (opts, _guard) = store
|
||||
.guard_multipart_bucket_incarnation(upload_bucket, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pin upload incarnation");
|
||||
store.pools[0]
|
||||
.new_multipart_upload(upload_bucket, &encode_dir_object(key), &opts)
|
||||
.await
|
||||
.expect("seed nonmatching upload in first pool");
|
||||
}
|
||||
let (opts, _guard) = store
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pin target incarnation");
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", opts.expected_bucket_incarnation_id)
|
||||
.await
|
||||
.expect("first pool contains only other keys")
|
||||
);
|
||||
store.pools[1]
|
||||
.new_multipart_upload(bucket, "report", &opts)
|
||||
.await
|
||||
.expect("seed exact upload in second pool");
|
||||
let (created, pool_index, incarnation) = store
|
||||
.handle_new_multipart_upload_with_pool_idx(bucket, "report", &ObjectOptions::default(), None)
|
||||
.await
|
||||
.expect("create another upload for the exact key");
|
||||
assert_eq!(pool_index, 1, "prefix matches in the first pool must not change placement");
|
||||
assert_eq!(incarnation, opts.expected_bucket_incarnation_id);
|
||||
assert!(!created.upload_id.is_empty());
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", incarnation)
|
||||
.await
|
||||
.expect("new upload must not appear in first pool")
|
||||
);
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[1], bucket, "report", Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("old bucket incarnations must not affect placement")
|
||||
);
|
||||
cancel.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_directory_keys_preserve_the_bare_object_and_round_trip() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use sha2::Digest;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
for versioned in [false, true] {
|
||||
let bucket = if versioned {
|
||||
"directory-mpu-versioned"
|
||||
} else {
|
||||
"directory-mpu-unversioned"
|
||||
};
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
if versioned {
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
bucket,
|
||||
"versioning.xml",
|
||||
br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
versioned,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
let original = store
|
||||
.put_object(
|
||||
bucket,
|
||||
"protected",
|
||||
&mut PutObjReader::from_vec(b"private original".to_vec()),
|
||||
&ObjectOptions {
|
||||
user_defined: HashMap::from([(
|
||||
rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(),
|
||||
"classification=secret".to_string(),
|
||||
)]),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write protected bare object");
|
||||
let upload = store
|
||||
.new_multipart_upload(bucket, "protected/", &opts)
|
||||
.await
|
||||
.expect("create directory upload");
|
||||
let part = store
|
||||
.put_object_part(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut PutObjReader::from_vec(b"directory payload".to_vec()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("write directory part");
|
||||
store
|
||||
.put_object_part(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
2,
|
||||
&mut PutObjReader::from_vec(b"unselected private bytes".to_vec()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("write an unselected part");
|
||||
let upload_sha = hex_simd::encode_to_string(
|
||||
sha2::Sha256::digest(format!("{bucket}/protected__XLDIR__").as_bytes()),
|
||||
hex_simd::AsciiCase::Lower,
|
||||
);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
for dir in &dirs {
|
||||
let staging = dir
|
||||
.path()
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid);
|
||||
let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(staging.join("xl.meta")).expect("upload metadata"))
|
||||
.expect("decode upload metadata");
|
||||
let versions = metadata
|
||||
.get_file_info_versions(bucket, "protected__XLDIR__", true)
|
||||
.expect("upload file info");
|
||||
let data_dir = versions
|
||||
.versions
|
||||
.first()
|
||||
.expect("upload version")
|
||||
.data_dir
|
||||
.expect("staging data directory");
|
||||
assert!(
|
||||
staging.join(data_dir.to_string()).join("part.2").is_file(),
|
||||
"unselected part exists before completion"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
store
|
||||
.get_multipart_info(bucket, "protected/", &upload.upload_id, &opts)
|
||||
.await
|
||||
.expect("upload info")
|
||||
.object,
|
||||
"protected/"
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.list_object_parts(bucket, "protected/", &upload.upload_id, None, 10, &opts)
|
||||
.await
|
||||
.expect("list parts")
|
||||
.object,
|
||||
"protected/"
|
||||
);
|
||||
let uploads = store
|
||||
.list_multipart_uploads(bucket, "protected/", None, None, None, 10)
|
||||
.await
|
||||
.expect("list logical prefix");
|
||||
assert_eq!(uploads.uploads.len(), 1);
|
||||
assert_eq!(uploads.uploads[0].object, "protected/");
|
||||
assert_eq!(crate::runtime::sources::upload_uuid_suffix(&uploads.uploads[0].upload_id), upload_uuid,);
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete directory upload");
|
||||
assert_eq!(completed.name, "protected/");
|
||||
for (key, expected) in [
|
||||
("protected", b"private original".as_slice()),
|
||||
("protected/", b"directory payload".as_slice()),
|
||||
] {
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read distinct object");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read body");
|
||||
assert_eq!(bytes, expected);
|
||||
}
|
||||
let preserved = store
|
||||
.get_object_info(bucket, "protected", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("bare metadata");
|
||||
assert_eq!(preserved.version_id, original.version_id);
|
||||
assert_eq!(preserved.etag, original.etag);
|
||||
assert_eq!(preserved.user_tags.as_str(), "classification=secret");
|
||||
for dir in &dirs {
|
||||
assert!(dir.path().join(bucket).join("protected/xl.meta").is_file());
|
||||
assert!(dir.path().join(bucket).join("protected__XLDIR__/xl.meta").is_file());
|
||||
let data = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("protected__XLDIR__")
|
||||
.join(completed.data_dir.expect("multipart data directory").to_string());
|
||||
assert!(data.join("part.1").is_file());
|
||||
assert!(!data.join("part.2").exists(), "unselected part must never be published");
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
.exists(),
|
||||
"original staging must be cleaned"
|
||||
);
|
||||
}
|
||||
let abandoned = store
|
||||
.new_multipart_upload(bucket, "protected/", &opts)
|
||||
.await
|
||||
.expect("create abort control");
|
||||
store
|
||||
.abort_multipart_upload(bucket, "protected/", &abandoned.upload_id, &opts)
|
||||
.await
|
||||
.expect("abort encoded upload");
|
||||
assert!(
|
||||
store
|
||||
.get_multipart_info(bucket, "protected/", &abandoned.upload_id, &opts)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, "protected/", &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn new_control_character_keys_are_rejected_without_trapping_legacy_objects() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "xml-key-admission";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) {
|
||||
let key = format!("bad-{}", char::from(byte));
|
||||
assert!(
|
||||
store
|
||||
.put_object(bucket, &key, &mut PutObjReader::from_vec(b"body".to_vec()), &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.new_multipart_upload(bucket, &key, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for dir in &dirs {
|
||||
assert!(!dir.path().join(bucket).join(&key).exists());
|
||||
}
|
||||
}
|
||||
let legacy_key = "legacy-\u{1}";
|
||||
store.pools[0]
|
||||
.get_disks_by_key(legacy_key)
|
||||
.put_object(
|
||||
bucket,
|
||||
legacy_key,
|
||||
&mut PutObjReader::from_vec(b"legacy payload".to_vec()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed a pre-upgrade object below new-key admission");
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, legacy_key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy GET stays usable");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("legacy bytes");
|
||||
assert_eq!(bytes, b"legacy payload");
|
||||
drop(reader);
|
||||
store
|
||||
.delete_object(bucket, legacy_key, ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy DELETE stays usable");
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, legacy_key, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
fn mp(object: &str, upload_id: &str) -> MultipartInfo {
|
||||
MultipartInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
|
||||
+148
-417
@@ -522,29 +522,16 @@ async fn pause_prefix_delete_after_journal_prepare(bucket: &str, prefix: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
// Construct only the selected operation off the caller's poll frame. A plain prefix
|
||||
// delete must not carry the journal state machine into the nested per-disk delete.
|
||||
#[inline(never)]
|
||||
fn delete_prefix_with_tier_delete_journal<'a>(
|
||||
store: &'a ECStore,
|
||||
bucket: &'a str,
|
||||
object: &'a str,
|
||||
opts: &'a ObjectOptions,
|
||||
tier_journal_api: Option<&'a Arc<ECStore>>,
|
||||
) -> futures::future::BoxFuture<'a, Result<()>> {
|
||||
match tier_journal_api {
|
||||
Some(api) => Box::pin(delete_prefix_with_active_tier_journal(store, bucket, object, opts, api)),
|
||||
None => Box::pin(store.delete_prefix(bucket, object, opts)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_prefix_with_active_tier_journal(
|
||||
async fn delete_prefix_with_tier_delete_journal(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
api: &Arc<ECStore>,
|
||||
tier_journal_api: Option<&Arc<ECStore>>,
|
||||
) -> Result<()> {
|
||||
let Some(api) = tier_journal_api else {
|
||||
return store.delete_prefix(bucket, object, opts).await;
|
||||
};
|
||||
let PreparedPrefixTierDelete {
|
||||
dispatch,
|
||||
chunk_parent_active,
|
||||
@@ -1398,7 +1385,7 @@ impl Drop for ObjectLockDiagGuard {
|
||||
}
|
||||
|
||||
/// A failure to preserve one object generation for a SelectObjectContent read.
|
||||
#[derive(Debug, Clone, Copy, thiserror::Error)]
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[non_exhaustive]
|
||||
pub enum SnapshotConsistencyError {
|
||||
#[error("namespace locking is disabled for SelectObjectContent")]
|
||||
@@ -1512,11 +1499,9 @@ impl SelectObjectSnapshotLease {
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque object generation used by SelectObjectContent. Each range reacquires
|
||||
/// short namespace read fences and rejects changed read semantics before opening
|
||||
/// its generation-bound reader. An idle snapshot never blocks object mutations.
|
||||
/// Opaque, lock-backed object generation used by SelectObjectContent.
|
||||
pub struct SelectObjectSnapshot {
|
||||
store: Arc<ECStore>,
|
||||
pool: Arc<Sets>,
|
||||
bucket: String,
|
||||
object: String,
|
||||
headers: HeaderMap,
|
||||
@@ -1524,7 +1509,7 @@ pub struct SelectObjectSnapshot {
|
||||
object_info: ObjectInfo,
|
||||
logical_size: u64,
|
||||
read_semantics_identity: [u8; 32],
|
||||
failure: ParkingMutex<Option<SnapshotConsistencyError>>,
|
||||
first_metadata: ParkingMutex<Option<crate::set_disk::PreparedGetObjectMetadata>>,
|
||||
lease: Arc<SelectObjectSnapshotLease>,
|
||||
}
|
||||
|
||||
@@ -1557,9 +1542,6 @@ impl SelectObjectSnapshot {
|
||||
}
|
||||
|
||||
pub fn ensure_valid(&self) -> std::result::Result<(), SnapshotConsistencyError> {
|
||||
if let Some(failure) = *self.failure.lock() {
|
||||
return Err(failure);
|
||||
}
|
||||
self.lease.check()
|
||||
}
|
||||
|
||||
@@ -1573,72 +1555,38 @@ impl SelectObjectSnapshot {
|
||||
&self,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
) -> std::result::Result<GetObjectReader, SelectObjectSnapshotReadError> {
|
||||
let result = self.open_generation_reader(range).await;
|
||||
if let Err(SelectObjectSnapshotReadError::Consistency(failure)) = &result {
|
||||
*self.failure.lock() = Some(*failure);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn open_generation_reader(
|
||||
&self,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
) -> std::result::Result<GetObjectReader, SelectObjectSnapshotReadError> {
|
||||
self.ensure_valid()?;
|
||||
if self.store.ctx.lock_manager().is_disabled() {
|
||||
return Err(SnapshotConsistencyError::LockingDisabled.into());
|
||||
}
|
||||
let mut opts = self.opts.clone();
|
||||
opts.no_lock = false;
|
||||
let guards = self
|
||||
.store
|
||||
.acquire_all_object_read_locks("select_object_range", &self.bucket, &self.object, &mut opts)
|
||||
.await?;
|
||||
let pin_lease = SelectObjectSnapshotLease::new(guards);
|
||||
pin_lease.check()?;
|
||||
#[cfg(test)]
|
||||
tests::pause_select_range_after_lock(&self.bucket, &self.object).await;
|
||||
// Re-select across all pools, including on the first range. Metadata
|
||||
// captured during prepare is no longer protected after that call returns.
|
||||
let (metadata, pool) = self
|
||||
.store
|
||||
.prepare_select_object_metadata(&self.bucket, &self.object, &opts)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if is_err_object_not_found(&error)
|
||||
|| is_err_version_not_found(&error)
|
||||
|| matches!(&error, StorageError::BucketNotFound(_) | StorageError::VolumeNotFound)
|
||||
{
|
||||
SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged)
|
||||
} else {
|
||||
SelectObjectSnapshotReadError::Storage(error)
|
||||
}
|
||||
})?;
|
||||
pin_lease.check()?;
|
||||
self.lease.check()?;
|
||||
let first_metadata = self.first_metadata.lock().take();
|
||||
let metadata = match first_metadata {
|
||||
Some(metadata) => metadata,
|
||||
None => {
|
||||
self.pool
|
||||
.prepare_get_object_reader_metadata(&self.bucket, &self.object, &self.opts)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
self.lease.check()?;
|
||||
if metadata.read_semantics_identity() != self.read_semantics_identity {
|
||||
return Err(SnapshotConsistencyError::ObjectChanged.into());
|
||||
}
|
||||
let mut reader = crate::object_api::without_get_object_body_cache_hook(pool.get_object_reader_with_prepared_metadata(
|
||||
&self.bucket,
|
||||
&self.object,
|
||||
range,
|
||||
self.headers.clone(),
|
||||
&opts,
|
||||
metadata,
|
||||
))
|
||||
.await?;
|
||||
pin_lease.check()?;
|
||||
self.ensure_valid()?;
|
||||
|
||||
let mut reader =
|
||||
crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
|
||||
&self.bucket,
|
||||
&self.object,
|
||||
range,
|
||||
self.headers.clone(),
|
||||
&self.opts,
|
||||
metadata,
|
||||
))
|
||||
.await?;
|
||||
self.lease.check()?;
|
||||
reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
|
||||
reader.stream = Box::new(SelectObjectSnapshotReader {
|
||||
inner: reader.stream,
|
||||
lock_loss_wake: SelectObjectSnapshotLockLossWake::new(self.lease.subscribe_lock_loss()),
|
||||
lease: Arc::clone(&self.lease),
|
||||
});
|
||||
// Streaming/lazy readers retain old FileInfo, inline bytes and data_dir
|
||||
// paths, never a current-key lookup. Cleanup can fail the read, but may
|
||||
// not select replacement bytes. This is independent of GET's lock knob.
|
||||
drop(pin_lease);
|
||||
Ok(reader)
|
||||
}
|
||||
}
|
||||
@@ -3235,7 +3183,7 @@ impl ECStore {
|
||||
)))
|
||||
}
|
||||
|
||||
pub(crate) async fn object_lock_config_snapshot_for_single_put(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
async fn object_lock_config_snapshot_for_single_put(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
check_valid_bucket_name(bucket)?;
|
||||
let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
@@ -3284,46 +3232,13 @@ impl ECStore {
|
||||
)))
|
||||
}
|
||||
|
||||
/// Selects authoritative metadata across all pools under the caller's locks.
|
||||
async fn prepare_select_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(crate::set_disk::PreparedGetObjectMetadata, Arc<Sets>)> {
|
||||
// Acquiring the namespace locks marks ordinary GETs cache-safe. A
|
||||
// SELECT generation check must instead consult the disks: another node
|
||||
// (or another SetDisks owner) can commit without invalidating this cache.
|
||||
let mut opts = opts.clone();
|
||||
opts.metadata_cache_safe = false;
|
||||
Ok(if self.single_pool() {
|
||||
let pool = Arc::clone(&self.pools[0]);
|
||||
let metadata = pool.prepare_get_object_reader_metadata(bucket, object, &opts).await?;
|
||||
(metadata, pool)
|
||||
} else {
|
||||
// Keep the large multi-pool selection future off the caller stack.
|
||||
// Debug builds otherwise exceed the common 2 MiB worker stack.
|
||||
Box::pin(async {
|
||||
let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, object, &opts).await?;
|
||||
let pool = self.pools.get(pool_idx).cloned().ok_or_else(|| {
|
||||
StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds"))
|
||||
})?;
|
||||
Ok::<_, StorageError>((metadata, pool))
|
||||
})
|
||||
.await?
|
||||
})
|
||||
}
|
||||
|
||||
/// Captures the initial generation for a SelectObjectContent query under short namespace locks.
|
||||
/// Resolves a GET's object identity without constructing its body reader.
|
||||
///
|
||||
/// Each range, including the first, reacquires these locks, revalidates the
|
||||
/// initial read semantics, and constructs a generation-bound reader before
|
||||
/// releasing them. Concurrent mutation may cause a consistency or storage
|
||||
/// error; callers must propagate it instead of reporting successful query completion.
|
||||
/// Cancelling preparation or a range open drops its guards. Returned snapshots
|
||||
/// and readers never retain namespace locks while waiting for client input.
|
||||
/// This is an additive two-stage counterpart to `get_object_reader`. The
|
||||
/// existing method remains the compatibility path for callers that do not
|
||||
/// need a pre-reader decision point.
|
||||
pub async fn prepare_select_object_snapshot(
|
||||
self: &Arc<Self>,
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
headers: &HeaderMap,
|
||||
@@ -3345,7 +3260,22 @@ impl ECStore {
|
||||
return Err(SnapshotConsistencyError::LockLost.into());
|
||||
}
|
||||
|
||||
let (mut metadata, _pool) = self.prepare_select_object_metadata(bucket, &object, &opts).await?;
|
||||
let (mut metadata, pool) = if self.single_pool() {
|
||||
let pool = Arc::clone(&self.pools[0]);
|
||||
let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
|
||||
(metadata, pool)
|
||||
} else {
|
||||
// Keep the large multi-pool selection future off the caller stack.
|
||||
// Debug builds otherwise exceed the common 2 MiB worker stack.
|
||||
Box::pin(async {
|
||||
let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
|
||||
let pool = self.pools.get(pool_idx).cloned().ok_or_else(|| {
|
||||
StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds"))
|
||||
})?;
|
||||
Ok::<_, StorageError>((metadata, pool))
|
||||
})
|
||||
.await?
|
||||
};
|
||||
if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(SnapshotConsistencyError::LockLost.into());
|
||||
}
|
||||
@@ -3362,12 +3292,8 @@ impl ECStore {
|
||||
return Err(SnapshotConsistencyError::LockLost.into());
|
||||
}
|
||||
|
||||
// Snapshot identity survives, but preparation metadata and all namespace
|
||||
// fences are released before a caller can wait on client backpressure.
|
||||
drop(metadata);
|
||||
drop(read_lock_guards);
|
||||
Ok(SelectObjectSnapshot {
|
||||
store: Arc::clone(self),
|
||||
pool,
|
||||
bucket: bucket.to_owned(),
|
||||
object,
|
||||
headers: rustfs_utils::http::project_ssec_transport_headers(headers),
|
||||
@@ -3375,8 +3301,8 @@ impl ECStore {
|
||||
object_info,
|
||||
logical_size,
|
||||
read_semantics_identity,
|
||||
failure: ParkingMutex::new(None),
|
||||
lease: Arc::new(SelectObjectSnapshotLease::new(Vec::new())),
|
||||
first_metadata: ParkingMutex::new(Some(metadata)),
|
||||
lease: Arc::new(SelectObjectSnapshotLease::new(read_lock_guards)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4783,12 +4709,6 @@ impl ECStore {
|
||||
opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
|
||||
}
|
||||
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||
if opts.object_lock_config_snapshot.is_none() && !opts.no_lock && !opts.data_movement {
|
||||
// The request body is client-paced. Acquire configuration and
|
||||
// lifecycle guards in SetDisks only once its staging has finished.
|
||||
opts.defer_put_object_lock_snapshot = true;
|
||||
return Ok((object, opts));
|
||||
}
|
||||
let expected_incarnation_id = opts
|
||||
.expected_bucket_incarnation_id
|
||||
.ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
|
||||
@@ -5439,12 +5359,6 @@ impl ECStore {
|
||||
) -> Result<ObjectInfo> {
|
||||
Box::pin(async move {
|
||||
let mut opts = opts;
|
||||
if opts.delete_recovery_work_budget.is_none() {
|
||||
// Reservations are taken up front per disk, so size the request
|
||||
// budget for every drive the delete can fan out to.
|
||||
let disks = self.pools.iter().map(|pool| pool.set_count * pool.set_drive_count).sum();
|
||||
opts.delete_recovery_work_budget = Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::for_disks(disks)));
|
||||
}
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result = self
|
||||
.handle_delete_object_with_journal_inner_boxed(bucket, object, opts, tier_journal_api)
|
||||
@@ -6004,12 +5918,6 @@ impl ECStore {
|
||||
return (deleted, errors, accounting);
|
||||
}
|
||||
let mut opts = opts;
|
||||
if opts.delete_recovery_work_budget.is_none() {
|
||||
// Reservations are taken up front per disk, so size the request
|
||||
// budget for every drive the delete can fan out to.
|
||||
let disks = self.pools.iter().map(|pool| pool.set_count * pool.set_drive_count).sum();
|
||||
opts.delete_recovery_work_budget = Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::for_disks(disks)));
|
||||
}
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let (valid_deleted, valid_errors, valid_accounting) = self
|
||||
.handle_delete_objects_with_journal_and_accounting_inner(bucket, valid_objects, opts, tier_journal_api)
|
||||
@@ -12040,63 +11948,6 @@ mod tests {
|
||||
assert!(other.is_empty());
|
||||
}
|
||||
|
||||
struct SelectRangeLockBarrier {
|
||||
bucket: String,
|
||||
object: String,
|
||||
reached: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
type SelectRangeBarriers = ParkingMutex<HashMap<(String, String), Arc<SelectRangeLockBarrier>>>;
|
||||
|
||||
fn select_range_barriers() -> &'static SelectRangeBarriers {
|
||||
static BARRIERS: std::sync::OnceLock<SelectRangeBarriers> = std::sync::OnceLock::new();
|
||||
BARRIERS.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
struct SelectRangeBarrierGuard(Arc<SelectRangeLockBarrier>);
|
||||
|
||||
impl SelectRangeBarrierGuard {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let barrier = Arc::new(SelectRangeLockBarrier {
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
reached: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
select_range_barriers()
|
||||
.lock()
|
||||
.insert((bucket.to_owned(), object.to_owned()), Arc::clone(&barrier));
|
||||
Self(barrier)
|
||||
}
|
||||
|
||||
async fn wait(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
|
||||
.await
|
||||
.expect("range acquired its namespace locks");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SelectRangeBarrierGuard {
|
||||
fn drop(&mut self) {
|
||||
select_range_barriers()
|
||||
.lock()
|
||||
.remove(&(self.0.bucket.clone(), self.0.object.clone()));
|
||||
self.0.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_select_range_after_lock(bucket: &str, object: &str) {
|
||||
let barrier = select_range_barriers()
|
||||
.lock()
|
||||
.get(&(bucket.to_owned(), object.to_owned()))
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.reached.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) {
|
||||
assert_pool_writer_is_blocked(store, 0, bucket, object).await;
|
||||
}
|
||||
@@ -12237,175 +12088,76 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn select_snapshot_releases_namespace_locks_independent_of_get_optimization() {
|
||||
for optimization in ["true", "false"] {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some(optimization))], async {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store(&[set_disks]).await);
|
||||
let bucket = "select-snapshot-lock-lifetime";
|
||||
let object = "object.bin";
|
||||
let payload = b"select-snapshot-lock-lifetime-payload-".repeat(40_000);
|
||||
let put_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
async fn select_snapshot_holds_namespace_lock_independent_of_get_optimization() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = new_prepared_reader_test_store(&[set_disks]).await;
|
||||
let bucket = "select-snapshot-lock-lifetime";
|
||||
let object = "object.bin";
|
||||
let payload = b"select-snapshot-lock-lifetime-payload-".repeat(40_000);
|
||||
let put_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
store.pools[0]
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
store.pools[0]
|
||||
.put_object(bucket, object, &mut put_reader, &put_opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
|
||||
let mut request_headers = HeaderMap::new();
|
||||
request_headers.insert(SSEC_ALGORITHM_HEADER, "AES256".parse().expect("valid SSE-C algorithm header"));
|
||||
request_headers.insert(SSEC_KEY_HEADER, "secret-key".parse().expect("valid SSE-C key header"));
|
||||
request_headers.insert(SSEC_KEY_MD5_HEADER, "key-md5".parse().expect("valid SSE-C key digest header"));
|
||||
request_headers.insert("authorization", "credential".parse().expect("valid authorization header"));
|
||||
let snapshot = store
|
||||
.prepare_select_object_snapshot(bucket, object, &request_headers, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("SelectObjectContent snapshot should be prepared");
|
||||
assert_eq!(snapshot.headers.len(), 3);
|
||||
assert_eq!(snapshot.headers.get(SSEC_ALGORITHM_HEADER), request_headers.get(SSEC_ALGORITHM_HEADER));
|
||||
assert_eq!(snapshot.headers.get(SSEC_KEY_HEADER), request_headers.get(SSEC_KEY_HEADER));
|
||||
assert_eq!(snapshot.headers.get(SSEC_KEY_MD5_HEADER), request_headers.get(SSEC_KEY_MD5_HEADER));
|
||||
assert!(snapshot.headers.get("authorization").is_none());
|
||||
assert!(snapshot.headers.values().all(http::HeaderValue::is_sensitive));
|
||||
assert!(!format!("{:?}", snapshot.headers).contains("secret-key"));
|
||||
assert_eq!(
|
||||
snapshot.logical_size(),
|
||||
u64::try_from(payload.len()).expect("test payload length should fit in u64")
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.object_info().size,
|
||||
i64::try_from(payload.len()).expect("test payload length should fit in i64")
|
||||
);
|
||||
drop(acquire_prepared_reader_writer(&store, bucket, object).await);
|
||||
|
||||
let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("snapshot body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
drop(reader);
|
||||
drop(acquire_prepared_reader_writer(&store, bucket, object).await);
|
||||
|
||||
let mut unread = snapshot.open_reader(None).await.expect("unread SELECT range");
|
||||
let mut replacement = PutObjReader::from_vec(vec![b'z'; payload.len()]);
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
store.pools[0].put_object(bucket, object, &mut replacement, &ObjectOptions::default()),
|
||||
)
|
||||
store.pools[0]
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unread SELECT cannot block an overwrite")
|
||||
.expect("overwrite");
|
||||
let mut old_bytes = Vec::new();
|
||||
match unread.stream.read_to_end(&mut old_bytes).await {
|
||||
Ok(_) => assert_eq!(old_bytes, payload, "an opened range must never switch generations"),
|
||||
Err(_) => assert!(payload.starts_with(&old_bytes), "failed reads may emit only original bytes"),
|
||||
}
|
||||
assert!(matches!(
|
||||
snapshot.open_reader(None).await,
|
||||
Err(SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged))
|
||||
));
|
||||
assert!(
|
||||
snapshot.ensure_valid().is_err(),
|
||||
"consistency failure prevents successful terminal events"
|
||||
);
|
||||
drop(snapshot);
|
||||
drop(acquire_prepared_reader_writer(&store, bucket, object).await);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
store.pools[0]
|
||||
.put_object(bucket, object, &mut put_reader, &put_opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn select_snapshot_lazy_multipart_reader_never_switches_to_replacement_parts() {
|
||||
use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations};
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_CODEC_STREAMING_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", Some("true")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", Some("true")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT", Some("on")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_MIN_SIZE", Some("1")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true")),
|
||||
],
|
||||
async {
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set)]).await);
|
||||
let bucket = "select-lazy-multipart-generation";
|
||||
let object = "multipart.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default()).await.expect("bucket");
|
||||
let opts = ObjectOptions::default();
|
||||
let upload = set.new_multipart_upload(bucket, object, &opts).await.expect("upload");
|
||||
let mut expected = Vec::new();
|
||||
let mut parts = Vec::new();
|
||||
for (number, length, byte) in [(1, 5 * 1024 * 1024, b'a'), (2, 1024, b'b')] {
|
||||
let bytes = vec![byte; length];
|
||||
expected.extend_from_slice(&bytes);
|
||||
let mut body = PutObjReader::from_vec(bytes);
|
||||
let part = set
|
||||
.put_object_part(bucket, object, &upload.upload_id, number, &mut body, &opts)
|
||||
.await
|
||||
.expect("part");
|
||||
parts.push(CompletePart {
|
||||
part_num: number,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
Arc::clone(&set)
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, parts, &opts)
|
||||
.await
|
||||
.expect("complete upload");
|
||||
let snapshot = store
|
||||
.prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("snapshot");
|
||||
crate::set_disk::reset_test_get_object_reader_path();
|
||||
let mut reader = snapshot.open_reader(None).await.expect("lazy multipart reader");
|
||||
assert_eq!(
|
||||
crate::set_disk::test_get_object_reader_path_id(),
|
||||
5,
|
||||
"exercise codec streaming rather than a materialized read"
|
||||
);
|
||||
let mut first = [0u8; 1];
|
||||
reader.stream.read_exact(&mut first).await.expect("first original byte");
|
||||
assert_eq!(first, [b'a']);
|
||||
let mut replacement = PutObjReader::from_vec(vec![b'z'; expected.len()]);
|
||||
tokio::time::timeout(Duration::from_secs(10), set.put_object(bucket, object, &mut replacement, &opts))
|
||||
.await
|
||||
.expect("unread parts cannot block overwrite")
|
||||
.expect("overwrite");
|
||||
let mut body = first.to_vec();
|
||||
match reader.stream.read_to_end(&mut body).await {
|
||||
Ok(_) => assert_eq!(body, expected, "all successful bytes belong to the original MPU"),
|
||||
Err(_) => assert!(expected.starts_with(&body), "even a failed stream cannot contain replacement bytes"),
|
||||
}
|
||||
assert!(matches!(
|
||||
snapshot.open_reader(None).await,
|
||||
Err(SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged))
|
||||
));
|
||||
},
|
||||
)
|
||||
use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
|
||||
let mut request_headers = HeaderMap::new();
|
||||
request_headers.insert(SSEC_ALGORITHM_HEADER, "AES256".parse().expect("valid SSE-C algorithm header"));
|
||||
request_headers.insert(SSEC_KEY_HEADER, "secret-key".parse().expect("valid SSE-C key header"));
|
||||
request_headers.insert(SSEC_KEY_MD5_HEADER, "key-md5".parse().expect("valid SSE-C key digest header"));
|
||||
request_headers.insert("authorization", "credential".parse().expect("valid authorization header"));
|
||||
let snapshot = store
|
||||
.prepare_select_object_snapshot(bucket, object, &request_headers, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("SelectObjectContent snapshot should be prepared");
|
||||
assert_eq!(snapshot.headers.len(), 3);
|
||||
assert_eq!(snapshot.headers.get(SSEC_ALGORITHM_HEADER), request_headers.get(SSEC_ALGORITHM_HEADER));
|
||||
assert_eq!(snapshot.headers.get(SSEC_KEY_HEADER), request_headers.get(SSEC_KEY_HEADER));
|
||||
assert_eq!(snapshot.headers.get(SSEC_KEY_MD5_HEADER), request_headers.get(SSEC_KEY_MD5_HEADER));
|
||||
assert!(snapshot.headers.get("authorization").is_none());
|
||||
assert!(snapshot.headers.values().all(http::HeaderValue::is_sensitive));
|
||||
assert!(!format!("{:?}", snapshot.headers).contains("secret-key"));
|
||||
assert_eq!(
|
||||
snapshot.logical_size(),
|
||||
u64::try_from(payload.len()).expect("test payload length should fit in u64")
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.object_info().size,
|
||||
i64::try_from(payload.len()).expect("test payload length should fit in i64")
|
||||
);
|
||||
assert_prepared_reader_blocks_writer(&store, bucket, object).await;
|
||||
|
||||
let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("snapshot body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
drop(reader);
|
||||
assert_prepared_reader_blocks_writer(&store, bucket, object).await;
|
||||
|
||||
drop(snapshot);
|
||||
drop(acquire_prepared_reader_writer(&store, bucket, object).await);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn select_snapshot_identity_compares_encoded_directory_object_key() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store(&[set_disks]).await);
|
||||
let store = new_prepared_reader_test_store(&[set_disks]).await;
|
||||
let bucket = "select-snapshot-directory-identity";
|
||||
let source_object = "source.bin";
|
||||
|
||||
@@ -12439,9 +12191,9 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn select_snapshot_revalidates_initial_metadata_before_first_reader() {
|
||||
async fn select_snapshot_reuses_initial_metadata_fanout_for_first_reader() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store(&[set_disks]).await);
|
||||
let store = new_prepared_reader_test_store(&[set_disks]).await;
|
||||
let bucket = "select-snapshot-initial-metadata";
|
||||
let object = "initial-metadata.bin";
|
||||
let payload = b"select snapshot initial metadata".repeat(4_000);
|
||||
@@ -12474,8 +12226,8 @@ mod tests {
|
||||
let mut reader = snapshot.open_reader(None).await.expect("first snapshot reader should open");
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
8,
|
||||
"first reader must revalidate metadata after preparation released its locks"
|
||||
4,
|
||||
"first reader should consume the metadata captured during preparation"
|
||||
);
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
@@ -12490,7 +12242,7 @@ mod tests {
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn select_snapshot_rejects_identity_change_before_second_reader() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks)]).await);
|
||||
let store = new_prepared_reader_test_store(&[Arc::clone(&set_disks)]).await;
|
||||
let bucket = "select-snapshot-identity-change";
|
||||
let object = "identity-change.bin";
|
||||
let no_lock_opts = ObjectOptions {
|
||||
@@ -12507,7 +12259,7 @@ mod tests {
|
||||
.put_object(bucket, object, &mut initial_reader, &no_lock_opts)
|
||||
.await
|
||||
.expect("initial generation should be written");
|
||||
let snapshot = store
|
||||
let mut snapshot = store
|
||||
.prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("snapshot should be prepared");
|
||||
@@ -12517,12 +12269,13 @@ mod tests {
|
||||
.await
|
||||
.expect("first snapshot reader should consume captured metadata"),
|
||||
);
|
||||
snapshot.opts.metadata_cache_safe = false;
|
||||
|
||||
let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut replacement_reader, &ObjectOptions::default())
|
||||
.put_object(bucket, object, &mut replacement_reader, &no_lock_opts)
|
||||
.await
|
||||
.expect("normal writer should replace the object while the snapshot is idle");
|
||||
.expect("test-only no-lock write should replace the object");
|
||||
let error = match snapshot.open_reader(None).await {
|
||||
Ok(_) => panic!("a later reader must reject the replacement generation"),
|
||||
Err(error) => error,
|
||||
@@ -12539,8 +12292,7 @@ mod tests {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store =
|
||||
Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
|
||||
let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
|
||||
let bucket = "select-snapshot-latest-delete-marker";
|
||||
let object = "versioned-object.bin";
|
||||
let versioned_opts = ObjectOptions {
|
||||
@@ -12593,7 +12345,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn select_snapshot_allows_store_delete_for_object_in_nonzero_set() {
|
||||
async fn select_snapshot_blocks_store_delete_for_object_in_nonzero_set() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
@@ -12633,35 +12385,37 @@ mod tests {
|
||||
sets.disk_set[1].lockers.clone(),
|
||||
2,
|
||||
);
|
||||
let writer_guard = hashed_set_writer
|
||||
let writer_error = hashed_set_writer
|
||||
.get_write_lock(
|
||||
rustfs_lock::ObjectKey::new(bucket, object.as_str()),
|
||||
"competing-hashed-set-writer",
|
||||
Duration::from_millis(50),
|
||||
)
|
||||
.await
|
||||
.expect("idle Select must release the nonzero hashed-set read lock");
|
||||
drop(writer_guard);
|
||||
.expect_err("Select must hold the nonzero hashed-set read lock");
|
||||
assert!(matches!(writer_error, rustfs_lock::LockError::Timeout { .. }));
|
||||
let barrier = DeleteAfterObjectLockSnapshotBarrier::install(bucket);
|
||||
let delete_store = Arc::clone(&store);
|
||||
let delete_object = object.clone();
|
||||
let delete = tokio::spawn(async move {
|
||||
let mut delete = tokio::spawn(async move {
|
||||
delete_store
|
||||
.delete_object(bucket, &delete_object, ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
barrier.release();
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(),
|
||||
"store DELETE must wait for the fixed-domain Select read lock"
|
||||
);
|
||||
|
||||
drop(snapshot);
|
||||
let deleted = tokio::time::timeout(Duration::from_secs(10), delete)
|
||||
.await
|
||||
.expect("DELETE should complete while the snapshot remains alive")
|
||||
.expect("DELETE should resume after the snapshot is dropped")
|
||||
.expect("DELETE task should not panic")
|
||||
.expect("DELETE should complete");
|
||||
assert_eq!(deleted.name, object);
|
||||
assert!(matches!(
|
||||
snapshot.open_reader(None).await,
|
||||
Err(SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -12671,7 +12425,7 @@ mod tests {
|
||||
rustfs_lock::GlobalLockManager::Disabled(rustfs_lock::DisabledLockManager::new()),
|
||||
)));
|
||||
let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
|
||||
let store = new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await;
|
||||
let bucket = "select-snapshot-lock-disabled";
|
||||
let object = "object.bin";
|
||||
let mut put_reader = PutObjReader::from_vec(b"payload".to_vec());
|
||||
@@ -12783,11 +12537,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn select_snapshot_revalidates_metadata_across_three_pools() {
|
||||
async fn select_snapshot_reuses_metadata_across_three_pools() {
|
||||
let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
|
||||
let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
|
||||
let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store(&[first_set, second_set, third_set]).await);
|
||||
let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
|
||||
let bucket = "select-snapshot-three-pools";
|
||||
let object = "object.bin";
|
||||
let payload = b"select-snapshot-three-pool-payload-".repeat(40_000);
|
||||
@@ -12819,8 +12573,8 @@ mod tests {
|
||||
let mut reader = snapshot.open_reader(None).await.expect("snapshot body reader should open");
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
metadata_calls * 2,
|
||||
"each range must select the authoritative winner across all pools"
|
||||
metadata_calls,
|
||||
"SelectObjectContent must consume the prepared winner without a second fanout"
|
||||
);
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
@@ -12829,18 +12583,6 @@ mod tests {
|
||||
.await
|
||||
.expect("snapshot body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
|
||||
// A later authoritative winner can be in a different pool. Reopening
|
||||
// only the initially selected pool would silently keep stale metadata.
|
||||
let mut replacement = PutObjReader::from_vec(b"new-pool-winner".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(bucket, object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("normal overwrite in another pool");
|
||||
assert!(matches!(
|
||||
snapshot.open_reader(None).await,
|
||||
Err(SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -13157,7 +12899,7 @@ mod tests {
|
||||
async fn select_snapshot_locks_the_hashed_set_in_every_pool() {
|
||||
let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
|
||||
let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store(&[first_set, second_set]).await);
|
||||
let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
|
||||
let bucket = "select-snapshot-all-pools";
|
||||
let object = "object.bin";
|
||||
let payload = b"second-pool-snapshot".to_vec();
|
||||
@@ -13181,21 +12923,10 @@ mod tests {
|
||||
.prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("snapshot should resolve the second-pool object");
|
||||
drop(acquire_pool_writer(&store, 0, bucket, object).await);
|
||||
drop(acquire_pool_writer(&store, 1, bucket, object).await);
|
||||
let snapshot = Arc::new(snapshot);
|
||||
let barrier = SelectRangeBarrierGuard::install(bucket, object);
|
||||
let opening = tokio::spawn({
|
||||
let snapshot = Arc::clone(&snapshot);
|
||||
async move { snapshot.open_reader(None).await }
|
||||
});
|
||||
barrier.wait().await;
|
||||
assert_pool_writer_is_blocked(&store, 0, bucket, object).await;
|
||||
assert_pool_writer_is_blocked(&store, 1, bucket, object).await;
|
||||
drop(barrier);
|
||||
let mut reader = opening.await.expect("range task").expect("range opens");
|
||||
drop(acquire_pool_writer(&store, 0, bucket, object).await);
|
||||
drop(acquire_pool_writer(&store, 1, bucket, object).await);
|
||||
|
||||
let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
|
||||
@@ -20,230 +20,14 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::contract_compat::ECStore;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, DeleteBucketOptions, Error, HashReader, MakeBucketOptions, MultipartOperations,
|
||||
NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause,
|
||||
init_bucket_metadata_sys, isolated_store_over_temp_disks,
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
isolated_store_over_temp_disks,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct ObservedPutBody {
|
||||
inner: tokio::io::DuplexStream,
|
||||
polled: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncRead for ObservedPutBody {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
self.polled.notify_one();
|
||||
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn destructive_put_takes_bucket_locks_only_after_its_body_is_staged() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
for (object, version_id) in [("object", None), ("directory/", Some(Uuid::nil().to_string()))] {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-locks";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let encoded_object = if object.ends_with('/') { "directory__XLDIR__" } else { object };
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, encoded_object, PutObjectCommitPause::BeforeQuotaRename);
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
version_id,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified())
|
||||
.await
|
||||
.expect("PUT must be waiting for client data");
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("metadata lock");
|
||||
drop(
|
||||
timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5)))
|
||||
.await
|
||||
.expect("body must not block metadata writers")
|
||||
.expect("metadata write guard"),
|
||||
);
|
||||
drop(
|
||||
timeout(Duration::from_secs(2), store.lock_bucket_for_recursive_delete(bucket))
|
||||
.await
|
||||
.expect("body must not retain lifecycle guard")
|
||||
.expect("lifecycle writer"),
|
||||
);
|
||||
sender.write_all(b"body").await.expect("finish body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
timeout(Duration::from_secs(5), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("staged PUT reaches commit");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending(), "commit must fence metadata updates");
|
||||
let lifecycle = store.lock_bucket_for_recursive_delete(bucket);
|
||||
tokio::pin!(lifecycle);
|
||||
assert!(futures::poll!(&mut lifecycle).is_pending(), "commit must fence bucket deletion");
|
||||
barrier.release();
|
||||
let release_metadata = async {
|
||||
drop(writer.await.expect("metadata writer"));
|
||||
};
|
||||
let release_lifecycle = async {
|
||||
drop(lifecycle.await.expect("lifecycle writer"));
|
||||
};
|
||||
timeout(Duration::from_secs(5), async {
|
||||
let (put, (), ()) = tokio::join!(put, release_metadata, release_lifecycle);
|
||||
put.expect("owner task").expect("commit");
|
||||
})
|
||||
.await
|
||||
.expect("PUT and queued writers finish");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_put_cannot_publish_into_a_recreated_bucket() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-generation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
|
||||
store
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove old bucket while PUT stalls");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
sender.write_all(b"body").await.expect("finish body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
let error = timeout(Duration::from_secs(5), put)
|
||||
.await
|
||||
.expect("commit gate")
|
||||
.expect("owner")
|
||||
.expect_err("old request generation must not publish");
|
||||
assert!(
|
||||
error.to_string().contains("generation") || matches!(error, Error::BucketNotFound(..)),
|
||||
"{error:?}"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_put_rechecks_current_object_lock_at_commit() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-retention";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("locked bucket");
|
||||
let mut old = PutObjReader::from_vec(b"old".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut old, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("unlocked predecessor");
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
|
||||
let mut protected = PutObjReader::from_vec(b"protected".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut protected,
|
||||
&ObjectOptions {
|
||||
user_defined: std::collections::HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("install a protected predecessor while the client stalls");
|
||||
sender.write_all(b"body").await.expect("finish stale body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
let error = timeout(Duration::from_secs(5), put)
|
||||
.await
|
||||
.expect("commit gate")
|
||||
.expect("task")
|
||||
.expect_err("pre-body state cannot admit an overwrite of the newly protected version");
|
||||
assert!(matches!(error, Error::PrefixAccessDenied(..)), "{error:?}");
|
||||
let info = store
|
||||
.get_object_info(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("protected object remains");
|
||||
assert_eq!(info.size, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(false).await;
|
||||
|
||||
@@ -9,14 +9,12 @@ use rustfs_storage_api as storage_contracts;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) mod metadata_lock {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketOperations, CompletePart, DeleteBucketOptions, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO,
|
||||
ObjectOperations,
|
||||
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
};
|
||||
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rio::HashReader;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
|
||||
|
||||
@@ -3930,34 +3930,6 @@ mod test {
|
||||
assert!(versions_with_free.versions.iter().all(|version| version.num_versions == 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_rejects_mod_times_that_cannot_round_trip() {
|
||||
for nanos in [0, i128::from(i64::MAX) + 1, i128::from(i64::MIN) - 1, 1_i128 << 64] {
|
||||
for deleted in [false, true] {
|
||||
let mut meta = FileMeta::new();
|
||||
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
|
||||
info.deleted = deleted;
|
||||
info.mod_time = Some(OffsetDateTime::from_unix_timestamp_nanos(nanos).expect("time within supported date range"));
|
||||
assert!(
|
||||
meta.add_version(info).is_err(),
|
||||
"unrepresentable time {nanos} must be rejected before publication"
|
||||
);
|
||||
}
|
||||
}
|
||||
for nanos in [-1, 1, i64::MIN, i64::MAX] {
|
||||
let mut meta = FileMeta::new();
|
||||
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
|
||||
let time = OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).expect("valid codec time");
|
||||
info.mod_time = Some(time);
|
||||
meta.add_version(info).expect("representable times remain supported");
|
||||
let reopened = FileMeta::load(&meta.marshal_msg().expect("serialize version")).expect("reopen version");
|
||||
let versions = reopened
|
||||
.get_file_info_versions("bucket", "object", false)
|
||||
.expect("read version history");
|
||||
assert_eq!(versions.versions[0].mod_time, Some(time));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_integrity_validation() {
|
||||
// Test data integrity checks
|
||||
|
||||
@@ -858,24 +858,13 @@ impl FileMetaVersion {
|
||||
VersionType::Object => self
|
||||
.object
|
||||
.as_ref()
|
||||
.map(|v| {
|
||||
v.erasure_algorithm.valid()
|
||||
&& v.bitrot_checksum_algo.valid()
|
||||
&& v.mod_time.is_some_and(|time| {
|
||||
let nanos = time.unix_timestamp_nanos();
|
||||
nanos != 0 && i64::try_from(nanos).is_ok()
|
||||
})
|
||||
})
|
||||
.map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some())
|
||||
.unwrap_or_default(),
|
||||
VersionType::Legacy => self.legacy_object.as_ref().map(MetaObjectV1::valid).unwrap_or_default(),
|
||||
VersionType::Delete => self
|
||||
.delete_marker
|
||||
.as_ref()
|
||||
.map(|v| {
|
||||
v.mod_time.is_some_and(|time| {
|
||||
time > OffsetDateTime::UNIX_EPOCH && i64::try_from(time.unix_timestamp_nanos()).is_ok()
|
||||
})
|
||||
})
|
||||
.map(|v| v.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) > OffsetDateTime::UNIX_EPOCH)
|
||||
.unwrap_or_default(),
|
||||
_ => false,
|
||||
}
|
||||
|
||||
@@ -77,9 +77,6 @@ pub enum Error {
|
||||
#[error("Replacement target is not ready: {0}")]
|
||||
ReplacementTargetNotReady(String),
|
||||
|
||||
#[error("Replacement bucket namespace changed; a fresh identity-bound pass is required")]
|
||||
ReplacementNamespaceChanged,
|
||||
|
||||
#[error("replacement recovery retry budget exhausted")]
|
||||
ReplacementRetryBudgetExhausted,
|
||||
|
||||
@@ -135,10 +132,7 @@ impl Error {
|
||||
/// Callers must cut their own concurrency instead of retrying at full speed.
|
||||
pub(crate) fn is_slow_down(&self) -> bool {
|
||||
match self {
|
||||
Error::Storage(error) => matches!(
|
||||
error,
|
||||
EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles | EcstoreError::DeleteRecoveryWorkLimit
|
||||
),
|
||||
Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
|
||||
Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
|
||||
Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
|
||||
message.contains("Please reduce your request rate")
|
||||
@@ -235,10 +229,9 @@ pub(crate) fn classify_heal_error(error: &Error) -> HealErrorClass {
|
||||
Error::DanglingDeleteDeferred { retry_not_before } => HealErrorClass::DanglingGrace {
|
||||
retry_not_before: Some(*retry_not_before),
|
||||
},
|
||||
Error::ReplacementTargetNotReady(_)
|
||||
| Error::ReplacementNamespaceChanged
|
||||
| Error::TransientSkip { .. }
|
||||
| Error::BucketDiscoveryIncomplete => HealErrorClass::Transient,
|
||||
Error::ReplacementTargetNotReady(_) | Error::TransientSkip { .. } | Error::BucketDiscoveryIncomplete => {
|
||||
HealErrorClass::Transient
|
||||
}
|
||||
Error::LegacyMrfUnverified { .. } => HealErrorClass::Permanent,
|
||||
Error::Storage(error) => classify_storage_error(error),
|
||||
Error::Disk(error) => classify_disk_error(error),
|
||||
@@ -292,7 +285,6 @@ pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
|
||||
| EcstoreError::FaultyDisk
|
||||
| EcstoreError::FaultyRemoteDisk
|
||||
| EcstoreError::SlowDown
|
||||
| EcstoreError::DeleteRecoveryWorkLimit
|
||||
| EcstoreError::OperationCanceled
|
||||
| EcstoreError::RemoteClientUnavailable(_) => HealErrorClass::Transient,
|
||||
_ if error.is_quorum_error() => HealErrorClass::Transient,
|
||||
|
||||
@@ -468,16 +468,7 @@ impl ErasureSetHealer {
|
||||
let task_id = self.get_or_create_task_id(set_disk_id).await?;
|
||||
|
||||
// 2. initialize or resume resume state
|
||||
let (resume_manager, mut checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
|
||||
let replacement_buckets = if self.replacement_task_id.is_some() {
|
||||
let current = resume_manager.prepare_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
// Preparation may have durably rewound the companion checkpoint.
|
||||
checkpoint_manager = CheckpointManager::load_from_disk(self.disk.clone(), &task_id).await?;
|
||||
Some(current)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let buckets = replacement_buckets.as_deref().unwrap_or(buckets);
|
||||
let (resume_manager, checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
|
||||
|
||||
let admin_buckets = if self.admin_task.is_some() {
|
||||
Some(
|
||||
@@ -498,9 +489,6 @@ impl ErasureSetHealer {
|
||||
.execute_heal_with_resume(buckets, set_disk_id, &resume_manager, &checkpoint_manager)
|
||||
.await;
|
||||
|
||||
if self.replacement_task_id.is_some() {
|
||||
resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
}
|
||||
result?;
|
||||
self.verify_replacement_identity_fence("completion").await?;
|
||||
|
||||
@@ -979,16 +967,9 @@ impl ErasureSetHealer {
|
||||
// Recheck even resumed buckets: their user-object cursor does not
|
||||
// prove that the replacement holds the internal bucket records.
|
||||
for bucket in buckets {
|
||||
if self.replacement_task_id.is_some() {
|
||||
let expected = resume_manager.replacement_bucket_incarnation(bucket).await?;
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, &self.heal_opts, targets)
|
||||
.await?;
|
||||
} else {
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
|
||||
.await?;
|
||||
}
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
self.heal_replacement_pool_metadata(
|
||||
@@ -1038,7 +1019,6 @@ impl ErasureSetHealer {
|
||||
// never hot-retried in place here.
|
||||
if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
|
||||
if self.replacement_task_id.is_some() {
|
||||
resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
let state = resume_manager.get_state().await;
|
||||
let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
|
||||
if state.can_retry()
|
||||
@@ -1472,11 +1452,7 @@ impl ErasureSetHealer {
|
||||
"Erasure set bucket started"
|
||||
);
|
||||
|
||||
let incarnation = if self.replacement_task_id.is_some() {
|
||||
Some(resume_manager.replacement_bucket_incarnation(bucket).await?)
|
||||
} else {
|
||||
self.admin_bucket_incarnation(checkpoint_manager, bucket).await?
|
||||
};
|
||||
let incarnation = self.admin_bucket_incarnation(checkpoint_manager, bucket).await?;
|
||||
let unresolved_delete_recovery = self.heal_admin_bucket_delete_recovery(bucket, incarnation).await?;
|
||||
if unresolved_delete_recovery && !self.heal_opts.dry_run {
|
||||
resume_manager.set_delete_recovery_rescan_pending(true).await?;
|
||||
@@ -1768,20 +1744,15 @@ impl ErasureSetHealer {
|
||||
(0, Err(Error::TaskCancelled))
|
||||
} else {
|
||||
let result = if replacement_object {
|
||||
match incarnation {
|
||||
Some(expected) => storage
|
||||
.heal_replacement_object_at_incarnation(
|
||||
&bucket_name,
|
||||
&object_name,
|
||||
version_id.as_deref(),
|
||||
expected,
|
||||
&heal_opts,
|
||||
&cancel_token,
|
||||
)
|
||||
.await
|
||||
.map(|result| (result.item, result.error)),
|
||||
None => Err(Error::ReplacementNamespaceChanged),
|
||||
}
|
||||
storage
|
||||
.heal_object_with_task_cancel(
|
||||
&bucket_name,
|
||||
&object_name,
|
||||
version_id.as_deref(),
|
||||
&heal_opts,
|
||||
&cancel_token,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
storage
|
||||
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
|
||||
@@ -2667,7 +2638,6 @@ mod resume_loop_tests {
|
||||
struct FakeStorage {
|
||||
admin_disk: Mutex<Option<DiskStore>>,
|
||||
bucket_discovery_incomplete: AtomicBool,
|
||||
bucket_discovery_error: Mutex<Option<EcstoreError>>,
|
||||
bucket_discovery_scopes: Mutex<Vec<Option<crate::heal::storage_api::storage::DiskSetSelector>>>,
|
||||
bucket_pages: Mutex<HashMap<String, Page>>,
|
||||
incarnations: Mutex<HashMap<String, uuid::Uuid>>,
|
||||
@@ -2759,9 +2729,6 @@ mod resume_loop_tests {
|
||||
if self.bucket_discovery_incomplete.load(Ordering::SeqCst) {
|
||||
return Err(Error::BucketDiscoveryIncomplete);
|
||||
}
|
||||
if let Some(error) = self.bucket_discovery_error.lock().expect("discovery failure").clone() {
|
||||
return Err(Error::Storage(error));
|
||||
}
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
@@ -2938,16 +2905,6 @@ mod resume_loop_tests {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
self.validate_bucket_incarnation(bucket, Some(expected)).await?;
|
||||
self.heal_replacement_bucket_metadata(bucket, opts, targets).await
|
||||
}
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -3064,16 +3021,6 @@ mod resume_loop_tests {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let disk = make_disk(&temp).await;
|
||||
let storage = Arc::new(FakeStorage::default());
|
||||
if !target_endpoints.is_empty() {
|
||||
storage.bucket_pages.lock().expect("replacement inventory").insert(
|
||||
"b".to_string(),
|
||||
Page {
|
||||
items: Vec::new(),
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
}
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let healer = ErasureSetHealer::new(
|
||||
storage.clone(),
|
||||
@@ -3093,17 +3040,7 @@ mod resume_loop_tests {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
if !healer.target_endpoints.is_empty() {
|
||||
resume
|
||||
.prepare_replacement_bucket_pass(storage.as_ref())
|
||||
.await
|
||||
.expect("replacement fixture identity admission");
|
||||
}
|
||||
let checkpoint = if CheckpointManager::has_checkpoint(&disk, &task_id).await {
|
||||
CheckpointManager::load_from_disk(disk, &task_id).await.unwrap()
|
||||
} else {
|
||||
CheckpointManager::new(disk, task_id.clone()).await.unwrap()
|
||||
};
|
||||
let checkpoint = CheckpointManager::new(disk, task_id.clone()).await.unwrap();
|
||||
Env {
|
||||
healer,
|
||||
storage,
|
||||
@@ -3143,60 +3080,6 @@ mod resume_loop_tests {
|
||||
(processed, successful, failed, skipped, result.map(|_| ()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_bucket_pass_rejects_incomplete_inventory_without_reusing_or_clearing_progress() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.resume
|
||||
.set_resume_cursor(Some("held-cursor".to_string()))
|
||||
.await
|
||||
.expect("cursor before discovery failure");
|
||||
env.checkpoint
|
||||
.add_processed_object("held-key".to_string())
|
||||
.await
|
||||
.expect("dedup before discovery failure");
|
||||
// Model a page whose dedup identities must survive until cursor
|
||||
// publication. complete_page deliberately prunes that identity set.
|
||||
env.checkpoint
|
||||
.advance_page(0, 7)
|
||||
.await
|
||||
.expect("persist the retained page identities");
|
||||
let before = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("durable checkpoint before discovery failure")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(before.current_object_index, 7);
|
||||
assert!(before.processed_objects.contains("held-key"));
|
||||
let state = env.resume.get_state().await;
|
||||
env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst);
|
||||
assert!(matches!(
|
||||
env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
|
||||
Err(Error::BucketDiscoveryIncomplete)
|
||||
));
|
||||
assert_eq!(env.resume.get_state().await, state, "partial discovery must never publish a fresh plan");
|
||||
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("durable checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(checkpoint.current_object_index, 7);
|
||||
assert!(checkpoint.processed_objects.contains("held-key"));
|
||||
env.storage.bucket_discovery_incomplete.store(false, Ordering::SeqCst);
|
||||
*env.storage.bucket_discovery_error.lock().expect("inject quorum failure") =
|
||||
Some(EcstoreError::InsufficientReadQuorum("bucket".to_string(), "inventory".to_string()));
|
||||
assert!(matches!(
|
||||
env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
|
||||
Err(Error::Storage(EcstoreError::InsufficientReadQuorum(..)))
|
||||
));
|
||||
assert_eq!(env.resume.get_state().await, state, "quorum failure is not an empty namespace");
|
||||
*env.storage.bucket_discovery_error.lock().expect("restore inventory") = None;
|
||||
env.resume
|
||||
.prepare_replacement_bucket_pass(env.storage.as_ref())
|
||||
.await
|
||||
.expect("unchanged complete inventory resumes existing pass");
|
||||
assert_eq!(env.resume.get_state().await.resume_cursor.as_deref(), Some("held-cursor"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_empty_bucket_no_panic() {
|
||||
let env = make_env().await;
|
||||
@@ -4172,7 +4055,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4227,7 +4110,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer.heal_opts.no_lock = true;
|
||||
env.healer = env
|
||||
@@ -4284,7 +4167,7 @@ mod resume_loop_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_replacement_retries_a_transient_object_before_rewinding_the_scan() {
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4316,7 +4199,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("1")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4348,7 +4231,7 @@ mod resume_loop_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_replacement_cancellation_interrupts_object_retry() {
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4364,18 +4247,14 @@ mod resume_loop_tests {
|
||||
env.storage.set_outcome("retry", None, HealOutcome::Transient);
|
||||
let cancel = env.healer.cancel_token.clone();
|
||||
let observe = async {
|
||||
let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
let notified = env.storage.heal_started.notified();
|
||||
if !env.storage.calls().is_empty() {
|
||||
break;
|
||||
}
|
||||
notified.await;
|
||||
loop {
|
||||
let notified = env.storage.heal_started.notified();
|
||||
if !env.storage.calls().is_empty() {
|
||||
break;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
notified.await;
|
||||
}
|
||||
cancel.cancel();
|
||||
started.expect("replacement object retry must start");
|
||||
};
|
||||
|
||||
let ((_, successful, _, _, result), ()) = tokio::join!(run(&env), observe);
|
||||
@@ -4393,7 +4272,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
let execution = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
|
||||
env.healer = env
|
||||
|
||||
@@ -51,25 +51,6 @@ async fn replacement_env() -> Env {
|
||||
)
|
||||
.await
|
||||
.expect("replacement intent should persist before scanning");
|
||||
env.resume
|
||||
.prepare_replacement_bucket_pass(env.storage.as_ref())
|
||||
.await
|
||||
.expect("the new replacement generation must admit its bucket identities before direct execution");
|
||||
// Admission durably resets the companion checkpoint. Reload its current
|
||||
// revision instead of retaining the previous generation's CAS instance.
|
||||
env.checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("reload the admitted replacement pass checkpoint");
|
||||
assert_eq!(
|
||||
env.resume
|
||||
.replacement_bucket_incarnation("b")
|
||||
.await
|
||||
.expect("the fixture bucket must belong to the replacement pass"),
|
||||
env.storage
|
||||
.admit_bucket_incarnation("b")
|
||||
.await
|
||||
.expect("fixture bucket identity")
|
||||
);
|
||||
env.healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
|
||||
@@ -881,13 +881,6 @@ async fn auto_format_fixture(
|
||||
// Preserve the constructor's explicit open/closed state before any fixture await.
|
||||
let node_limit = node.as_ref().map(|node| node.snapshot().limit);
|
||||
let fixture = resume_admission_fixture(node.clone()).await;
|
||||
// The endpoint monitor independently probes format.json after five seconds.
|
||||
// Drain it before aging the cache so the path-only pause observes this scanner.
|
||||
for pool in &fixture._store.ecstore.pools {
|
||||
tokio::time::timeout(Duration::from_secs(30), pool.stop_endpoint_monitor_for_test())
|
||||
.await
|
||||
.expect("endpoint reconnection must stop before the scanner owns format probes");
|
||||
}
|
||||
let disks = crate::heal::local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
|
||||
@@ -27,7 +27,6 @@ use super::{
|
||||
BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET, storage_api::owner::EcstoreDiskBytes,
|
||||
};
|
||||
|
||||
mod bucket_pass;
|
||||
mod checkpoint;
|
||||
mod gc;
|
||||
mod handoff;
|
||||
@@ -346,16 +345,6 @@ pub struct ResumeState {
|
||||
/// input after restart.
|
||||
#[serde(default)]
|
||||
pub replacement_buckets: Vec<String>,
|
||||
/// Current namespace pass, separate from the immutable generation plan.
|
||||
/// Legacy snapshots have no binding and must discard every old cursor.
|
||||
#[serde(default)]
|
||||
pub replacement_bucket_incarnations: Option<std::collections::BTreeMap<String, Uuid>>,
|
||||
/// Last proven retired identity per name, retained for returning stale volumes.
|
||||
#[serde(default)]
|
||||
pub replacement_retired_bucket_incarnations: std::collections::BTreeMap<String, Uuid>,
|
||||
/// Reapply the checkpoint rewind after any interruption between the two writes.
|
||||
#[serde(default)]
|
||||
pub replacement_bucket_pass_reset_pending: bool,
|
||||
/// A task-scoped generation assigned before formatting an automatic
|
||||
/// replacement. A new unformatted replacement receives a new generation
|
||||
/// and therefore cannot reuse an older disk's cursor or checkpoint.
|
||||
@@ -465,9 +454,6 @@ impl ResumeState {
|
||||
replacement_targets: Vec::new(),
|
||||
replacement_target_identities: Vec::new(),
|
||||
replacement_buckets: Vec::new(),
|
||||
replacement_bucket_incarnations: None,
|
||||
replacement_retired_bucket_incarnations: std::collections::BTreeMap::new(),
|
||||
replacement_bucket_pass_reset_pending: false,
|
||||
replacement_generation: None,
|
||||
replacement_phase: ReplacementPhase::None,
|
||||
replacement_execution_protocol: 0,
|
||||
@@ -999,16 +985,7 @@ impl ResumeManager {
|
||||
return Err(replacement_recovery_conflict("invalid retired legacy generation"));
|
||||
}
|
||||
}
|
||||
let valid_execution_protocol = match state.replacement_execution_protocol {
|
||||
1 => state.replacement_bucket_incarnations.is_none(),
|
||||
2 => state
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.is_some_and(|identities| identities.values().all(|incarnation| !incarnation.is_nil())),
|
||||
_ => false,
|
||||
};
|
||||
if !valid_execution_protocol
|
||||
|| state.replacement_retired_bucket_incarnations.values().any(Uuid::is_nil)
|
||||
if state.replacement_execution_protocol != 1
|
||||
|| state.replacement_predecessor.as_deref()
|
||||
!= state.replacement_lineage.last().map(|link| link.predecessor.as_str())
|
||||
{
|
||||
|
||||
@@ -1,411 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::storage::HealStorageAPI;
|
||||
use crate::heal::storage_api::storage::DiskSetSelector;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
async fn namespace_snapshot(storage: &dyn HealStorageAPI, state: &ResumeState) -> Result<BTreeMap<String, Uuid>> {
|
||||
let (pool, set) = crate::heal::utils::parse_set_disk_id(&state.set_disk_id)?;
|
||||
// This API rejects partial topology and propagates read-quorum failures.
|
||||
let buckets = storage
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(pool, set)))
|
||||
.await?;
|
||||
let mut identities = BTreeMap::new();
|
||||
for bucket in buckets {
|
||||
let Some(incarnation) = storage.replacement_bucket_incarnation(&bucket.name).await? else {
|
||||
// A returning disk may still list a retired volume. Only the admitted
|
||||
// old UUID plus its durable receipt can discharge that obligation.
|
||||
if let Some(expected) = state
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|map| map.get(&bucket.name))
|
||||
.or_else(|| state.replacement_retired_bucket_incarnations.get(&bucket.name))
|
||||
&& storage.mrf_bucket_incarnation_retired(&bucket.name, *expected).await?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
return Err(Error::ReplacementNamespaceChanged);
|
||||
};
|
||||
if incarnation.is_nil() {
|
||||
return Err(Error::InvalidBucketIncarnation);
|
||||
}
|
||||
identities.insert(bucket.name, incarnation);
|
||||
}
|
||||
Ok(identities)
|
||||
}
|
||||
|
||||
impl ResumeManager {
|
||||
pub(crate) async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Uuid> {
|
||||
self.state
|
||||
.read()
|
||||
.await
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|identities| identities.get(bucket).copied())
|
||||
.ok_or(Error::ReplacementNamespaceChanged)
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<Vec<String>> {
|
||||
let previous = self.get_state().await;
|
||||
let current = namespace_snapshot(storage, &previous).await?;
|
||||
if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
|
||||
for (bucket, expected) in identities {
|
||||
if current.get(bucket) != Some(expected) && !storage.mrf_bucket_incarnation_retired(bucket, *expected).await? {
|
||||
// Absence or a different current identity alone is not a deletion receipt.
|
||||
return Err(Error::ReplacementNamespaceChanged);
|
||||
}
|
||||
}
|
||||
}
|
||||
if previous.replacement_bucket_incarnations.as_ref() != Some(¤t) {
|
||||
let mut state = self.state.write().await;
|
||||
if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
|
||||
for (bucket, expected) in identities {
|
||||
if current.get(bucket) != Some(expected) {
|
||||
state
|
||||
.replacement_retired_bucket_incarnations
|
||||
.insert(bucket.clone(), *expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
state.replacement_bucket_incarnations = Some(current.clone());
|
||||
if state.replacement_generation.is_some() {
|
||||
// Protocol-1 readers reject this before interpreting a new pass cursor.
|
||||
state.replacement_execution_protocol = 2;
|
||||
}
|
||||
state.replacement_bucket_pass_reset_pending = true;
|
||||
state.reset_for_retry();
|
||||
state.pending_buckets = current.keys().cloned().collect();
|
||||
state.total_objects = 0;
|
||||
state.total_bytes = 0;
|
||||
state.baseline_generation = None;
|
||||
state.baseline_known = false;
|
||||
drop(state);
|
||||
self.save_state_strict().await?;
|
||||
}
|
||||
if self.get_state().await.replacement_bucket_pass_reset_pending {
|
||||
let checkpoint = if CheckpointManager::has_checkpoint(&self.disk, &previous.task_id).await {
|
||||
CheckpointManager::load_from_disk(self.disk.clone(), &previous.task_id).await?
|
||||
} else {
|
||||
CheckpointManager::new(self.disk.clone(), previous.task_id.clone()).await?
|
||||
};
|
||||
// Unlike the throttled retry writer this is durable before clearing the intent.
|
||||
checkpoint.reset_after_delete_recovery().await?;
|
||||
self.state.write().await.replacement_bucket_pass_reset_pending = false;
|
||||
self.save_state_strict().await?;
|
||||
}
|
||||
Ok(current.into_keys().collect())
|
||||
}
|
||||
|
||||
pub(crate) async fn verify_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<()> {
|
||||
let state = self.get_state().await;
|
||||
let current = namespace_snapshot(storage, &state).await?;
|
||||
if state.replacement_bucket_incarnations.as_ref() != Some(¤t) {
|
||||
return Err(Error::ReplacementNamespaceChanged);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::storage::{ECStoreHealStorage, HealStorageAPI};
|
||||
use crate::heal::storage_api::{
|
||||
storage::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _},
|
||||
test_support::{
|
||||
ObjectOptions, PutObjReader, delete_config, init_bucket_metadata_sys, isolated_store_over_temp_disks, read_config,
|
||||
save_config,
|
||||
},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_bucket_pass_retires_deleted_identity_and_restarts_recreated_bucket() {
|
||||
// Keep the concrete storage futures off the test executor stack.
|
||||
let (directories, store) = Box::pin(isolated_store_over_temp_disks()).await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "replacement-pass-bucket";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original generation");
|
||||
let old = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("original identity");
|
||||
let storage = ECStoreHealStorage::new(store.clone());
|
||||
let (_anchor_dir, disk) = super::super::tests::schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = ResumeManager::new_replacement_intent(
|
||||
disk.clone(),
|
||||
task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec![bucket.to_owned()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("persist generation");
|
||||
// A legacy name-only cursor cannot be attributed to the currently live bucket.
|
||||
manager
|
||||
.set_resume_cursor(Some("unbound-legacy-cursor".to_string()))
|
||||
.await
|
||||
.expect("legacy cursor");
|
||||
let legacy_checkpoint = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("legacy checkpoint");
|
||||
legacy_checkpoint
|
||||
.add_processed_object("legacy-key".to_string())
|
||||
.await
|
||||
.expect("legacy dedup");
|
||||
legacy_checkpoint.complete_page(0, 99).await.expect("persist legacy cursor");
|
||||
assert_eq!(
|
||||
manager
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("admit original pass"),
|
||||
[bucket]
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_state()
|
||||
.await
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|m| m.get(bucket))
|
||||
.copied(),
|
||||
Some(old)
|
||||
);
|
||||
assert!(
|
||||
manager.get_state().await.resume_cursor.is_none(),
|
||||
"legacy cursor must not enter the fresh pass"
|
||||
);
|
||||
let fresh_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("fresh checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(fresh_checkpoint.current_object_index, 0);
|
||||
assert!(fresh_checkpoint.processed_objects.is_empty());
|
||||
manager
|
||||
.set_resume_cursor(Some("old-generation-cursor".to_string()))
|
||||
.await
|
||||
.expect("persist old cursor");
|
||||
let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("checkpoint");
|
||||
checkpoint
|
||||
.add_processed_object("same-key".to_string())
|
||||
.await
|
||||
.expect("old dedup entry");
|
||||
checkpoint.complete_page(0, 7).await.expect("persist old position and dedup");
|
||||
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original generation");
|
||||
assert!(matches!(
|
||||
manager.verify_replacement_bucket_pass(&storage).await,
|
||||
Err(Error::ReplacementNamespaceChanged)
|
||||
));
|
||||
manager
|
||||
.record_replacement_failure(&Error::ReplacementNamespaceChanged, 99)
|
||||
.await
|
||||
.expect("defer without exhausting replacement");
|
||||
assert_eq!(manager.get_state().await.retry_count, 0);
|
||||
let resumed = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restart replacement manager");
|
||||
let retirement_path = format!("bucket-retirements/{bucket}/{old}.json");
|
||||
let retirement = Box::pin(read_config(store.clone(), &retirement_path))
|
||||
.await
|
||||
.expect("authoritative retirement");
|
||||
Box::pin(delete_config(store.clone(), &retirement_path))
|
||||
.await
|
||||
.expect("simulate missing retirement evidence");
|
||||
assert!(
|
||||
matches!(
|
||||
resumed.prepare_replacement_bucket_pass(&storage).await,
|
||||
Err(Error::ReplacementNamespaceChanged)
|
||||
),
|
||||
"physical absence alone cannot retire an admitted identity"
|
||||
);
|
||||
Box::pin(save_config(store.clone(), &retirement_path, b"corrupt retirement".to_vec()))
|
||||
.await
|
||||
.expect("inject unreadable proof");
|
||||
assert!(
|
||||
matches!(resumed.prepare_replacement_bucket_pass(&storage).await, Err(Error::Storage(_))),
|
||||
"a corrupt proof cannot be treated as an absent bucket"
|
||||
);
|
||||
assert_eq!(resumed.get_state().await.resume_cursor.as_deref(), Some("old-generation-cursor"));
|
||||
Box::pin(save_config(store.clone(), &retirement_path, retirement))
|
||||
.await
|
||||
.expect("restore proof");
|
||||
assert!(
|
||||
resumed
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("retirement proves deleted plan item")
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// A disk returning after DELETE may expose the retired physical volume
|
||||
// again; the persisted receipt identity still discharges that old name.
|
||||
tokio::fs::create_dir(directories[0].path().join(bucket))
|
||||
.await
|
||||
.expect("return stale retired volume");
|
||||
let returned = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restart after retirement");
|
||||
assert!(
|
||||
returned
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("retired stale volume is not a new bucket")
|
||||
.is_empty()
|
||||
);
|
||||
tokio::fs::remove_dir(directories[0].path().join(bucket))
|
||||
.await
|
||||
.expect("remove stale test volume");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate same name");
|
||||
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
|
||||
assert_ne!(new, old);
|
||||
Box::pin(store.put_object(
|
||||
bucket,
|
||||
"same-key",
|
||||
&mut PutObjReader::from_vec(b"new payload".to_vec()),
|
||||
&ObjectOptions::default(),
|
||||
))
|
||||
.await
|
||||
.expect("new generation payload");
|
||||
assert_eq!(
|
||||
resumed
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("new generation gets independent pass"),
|
||||
[bucket]
|
||||
);
|
||||
let state = resumed.get_state().await;
|
||||
assert_eq!(
|
||||
state.replacement_execution_protocol, 2,
|
||||
"older binaries must reject identity-bound cursors"
|
||||
);
|
||||
assert_eq!(
|
||||
state
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|m| m.get(bucket))
|
||||
.copied(),
|
||||
Some(new)
|
||||
);
|
||||
assert!(state.resume_cursor.is_none());
|
||||
let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restarted checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!((checkpoint.current_bucket_index, checkpoint.current_object_index), (0, 0));
|
||||
assert!(checkpoint.processed_objects.is_empty());
|
||||
|
||||
let opts = HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
storage
|
||||
.heal_replacement_object_at_incarnation(bucket, "same-key", None, old, &opts, &CancellationToken::new())
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale worker cannot mutate a recreated bucket"
|
||||
);
|
||||
let targets = directories
|
||||
.iter()
|
||||
.map(|dir| dir.path().to_string_lossy().into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
storage
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, old, &opts, &targets)
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale metadata pass cannot certify the new incarnation"
|
||||
);
|
||||
storage
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, new, &opts, &targets)
|
||||
.await
|
||||
.expect("current incarnation can repair metadata");
|
||||
let mut reader =
|
||||
Box::pin(store.get_object_reader(bucket, "same-key", None, Default::default(), &ObjectOptions::default()))
|
||||
.await
|
||||
.expect("open new generation payload");
|
||||
assert_eq!(reader.read_all().await.expect("read new payload"), b"new payload");
|
||||
drop(reader);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("new identity remains"),
|
||||
new
|
||||
);
|
||||
// Simulate interruption after publishing the reset obligation, before
|
||||
// replacing the old checkpoint.
|
||||
let stale_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("checkpoint before interruption");
|
||||
stale_checkpoint
|
||||
.add_processed_object("stale-before-reset".to_string())
|
||||
.await
|
||||
.expect("stale dedup");
|
||||
stale_checkpoint
|
||||
.complete_page(0, 42)
|
||||
.await
|
||||
.expect("persist interrupted checkpoint");
|
||||
resumed.state.write().await.replacement_bucket_pass_reset_pending = true;
|
||||
resumed.save_state_strict().await.expect("persist reset obligation");
|
||||
let interrupted = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restart interrupted pass");
|
||||
interrupted
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("finish interrupted checkpoint rewind");
|
||||
let repaired_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("repaired checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(repaired_checkpoint.current_object_index, 0);
|
||||
assert!(repaired_checkpoint.processed_objects.is_empty());
|
||||
assert!(!interrupted.get_state().await.replacement_bucket_pass_reset_pending);
|
||||
let loaded = ResumeManager::load_replacement_intent(disk, &task_id)
|
||||
.await
|
||||
.expect("new pass survives restart");
|
||||
loaded
|
||||
.verify_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("new pass matches current namespace");
|
||||
}
|
||||
}
|
||||
@@ -277,7 +277,7 @@ impl ResumeManager {
|
||||
}
|
||||
return self.ensure_handoff_successor().await;
|
||||
}
|
||||
if !matches!(state.replacement_execution_protocol, 1 | 2)
|
||||
if state.replacement_execution_protocol != 1
|
||||
|| !matches!(
|
||||
state.replacement_phase,
|
||||
ReplacementPhase::Intent
|
||||
@@ -458,12 +458,6 @@ impl ResumeManager {
|
||||
predecessor.replacement_targets.clone(),
|
||||
link.targets.clone(),
|
||||
);
|
||||
if predecessor.replacement_execution_protocol == 2 {
|
||||
state.replacement_execution_protocol = 2;
|
||||
state.replacement_bucket_incarnations = predecessor.replacement_bucket_incarnations.clone();
|
||||
state.replacement_retired_bucket_incarnations = predecessor.replacement_retired_bucket_incarnations.clone();
|
||||
state.replacement_bucket_pass_reset_pending = true;
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::OwnershipPending;
|
||||
// The constructor marks this compatibility grant consumed for every
|
||||
// successor, including descendants of a recovered older generation.
|
||||
@@ -586,7 +580,7 @@ fn preformat_handoff_retry_candidate(state: &ResumeState) -> bool {
|
||||
&& state.replacement_generation.as_deref() == Some(state.task_id.as_str())
|
||||
&& !state.replacement_preformat_handoff_retry_done
|
||||
&& state.replacement_phase == ReplacementPhase::Intent
|
||||
&& matches!(state.replacement_execution_protocol, 1 | 2)
|
||||
&& state.replacement_execution_protocol == 1
|
||||
&& state.max_retries >= 2
|
||||
&& state.replacement_predecessor.is_none()
|
||||
&& state.replacement_handoff.is_none()
|
||||
|
||||
@@ -918,33 +918,3 @@ async fn committed_predecessor_is_collectible_once_its_successor_is_verified() {
|
||||
"the verified successor keeps its own record until cleanup"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handoff_preserves_bucket_pass_protocol_and_rewinds_successor_progress() {
|
||||
let (_dirs, parent, execution) = fixture(1).await;
|
||||
let identities = [("bucket-a".to_string(), Uuid::new_v4())].into_iter().collect();
|
||||
{
|
||||
let mut state = parent.state.write().await;
|
||||
state.replacement_execution_protocol = 2;
|
||||
state.replacement_bucket_incarnations = Some(identities);
|
||||
state.resume_cursor = Some("old-pass-cursor".to_string());
|
||||
}
|
||||
parent.save_state_strict().await.expect("persist identity-bound predecessor");
|
||||
let child = parent
|
||||
.prepare_replacement_handoff(&execution, Vec::new())
|
||||
.await
|
||||
.expect("handoff protocol two");
|
||||
let state = child.get_state().await;
|
||||
assert_eq!(state.replacement_execution_protocol, 2);
|
||||
assert_eq!(
|
||||
state.replacement_bucket_incarnations,
|
||||
parent.get_state().await.replacement_bucket_incarnations
|
||||
);
|
||||
assert!(state.replacement_bucket_pass_reset_pending);
|
||||
assert!(state.resume_cursor.is_none());
|
||||
assert!(state.completed_buckets.is_empty());
|
||||
let recovered = ResumeManager::load_replacement_intent(child.disk.clone(), &state.task_id)
|
||||
.await
|
||||
.expect("reload protocol-two successor");
|
||||
assert_eq!(recovered.get_state().await, state);
|
||||
}
|
||||
|
||||
@@ -345,16 +345,6 @@ pub(crate) fn merge_replacement_recovery_candidate(
|
||||
|
||||
let candidate_revision = candidate.state.replacement_revision;
|
||||
let current_revision = current.state.replacement_revision;
|
||||
let candidate_protocol = candidate.state.replacement_execution_protocol;
|
||||
let current_protocol = current.state.replacement_execution_protocol;
|
||||
if (candidate_protocol > current_protocol && candidate_revision <= current_revision)
|
||||
|| (candidate_protocol < current_protocol && candidate_revision >= current_revision)
|
||||
{
|
||||
return Err(Error::ReplacementGenerationConflict {
|
||||
task_id: candidate.state.task_id,
|
||||
reason: "replacement execution protocol regresses with durable revision".to_string(),
|
||||
});
|
||||
}
|
||||
if candidate_revision == current_revision && candidate.state != current.state {
|
||||
return Err(Error::ReplacementGenerationConflict {
|
||||
task_id: candidate.state.task_id,
|
||||
@@ -384,10 +374,7 @@ fn same_replacement_generation_binding(left: &ResumeState, right: &ResumeState)
|
||||
&& left.replacement_target_identities == right.replacement_target_identities
|
||||
&& left.replacement_buckets == right.replacement_buckets
|
||||
&& left.replacement_generation == right.replacement_generation
|
||||
// A revision-ordered protocol-1 -> protocol-2 pass admission preserves
|
||||
// the generation's target ownership and immutable lineage.
|
||||
&& matches!(left.replacement_execution_protocol, 1 | 2)
|
||||
&& matches!(right.replacement_execution_protocol, 1 | 2)
|
||||
&& left.replacement_execution_protocol == right.replacement_execution_protocol
|
||||
&& left.replacement_predecessor == right.replacement_predecessor
|
||||
&& left.replacement_lineage == right.replacement_lineage
|
||||
&& left.replacement_legacy_import == right.replacement_legacy_import
|
||||
@@ -605,12 +592,6 @@ impl ResumeManager {
|
||||
if state.replacement_generation.as_deref() != Some(state.task_id.as_str()) {
|
||||
return Err(replacement_recovery_conflict("replacement failure has no matching generation"));
|
||||
}
|
||||
if matches!(error, Error::ReplacementNamespaceChanged | Error::BucketDiscoveryIncomplete) {
|
||||
// Namespace churn is not evidence that the replacement is irreparable.
|
||||
state.error_message = Some(error.to_string());
|
||||
drop(state);
|
||||
return self.save_state_strict().await;
|
||||
}
|
||||
if let Error::DanglingDeleteDeferred { retry_not_before } = error {
|
||||
if !super::dangling_delete_deadline_is_representable(*retry_not_before) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
|
||||
@@ -3629,129 +3629,3 @@ async fn blocked_checkpoint_is_not_revalidated_before_its_schedule() {
|
||||
assert_eq!(serde_json::from_slice::<serde_json::Value>(&retained).expect("marker JSON"), marker);
|
||||
temp_dir.close().expect("remove scheduled revalidation fixture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_candidate_merge_rejects_protocol_downgrade_in_either_read_order() {
|
||||
for downgrade in [false, true] {
|
||||
for reverse in [false, true] {
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let mut protocol_one = replacement_candidate_state(&task_id);
|
||||
protocol_one.replacement_revision = if downgrade { 11 } else { 9 };
|
||||
let mut protocol_two = replacement_candidate_state(&task_id);
|
||||
protocol_two.replacement_execution_protocol = 2;
|
||||
protocol_two.replacement_bucket_incarnations = Some(Default::default());
|
||||
protocol_two.replacement_revision = 10;
|
||||
let (first, second) = if reverse {
|
||||
(protocol_two, protocol_one)
|
||||
} else {
|
||||
(protocol_one, protocol_two)
|
||||
};
|
||||
let mut selected = None;
|
||||
merge_replacement_recovery_candidate(
|
||||
&mut selected,
|
||||
ReplacementRecoveryCandidate::new(first, "anchor-a", false).expect("first durable copy"),
|
||||
)
|
||||
.expect("initial observation");
|
||||
let result = merge_replacement_recovery_candidate(
|
||||
&mut selected,
|
||||
ReplacementRecoveryCandidate::new(second, "anchor-b", true).expect("second durable copy"),
|
||||
);
|
||||
if downgrade {
|
||||
assert!(
|
||||
matches!(result, Err(Error::ReplacementGenerationConflict { .. })),
|
||||
"protocol 1 at a newer revision must not erase protocol 2, reverse={reverse}"
|
||||
);
|
||||
} else {
|
||||
result.expect("protocol 1 at an older revision may advance to protocol 2");
|
||||
let state = selected.expect("selected upgraded generation").state;
|
||||
assert_eq!(state.replacement_execution_protocol, 2);
|
||||
assert_eq!(state.replacement_revision, 10);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_recovery_survives_same_named_bucket_deletion_and_recreation() {
|
||||
use crate::heal::storage_api::{
|
||||
storage::{BucketOperations as _, DeleteBucketOptions, DiskSetSelector, MakeBucketOptions, StorageAdminApi as _},
|
||||
test_support::{init_bucket_metadata_sys, isolated_store_over_temp_disks},
|
||||
};
|
||||
|
||||
let (_directories, store) = Box::pin(isolated_store_over_temp_disks()).await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "ahm-replacement";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create legal same-name bucket");
|
||||
let disk = store
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("survivor inventory")
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.expect("survivor disk");
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = ResumeManager::new_replacement_intent(
|
||||
disk.clone(),
|
||||
task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["other-bucket".to_string()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("persist sealed replacement intent");
|
||||
let expected = manager.get_state().await;
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete same-name user bucket");
|
||||
let resumed = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("sealed intent remains recoverable");
|
||||
assert_eq!(resumed.get_state().await, expected);
|
||||
assert_eq!(
|
||||
ResumeUtils::get_replacement_intent_tasks(&disk)
|
||||
.await
|
||||
.expect("scanner recovery enumeration"),
|
||||
std::slice::from_ref(&task_id)
|
||||
);
|
||||
resumed
|
||||
.mark_replacement_completed_and_verified()
|
||||
.await
|
||||
.expect("replacement can finish after deletion");
|
||||
resumed
|
||||
.ensure_replacement_completion_proof()
|
||||
.await
|
||||
.expect("persist completion proof");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate same-name user bucket");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete recreated bucket");
|
||||
let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("completion proof survives recreation");
|
||||
assert_eq!(proof.task_id, task_id);
|
||||
assert_eq!(
|
||||
ResumeManager::load_replacement_intent(disk, &task_id)
|
||||
.await
|
||||
.expect("verified intent survives")
|
||||
.get_state()
|
||||
.await
|
||||
.replacement_phase,
|
||||
ReplacementPhase::Verified
|
||||
);
|
||||
}
|
||||
|
||||
@@ -476,15 +476,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Published identity for a replacement pass; None proves missing metadata only.
|
||||
async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
match self.admit_bucket_incarnation(bucket).await {
|
||||
Ok(id) => Ok(Some(id)),
|
||||
Err(Error::StaleBucketIncarnation { .. }) => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the exact old bucket generation has a durable deletion record.
|
||||
/// Alternate backends must fail closed unless they can validate the same
|
||||
/// deployment, bucket and incarnation identity.
|
||||
@@ -607,23 +598,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
self.heal_object(bucket, object, version_id, opts).await
|
||||
}
|
||||
|
||||
/// Replacement writes retain both cancellation ownership and the admitted bucket fence.
|
||||
async fn heal_replacement_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
task_cancel: &CancellationToken,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
if task_cancel.is_cancelled() {
|
||||
return Err(Error::TaskCancelled);
|
||||
}
|
||||
self.heal_object_at_incarnation(bucket, object, version_id, expected, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -689,17 +663,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::Storage(StorageError::PreconditionFailed))
|
||||
}
|
||||
|
||||
async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_expected: Uuid,
|
||||
_opts: &HealOpts,
|
||||
_targets: &[String],
|
||||
) -> Result<()> {
|
||||
// An unfenced fallback could certify a different same-name bucket.
|
||||
Err(Error::Storage(StorageError::PreconditionFailed))
|
||||
}
|
||||
|
||||
/// Read target-specific physical evidence for one replacement version.
|
||||
///
|
||||
/// This is only used by automatic replacement healing after the normal
|
||||
@@ -1353,35 +1316,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_replacement_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
task_cancel: &CancellationToken,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
run_background_object_inner(
|
||||
self.admission.as_ref(),
|
||||
|| self.ecstore.physical_heal_plan(object, opts).map_err(Error::Storage),
|
||||
opts,
|
||||
Some(task_cancel),
|
||||
|result: &HealStorageObjectResult| result.error.is_none(),
|
||||
boxed_object_operation(|| async {
|
||||
let result = self
|
||||
.ecstore
|
||||
.heal_object_at_incarnation(bucket, object, version_id.unwrap_or_default(), expected, opts)
|
||||
.await
|
||||
.map_err(|error| incarnation_storage_error(bucket, expected, error))?;
|
||||
Ok(self
|
||||
.object_result_with_receipt(bucket, object, version_id, opts, result, Some(expected))
|
||||
.await)
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_mrf_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1586,13 +1520,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.ecstore
|
||||
.replacement_bucket_incarnation_from_disk(bucket)
|
||||
.await
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn mrf_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
|
||||
self.ecstore
|
||||
.is_bucket_incarnation_retired(bucket, expected)
|
||||
@@ -2032,19 +1959,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
self.ecstore
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
|
||||
.await
|
||||
.map_err(|error| incarnation_storage_error(bucket, expected, error))
|
||||
}
|
||||
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -98,19 +98,10 @@ pub(crate) mod storage {
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions};
|
||||
pub(crate) use super::storage_contracts::DeleteBucketOptions;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod status {
|
||||
pub(crate) use super::storage_contracts::BucketInfo;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::{delete_config, read_config, save_config};
|
||||
pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
|
||||
}
|
||||
|
||||
@@ -1310,22 +1310,7 @@ impl HealTask {
|
||||
.load(Ordering::Acquire)
|
||||
.max(self.retry_attempts)
|
||||
.saturating_add(1);
|
||||
// A deletion can race any stage, including hidden-transaction recovery.
|
||||
// Only a successful complete inventory may reclassify that failure;
|
||||
// lookup/quorum failures remain the original storage failure.
|
||||
if manager.get_state().await.replacement_bucket_incarnations.is_some()
|
||||
&& !matches!(failure, Error::TaskCancelled | Error::TaskTimeout)
|
||||
&& matches!(
|
||||
manager.verify_replacement_bucket_pass(self.storage.as_ref()).await,
|
||||
Err(Error::ReplacementNamespaceChanged)
|
||||
)
|
||||
{
|
||||
manager
|
||||
.record_replacement_failure(&Error::ReplacementNamespaceChanged, attempt)
|
||||
.await
|
||||
} else {
|
||||
manager.record_replacement_failure(&failure, attempt).await
|
||||
}
|
||||
manager.record_replacement_failure(&failure, attempt).await
|
||||
}
|
||||
.await;
|
||||
match persisted {
|
||||
|
||||
@@ -247,12 +247,7 @@ impl HealTask {
|
||||
*self.replacement_resume_disk.write().await = Some(disk.clone());
|
||||
let state = manager.get_state().await;
|
||||
self.replacement_start_retry_count.store(state.retry_count, Ordering::Release);
|
||||
buckets = if state.completed {
|
||||
state.replacement_buckets
|
||||
} else {
|
||||
self.await_with_control(manager.prepare_replacement_bucket_pass(self.storage.as_ref()))
|
||||
.await?
|
||||
};
|
||||
buckets = state.replacement_buckets;
|
||||
Some((disk, manager, identities))
|
||||
} else {
|
||||
None
|
||||
@@ -483,27 +478,14 @@ impl HealTask {
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "bucket prepass")
|
||||
.await?;
|
||||
}
|
||||
let expected = if let Some((_, manager, _)) = replacement_resume.as_ref() {
|
||||
Some(manager.replacement_bucket_incarnation(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let heal_result = self
|
||||
.await_with_control(async {
|
||||
match expected {
|
||||
Some(id) => self.storage.heal_bucket_at_incarnation(bucket, id, &bucket_heal_opts).await,
|
||||
None => self.storage.heal_bucket(bucket, &bucket_heal_opts).await,
|
||||
}
|
||||
})
|
||||
.await_with_control(self.storage.heal_bucket(bucket, &bucket_heal_opts))
|
||||
.await;
|
||||
match heal_result {
|
||||
Ok(result) => {
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some((_, manager, _)) = replacement_resume.as_ref() {
|
||||
manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
@@ -523,7 +505,7 @@ impl HealTask {
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.recover_delete_transactions(bucket, None, expected, &bucket_heal_opts),
|
||||
.recover_delete_transactions(bucket, None, None, &bucket_heal_opts),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -1056,7 +1056,6 @@ async fn automatic_replacement_uses_target_scoped_format() {
|
||||
let temp = TempDir::new().expect("temporary resume disk directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
..Default::default()
|
||||
@@ -1265,7 +1264,6 @@ async fn automatic_replacement_rejects_a_new_identity_after_format() {
|
||||
let first_identity = replacement_identity("replacement-a", "device-a", "filesystem-a");
|
||||
let second_identity = replacement_identity("replacement-a", "device-b", "filesystem-b");
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([
|
||||
vec![first_identity.clone()],
|
||||
@@ -1332,7 +1330,6 @@ async fn automatic_replacement_reuses_an_existing_non_target_resume_anchor() {
|
||||
.await
|
||||
.expect("existing intent should be stored on the non-target anchor");
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_resume_disk: Mutex::new(Some(anchor.clone())),
|
||||
..Default::default()
|
||||
@@ -3001,7 +2998,6 @@ async fn replacement_dropped_waiter_keeps_executor_until_format_and_failure_pers
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![identity.clone()], vec![identity]])),
|
||||
replacement_execution_fixture: Mutex::new(Some(execution)),
|
||||
@@ -3016,9 +3012,7 @@ async fn replacement_dropped_waiter_keeps_executor_until_format_and_failure_pers
|
||||
let task = task.clone();
|
||||
async move { task.execute().await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("owned format I/O must start");
|
||||
entered.notified().await;
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
|
||||
task.cancel_token.cancel();
|
||||
@@ -3128,7 +3122,6 @@ async fn replacement_format_failures_keep_the_durable_budget_across_new_executor
|
||||
request.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
for _attempt in 1..=3 {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
format_error: Mutex::new(Some(Error::other("injected format failure"))),
|
||||
@@ -5690,7 +5683,6 @@ async fn replacement_owned_executor_keeps_manager_drain_pending_through_failure_
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![identity.clone()], vec![identity]])),
|
||||
replacement_execution_fixture: Mutex::new(Some(execution)),
|
||||
@@ -5707,9 +5699,7 @@ async fn replacement_owned_executor_keeps_manager_drain_pending_through_failure_
|
||||
let task = task.clone();
|
||||
async move { task.execute_owned(Some(parent)).await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("owned format I/O must start");
|
||||
entered.notified().await;
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
|
||||
task.cancel_token.cancel();
|
||||
|
||||
@@ -42,6 +42,7 @@ rustfs-io-metrics = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
metrics = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs"] }
|
||||
|
||||
[lib]
|
||||
|
||||
+79
-68
@@ -83,7 +83,10 @@ struct PoolTier {
|
||||
/// Queue of available buffers for reuse
|
||||
available_buffers: Mutex<Vec<BytesMut>>,
|
||||
/// Metrics for tracking this tier
|
||||
metrics: Mutex<Option<Arc<BytesPoolMetrics>>>,
|
||||
metrics: Arc<BytesPoolMetrics>,
|
||||
/// Serialize acquisitions before taking buffers so a slow recorder does not
|
||||
/// accumulate allocated buffers behind hit-rate publication.
|
||||
acquire_metrics_lock: Mutex<()>,
|
||||
/// Total acquisitions for this tier
|
||||
tier_total_acquires: AtomicU64,
|
||||
/// Total hits for this tier
|
||||
@@ -185,16 +188,10 @@ impl BytesPool {
|
||||
/// ```
|
||||
pub fn with_config(config: BytesPoolConfig) -> Self {
|
||||
let metrics = Arc::new(BytesPoolMetrics::default());
|
||||
let small_pool = Arc::new(PoolTier::new(config.small_size, config.small_max, "small"));
|
||||
let medium_pool = Arc::new(PoolTier::new(config.medium_size, config.medium_max, "medium"));
|
||||
let large_pool = Arc::new(PoolTier::new(config.large_size, config.large_max, "large"));
|
||||
let xlarge_pool = Arc::new(PoolTier::new(config.xlarge_size, config.xlarge_max, "xlarge"));
|
||||
|
||||
// Set metrics reference in all tiers
|
||||
small_pool.set_metrics(Arc::clone(&metrics));
|
||||
medium_pool.set_metrics(Arc::clone(&metrics));
|
||||
large_pool.set_metrics(Arc::clone(&metrics));
|
||||
xlarge_pool.set_metrics(Arc::clone(&metrics));
|
||||
let small_pool = Arc::new(PoolTier::new(config.small_size, config.small_max, "small", Arc::clone(&metrics)));
|
||||
let medium_pool = Arc::new(PoolTier::new(config.medium_size, config.medium_max, "medium", Arc::clone(&metrics)));
|
||||
let large_pool = Arc::new(PoolTier::new(config.large_size, config.large_max, "large", Arc::clone(&metrics)));
|
||||
let xlarge_pool = Arc::new(PoolTier::new(config.xlarge_size, config.xlarge_max, "xlarge", Arc::clone(&metrics)));
|
||||
|
||||
Self {
|
||||
small_pool,
|
||||
@@ -225,7 +222,7 @@ impl BytesPool {
|
||||
/// ```
|
||||
pub async fn acquire_buffer(&self, size: usize) -> PooledBuffer {
|
||||
let tier = self.select_tier(size);
|
||||
let mut buffer = tier.acquire_buffer(size, &self.metrics).await;
|
||||
let mut buffer = tier.acquire_buffer(size).await;
|
||||
if buffer._permit.is_some() {
|
||||
buffer.tier = Some(Arc::clone(tier));
|
||||
}
|
||||
@@ -252,7 +249,7 @@ impl BytesPool {
|
||||
/// ```
|
||||
pub fn try_acquire_buffer(&self, size: usize) -> Option<PooledBuffer> {
|
||||
let tier = self.select_tier(size);
|
||||
let mut buffer = tier.try_acquire_buffer(size, &self.metrics)?;
|
||||
let mut buffer = tier.try_acquire_buffer(size)?;
|
||||
// Set tier reference for return on drop
|
||||
buffer.tier = Some(Arc::clone(tier));
|
||||
Some(buffer)
|
||||
@@ -290,43 +287,38 @@ impl BytesPool {
|
||||
}
|
||||
|
||||
impl PoolTier {
|
||||
fn new(buffer_size: usize, max_buffers: usize, name: &'static str) -> Self {
|
||||
fn new(buffer_size: usize, max_buffers: usize, name: &'static str, metrics: Arc<BytesPoolMetrics>) -> Self {
|
||||
Self {
|
||||
buffer_size,
|
||||
max_buffers,
|
||||
semaphore: Arc::new(Semaphore::new(max_buffers)),
|
||||
name,
|
||||
available_buffers: Mutex::new(Vec::new()),
|
||||
metrics: Mutex::new(None),
|
||||
metrics,
|
||||
acquire_metrics_lock: Mutex::new(()),
|
||||
tier_total_acquires: AtomicU64::new(0),
|
||||
tier_pool_hits: AtomicU64::new(0),
|
||||
tier_current_allocated_bytes: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_metrics(&self, metrics: Arc<BytesPoolMetrics>) {
|
||||
*self.metrics.lock().unwrap_or_else(|e| e.into_inner()) = Some(metrics);
|
||||
}
|
||||
|
||||
fn take_or_allocate_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> (BytesMut, bool) {
|
||||
fn take_or_allocate_buffer(&self, size: usize) -> (BytesMut, bool) {
|
||||
let pool_metrics = &self.metrics;
|
||||
let buffer_opt = {
|
||||
let mut available = self.available_buffers.lock().unwrap_or_else(|e| e.into_inner());
|
||||
available.pop()
|
||||
let buffer = available.pop();
|
||||
if buffer.is_some() {
|
||||
pool_metrics.available_buffers.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
buffer
|
||||
};
|
||||
let was_reused = buffer_opt.is_some();
|
||||
if was_reused {
|
||||
// A reused buffer leaves the available pool. `return_buffer` does a
|
||||
// matching `fetch_add(1)` on push, so without this the
|
||||
// `available_buffers` gauge only ever grows and never reflects the real
|
||||
// pool size (backlog#806).
|
||||
pool_metrics.available_buffers.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
let buffer = if let Some(mut buf) = buffer_opt {
|
||||
let previous_capacity = buf.capacity();
|
||||
buf.clear();
|
||||
if previous_capacity < size {
|
||||
buf.reserve(size - previous_capacity);
|
||||
buf.reserve(size);
|
||||
}
|
||||
let current_capacity = buf.capacity();
|
||||
if current_capacity > previous_capacity {
|
||||
@@ -353,7 +345,10 @@ impl PoolTier {
|
||||
(buffer, was_reused)
|
||||
}
|
||||
|
||||
fn record_acquire_metrics(&self, pool_metrics: &BytesPoolMetrics, buffer_capacity: usize, was_reused: bool) {
|
||||
fn record_acquire_metrics(&self, buffer_capacity: usize, was_reused: bool) {
|
||||
let pool_metrics = &self.metrics;
|
||||
pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
|
||||
self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
|
||||
rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
|
||||
|
||||
if was_reused {
|
||||
@@ -374,7 +369,7 @@ impl PoolTier {
|
||||
rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
|
||||
}
|
||||
|
||||
async fn acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> PooledBuffer {
|
||||
async fn acquire_buffer(&self, size: usize) -> PooledBuffer {
|
||||
// Acquire semaphore permit (owned for storage in PooledBuffer)
|
||||
let permit = match Arc::clone(&self.semaphore).acquire_owned().await {
|
||||
Ok(p) => p,
|
||||
@@ -388,17 +383,10 @@ impl PoolTier {
|
||||
}
|
||||
};
|
||||
|
||||
// Use the pool's shared metrics for recording
|
||||
let _metrics_lock = self.metrics.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let _metrics = _metrics_lock.as_ref().expect("operation should succeed");
|
||||
|
||||
// Record acquisition
|
||||
pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
|
||||
self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
|
||||
let _guard = self.acquire_metrics_lock.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let (buffer, was_reused) = self.take_or_allocate_buffer(size);
|
||||
let buffer_capacity = buffer.capacity();
|
||||
self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
|
||||
self.record_acquire_metrics(buffer_capacity, was_reused);
|
||||
|
||||
PooledBuffer {
|
||||
buffer: ManuallyDrop::new(buffer),
|
||||
@@ -407,21 +395,14 @@ impl PoolTier {
|
||||
}
|
||||
}
|
||||
|
||||
fn try_acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> Option<PooledBuffer> {
|
||||
fn try_acquire_buffer(&self, size: usize) -> Option<PooledBuffer> {
|
||||
// Try to acquire permit without blocking
|
||||
let permit = Arc::clone(&self.semaphore).try_acquire_owned().ok()?;
|
||||
|
||||
// Use the pool's shared metrics for recording
|
||||
let _metrics_lock = self.metrics.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let _metrics = _metrics_lock.as_ref().expect("operation should succeed");
|
||||
|
||||
// Record acquisition
|
||||
pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
|
||||
self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
|
||||
let _guard = self.acquire_metrics_lock.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let (buffer, was_reused) = self.take_or_allocate_buffer(size);
|
||||
let buffer_capacity = buffer.capacity();
|
||||
self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
|
||||
self.record_acquire_metrics(buffer_capacity, was_reused);
|
||||
|
||||
Some(PooledBuffer {
|
||||
buffer: ManuallyDrop::new(buffer),
|
||||
@@ -430,7 +411,7 @@ impl PoolTier {
|
||||
})
|
||||
}
|
||||
|
||||
/// Return a buffer to the pool for reuse without ever blocking the caller.
|
||||
/// Return a buffer to the pool for reuse without waiting for the queue lock.
|
||||
fn return_buffer(&self, buffer: BytesMut) {
|
||||
let mut buffer = Some(buffer);
|
||||
|
||||
@@ -438,11 +419,7 @@ impl PoolTier {
|
||||
&& available.len() < self.max_buffers
|
||||
{
|
||||
available.push(buffer.take().expect("buffer should be present until returned"));
|
||||
if let Ok(metrics) = self.metrics.try_lock()
|
||||
&& let Some(metrics) = metrics.as_ref()
|
||||
{
|
||||
metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
if let Some(buffer) = buffer {
|
||||
@@ -452,16 +429,12 @@ impl PoolTier {
|
||||
Some(current.saturating_sub(released_bytes))
|
||||
})
|
||||
.ok();
|
||||
if let Ok(metrics) = self.metrics.try_lock()
|
||||
&& let Some(metrics) = metrics.as_ref()
|
||||
{
|
||||
metrics
|
||||
.current_allocated_bytes
|
||||
.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_sub(released_bytes))
|
||||
})
|
||||
.ok();
|
||||
}
|
||||
self.metrics
|
||||
.current_allocated_bytes
|
||||
.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_sub(released_bytes))
|
||||
})
|
||||
.ok();
|
||||
}
|
||||
rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
|
||||
}
|
||||
@@ -551,6 +524,44 @@ mod tests {
|
||||
assert!(buffer.capacity() >= 2048);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reused_buffer_satisfies_larger_async_request() {
|
||||
let pool = BytesPool::with_config(BytesPoolConfig {
|
||||
small_max: 1,
|
||||
..Default::default()
|
||||
});
|
||||
let mut first = pool.acquire_buffer(32 * 1024).await;
|
||||
first.extend_from_slice(b"previous contents");
|
||||
assert!(first.capacity() < 48 * 1024, "fixture must require buffer growth");
|
||||
drop(first);
|
||||
|
||||
let reused = pool.acquire_buffer(48 * 1024).await;
|
||||
assert!(reused.is_empty());
|
||||
assert!(reused.capacity() >= 48 * 1024);
|
||||
assert_eq!(pool.metrics().pool_hits.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(
|
||||
pool.metrics().current_allocated_bytes.load(Ordering::Relaxed),
|
||||
u64::try_from(reused.capacity()).expect("buffer capacity fits in metrics")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reused_buffer_satisfies_larger_try_request() {
|
||||
let pool = BytesPool::with_config(BytesPoolConfig {
|
||||
small_max: 1,
|
||||
..Default::default()
|
||||
});
|
||||
let mut first = pool.try_acquire_buffer(32 * 1024).expect("first buffer is available");
|
||||
first.extend_from_slice(b"previous contents");
|
||||
assert!(first.capacity() < 48 * 1024, "fixture must require buffer growth");
|
||||
drop(first);
|
||||
|
||||
let reused = pool.try_acquire_buffer(48 * 1024).expect("returned buffer is available");
|
||||
assert!(reused.is_empty());
|
||||
assert!(reused.capacity() >= 48 * 1024);
|
||||
assert_eq!(pool.metrics().pool_hits.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn available_buffers_gauge_decrements_on_reuse() {
|
||||
// Regression (backlog#806): acquiring a pooled (reused) buffer must
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::{Mutex, mpsc};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit};
|
||||
use rustfs_io_core::{BytesPool, BytesPoolConfig};
|
||||
|
||||
const WAIT: Duration = Duration::from_secs(5);
|
||||
const GATE_WAIT: Duration = Duration::from_secs(20);
|
||||
|
||||
struct AcquisitionGate {
|
||||
entered: mpsc::Sender<()>,
|
||||
release: Mutex<mpsc::Receiver<()>>,
|
||||
}
|
||||
|
||||
impl Recorder for AcquisitionGate {
|
||||
fn describe_counter(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
|
||||
|
||||
fn describe_gauge(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
|
||||
|
||||
fn describe_histogram(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
|
||||
|
||||
fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
|
||||
if key.name() == "rustfs_bytes_pool_acquisitions_total" {
|
||||
self.entered
|
||||
.send(())
|
||||
.expect("notify test that acquisition reached the recorder");
|
||||
self.release
|
||||
.lock()
|
||||
.expect("lock acquisition gate")
|
||||
.recv_timeout(GATE_WAIT)
|
||||
.expect("release the acquisition recorder gate");
|
||||
}
|
||||
Counter::noop()
|
||||
}
|
||||
|
||||
fn register_gauge(&self, _key: &Key, _metadata: &Metadata<'_>) -> Gauge {
|
||||
Gauge::noop()
|
||||
}
|
||||
|
||||
fn register_histogram(&self, _key: &Key, _metadata: &Metadata<'_>) -> Histogram {
|
||||
Histogram::noop()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returned_buffer_is_counted_while_an_acquisition_recorder_is_busy() {
|
||||
let pool = BytesPool::with_config(BytesPoolConfig {
|
||||
small_max: 2,
|
||||
..Default::default()
|
||||
});
|
||||
let held = pool.try_acquire_buffer(8 * 1024).expect("acquire the buffer to return");
|
||||
assert_eq!(pool.available_buffers(), 0);
|
||||
|
||||
// This integration test is a dedicated test process: the recorder is local
|
||||
// to the worker, while the metrics enablement flag is process-wide.
|
||||
rustfs_io_metrics::set_metrics_enabled(true);
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let worker_pool = pool.clone();
|
||||
let acquirer = thread::spawn(move || {
|
||||
let gate = AcquisitionGate {
|
||||
entered: entered_tx,
|
||||
release: Mutex::new(release_rx),
|
||||
};
|
||||
metrics::with_local_recorder(&gate, || {
|
||||
worker_pool
|
||||
.try_acquire_buffer(8 * 1024)
|
||||
.expect("acquire a second same-tier buffer")
|
||||
})
|
||||
});
|
||||
|
||||
let entered = entered_rx.recv_timeout(WAIT);
|
||||
if entered.is_err() {
|
||||
let _ = release_tx.send(());
|
||||
let _ = acquirer.join();
|
||||
panic!("acquisition did not reach the recorder gate: {entered:?}");
|
||||
}
|
||||
|
||||
let (dropped_tx, dropped_rx) = mpsc::channel();
|
||||
let returner = thread::spawn(move || {
|
||||
drop(held);
|
||||
let _ = dropped_tx.send(());
|
||||
});
|
||||
let returned_while_blocked = dropped_rx.recv_timeout(WAIT);
|
||||
|
||||
// Always release before joining so a blocking Drop cannot deadlock the test.
|
||||
let _ = release_tx.send(());
|
||||
let returned = returner.join();
|
||||
let acquired = acquirer.join();
|
||||
rustfs_io_metrics::set_metrics_enabled(false);
|
||||
|
||||
assert!(returned_while_blocked.is_ok(), "returning a buffer must not wait on the recorder");
|
||||
returned.expect("return worker should finish");
|
||||
let acquired = acquired.expect("acquisition worker should finish");
|
||||
let after_return = pool.available_buffers();
|
||||
let reused = pool.try_acquire_buffer(8 * 1024).expect("reuse the returned buffer");
|
||||
let after_reuse = pool.available_buffers();
|
||||
drop(reused);
|
||||
drop(acquired);
|
||||
let after_both_return = pool.available_buffers();
|
||||
assert_eq!(
|
||||
(after_return, after_reuse, after_both_return),
|
||||
(1, 0, 2),
|
||||
"returned buffers must be counted, and reuse must not underflow the available gauge"
|
||||
);
|
||||
}
|
||||
@@ -45,7 +45,6 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"
|
||||
hotpath.workspace = true
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
futures.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -31,13 +31,10 @@ pub mod manager_trait;
|
||||
pub mod metrics;
|
||||
pub mod object_pool;
|
||||
pub mod optimized_notify;
|
||||
mod publication;
|
||||
pub mod shard;
|
||||
pub mod state;
|
||||
pub mod types;
|
||||
|
||||
#[cfg(test)]
|
||||
mod publication_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
|
||||
@@ -95,8 +95,6 @@ impl ObjectLockState {
|
||||
// Reset atomic state
|
||||
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
|
||||
|
||||
self.publication = std::sync::OnceLock::new();
|
||||
|
||||
// Clear owners
|
||||
*self.current_owner.write() = None;
|
||||
self.shared_owners.write().clear();
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! FIFO admission only for the bucket publication fence. Consecutive readers
|
||||
//! form a group; a later writer cannot overtake them, nor can later readers
|
||||
//! overtake a queued writer. This does not limit a current holder's I/O time.
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use std::{collections::VecDeque, sync::Arc};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::{
|
||||
state::ObjectLockState,
|
||||
types::{LockMode, ObjectKey, ObjectLockRequest},
|
||||
};
|
||||
|
||||
pub(super) fn applies(key: &ObjectKey) -> bool {
|
||||
key.version.is_none() && key.object.as_ref() == rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct PublicationQueue {
|
||||
entries: Mutex<VecDeque<(Arc<()>, LockMode)>>,
|
||||
pub(super) changed: Notify,
|
||||
}
|
||||
|
||||
pub(super) struct PublicationTicket {
|
||||
pub(super) state: Arc<ObjectLockState>,
|
||||
pub(super) queue: Arc<PublicationQueue>,
|
||||
token: Arc<()>,
|
||||
mode: LockMode,
|
||||
}
|
||||
|
||||
impl PublicationTicket {
|
||||
// Called while the shard map is locked. The waiter counter pins the map
|
||||
// entry until ticket destruction, including cancellation between polls.
|
||||
pub(super) fn new(state: Arc<ObjectLockState>, mode: LockMode) -> Option<Self> {
|
||||
let counted = match mode {
|
||||
LockMode::Shared => state.atomic_state.inc_readers_waiting(),
|
||||
LockMode::Exclusive => state.atomic_state.inc_writers_waiting(),
|
||||
};
|
||||
if !counted {
|
||||
return None;
|
||||
}
|
||||
let queue = state
|
||||
.publication
|
||||
.get_or_init(|| Arc::new(PublicationQueue::default()))
|
||||
.clone();
|
||||
let token = Arc::new(());
|
||||
queue.entries.lock().push_back((token.clone(), mode));
|
||||
Some(Self {
|
||||
state,
|
||||
queue,
|
||||
token,
|
||||
mode,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn try_acquire(&self, request: &ObjectLockRequest) -> bool {
|
||||
// Lock order: shard map -> queue -> owner metadata. Release never
|
||||
// locks the queue; ticket destruction only locks the queue.
|
||||
let entries = self.queue.entries.lock();
|
||||
let mut found = false;
|
||||
for (token, mode) in entries.iter() {
|
||||
if Arc::ptr_eq(token, &self.token) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
if self.mode == LockMode::Exclusive || *mode == LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false;
|
||||
}
|
||||
match self.mode {
|
||||
LockMode::Shared => self.state.try_acquire_admitted_shared(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => self.state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PublicationTicket {
|
||||
fn drop(&mut self) {
|
||||
{
|
||||
let mut entries = self.queue.entries.lock();
|
||||
if let Some(index) = entries.iter().position(|(token, _)| Arc::ptr_eq(token, &self.token)) {
|
||||
entries.remove(index);
|
||||
}
|
||||
}
|
||||
match self.mode {
|
||||
LockMode::Shared => self.state.atomic_state.dec_readers_waiting(),
|
||||
LockMode::Exclusive => self.state.atomic_state.dec_writers_waiting(),
|
||||
}
|
||||
self.queue.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
@@ -1,327 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use tokio::{task::JoinHandle, time::timeout};
|
||||
|
||||
fn manager() -> Arc<FastObjectLockManager> {
|
||||
Arc::new(FastObjectLockManager::with_config(LockConfig {
|
||||
shard_count: 1,
|
||||
max_idle_time: Duration::ZERO,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
fn key() -> ObjectKey {
|
||||
ObjectKey::new("publication-bucket", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
}
|
||||
|
||||
fn queued(manager: &Arc<FastObjectLockManager>, owner: &str, mode: LockMode) -> JoinHandle<FastLockGuard> {
|
||||
let manager = manager.clone();
|
||||
let owner = owner.to_owned();
|
||||
tokio::spawn(async move {
|
||||
let mut request = ObjectLockRequest::new_read(key(), owner);
|
||||
request.mode = mode;
|
||||
manager
|
||||
.acquire_lock(request.with_acquire_timeout(Duration::from_secs(5)))
|
||||
.await
|
||||
.expect("queued publication lock")
|
||||
})
|
||||
}
|
||||
|
||||
async fn waiters(manager: &FastObjectLockManager, count: u64) {
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while manager.current_lock_counts().1 != count {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiter admission must be observable");
|
||||
}
|
||||
|
||||
async fn acquired(task: JoinHandle<FastLockGuard>) -> FastLockGuard {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("queue must progress")
|
||||
.expect("acquire task")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_serves_read_groups_and_writers_in_arrival_order() {
|
||||
let manager = manager();
|
||||
let holder = manager.acquire_write_lock(key(), "holder").await.expect("initial holder");
|
||||
let w1 = queued(&manager, "w1", LockMode::Exclusive);
|
||||
waiters(&manager, 1).await;
|
||||
let r1 = queued(&manager, "r1", LockMode::Shared);
|
||||
waiters(&manager, 2).await;
|
||||
let r2 = queued(&manager, "r2", LockMode::Shared);
|
||||
waiters(&manager, 3).await;
|
||||
let w2 = queued(&manager, "w2", LockMode::Exclusive);
|
||||
waiters(&manager, 4).await;
|
||||
let r3 = queued(&manager, "r3", LockMode::Shared);
|
||||
waiters(&manager, 5).await;
|
||||
assert_eq!(manager.cleanup_expired_traditional().await, 0, "queued state must stay mapped");
|
||||
drop(holder);
|
||||
assert_eq!(
|
||||
manager.cleanup_expired_traditional().await,
|
||||
0,
|
||||
"an unlocked state with queued tickets must stay mapped"
|
||||
);
|
||||
let w1 = acquired(w1).await;
|
||||
assert!(!r1.is_finished() && !r2.is_finished() && !w2.is_finished() && !r3.is_finished());
|
||||
drop(w1);
|
||||
let r1 = acquired(r1).await;
|
||||
let r2 = acquired(r2).await;
|
||||
assert!(!w2.is_finished() && !r3.is_finished(), "later requests cannot pass the reader group");
|
||||
drop(r1);
|
||||
assert!(!w2.is_finished(), "the second reader still holds the fence");
|
||||
drop(r2);
|
||||
let w2 = acquired(w2).await;
|
||||
assert!(!r3.is_finished(), "a later reader cannot starve the queued writer");
|
||||
drop(w2);
|
||||
drop(acquired(r3).await);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_cancellation_and_timeout_remove_their_turns() {
|
||||
let manager = manager();
|
||||
let holder = manager.acquire_write_lock(key(), "holder").await.expect("initial holder");
|
||||
let cancelled = queued(&manager, "cancelled", LockMode::Exclusive);
|
||||
waiters(&manager, 1).await;
|
||||
let expiring = tokio::spawn({
|
||||
let manager = manager.clone();
|
||||
async move {
|
||||
manager
|
||||
.acquire_lock(ObjectLockRequest::new_write(key(), "expires").with_acquire_timeout(Duration::from_millis(500)))
|
||||
.await
|
||||
}
|
||||
});
|
||||
waiters(&manager, 2).await;
|
||||
let reader = queued(&manager, "survivor", LockMode::Shared);
|
||||
waiters(&manager, 3).await;
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.expect_err("cancel waiter").is_cancelled());
|
||||
assert!(matches!(expiring.await.expect("timeout task"), Err(LockResult::Timeout)));
|
||||
waiters(&manager, 1).await;
|
||||
assert_eq!(manager.cleanup_expired_traditional().await, 0);
|
||||
drop(holder);
|
||||
drop(acquired(reader).await);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while manager.cleanup_expired_traditional().await == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("an empty queue must allow map cleanup");
|
||||
// An empty queue neither keeps a stale turn nor prevents subsequent writers.
|
||||
drop(manager.acquire_write_lock(key(), "next").await.expect("queue recovered"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_batch_fast_path_cannot_bypass_a_reader() {
|
||||
for all_or_nothing in [false, true] {
|
||||
let manager = manager();
|
||||
let holder = manager.acquire_write_lock(key(), "holder").await.expect("initial holder");
|
||||
let reader = queued(&manager, "queued-reader", LockMode::Shared);
|
||||
waiters(&manager, 1).await;
|
||||
drop(holder);
|
||||
// This future can try the batch fast path before the reader runs again.
|
||||
let batch = manager.acquire_locks_batch(
|
||||
BatchLockRequest::new("batch")
|
||||
.add_write_lock(key())
|
||||
.with_all_or_nothing(all_or_nothing),
|
||||
);
|
||||
tokio::pin!(batch);
|
||||
let reader = tokio::select! {
|
||||
biased;
|
||||
_ = &mut batch => panic!("batch overtook an already queued reader"),
|
||||
reader = acquired(reader) => reader,
|
||||
};
|
||||
drop(reader);
|
||||
let batch = timeout(Duration::from_secs(3), batch)
|
||||
.await
|
||||
.expect("batch progresses after reader");
|
||||
assert!(batch.all_acquired);
|
||||
drop(batch);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_queue_is_scoped_to_the_unversioned_bucket_fence() {
|
||||
assert!(publication::applies(&key()));
|
||||
assert!(!publication::applies(&ObjectKey::new("publication-bucket", "plain-object")));
|
||||
assert!(!publication::applies(&ObjectKey::with_version(
|
||||
"publication-bucket",
|
||||
rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH,
|
||||
"version"
|
||||
)));
|
||||
}
|
||||
|
||||
async fn cluster_waiters(managers: &[Arc<FastObjectLockManager>], minimum: u64) {
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while !managers.iter().all(|manager| manager.current_lock_counts().1 >= minimum) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all real local peers registered the waiter");
|
||||
}
|
||||
|
||||
fn cluster_queued(lock: &Arc<crate::NamespaceLock>, owner: &str, kind: crate::LockType) -> JoinHandle<crate::NamespaceLockGuard> {
|
||||
let lock = lock.clone();
|
||||
let owner = owner.to_owned();
|
||||
tokio::spawn(async move {
|
||||
lock.acquire_guard(
|
||||
&crate::LockRequest::new(key(), kind, owner)
|
||||
.with_acquire_timeout(Duration::from_secs(5))
|
||||
.with_ttl(Duration::from_secs(30)),
|
||||
)
|
||||
.await
|
||||
.expect("distributed acquire")
|
||||
.expect("distributed quorum")
|
||||
})
|
||||
}
|
||||
|
||||
async fn cluster_acquired(task: JoinHandle<crate::NamespaceLockGuard>) -> crate::NamespaceLockGuard {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("quorum must progress")
|
||||
.expect("cluster task")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_distributed_quorum_preserves_bidirectional_progress() {
|
||||
let managers = (0..3).map(|_| manager()).collect::<Vec<_>>();
|
||||
let clients = managers
|
||||
.iter()
|
||||
.map(|manager| {
|
||||
Arc::new(crate::LocalClient::with_manager(Arc::new(crate::GlobalLockManager::Enabled(
|
||||
manager.clone(),
|
||||
)))) as Arc<dyn crate::LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let lock = Arc::new(crate::NamespaceLock::with_clients("publication-fairness".to_owned(), clients));
|
||||
// Hold every voter while establishing a deterministic arrival order.
|
||||
// A distributed holder may return at quorum and release its spare vote.
|
||||
let mut holders = Vec::new();
|
||||
for manager in &managers {
|
||||
holders.push(manager.acquire_write_lock(key(), "holder").await.expect("hold voter"));
|
||||
}
|
||||
let w1 = cluster_queued(&lock, "w1", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 1).await;
|
||||
let r1 = cluster_queued(&lock, "r1", crate::LockType::Shared);
|
||||
cluster_waiters(&managers, 2).await;
|
||||
let w2 = cluster_queued(&lock, "w2", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 3).await;
|
||||
let r2 = cluster_queued(&lock, "r2", crate::LockType::Shared);
|
||||
cluster_waiters(&managers, 4).await;
|
||||
drop(holders);
|
||||
let w1 = cluster_acquired(w1).await;
|
||||
assert!(!r1.is_finished() && !w2.is_finished() && !r2.is_finished());
|
||||
drop(w1);
|
||||
let r1 = cluster_acquired(r1).await;
|
||||
assert!(!w2.is_finished() && !r2.is_finished());
|
||||
drop(r1);
|
||||
let w2 = cluster_acquired(w2).await;
|
||||
assert!(!r2.is_finished());
|
||||
drop(w2);
|
||||
drop(cluster_acquired(r2).await);
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts() != (0, 0)) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all quorum guards and pending cleanup must release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_distributed_cancel_and_timeout_leave_no_orphan_turns() {
|
||||
let managers = (0..3).map(|_| manager()).collect::<Vec<_>>();
|
||||
let clients = managers
|
||||
.iter()
|
||||
.map(|manager| {
|
||||
Arc::new(crate::LocalClient::with_manager(Arc::new(crate::GlobalLockManager::Enabled(
|
||||
manager.clone(),
|
||||
)))) as Arc<dyn crate::LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let lock = Arc::new(crate::NamespaceLock::with_clients("publication-cancellation".to_owned(), clients));
|
||||
let mut holders = Vec::new();
|
||||
for manager in &managers {
|
||||
holders.push(manager.acquire_write_lock(key(), "holder").await.expect("hold voter"));
|
||||
}
|
||||
let cancelled = cluster_queued(&lock, "cancelled", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 1).await;
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.expect_err("cancel distributed waiter").is_cancelled());
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts().1 != 0) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("aborted peer attempts must remove their tickets");
|
||||
let timed_out = lock
|
||||
.acquire_guard(
|
||||
&crate::LockRequest::new(key(), crate::LockType::Exclusive, "timeout")
|
||||
.with_acquire_timeout(Duration::from_millis(100)),
|
||||
)
|
||||
.await
|
||||
.expect("contention is a failed acquire");
|
||||
assert!(timed_out.is_none());
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts().1 != 0) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("timed out peer attempts must remove their tickets");
|
||||
let reader = cluster_queued(&lock, "next-reader", crate::LockType::Shared);
|
||||
cluster_waiters(&managers, 1).await;
|
||||
let writer = cluster_queued(&lock, "next-writer", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 2).await;
|
||||
drop(holders);
|
||||
let reader = cluster_acquired(reader).await;
|
||||
assert!(!writer.is_finished());
|
||||
drop(reader);
|
||||
drop(cluster_acquired(writer).await);
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts() != (0, 0)) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("distributed holders and queues must drain");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_does_not_change_ordinary_key_writer_preference() {
|
||||
let manager = manager();
|
||||
let ordinary = ObjectKey::new("publication-bucket", "ordinary-key");
|
||||
let holder = manager
|
||||
.acquire_write_lock(ordinary.clone(), "holder")
|
||||
.await
|
||||
.expect("ordinary holder");
|
||||
let reader = tokio::spawn({
|
||||
let manager = manager.clone();
|
||||
let ordinary = ordinary.clone();
|
||||
async move { manager.acquire_read_lock(ordinary, "reader").await.expect("ordinary reader") }
|
||||
});
|
||||
waiters(&manager, 1).await;
|
||||
let writer = tokio::spawn({
|
||||
let manager = manager.clone();
|
||||
async move { manager.acquire_write_lock(ordinary, "writer").await.expect("ordinary writer") }
|
||||
});
|
||||
waiters(&manager, 2).await;
|
||||
drop(holder);
|
||||
let writer = acquired(writer).await;
|
||||
assert!(!reader.is_finished(), "ordinary keys retain writer preference");
|
||||
drop(writer);
|
||||
drop(acquired(reader).await);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
}
|
||||
@@ -21,7 +21,6 @@ use tokio::time::timeout;
|
||||
use crate::fast_lock::{
|
||||
metrics::ShardMetrics,
|
||||
object_pool::ObjectStatePool,
|
||||
publication::{self, PublicationTicket},
|
||||
state::ObjectLockState,
|
||||
types::{LockMode, LockResult, ObjectKey, ObjectLockRequest},
|
||||
};
|
||||
@@ -135,9 +134,6 @@ impl LockShard {
|
||||
/// Acquire lock with fast path optimization
|
||||
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
|
||||
let start_time = Instant::now();
|
||||
if publication::applies(&request.key) {
|
||||
return self.acquire_publication(request, start_time).await;
|
||||
}
|
||||
|
||||
// Try fast path first
|
||||
if let Some(_state) = self.try_fast_path(request) {
|
||||
@@ -149,50 +145,8 @@ impl LockShard {
|
||||
self.acquire_lock_slow_path(request, start_time).await
|
||||
}
|
||||
|
||||
fn publication_ticket(&self, request: &ObjectLockRequest) -> Option<PublicationTicket> {
|
||||
let mut objects = self.objects.write();
|
||||
let state = objects
|
||||
.entry(request.key.clone())
|
||||
.or_insert_with(|| Arc::new(*self.object_pool.acquire()))
|
||||
.clone();
|
||||
PublicationTicket::new(state, request.mode)
|
||||
}
|
||||
|
||||
async fn acquire_publication(&self, request: &ObjectLockRequest, started: Instant) -> Result<(), LockResult> {
|
||||
let Some(ticket) = self.publication_ticket(request) else {
|
||||
return Err(LockResult::Timeout);
|
||||
};
|
||||
let mut first_attempt = true;
|
||||
loop {
|
||||
// Register before checking so a release/cancellation cannot be lost
|
||||
// between the eligibility check and the notification wait.
|
||||
let notified = ticket.queue.changed.notified();
|
||||
let remaining = request.acquire_timeout.saturating_sub(started.elapsed());
|
||||
if !first_attempt && remaining.is_zero() {
|
||||
self.metrics.record_timeout();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
if ticket.try_acquire(request) {
|
||||
self.metrics.record_slow_path_success();
|
||||
return Ok(());
|
||||
}
|
||||
first_attempt = false;
|
||||
if remaining.is_zero() || timeout(remaining, notified).await.is_err() {
|
||||
self.metrics.record_timeout();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try fast path only (without fallback to slow path)
|
||||
pub fn try_fast_path_only(&self, request: &ObjectLockRequest) -> bool {
|
||||
if publication::applies(&request.key) {
|
||||
// Batch acquisition must join the same queue, even for a no-wait
|
||||
// attempt. Dropping a failed ticket does not consume a queue turn.
|
||||
return self
|
||||
.publication_ticket(request)
|
||||
.is_some_and(|ticket| ticket.try_acquire(request));
|
||||
}
|
||||
// Early check to avoid unnecessary lock contention
|
||||
if let Some(state) = self.objects.read().get(&request.key)
|
||||
&& !state.atomic_state.is_fast_path_available(request.mode)
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
@@ -78,21 +78,13 @@ impl AtomicLockState {
|
||||
|
||||
/// Try to acquire shared lock (fast path)
|
||||
pub fn try_acquire_shared(&self) -> bool {
|
||||
self.try_acquire_shared_with_mask(NO_WRITER_AND_NO_WAITING_WRITERS)
|
||||
}
|
||||
|
||||
// Publication admission already orders readers against waiting writers.
|
||||
fn try_acquire_admitted_shared(&self) -> bool {
|
||||
self.try_acquire_shared_with_mask(WRITER_FLAG_MASK)
|
||||
}
|
||||
|
||||
fn try_acquire_shared_with_mask(&self, blocked_mask: u64) -> bool {
|
||||
self.update_access_time();
|
||||
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
|
||||
if (current & blocked_mask) != 0 {
|
||||
// Fast path check - cannot acquire if there's a writer or writers waiting
|
||||
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -342,7 +334,6 @@ pub struct ObjectLockState {
|
||||
// First cache line: Most frequently accessed data
|
||||
/// Atomic state for fast operations
|
||||
pub atomic_state: AtomicLockState,
|
||||
pub(super) publication: OnceLock<Arc<super::publication::PublicationQueue>>,
|
||||
|
||||
// Second cache line: Notification mechanisms
|
||||
/// Notification for readers (traditional)
|
||||
@@ -386,7 +377,6 @@ impl ObjectLockState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
atomic_state: AtomicLockState::new(),
|
||||
publication: OnceLock::new(),
|
||||
read_notify: Notify::new(),
|
||||
write_notify: Notify::new(),
|
||||
optimized_notify: OptimizedNotify::new(),
|
||||
@@ -402,19 +392,6 @@ impl ObjectLockState {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.record_shared_owner(owner, lock_timeout);
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn try_acquire_admitted_shared(&self, owner: &Arc<str>, lock_timeout: Duration) -> bool {
|
||||
if !self.atomic_state.try_acquire_admitted_shared() {
|
||||
return false;
|
||||
}
|
||||
self.record_shared_owner(owner, lock_timeout);
|
||||
true
|
||||
}
|
||||
|
||||
fn record_shared_owner(&self, owner: &Arc<str>, lock_timeout: Duration) {
|
||||
self.atomic_state.update_access_time();
|
||||
let mut shared = self.shared_owners.write();
|
||||
if let Some(entry) = shared.iter_mut().find(|entry| entry.owner.as_ref() == owner.as_ref()) {
|
||||
@@ -429,6 +406,7 @@ impl ObjectLockState {
|
||||
lock_timeout,
|
||||
});
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Try fast path exclusive lock acquisition
|
||||
@@ -459,9 +437,6 @@ impl ObjectLockState {
|
||||
Some(shared.remove(pos))
|
||||
};
|
||||
if self.atomic_state.release_shared() {
|
||||
if let Some(queue) = self.publication.get() {
|
||||
queue.changed.notify_waiters();
|
||||
}
|
||||
if shared.is_empty() {
|
||||
drop(shared);
|
||||
self.optimized_notify.notify_writer();
|
||||
@@ -505,9 +480,6 @@ impl ObjectLockState {
|
||||
if self.atomic_state.release_exclusive() {
|
||||
*current = None;
|
||||
drop(current);
|
||||
if let Some(queue) = self.publication.get() {
|
||||
queue.changed.notify_waiters();
|
||||
}
|
||||
// Notify waiters using optimized system - prefer writers over readers
|
||||
if self
|
||||
.atomic_state
|
||||
|
||||
@@ -166,6 +166,7 @@ libc = { workspace = true }
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
opentelemetry_sdk = { workspace = true, features = ["testing"] }
|
||||
tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
log.workspace = true
|
||||
|
||||
@@ -93,6 +93,17 @@ const LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||
];
|
||||
|
||||
const OBJECT_LOCK_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
||||
"rustfs_object_lock_diag_acquire_duration_seconds",
|
||||
"rustfs_object_lock_diag_hold_duration_seconds",
|
||||
];
|
||||
|
||||
// Resolve millisecond acquisitions without losing long-held delete and recovery locks.
|
||||
const OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002, 0.003, 0.004, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03, 0.05, 0.075,
|
||||
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0,
|
||||
];
|
||||
|
||||
const DEFAULT_OTLP_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const OTEL_EXPORTER_OTLP_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TIMEOUT";
|
||||
const OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT";
|
||||
@@ -476,7 +487,7 @@ fn build_meter_provider(
|
||||
.with_interval(Duration::from_secs(meter_interval))
|
||||
.build(),
|
||||
)
|
||||
.with_view(get_object_duration_histogram_view);
|
||||
.with_view(duration_histogram_view);
|
||||
if use_stdout {
|
||||
b.with_reader(create_periodic_reader(meter_interval))
|
||||
} else {
|
||||
@@ -491,8 +502,8 @@ fn build_meter_provider(
|
||||
Ok(Some(provider))
|
||||
}
|
||||
|
||||
fn get_object_duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
|
||||
let boundaries = get_object_duration_histogram_buckets(instrument.name())?;
|
||||
fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
|
||||
let boundaries = duration_histogram_buckets(instrument.name())?;
|
||||
|
||||
Stream::builder()
|
||||
.with_aggregation(Aggregation::ExplicitBucketHistogram {
|
||||
@@ -503,11 +514,13 @@ fn get_object_duration_histogram_view(instrument: &Instrument) -> Option<Stream>
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn get_object_duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
|
||||
fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
|
||||
if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -1112,25 +1125,86 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_object_duration_histogram_metric_match_is_scoped() {
|
||||
fn test_duration_histogram_metric_match_is_scoped() {
|
||||
for name in [
|
||||
"rustfs_io_get_object_stage_duration_seconds",
|
||||
"rustfs_io_get_object_request_duration_seconds",
|
||||
] {
|
||||
assert_eq!(get_object_duration_histogram_buckets(name), Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS));
|
||||
assert_eq!(duration_histogram_buckets(name), Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS));
|
||||
}
|
||||
for name in LOCAL_BATCH_DURATION_HISTOGRAM_METRICS {
|
||||
assert_eq!(get_object_duration_histogram_buckets(name), Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS));
|
||||
assert_eq!(duration_histogram_buckets(name), Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS));
|
||||
}
|
||||
for name in OBJECT_LOCK_DURATION_HISTOGRAM_METRICS {
|
||||
assert_eq!(duration_histogram_buckets(name), Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS));
|
||||
}
|
||||
for name in [
|
||||
"rustfs_io_put_object_request_duration_seconds",
|
||||
"rustfs_io_get_object_response_size_bytes",
|
||||
"rustfs_io_get_object_local_batch_shards",
|
||||
"rustfs_object_lock_diag_slow_acquire_total",
|
||||
"rustfs_object_lock_diag_acquire_duration_seconds_bucket",
|
||||
] {
|
||||
assert!(get_object_duration_histogram_buckets(name).is_none());
|
||||
assert!(duration_histogram_buckets(name).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
|
||||
use metrics::Recorder as _;
|
||||
use opentelemetry_sdk::metrics::{
|
||||
InMemoryMetricExporter,
|
||||
data::{AggregatedMetrics, MetricData},
|
||||
};
|
||||
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let reader = PeriodicReader::builder(exporter.clone())
|
||||
.with_interval(Duration::from_secs(60))
|
||||
.build();
|
||||
let (provider, recorder) = Recorder::builder("lock-histogram-test")
|
||||
.with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
|
||||
.build();
|
||||
let metadata = metrics::Metadata::new(module_path!(), metrics::Level::INFO, None);
|
||||
|
||||
for (name, observations) in [
|
||||
(OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[0], [0.002, 0.004, 0.012]),
|
||||
(OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[1], [0.002, 12.9, 31.0]),
|
||||
] {
|
||||
let histogram = recorder.register_histogram(&metrics::Key::from_name(name), &metadata);
|
||||
for seconds in observations {
|
||||
histogram.record(seconds);
|
||||
}
|
||||
}
|
||||
provider.force_flush().expect("export lock duration observations");
|
||||
let exported = exporter.get_finished_metrics().expect("read exported lock histograms");
|
||||
let mut checked = 0;
|
||||
for metric in exported
|
||||
.iter()
|
||||
.flat_map(|resource| resource.scope_metrics())
|
||||
.flat_map(|scope| scope.metrics())
|
||||
{
|
||||
let expected = match metric.name() {
|
||||
"rustfs_object_lock_diag_acquire_duration_seconds" => [(0.002, 1), (0.004, 1), (0.015, 1)],
|
||||
"rustfs_object_lock_diag_hold_duration_seconds" => [(0.002, 1), (15.0, 1), (60.0, 1)],
|
||||
_ => continue,
|
||||
};
|
||||
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
|
||||
panic!("lock duration must export as an f64 histogram");
|
||||
};
|
||||
let points: Vec<_> = histogram.data_points().collect();
|
||||
assert_eq!(points.len(), 1, "one time series for each lock duration instrument");
|
||||
let point = points[0];
|
||||
assert_eq!(point.count(), 3, "all observations reach the exporter");
|
||||
let counts: Vec<_> = point.bucket_counts().collect();
|
||||
assert_eq!(counts.last(), Some(&0), "observations do not fall into the overflow bucket");
|
||||
let occupied: Vec<_> = point.bounds().zip(counts).filter(|(_, count)| *count != 0).collect();
|
||||
assert_eq!(occupied, expected, "seconds remain distinct at millisecond and long-hold scales");
|
||||
checked += 1;
|
||||
}
|
||||
assert_eq!(checked, 2, "both acquisition and hold histograms were exported");
|
||||
provider.shutdown().expect("stop lock histogram test provider");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_object_duration_histogram_buckets_are_sorted() {
|
||||
assert!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.windows(2).all(|pair| pair[0] < pair[1]));
|
||||
|
||||
@@ -881,7 +881,7 @@ pub fn parse_replicate_decision(_bucket: &str, s: &str) -> std::io::Result<Repli
|
||||
/// writers may use different textual representations. `invalid` preserves the
|
||||
/// distinction between truly absent legacy metadata and corrupt/conflicting
|
||||
/// compatibility aliases.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ReplicationGenerationSnapshot {
|
||||
pub timestamp: Option<String>,
|
||||
pub mutation_id: Option<String>,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user